Merge pull request ntop#947 from hamishcoleman/json_pubsub

Add JSON pubsub framework for edge

Author: hamishcoleman

.github/workflows/cmake.yml

@@ -66,3 +66,17 @@ jobs:
         # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more
         # detail
         run: ctest -C $BUILD_TYPE
+
+      - if: ${{ failure() }}
+        name: Upload test data
+        uses: actions/upload-artifact@v2
+        with:
+          name: tests-out
+          path: tests
+
+      - if: ${{ failure() }}
+        name: Upload cmake test output
+        uses: actions/upload-artifact@v2
+        with:
+          name: tests-out
+          path: build/Testing

.github/workflows/tests.yml

@@ -91,18 +91,18 @@ jobs:
           ./configure
         shell: bash
 
-      - name: Run embedded tests
-        run: make test
+      - name: Run embedded unit tests
+        run: make test.units
         shell: bash
 
-      - if: ${{ always() }}
+      - if: ${{ failure() }}
         name: Move test outputs to an arch specific location
         shell: bash
         run: |
           mkdir -p tests/${{ matrix.os }}
           mv tests/*.out tests/${{ matrix.os }}
 
-      - if: ${{ always() }}
+      - if: ${{ failure() }}
         name: Upload tests output
         uses: actions/upload-artifact@v2
         with:
@@ -162,18 +162,18 @@ jobs:
           ./configure
         shell: bash
 
-      - name: Run embedded tests
-        run: make test
+      - name: Run embedded unit tests
+        run: make test.units
         shell: bash
 
-      - if: ${{ always() }}
+      - if: ${{ failure() }}
         name: Move test outputs to an arch specific location
         shell: bash
         run: |
           mkdir -p tests/${{ matrix.os }}
           mv tests/*.out tests/${{ matrix.os }}
 
-      - if: ${{ always() }}
+      - if: ${{ failure() }}
         name: Upload tests output
         uses: actions/upload-artifact@v2
         with:
@@ -237,18 +237,18 @@ jobs:
           ./scripts/hack_fakeautoconf.sh
         shell: bash
 
-      - name: Run embedded tests
-        run: make test
+      - name: Run embedded unit tests
+        run: make test.units
         shell: bash
 
-      - if: ${{ always() }}
+      - if: ${{ failure() }}
         name: Move test outputs to an arch specific location
         shell: bash
         run: |
           mkdir -p tests/${{ matrix.os }}
           mv tests/*.out tests/${{ matrix.os }}
 
-      - if: ${{ always() }}
+      - if: ${{ failure() }}
         name: Upload tests output
         uses: actions/upload-artifact@v2
         with:

CMakeLists.txt

@@ -309,10 +309,21 @@ install(FILES ${PROJECT_BINARY_DIR}/doc/supernode.1.gz
         DESTINATION /usr/share/man/man1)
 install(FILES ${PROJECT_BINARY_DIR}/doc/n2n.7.gz
         DESTINATION /usr/share/man/man7)
+endif(DEFINED UNIX)
+
+if (CMAKE_SYSTEM_NAME STREQUAL Linux)
 
 # TODO:
 # - Add the right dependancy so that the tests binaries get built first
 enable_testing()
-add_test(tests ${PROJECT_SOURCE_DIR}/scripts/test_harness.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/tests)
-
-endif(DEFINED UNIX)
+add_test(NAME unit
+    COMMAND ${CMAKE_COMMAND} -E env
+        TOPDIR=${PROJECT_SOURCE_DIR} BINDIR=${PROJECT_BINARY_DIR}
+        ${PROJECT_SOURCE_DIR}/scripts/test_harness.sh ${PROJECT_SOURCE_DIR}/tests/tests_units.list
+)
+add_test(NAME integration
+    COMMAND ${CMAKE_COMMAND} -E env
+        TOPDIR=${PROJECT_SOURCE_DIR} BINDIR=${PROJECT_BINARY_DIR}
+        ${PROJECT_SOURCE_DIR}/scripts/test_harness.sh ${PROJECT_SOURCE_DIR}/tests/tests_integration.list
+)
+endif()

Makefile.in

@@ -84,16 +84,30 @@ N2N_DEPS=$(wildcard include/*.h) $(wildcard src/*.c) Makefile
 # As source files pass the linter, they can be added here (If all the source
 # is passing the linter tests, this can be refactored)
 LINT_CCODE=\
+	include/curve25519.h \
 	include/edge_utils_win32.h \
+	include/header_encryption.h \
+	include/hexdump.h \
 	include/n2n_define.h \
+	include/n2n_wire.h \
+	include/network_traffic_filter.h \
 	include/pearson.h \
+	include/random_numbers.h \
+	include/sn_selection.h \
 	include/speck.h \
+	include/tf.h \
+	src/edge_management.c \
 	src/edge_utils_win32.c \
 	src/example_edge_embed_quick_edge_init.c \
 	src/header_encryption.c \
+	src/sn_management.c \
 	src/sn_selection.c \
 	src/transform_cc20.c \
+	src/transform_null.c \
+	src/tuntap_freebsd.c \
 	src/tuntap_linux.c \
+	src/tuntap_netbsd.c \
+	src/tuntap_osx.c \
 	src/wire.c \
 	tools/tests-auth.c \
 	tools/tests-compress.c \
@@ -146,9 +160,8 @@ SUBDIRS+=tools
 COVERAGEDIR?=coverage
 
 .PHONY: $(SUBDIRS)
-.PHONY: steps build push all clean distclean install test cover gcov build-dep
-.PHONY: lint lint.python lint.ccode lint.shell lint.yaml
 
+.PHONY: all
 all: version $(APPS) $(DOCS) $(SUBDIRS)
 
 # This allows breaking the build if the version.sh script discovers
@@ -188,9 +201,16 @@ $(N2N_LIB): $(N2N_OBJS)
 
 win32/n2n_win32.a: win32
 
-test: tools
-	scripts/test_harness.sh
+.PHONY: test test.units test.integration
+test: test.units test.integration
+
+test.units: tools
+	scripts/test_harness.sh tests/tests_units.list
 
+test.integration: $(APPS)
+	scripts/test_harness.sh tests/tests_integration.list
+
+.PHONY: lint lint.python lint.ccode lint.shell lint.yaml
 lint: lint.python lint.ccode lint.shell lint.yaml
 
 lint.python:
@@ -209,6 +229,7 @@ lint.yaml:
 # CFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS="--coverage"
 # and run the desired tests.  Ensure that package gcovr is installed
 # and then run "make cover"
+.PHONY: cover
 cover:
 	mkdir -p $(COVERAGEDIR)
 	gcovr -s --html --html-details --output=$(COVERAGEDIR)/index.html
@@ -217,11 +238,13 @@ cover:
 # Unfortunately, these end up in the wrong directory due to the
 # makefile layout
 # The steps to use this are similar to the "make cover" above
+.PHONY: gcov
 gcov:
 	gcov $(N2N_OBJS)
 	$(MAKE) -C tools gcov
 
 # This is a convinent target to use during development or from a CI/CD system
+.PHONY: build-dep
 build-dep:
 ifeq ($(CONFIG_TARGET),generic)
 	sudo apt install $(BUILD_DEP)
@@ -231,11 +254,13 @@ else
 	echo Not attempting to install dependancies for system $(CONFIG_TARGET)
 endif
 
+.PHONY: clean
 clean:
 	rm -rf $(N2N_OBJS) $(N2N_LIB) $(APPS) $(DOCS) $(COVERAGEDIR)/ *.dSYM *~
 	rm -f tests/*.out src/*.gcno src/*.gcda
 	for dir in $(SUBDIRS); do $(MAKE) -C $$dir clean; done
 
+.PHONY: distclean
 distclean:
 	rm -f tests/*.out src/*.gcno src/*.gcda src/*.indent src/*.unc-backup*
 	rm -rf autom4te.cache/
@@ -246,6 +271,7 @@ distclean:
 	rm -f packages/rpm/config.log packages/rpm/config.status
 	rm -f $(addprefix src/,$(APPS))
 
+.PHONY: install
 install: edge supernode edge.8.gz supernode.1.gz n2n.7.gz
 	echo "MANDIR=$(MANDIR)"
 	$(MKDIR) $(SBINDIR) $(MAN1DIR) $(MAN7DIR) $(MAN8DIR)
@@ -261,6 +287,7 @@ DOCKER_IMAGE_NAME=ntop/supernode
 DOCKER_IMAGE_VERSION=$N2N_VERSION_SHORT
 N2N_COMMIT_HASH=$(shell scripts/version.sh hash)
 
+.PHONY: default steps build push
 default: steps
 
 steps:

doc/ManagementAPI.md

@@ -1,6 +1,6 @@
 # Management API
 
-This document is focused on the machine readable API interfaces. 
+This document is focused on the machine readable API interfaces.
 
 Both the edge and the supernode provide a management interface UDP port.
 These interfaces have some documentation on their non machine readable
@@ -48,7 +48,7 @@ but this is intended for debugging.
 
 The request is a single UDP packet containing one line of text with at least
 three space separated fields.  Any text after the third field is available for
-the API method to use for additional parameters 
+the API method to use for additional parameters
 
 Fields:
 - Message Type
@@ -58,15 +58,23 @@ Fields:
 
 The maximum length of the entire line of text is 80 octets.
 
+All request packets should generate a reply.  However, this reply may simply
+be an error.
+
 ### Message Type
 
-This is a single octet that is either "r" for a read (or query) method
-call or "w" for a write (or change) method call.
+This is a single octet specifying the type:
+
+- "r" for a read-only method (or one that does not need change permissions)
+- "w" for a write method (or one that makes changes)
+- "s" for a subscribe method to request this socket receive some events
 
 To simplify the interface, the reply from both read and write calls to the
 same method is expected to contain the same data.  In the case of a write
 call, the reply will contain the new state after making the requested change.
 
+The subscribe and events message flow works with a different set of messages.
+
 ### Options
 
 The options field is a colon separated set of options for this request.  Only
@@ -83,14 +91,16 @@ SubFields:
 
 Each request provides a tag value.  Any non error reply associated with this
 request will include this tag value, allowing all related messages to be
-collected within the client.
+collected within the client.  The tag will be truncated if needed by the
+daemon, but there will be at least 8 octets of space available.
 
 Where possible, the error replies will also include this tag, however some
 errors occur before the tag is parsed.
 
 The tag is not interpreted by the daemon, it is simply echoed back in all
-the replies.  It is expected to be a short string that the client chooses
-to be unique amongst all recent or still outstanding requests.
+the replies.  It is expected to be a short string that the client knows
+will be unique amongst all recent, still outstanding or subscription requests
+on a given socket.
 
 One possible client implementation is a number between 0 and 999, incremented
 for each request and wrapping around to zero when it is greater than 999.
@@ -127,11 +137,14 @@ e.g:
 Each UDP packet in the reply is a complete and valid JSON dictionary
 containing a fragment of information related to the entire reply.
 
+Reply packets are generated both in response to requests and whenever
+an event is published to a subscribed channel.
+
 ### Common metadata
 
 There are two keys in each dictionary containing metadata.  First
 is the `_tag`, containing the Message Tag from the original request.
-Second is the `_type` whic identifies the expected contents of this
+Second is the `_type` which identifies the expected contents of this
 packet.
 
 ### `_type: error`
@@ -176,6 +189,47 @@ packets will be required.
 e.g:
     `{"_tag":"108","_type":"row","mode":"p2p","ip4addr":"10.135.98.84","macaddr":"86:56:21:E4:AA:39","sockaddr":"192.168.7.191:41701","desc":"client4","lastseen":1584682200}`
 
+### `_type: subscribed`
+
+Signals that the subscription request has been successfully completed.
+Any future events on the requested channel will be asynchronously sent
+as `event` packets using the same tag as the subscribe request.
+
+### `_type: unsubscribed`
+
+Only one management client can be subscribed to any given event topic, so if
+another subscribe request arrives, the older client will be sent this message
+to let them know that they have been replaced.
+
+(In the future, this may also be sent as a reply to a explicit unsubscribe
+request)
+
+### `_type: replacing`
+
+If a new subscription request will replace an existing one, this message is
+sent to the new client to inform them that they have replaced an older
+connection.
+
+### `_type: event`
+
+Asynchronous events will arrive with this message type, using the same tag as
+the original subscribe request.  Just like with the `row` packets, the non
+metadata contents are entirely defined by the topic and the specific n2n
+version.
+
+## Subscribe API
+
+A client can subscribe to events using a request with the type of "s".
+Once a subscribe has been successfully completed, any events published
+on that channel will be forwarded to the client.
+
+Only one management client can be subscribed to any given event topic,
+with newer subscriptions replacing older ones.
+
+The special channel "debug" will receive copies of all events published.
+Note that this is for debugging of events and the packets may not have
+the same tag as the debug subscription.
+
 ## Authentication
 
 Some API requests will make global changes to the running daemon and may

doc/Scripts.md

@@ -51,10 +51,6 @@ This shell script is a wrapper for the `uncrustify` C code style checker
 which checks or applies a set of rules to the code.  It is used during
 the automated lint checks.
 
-### `test_harness.sh`
-
-This shell script is used to run automated tests during development.
-
 ### `n2n-gateway.sh`
 
 A sample script to route all the host traffic towards a remote gateway,
@@ -96,3 +92,26 @@ Manually test fetching and config:
 /etc/munin/plugins/n2n_supernode_pkts
 /etc/munin/plugins/n2n_supernode_pkts config
 ```
+
+## Testing scripts
+
+### `test_harness.sh`
+
+This shell script is used to run automated tests during development.  It is
+run with a testlist filename - pointing at a file containing the list of
+tests to run.
+
+Each test needs a file containing the expected output `${TESTNAME}.expected`
+which is expected to exist in the same directory as the testlist (this dir is
+referred to as `${listdir}` below).
+
+Each test is a program, searched for in several locations, including the
+`${listdir}/../scripts` dir.
+
+Each test is run with its output being sent to `*.out` files in the `listdir`
+and compared with the expected output.
+
+### `scripts/test_integration_supernode.sh`
+
+This starts a supernode and runs an integration test on the Json API using
+the `n2n-ctl` command.

include/n2n.h

@@ -285,6 +285,7 @@ int assign_one_ip_subnet (n2n_sn_t *sss, struct sn_community *comm);
 const char* compression_str (uint8_t cmpr);
 const char* transop_str (enum n2n_transform tr);
 
-void handleMgmtJson (n2n_edge_t *eee, char *udp_buf, const struct sockaddr_in sender_sock);
-void handleMgmtJson_sn (n2n_sn_t *sss, char *udp_buf, const struct sockaddr_in sender_sock);
+void readFromMgmtSocket (n2n_edge_t *eee);
+
+void mgmt_event_post (enum n2n_event_topic topic, int data0, void *data1);
 #endif /* _N2N_H_ */