Merge branch 'feat/multi-arch'

.github/workflows/new-release-assets.yml

@@ -38,4 +38,4 @@ jobs:
 
       - name: Build binaries
         run: |
-          make V=1 LOG_LEVEL=DEBUG dist-amd4
+          make ${{ env.MAKEFLAGS }} CI=1 V=1 LOG_LEVEL=DEBUG dist-amd64

.gitignore

@@ -37,8 +37,11 @@ node_modules/
 
 # Ignore Jetbrains IDE files
 .idea/
+
+# RLN / keystore
 rlnCredentials.txt
-testPath.txt
+rlnKeystore.json
+*.tar.gz
 
 # Nimbus Build System
 nimbus-build-system.paths

.gitmodules

@@ -88,11 +88,6 @@
 	url = https://github.com/status-im/nim-http-utils.git
 	ignore = dirty
 	branch = master
-[submodule "vendor/news"]
-	path = vendor/news
-	url = https://github.com/tormund/news.git
-	ignore = dirty
-	branch = master
 [submodule "vendor/nim-bearssl"]
 	path = vendor/nim-bearssl
 	url = https://github.com/status-im/nim-bearssl.git

CHANGELOG.md

@@ -1,3 +1,102 @@
+## 2023-03-15 v0.16.0
+
+## What's Changed
+
+Release highlights:
+- a fix for an issue that prevented the node from generating high-resolution (up to nanosecond) timestamps
+- introduction of an application-defined `meta` attribute to the Waku Message. This can be quite valuable for network-wide deduplication, deterministic hashing, validity checking and other planned improvements to the protocol
+- many optimizations in RLN implementation and its underlying dependencies
+
+### Features
+
+- Integrated a new group manager for RLN-protected relay [1496](https://github.com/waku-org/nwaku/pull/1496)
+- Added application-defined meta attribute to Waku Message according to RFC [14/WAKU2-MESSAGE](https://rfc.vac.dev/spec/14/#message-attributes) [1581](https://github.com/waku-org/nwaku/pull/1581)
+- Implemented deterministic hashing scheme for Waku Messages according to RFC [14/WAKU2-MESSAGE](https://rfc.vac.dev/spec/14/#deterministic-message-hashing) [1586](https://github.com/waku-org/nwaku/pull/1586)
+
+### Changes
+
+- Upgraded nim-sqlite3-abi to the latest version [1565](https://github.com/waku-org/nwaku/pull/1565)
+- Better validation of protocol buffers [1563](https://github.com/waku-org/nwaku/pull/1563)
+- Improved underlying Zerokit performance and FFI [1571](https://github.com/waku-org/nwaku/pull/1571)
+- Node peer ID now logged with relay trace logging [1574](https://github.com/waku-org/nwaku/pull/1574)
+- Continued refactoring of several protocol implementations to improve maintainability and readability
+- Refactored and cleaned up peer manager [1539](https://github.com/waku-org/nwaku/pull/1539)
+- Removed unused and legacy websocket submodule [1580](https://github.com/waku-org/nwaku/pull/1580) [1582](https://github.com/waku-org/nwaku/pull/1582)
+- Use base64 URL-safe encoding for noise [1569](https://github.com/waku-org/nwaku/pull/1569)
+- Various general improvements to RLN implementation [1585](https://github.com/waku-org/nwaku/pull/1585) [1587](https://github.com/waku-org/nwaku/pull/1587)
+- Started on implementation for new and improved filter protocol [1584](https://github.com/waku-org/nwaku/pull/1584)
+- Updated pubsub and content topic namespacing to reflect latest changes in RFC [23/WAKU2-TOPICS](https://rfc.vac.dev/spec/23/) [1589](https://github.com/waku-org/nwaku/pull/1589)
+- Unified internal peer data models [1597](https://github.com/waku-org/nwaku/pull/1597)
+- Improved internal implementation of Waku ENR encoding and decoding [1598](https://github.com/waku-org/nwaku/pull/1598) [1599](https://github.com/waku-org/nwaku/pull/1599)
+- Underlying dependency for RLN implementation now loaded as a static library [1578](https://github.com/waku-org/nwaku/pull/1578)
+
+### Fixes
+
+- Fixed internally generated timestamps to allow higher resolution than seconds [1570](https://github.com/waku-org/nwaku/pull/1570)
+- Fixed padded base64 usage for encoding and decoding payloads on the JSON RPC API [1572](https://github.com/waku-org/nwaku/pull/1572)
+- Fixed incorrect relative module imports [1591](https://github.com/waku-org/nwaku/pull/1591)
+- Fixed RLN relay erroneously storing messages from multiple apps [1594](https://github.com/waku-org/nwaku/pull/1594)
+
+This release supports the following [libp2p protocols](https://docs.libp2p.io/concepts/protocols/):
+| Protocol | Spec status | Protocol id |
+| ---: | :---: | :--- |
+| [`11/WAKU2-RELAY`](https://rfc.vac.dev/spec/11/) | `stable` | `/vac/waku/relay/2.0.0` |
+| [`12/WAKU2-FILTER`](https://rfc.vac.dev/spec/12/) | `draft` | `/vac/waku/filter/2.0.0-beta1` |
+| [`13/WAKU2-STORE`](https://rfc.vac.dev/spec/13/) | `draft` | `/vac/waku/store/2.0.0-beta4` |
+| [`18/WAKU2-SWAP`](https://rfc.vac.dev/spec/18/) | `draft` | `/vac/waku/swap/2.0.0-beta1` |
+| [`19/WAKU2-LIGHTPUSH`](https://rfc.vac.dev/spec/19/) | `draft` | `/vac/waku/lightpush/2.0.0-beta1` |
+
+The Waku v1 implementation is stable but not under active development.
+
+## 2023-02-15 v0.15.0
+
+Release highlights:
+- Relay connectivity is now maintained by a management loop that selects from the peerstore
+- Ability to manually specify `multiaddrs` for the nwaku node to advertise
+- Two important fixes related to historical message queries:
+  - fixed archive bug that resulted in duplicate messages in store query response
+  - fixed query page size limit not being respected
+
+### Features
+
+- New connectivity loop to maintain relay connectivity from peerstore [1482](https://github.com/waku-org/nwaku/pull/1482) [1462](https://github.com/waku-org/nwaku/pull/1462)
+- Support for manually specifying `multiaddrs` to advertise [1509](https://github.com/waku-org/nwaku/pull/1509) [1512](https://github.com/waku-org/nwaku/pull/1512)
+- Added dynamic keystore for membership credential storage and management [1466](https://github.com/waku-org/nwaku/pull/1466)
+
+### Changes
+
+- Abstracted RLN relay group management into its own API [1465](https://github.com/waku-org/nwaku/pull/1465)
+- Prune peers from peerstore when exceeding capacity [1513](https://github.com/waku-org/nwaku/pull/1513)
+- Removed Kilic submodule [1517](https://github.com/waku-org/nwaku/pull/1517)
+- Continued refactoring of several protocol implementations to improve maintainability and readability
+- Refactored and improved JSON RPC API
+- Added safe default values for peer-store-capacity [1525](https://github.com/waku-org/nwaku/pull/1525)
+- Improvements in regular CI test reliability and repeatability
+- Improved archive query performance [1510](https://github.com/waku-org/nwaku/pull/1510)
+- Added better e2e trace logging for relay messages [1526](https://github.com/waku-org/nwaku/pull/1526)
+- Relay RPC API now encodes message payloads in base64 [572](https://github.com/vacp2p/rfc/pull/572) [1555](https://github.com/waku-org/nwaku/pull/1555)
+
+### Fixes
+
+- Fixed Waku archive queries returning duplicate messages due to incorrect reordering [1511](https://github.com/waku-org/nwaku/pull/1511)
+- Fixed Admin RPC API crashing on returning peer with no multiaddresses [1507](https://github.com/waku-org/nwaku/pull/1507)
+- Fixed page size limit not being respected in store query responses [1520](https://github.com/waku-org/nwaku/pull/1520)
+- Fixed nwaku subscribing to default pubsub topic even if not configured [1548](https://github.com/waku-org/nwaku/pull/1548)
+- Fixed underlying issue causing node to incorrectly report it's unreachable [1518](https://github.com/waku-org/nwaku/pull/1518) [1546](https://github.com/waku-org/nwaku/pull/1546)
+- Fixed Relay RPC API not adhering to RFC [1139](https://github.com/waku-org/nwaku/issues/1139)
+- Fixed message IDs in nwaku diverging from those in go-waku [1556](https://github.com/waku-org/nwaku/pull/1556)
+
+This release supports the following [libp2p protocols](https://docs.libp2p.io/concepts/protocols/):
+| Protocol | Spec status | Protocol id |
+| ---: | :---: | :--- |
+| [`11/WAKU2-RELAY`](https://rfc.vac.dev/spec/11/) | `stable` | `/vac/waku/relay/2.0.0` |
+| [`12/WAKU2-FILTER`](https://rfc.vac.dev/spec/12/) | `draft` | `/vac/waku/filter/2.0.0-beta1` |
+| [`13/WAKU2-STORE`](https://rfc.vac.dev/spec/13/) | `draft` | `/vac/waku/store/2.0.0-beta4` |
+| [`18/WAKU2-SWAP`](https://rfc.vac.dev/spec/18/) | `draft` | `/vac/waku/swap/2.0.0-beta1` |
+| [`19/WAKU2-LIGHTPUSH`](https://rfc.vac.dev/spec/19/) | `draft` | `/vac/waku/lightpush/2.0.0-beta1` |
+
+The Waku v1 implementation is stable but not under active development.
+
 ## 2023-01-16 v0.14.0
 
 Release highlights:

Dockerfile

@@ -1,10 +1,11 @@
-# BUILD IMAGE ------------------------------------------------------------------
+# BUILD NIM APP ----------------------------------------------------------------
 
 FROM alpine:edge AS nim-build
 
 ARG NIMFLAGS
 ARG MAKE_TARGET=wakunode2
 ARG EXPERIMENTAL=false
+ARG NIM_COMMIT
 
 # Get build tools and required header files
 RUN apk add --no-cache bash git cargo build-base pcre-dev linux-headers
@@ -16,10 +17,10 @@ COPY . .
 RUN git submodule update --init --recursive
 
 # Slowest build step for the sake of caching layers
-RUN make -j$(nproc) deps
+RUN make -j$(nproc) deps ${NIM_COMMIT}
 
 # Build the final node binary
-RUN make -j$(nproc) $MAKE_TARGET NIMFLAGS="${NIMFLAGS}" EXPERIMENTAL="${EXPERIMENTAL}"
+RUN make -j$(nproc) ${NIM_COMMIT} $MAKE_TARGET NIMFLAGS="${NIMFLAGS}" EXPERIMENTAL="${EXPERIMENTAL}"
 
 
 # PRODUCTION IMAGE -------------------------------------------------------------
@@ -52,13 +53,34 @@ COPY --from=nim-build /app/migrations/ /app/migrations/
 RUN ln -sv /usr/local/bin/$MAKE_TARGET /usr/bin/wakunode
 
 ENTRYPOINT ["/usr/bin/wakunode"]
+
 # By default just show help if called without arguments
 CMD ["--help"]
 
 
-# EXPERIMENTAL IMAGE -----------------------------------------------------------
+# DEBUG IMAGE ------------------------------------------------------------------
+
+# Build debug tools: heaptrack
+FROM alpine:edge AS heaptrack-build
+
+RUN apk update
+RUN apk add -- gdb git g++ make cmake zlib-dev boost-dev libunwind-dev
+RUN git clone https://github.com/KDE/heaptrack.git /heaptrack
+
+WORKDIR /heaptrack/build
+RUN cmake -DCMAKE_BUILD_TYPE=Release ..
+RUN make -j$(nproc)
+
+
+# Debug image
+FROM prod AS debug
+
+RUN apk add --no-cache gdb libunwind
+
+# Add heaptrack
+COPY --from=heaptrack-build /heaptrack/build/ /heaptrack/build/
 
-FROM prod AS experimental
+ENV LD_LIBRARY_PATH=/heaptrack/build/lib/heaptrack/
+RUN ln -s /heaptrack/build/bin/heaptrack /usr/local/bin/heaptrack
 
-# If RLN enabled, copy RLN resources (WASM ZK circuit, proving and verification keys) used by tests
-COPY --from=nim-build /app/vendor/zerokit/rln/resources/ vendor/zerokit/rln/resources/
+ENTRYPOINT ["/heaptrack/build/bin/heaptrack", "/usr/bin/wakunode"]