Merge pull request #58 from subquery/sync-main-20240410

Sync main 20240410

.github/actions/create-release/action.yml

@@ -12,18 +12,18 @@ inputs:
   npm-token:
     description: 'token to push to npm registry'
     required: true
-  
+
 runs:
   using: "composite"
   steps:
     - working-directory: ${{ inputs.package-path }}
-      run: echo "Changes exist in ${{ inputs.package-path }}" && yarn npm publish --access public 
+      run: echo "Changes exist in ${{ inputs.package-path }}" && yarn npm publish --access public
       env:
         NPM_TOKEN: ${{ inputs.npm-token }}
       shell: bash
 
     - working-directory: ${{ github.workspace }}
-      run: node ${{ github.action_path }}/gh-release-script.js ${{ github.workspace }}/${{ inputs.package-path }}
-      env: 
+      run: node ${{ github.action_path }}/gh-release-script.js ${{ github.workspace }}/${{ inputs.package-path }} ${{ github.sha }}
+      env:
         REPO_TOKEN: ${{ inputs.repo-token }}
       shell: bash

.github/actions/create-release/gh-release-script.js

@@ -8,76 +8,77 @@ const myArgs = process.argv.slice(2);
 const pJson = require(`${myArgs[0]}/package.json`)
 
 const version = pJson.version;
-const repoName = pJson.name; 
+const repoName = pJson.name;
 
 const packageName = repoName.split('/');
 
 function checkForBetaVersion(version) {
-    if (version.includes('-')){
-        exit(0); //skip this package but continue trying to release others
-    }
+  if (version.includes('-')){
+    exit(0); //skip this package but continue trying to release others
+  }
 }
 
 function gatherReleaseInfo(logPath) {
-    const changeLogs = fs.readFileSync(logPath, 'utf8');
-    const regex = /## \[([0-9]+(\.[0-9]+)+)] - [0-9]{4}-[0-9]{2}-[0-9]{2}/i;
-
-    let lines = changeLogs.split(/\n/);
-    let foundChangelog = false;
-    let releaseInfo = '';
-    let i = 0;
+  const changeLogs = fs.readFileSync(logPath, 'utf8');
+  const regex = /## \[([0-9]+(\.[0-9]+)+)] - [0-9]{4}-[0-9]{2}-[0-9]{2}/i;
 
-    for(let j = 0; j < lines.length; j++){
-       if(lines[j].includes(`[${version}]`)){
-           i = j;
-           j = lines.length;
-           foundChangelog = true;
-       } 
-    }
+  let lines = changeLogs.split(/\n/);
+  let foundChangelog = false;
+  let releaseInfo = '';
+  let i = 0;
 
-    lines = lines.slice(i);
-
-    if(foundChangelog){
-        for(let j = 0; j < lines.length; j++){
-            if(j == 0){
-               releaseInfo += `${lines[j]}`+ '\n';
-               continue;
-            }        
-
-            if(!regex.test(lines[j])){
-                releaseInfo += `${lines[j]}`+ '\n';
-            } else {
-                j = lines.length;
-            }
-        }
+  for(let j = 0; j < lines.length; j++){
+    if(lines[j].includes(`[${version}]`)){
+      i = j;
+      j = lines.length;
+      foundChangelog = true;
     }
-
-    if(releaseInfo === ''){
-        core.setFailed("No release info found, either missing in changelog or changelog is formatted incorrectly")
+  }
+
+  lines = lines.slice(i);
+
+  if(foundChangelog){
+    for(let j = 0; j < lines.length; j++){
+      if(j == 0){
+        releaseInfo += `${lines[j]}`+ '\n';
+        continue;
+      }
+
+      if(!regex.test(lines[j])){
+        releaseInfo += `${lines[j]}`+ '\n';
+      } else {
+        j = lines.length;
+      }
     }
+  }
+
+  if(releaseInfo === ''){
+    core.setFailed("No release info found, either missing in changelog or changelog is formatted incorrectly")
+  }
 
-    console.log("Gathered release info...")  
-    return releaseInfo;
+  console.log("Gathered release info...")
+  return releaseInfo;
 }
 
 async function publishRelease(releaseInfo) {
 
-    await request('POST /repos/{owner}/{repo}/releases', {
-        headers: {
-            authorization: `token ${process.env.REPO_TOKEN}`,
-        },
-        owner: 'subquery',
-        name: `[${version}] ${repoName}`,
-        repo: 'subql-stellar',
-        tag_name: `${packageName[1]}/${version}`,
-        body: releaseInfo
-    }).catch( err => {
-        core.setFailed(err)
-    })
-
-    console.log("Release Created...")  
+  await request('POST /repos/{owner}/{repo}/releases', {
+    headers: {
+      authorization: `token ${process.env.REPO_TOKEN}`,
+    },
+    owner: 'subquery',
+    name: `[${version}] ${repoName}`,
+    repo: 'subql-stellar',
+    tag_name: `${packageName[1]}/${version}`,
+    target_commitish: `${myArgs[1]}`,
+    body: releaseInfo
+  }).catch( err => {
+    core.setFailed(err)
+  })
+
+  console.log("Release Created...")
 }
- 
+
 checkForBetaVersion(version);
 
 const releaseInfo = gatherReleaseInfo(`${myArgs[0]}/CHANGELOG.md`);

.github/workflows/node-docker.yml

@@ -27,14 +27,14 @@ jobs:
             COMMIT_MESSAGE=$(git log -1 --pretty=format:"%s")
             if [[ $COMMIT_MESSAGE == "[release]"* ]] && git diff --name-only HEAD~1 HEAD -- './packages/node/package.json'
             then
-              echo "::set-output name=changes_found::true"
+              echo "::set-output name=changes_found::true" >> "$GITHUB_OUTPUT"
             else
-              echo "::set-output name=changes_found::false"
+              echo "::set-output name=changes_found::false" >> "$GITHUB_OUTPUT"
             fi
           else
-            echo "::set-output name=changes_found::true"
+            echo "::set-output name=changes_found::true" >> "$GITHUB_OUTPUT"
           fi
-          
+
   node-build-push-docker:
     needs: check
     if: needs.check.outputs.changes_found == 'true'
@@ -46,13 +46,13 @@ jobs:
           token: ${{ secrets.REPO_TOKEN }}
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: subquerynetwork
           password: ${{ secrets.SQ_DOCKERHUB_TOKEN }}
@@ -69,7 +69,7 @@ jobs:
 
       - name: Build and push
         if: github.event_name == 'workflow_dispatch' && github.event.inputs.isLatest == 'false'
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
@@ -80,7 +80,7 @@ jobs:
 
       - name: Build and push
         if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.isLatest == 'true')
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true

.github/workflows/prerelease.yml

@@ -6,6 +6,10 @@ on:
     paths-ignore:
       - '.github/workflows/**'
 
+concurrency:
+  group: publish
+  cancel-in-progress: false
+
 jobs:
   pre-ci:
     runs-on: ubuntu-latest

.github/workflows/release.yml

@@ -7,22 +7,37 @@ on:
       - '.github/workflows/**'
   workflow_dispatch:
 
+concurrency:
+  # Same group as prerelease
+  group: publish
+  cancel-in-progress: false
+
 jobs:
+  # This gets the commit message because workflow dispatch doesnt set: github.event.head_commit.message
   pre-ci:
     runs-on: ubuntu-latest
     timeout-minutes: 1
+    outputs:
+      commit-message: ${{ steps.step1.outputs.commit-message }}
     steps:
-      - name: 'Block Concurrent Executions'
-        uses: softprops/turnstyle@v1
+      - uses: actions/checkout@v4
         with:
-          poll-interval-seconds: 10
-        env:
-          GITHUB_TOKEN: ${{ secrets.REPO_TOKEN }}
+          fetch-depth: 100
+
+      # Get the commit message, workflow_dispatch doesn't contain it so we need to get it through git cli
+      - id: step1
+        run: |
+          if [ -n "${{ github.event.head_commit.message }}" ]
+          then
+            echo "commit-message=${{ github.event.head_commit.message }}" >> "$GITHUB_OUTPUT"
+          else
+            echo "commit-message=$(git log -1 --pretty=%B)" >> "$GITHUB_OUTPUT"
+          fi
 
   Build-Publish:
     name: Build-Publish
     needs: pre-ci
-    if: "!startsWith(github.event.head_commit.message, '[SKIP CI]') && startsWith(github.event.head_commit.message, '[release]') && github.repository == 'subquery/subql-stellar'"
+    if: "!startsWith(needs.pre-ci.outputs.commit-message, '[SKIP CI]') && startsWith(needs.pre-ci.outputs.commit-message, '[release]') && github.repository == 'subquery/subql-stellar'"
     runs-on: ubuntu-latest
     steps:
       #Check out

packages/common-stellar/CHANGELOG.md

@@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+- version bump with `@subql/common`
+
 ## [3.3.2] - 2024-02-23
 ### Changed
 - version bump with `@subql/common`

packages/common-stellar/package.json

@@ -14,7 +14,7 @@
   "main": "dist/index.js",
   "license": "GPL-3.0",
   "dependencies": {
-    "@subql/common": "^3.4.1",
+    "@subql/common": "^3.5.0",
     "@subql/types-stellar": "workspace:*",
     "js-yaml": "^4.1.0",
     "pino": "^6.13.3",

packages/node/CHANGELOG.md

@@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+
+### Changed
+- Updated with node-core. Now dictionary supports multiple endpoints, indexer will fetch and switch dictionaries based on available blocks
+### Fixed
+- Updated with node-core ,also fixed:
+  - Fix modulo block didn't apply correctly with multiple dataSources
+  - Now when `workers` set to 0, it will use block dispatcher instead of throw and exit
+
+
 ## [3.9.1] - 2024-03-14
 ### Changed
 - Update `@subql/node-core` to 4.7.2 with graphql comments escaping fix

packages/node/package.json

@@ -25,9 +25,9 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "^3.4.1",
+    "@subql/common": "^3.5.0",
     "@subql/common-stellar": "workspace:*",
-    "@subql/node-core": "^7.4.2",
+    "@subql/node-core": "^8.0.1",
     "@subql/testing": "^2.0.0",
     "@subql/types": "^2.2.1-1",
     "@subql/types-stellar": "workspace:*",

packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts

@@ -14,6 +14,7 @@ import {
   ApiService,
   IProjectUpgradeService,
   PoiSyncService,
+  IBlock,
 } from '@subql/node-core';
 import { StellarBlockWrapper } from '@subql/types-stellar';
 import {
@@ -67,11 +68,11 @@ export class BlockDispatcherService
   }
 
   protected async indexBlock(
-    block: StellarBlockWrapper,
+    block: IBlock<StellarBlockWrapper>,
   ): Promise<ProcessBlockResponse> {
     return this.indexerManager.indexBlock(
       block,
-      await this.projectService.getDataSources(this.getBlockHeight(block)),
+      await this.projectService.getDataSources(block.getHeader().blockHeight),
     );
   }
 }

packages/node/src/indexer/blockDispatcher/stellar-block-dispatcher.ts

@@ -2,7 +2,9 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import { IBlockDispatcher } from '@subql/node-core';
+import { StellarBlock } from '@subql/types-stellar';
 
-export interface IStellarBlockDispatcher extends IBlockDispatcher {
+export interface IStellarBlockDispatcher
+  extends IBlockDispatcher<StellarBlock> {
   init(onDynamicDsCreated: (height: number) => Promise<void>): Promise<void>;
 }

packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts

@@ -18,6 +18,7 @@ import {
   createIndexerWorker,
   InMemoryCacheService,
 } from '@subql/node-core';
+import { StellarBlockWrapper } from '@subql/types-stellar';
 import {
   StellarProjectDs,
   SubqueryProject,
@@ -36,7 +37,11 @@ type IndexerWorker = IIndexerWorker & {
 
 @Injectable()
 export class WorkerBlockDispatcherService
-  extends WorkerBlockDispatcher<StellarProjectDs, IndexerWorker>
+  extends WorkerBlockDispatcher<
+    StellarProjectDs,
+    IndexerWorker,
+    StellarBlockWrapper
+  >
   implements OnApplicationShutdown
 {
   constructor(