historical timestamp and db store (#92)

* historical timestamp and db store

* fix testing.module.ts

packages/common-stellar/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Bump `@subql/common` dependency
 
 ## [4.1.4] - 2024-10-23
 ### Changed

packages/common-stellar/package.json

@@ -15,7 +15,7 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@stellar/stellar-sdk": "^12.2.0",
-    "@subql/common": "^5.1.4",
+    "@subql/common": "^5.2.1",
     "@subql/types-stellar": "workspace:*",
     "js-yaml": "^4.1.0",
     "pino": "^6.13.3",

packages/node/CHANGELOG.md

@@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Support for historical indexing by timestamp as well as block height
+- Add an `--enable-cache` flag, allowing you to choose between DB or cache for IO operations.
 
 ## [4.0.4] - 2024-10-23
 ### Changed

packages/node/package.json

@@ -25,9 +25,9 @@
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
     "@stellar/stellar-sdk": "^12.2.0",
-    "@subql/common": "^5.1.4",
+    "@subql/common": "^5.2.1",
     "@subql/common-stellar": "workspace:*",
-    "@subql/node-core": "^14.1.6",
+    "@subql/node-core": "^15.0.3",
     "@subql/testing": "^2.2.0",
     "@subql/types-stellar": "workspace:*",
     "lodash": "^4.17.21",

packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts

@@ -5,7 +5,7 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import {
   NodeConfig,
-  StoreCacheService,
+  IStoreModelProvider,
   StoreService,
   IProjectService,
   BlockDispatcher,
@@ -37,7 +37,7 @@ export class BlockDispatcherService
     @Inject('IProjectUpgradeService')
     projectUpgradeService: IProjectUpgradeService,
     storeService: StoreService,
-    storeCacheService: StoreCacheService,
+    @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider,
     poiSyncService: PoiSyncService,
     @Inject('ISubqueryProject') project: SubqueryProject,
   ) {
@@ -47,7 +47,7 @@ export class BlockDispatcherService
       projectService,
       projectUpgradeService,
       storeService,
-      storeCacheService,
+      storeModelProvider,
       poiSyncService,
       project,
       apiService.fetchBlocks.bind(apiService),

packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts

@@ -9,14 +9,15 @@ import {
   NodeConfig,
   StoreService,
   PoiSyncService,
-  StoreCacheService,
+  IStoreModelProvider,
   IProjectService,
   WorkerBlockDispatcher,
   IProjectUpgradeService,
   ConnectionPoolStateManager,
   createIndexerWorker,
   InMemoryCacheService,
   MonitorServiceInterface,
+  Header,
 } from '@subql/node-core';
 import { StellarBlockWrapper, SubqlDatasource } from '@subql/types-stellar';
 import { SubqueryProject } from '../../configure/SubqueryProject';
@@ -50,7 +51,7 @@ export class WorkerBlockDispatcherService
     projectUpgradeService: IProjectUpgradeService,
     cacheService: InMemoryCacheService,
     storeService: StoreService,
-    storeCacheService: StoreCacheService,
+    @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider,
     poiSyncService: PoiSyncService,
     @Inject('ISubqueryProject') project: SubqueryProject,
     dynamicDsService: DynamicDsService,
@@ -64,7 +65,7 @@ export class WorkerBlockDispatcherService
       projectService,
       projectUpgradeService,
       storeService,
-      storeCacheService,
+      storeModelProvider,
       poiSyncService,
       project,
       () =>
@@ -89,10 +90,7 @@ export class WorkerBlockDispatcherService
     );
   }
 
-  protected async fetchBlock(
-    worker: IndexerWorker,
-    height: number,
-  ): Promise<void> {
-    await worker.fetchBlock(height, 0 /* Unused with stellar*/);
+  async fetchBlock(worker: IndexerWorker, height: number): Promise<Header> {
+    return worker.fetchBlock(height, 0 /* Unused with stellar*/);
   }
 }

packages/node/src/indexer/fetch.module.ts

@@ -8,13 +8,12 @@ import {
   PoiSyncService,
   ApiService,
   ConnectionPoolService,
-  StoreCacheService,
+  IStoreModelProvider,
   ConnectionPoolStateManager,
   NodeConfig,
   IProjectUpgradeService,
   ProjectUpgradeService,
   InMemoryCacheService,
-  SandboxService,
   MonitorService,
   CoreModule,
 } from '@subql/node-core';
@@ -74,7 +73,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         indexerManager: IndexerManager,
         cacheService: InMemoryCacheService,
         storeService: StoreService,
-        storeCacheService: StoreCacheService,
+        storeModelProvider: IStoreModelProvider,
         poiSyncService: PoiSyncService,
         project: SubqueryProject,
         dynamicDsService: DynamicDsService,
@@ -90,7 +89,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               projectUpgradeService,
               cacheService,
               storeService,
-              storeCacheService,
+              storeModelProvider,
               poiSyncService,
               project,
               dynamicDsService,
@@ -106,7 +105,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               projectService,
               projectUpgradeService,
               storeService,
-              storeCacheService,
+              storeModelProvider,
               poiSyncService,
               project,
             ),
@@ -119,7 +118,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         IndexerManager,
         InMemoryCacheService,
         StoreService,
-        StoreCacheService,
+        'IStoreModelProvider',
         PoiSyncService,
         'ISubqueryProject',
         DynamicDsService,

packages/node/src/indexer/fetch.service.ts

@@ -12,7 +12,7 @@ import {
   ApiService,
   getModulos,
   Header,
-  StoreCacheService,
+  IStoreModelProvider,
 } from '@subql/node-core';
 import { StellarBlock, SubqlDatasource } from '@subql/types-stellar';
 import { StellarApi } from '../stellar';
@@ -42,7 +42,7 @@ export class FetchService extends BaseFetchService<
     unfinalizedBlocksService: UnfinalizedBlocksService,
     eventEmitter: EventEmitter2,
     schedulerRegistry: SchedulerRegistry,
-    storeCacheService: StoreCacheService,
+    @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider,
   ) {
     super(
       nodeConfig,
@@ -52,7 +52,7 @@ export class FetchService extends BaseFetchService<
       eventEmitter,
       schedulerRegistry,
       unfinalizedBlocksService,
-      storeCacheService,
+      storeModelProvider,
     );
   }
 

packages/node/src/indexer/unfinalizedBlocks.service.ts

@@ -1,13 +1,13 @@
 // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 import {
   ApiService,
   BaseUnfinalizedBlocksService,
   Header,
   NodeConfig,
-  StoreCacheService,
+  IStoreModelProvider,
   getLogger,
   mainThreadOnly,
 } from '@subql/node-core';
@@ -21,9 +21,9 @@ export class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService<Block
   constructor(
     private readonly apiService: ApiService,
     nodeConfig: NodeConfig,
-    storeCache: StoreCacheService,
+    @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider,
   ) {
-    super(nodeConfig, storeCache);
+    super(nodeConfig, storeModelProvider);
   }
 
   @mainThreadOnly()
@@ -38,7 +38,7 @@ export class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService<Block
   }
 
   @mainThreadOnly()
-  protected async getHeaderForHeight(height: number): Promise<Header> {
+  async getHeaderForHeight(height: number): Promise<Header> {
     const block = (await this.apiService.api.fetchBlocks([height]))[0];
     return stellarBlockToHeader(block);
   }

packages/node/src/indexer/worker/worker.service.ts

@@ -13,11 +13,13 @@ import {
   BaseWorkerService,
   IProjectUpgradeService,
   IBlock,
+  Header,
 } from '@subql/node-core';
 import { BlockWrapper, SubqlDatasource } from '@subql/types-stellar';
+import { stellarBlockToHeader } from '../../stellar/utils.stellar';
 import { IndexerManager } from '../indexer.manager';
 
-export type FetchBlockResponse = { parentHash: string } | undefined;
+export type FetchBlockResponse = Header;
 
 export type WorkerStatusResponse = {
   threadId: number;
@@ -54,10 +56,8 @@ export class WorkerService extends BaseWorkerService<
     return block;
   }
 
-  protected toBlockResponse(block: BlockWrapper): { parentHash: string } {
-    return {
-      parentHash: block.block.prev_hash,
-    };
+  protected toBlockResponse(block: BlockWrapper): Header {
+    return stellarBlockToHeader(block.block);
   }
 
   protected async processFetchedBlock(

packages/node/src/subcommands/reindex.module.ts

@@ -2,24 +2,30 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import { Module } from '@nestjs/common';
-import { EventEmitterModule } from '@nestjs/event-emitter';
+import { EventEmitter2, EventEmitterModule } from '@nestjs/event-emitter';
 import { SchedulerRegistry } from '@nestjs/schedule';
 import {
   ApiService,
   DbModule,
   ForceCleanService,
-  StoreCacheService,
   StoreService,
   ReindexService,
+  storeModelFactory,
+  NodeConfig,
 } from '@subql/node-core';
+import { Sequelize } from '@subql/x-sequelize';
 import { ConfigureModule } from '../configure/configure.module';
 import { DsProcessorService } from '../indexer/ds-processor.service';
 import { DynamicDsService } from '../indexer/dynamic-ds.service';
 import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';
 
 @Module({
   providers: [
-    StoreCacheService,
+    {
+      provide: 'IStoreModelProvider',
+      useFactory: storeModelFactory,
+      inject: [NodeConfig, EventEmitter2, SchedulerRegistry, Sequelize],
+    },
     StoreService,
     ReindexService,
     ForceCleanService,

packages/node/src/subcommands/testing.module.ts

@@ -17,7 +17,9 @@ import {
   TestRunner,
   SandboxService,
   NodeConfig,
+  storeModelFactory,
 } from '@subql/node-core';
+import { Sequelize } from '@subql/x-sequelize';
 import { ConfigureModule } from '../configure/configure.module';
 import { SubqueryProject } from '../configure/SubqueryProject';
 import { DsProcessorService } from '../indexer/ds-processor.service';
@@ -33,6 +35,11 @@ import { TestingService } from './testing.service';
   providers: [
     InMemoryCacheService,
     StoreService,
+    {
+      provide: 'IStoreModelProvider',
+      useFactory: storeModelFactory,
+      inject: [NodeConfig, EventEmitter2, SchedulerRegistry, Sequelize],
+    },
     TestingService,
     EventEmitter2,
     PoiService,