Metadata Exploration in TVsCE

taqueria-plugin-flextesa/proxy.ts

@@ -293,11 +293,12 @@ const startInstance = async (sandboxName: string, sandbox: SandboxConfig.t, opts
 	if (sandbox.tzkt?.disableAutostartWithSandbox === true) {
 		return;
 	}
-	const { postgres, sync, api } = await getTzKtStartCommands(sandboxName, sandbox, opts);
+	const { postgres, sync, api, metadata } = await getTzKtStartCommands(sandboxName, sandbox, opts);
 	const tzKtContainers = [
 		{ name: 'postgresql', command: postgres },
 		{ name: 'TzKt.Sync', command: sync },
 		{ name: 'TzKt.Api', command: api },
+		// { name: 'metadata', command: metadata },
 	];
 	for (const container of tzKtContainers) {
 		await startContainer(container);
@@ -447,7 +448,7 @@ const stopTzKtContainers = async (
 	parsedArgs: ValidOpts,
 ): Promise<void> => {
 	const containerNames = await getTzKtContainerNames(sandboxName, parsedArgs);
-	const containersToStop = [containerNames.api, containerNames.sync, containerNames.postgres];
+	const containersToStop = [containerNames.api, containerNames.sync, containerNames.postgres, containerNames.metadata];
 	for (const container of containersToStop) {
 		try {
 			const result = await execCmd(`docker stop ${container}`);

taqueria-plugin-flextesa/tzkt-manager.ts

@@ -7,6 +7,7 @@ const getTzKtDockerImages = (opts: ValidOpts) => ({
 	postgres: `postgres:14.5-alpine`,
 	sync: `ghcr.io/ecadlabs/tzkt-sync:v1.11.0-taqueria`,
 	api: `ghcr.io/ecadlabs/tzkt-api:v1.11.0-taqueria`,
+	metadata: 'dipdup/metadata:latest',
 });
 
 export const getTzKtContainerNames = async (sandboxName: string, parsedArgs: ValidOpts) => {
@@ -15,6 +16,7 @@ export const getTzKtContainerNames = async (sandboxName: string, parsedArgs: Val
 		postgres: `taq-postgres-${uniqueSandboxName}`,
 		sync: `taq-tzkt-sync-${uniqueSandboxName}`,
 		api: `taq-tzkt-api-${uniqueSandboxName}`,
+		metadata: `taq-metadata-${uniqueSandboxName}`,
 	};
 };
 
@@ -25,9 +27,11 @@ const getTzKtContainerEnvironments = async (sandboxName: string, sandbox: Sandbo
 		`ConnectionStrings__DefaultConnection="host=${containerNames.postgres};port=5432;database=sandbox_data;username=tzkt;password=${sandboxName};"`;
 	return {
 		postgres: `--env POSTGRES_PASSWORD=${sandboxName} --env POSTGRES_USER=tzkt`,
-		sync: `--env ${connectionStringEnv} --env TezosNode__Endpoint="http://${sandboxContainerName}:20000/"`,
+		sync:
+			`--env ${connectionStringEnv} --env TezosNode__Endpoint="http://${sandboxContainerName}:20000/" --env Protocols__Fallback="PtLimaPtLMwfNinJi9rCfDPWea8dFgTZ1MeJ9f1m2SRic6ayiwW"`,
 		api:
 			`--env ${connectionStringEnv} --env Kestrel__Endpoints__Http__Url="http://*:5000" --env MaxAttemptsForMigrations=120`,
+		metadata: `--env POSTGRES_HOST=${containerNames.postgres} --env TZKT_URL="http://${containerNames.api}:5000"`,
 	};
 };
 
@@ -89,5 +93,7 @@ export const getTzKtStartCommands = async (sandboxName: string, sandbox: Sandbox
 			`docker run --network sandbox_${sandboxName}_net --name ${containerNames.sync} --rm --detach --platform ${arch} ${environmentVariables.sync} ${images.sync}`,
 		api:
 			`docker run --network sandbox_${sandboxName}_net --name ${containerNames.api} --rm --detach --platform ${arch} -p ${newAPIPort}:5000 ${environmentVariables.api} ${images.api}`,
+		metadata:
+			`docker run --network sandbox_${sandboxName}_net --name ${containerNames.metadata} --rm --detach --platform ${arch} -v /home/alireza/work/scrap/EcadLabs/DipdupExplorations:/work:Z ${environmentVariables.metadata} ${images.metadata} -c /work/dipdup.yml`,
 	};
 };

taqueria-vscode-extension/package.json

@@ -431,6 +431,13 @@
 				"title": "Invoke Entrypoint",
 				"shortTitle": "invoke",
 				"icon": "$(call-outgoing)"
+			},
+			{
+				"command": "taqueria.show_token_metadata",
+				"category": "Taqueria",
+				"title": "Show Token Metadata",
+				"shortTitle": "show token metadata",
+				"icon": "$(list-unordered)"
 			}
 		],
 		"menus": {
@@ -482,6 +489,10 @@
 				{
 					"command": "taqueria.originate",
 					"when": "false"
+				},
+				{
+					"command": "taqueria.show_token_metadata",
+					"when": "false"
 				}
 			],
 			"explorer/context": [
@@ -631,6 +642,11 @@
 					"command": "taqueria.show_operation_details",
 					"when": "view == taqueria-sandboxes && viewItem == operation",
 					"group": "inline"
+				},
+				{
+					"command": "taqueria.show_token_metadata",
+					"when": "view == taqueria-sandboxes && viewItem == token",
+					"group": "inline"
 				}
 			],
 			"view/title": [
@@ -750,7 +766,7 @@
 	},
 	"scripts": {
 		"vscode:prepublish": "npm run compile:base -- --minify",
-		"compile:base": "esbuild ./src/extension.ts --bundle --outfile=out/main.js --external:vscode --format=cjs --platform=node",
+		"compile:base": "esbuild ./src/extension.ts --bundle --outfile=out/main.js --external:vscode --external:pg-native --format=cjs --platform=node",
 		"compile": "npx tsc -noEmit && npm run compile:base -- --sourcemap",
 		"compile:watch": "npm run compile:base -- --sourcemap --watch",
 		"compile:test": "tsc -p tsconfig.test.json",
@@ -764,6 +780,7 @@
 		"@types/glob": "^7.2.0",
 		"@types/mocha": "^5.2.6",
 		"@types/node": "^17.0.17",
+		"@types/pg": "^8.6.6",
 		"@types/promise-memoize": "^1.2.1",
 		"@types/semver": "^7.3.10",
 		"@types/vscode": "1.63.0",
@@ -775,6 +792,7 @@
 		"fs-extra": "^10.1.0",
 		"glob": "^7.2.0",
 		"mocha": "^9.2.1",
+		"pg-types": "^4.0.0",
 		"typescript": "4.7.2",
 		"vsce": "^2.6.4"
 	},
@@ -785,6 +803,7 @@
 		"comment-json": "^4.1.1",
 		"eventsource": "^2.0.2",
 		"node-fetch-commonjs": "^3.1.1",
+		"pg": "^8.8.0",
 		"promise-memoize": "^1.2.1",
 		"rambda": "^7.0.1",
 		"rxjs": "^7.5.6",

taqueria-vscode-extension/src/lib/gui/CachedSandboxState.ts

@@ -149,4 +149,13 @@ export class CachedSandboxState {
 		let port = sandbox.tzkt?.apiPort ?? 5000;
 		return `http://127.0.0.1:${port}`;
 	}
+
+	getDipupConnectionString(sandboxName: string): string | undefined {
+		const sandbox = this.observableConfig.currentConfig.config?.config.sandbox?.[sandboxName];
+		if (!sandbox || typeof sandbox === 'string') {
+			return undefined;
+		}
+		let port = sandbox.tzkt?.postgresqlPort ?? 5432;
+		return `postgres://tzkt:${sandboxName}@127.0.0.1:${port}/sandbox_data`;
+	}
 }

taqueria-vscode-extension/src/lib/gui/DataModels.ts

@@ -0,0 +1,35 @@
+export type TokenInfo_TzKt = {
+	id: number;
+	contract: AccountReferenece;
+	tokenId: string;
+	standard: string;
+	firstMinter: AccountReferenece;
+	firstLevel: number;
+	firstTime: string;
+	lastLevel: number;
+	lastTime: string;
+	transfersCount: number;
+	balancesCount: number;
+	holdersCount: number;
+	totalMinted: string;
+	totalBurned: string;
+	totalSupply: string;
+	metadata: TokenMetadata;
+};
+
+export type TokenInfo_Dipdup = {
+	id: bigint;
+	token_id: bigint;
+	link: string | undefined | null;
+	metadata: TokenMetadata | undefined | null;
+	image_processed: boolean | undefined | null;
+};
+
+export type TokenMetadata = {
+	name?: string | undefined;
+};
+
+export type AccountReferenece = {
+	alias: string;
+	address: string;
+};

taqueria-vscode-extension/src/lib/gui/DbConnectionPools.ts

@@ -0,0 +1,17 @@
+import { Pool } from 'pg';
+
+export class DbConnectionPools {
+	private static pools: Map<string, Pool> = new Map();
+
+	getPool(connectionString: string): Pool {
+		let pool = DbConnectionPools.pools.get(connectionString);
+		if (pool) {
+			return pool;
+		}
+		pool = new Pool({
+			connectionString,
+		});
+		DbConnectionPools.pools.set(connectionString, pool);
+		return pool;
+	}
+}

taqueria-vscode-extension/src/lib/gui/SandboxTreeItemTypes.ts

@@ -1,5 +1,6 @@
 import * as vscode from 'vscode';
 import { SandboxState } from './CachedSandboxState';
+import { TokenMetadata } from './DataModels';
 
 export class SandboxTreeItemBase extends vscode.TreeItem {
 	constructor(
@@ -11,6 +12,7 @@ export class SandboxTreeItemBase extends vscode.TreeItem {
 			| 'smartContract'
 			| 'smartContractChild'
 			| 'smartContractEntryPoint'
+			| 'token'
 			| 'operation',
 		collapsibleState: vscode.TreeItemCollapsibleState,
 	) {
@@ -108,7 +110,7 @@ export class SandboxSmartContractTreeItem extends SandboxTreeItemBase {
 
 export class SmartContractChildrenTreeItem extends SandboxTreeItemBase {
 	constructor(
-		public kind: 'Operations' | 'Entrypoints',
+		public kind: 'Operations' | 'Entrypoints' | 'Tokens',
 		public readonly parent: SandboxSmartContractTreeItem,
 	) {
 		super(kind, 'smartContractChild', vscode.TreeItemCollapsibleState.Collapsed);
@@ -140,3 +142,14 @@ export class OperationTreeItem extends SandboxTreeItemBase {
 		this.contextValue = 'operation';
 	}
 }
+
+export class TokenTreeItem extends SandboxTreeItemBase {
+	constructor(
+		public readonly tokenId: string,
+		public readonly name: string | undefined,
+		public readonly metadata: TokenMetadata | undefined,
+	) {
+		super(name || tokenId.toString(), 'token', vscode.TreeItemCollapsibleState.None);
+		this.contextValue = 'token';
+	}
+}

taqueria-vscode-extension/src/lib/gui/SandboxesDataProvider.ts

@@ -7,6 +7,8 @@ import { OutputLevels } from '../LogHelper';
 import { getRunningContainerNames } from '../pure';
 import * as Util from '../pure';
 import { CachedSandboxState, SandboxState } from './CachedSandboxState';
+import { TokenInfo_Dipdup, TokenInfo_TzKt } from './DataModels';
+import { DbConnectionPools } from './DbConnectionPools';
 import { ObservableConfig } from './ObservableConfig';
 import { TzKtHead } from './SandboxDataModels';
 import {
@@ -18,6 +20,7 @@ import {
 	SandboxTreeItemBase,
 	SmartContractChildrenTreeItem,
 	SmartContractEntrypointTreeItem,
+	TokenTreeItem,
 } from './SandboxTreeItemTypes';
 import { TaqueriaDataProviderBase } from './TaqueriaDataProviderBase';
 
@@ -32,6 +35,7 @@ export class SandboxesDataProvider extends TaqueriaDataProviderBase
 	}
 
 	private sandboxStates: Record<string, CachedSandboxState> = {};
+	private pools = new DbConnectionPools();
 
 	refreshLevelInterval: NodeJS.Timer | undefined;
 	private sandboxTreeItems: SandboxTreeItem[] | undefined;
@@ -63,6 +67,8 @@ export class SandboxesDataProvider extends TaqueriaDataProviderBase
 					return await this.getSmartContractEntrypoints(element);
 				case 'Operations':
 					return await this.getAccountOperations(element);
+				case 'Tokens':
+					return await this.getTokens(element);
 				default:
 					return [];
 			}
@@ -280,6 +286,7 @@ export class SandboxesDataProvider extends TaqueriaDataProviderBase
 		return [
 			new SmartContractChildrenTreeItem('Entrypoints', element),
 			new SmartContractChildrenTreeItem('Operations', element),
+			new SmartContractChildrenTreeItem('Tokens', element),
 		];
 	}
 
@@ -291,7 +298,6 @@ export class SandboxesDataProvider extends TaqueriaDataProviderBase
 			return [];
 		}
 		const response = await fetch(`${tzktBaseUrl}/v1/accounts?type.ne=contract`);
-		tzktBaseUrl;
 		const data = await response.json();
 		const sandbox = this.observableConfig.currentConfig.config?.config.sandbox?.[element.parent.sandboxName];
 		const aliases = (sandbox === undefined || typeof sandbox === 'string' || !sandbox.accounts)
@@ -395,4 +401,48 @@ export class SandboxesDataProvider extends TaqueriaDataProviderBase
 		}
 		this._onDidChangeTreeData.fire();
 	}
+
+	async getTokens(element: SmartContractChildrenTreeItem): Promise<TokenTreeItem[]> {
+		const [dataFromTzKt, dataFromDipdup] = await Promise.all([
+			this.getTokensFromTzKt(element),
+			this.getTokensFromDipdup(element),
+		]);
+		const data = dataFromTzKt.map(tzKtItem => ({
+			tzKtItem,
+			dipDupItem: dataFromDipdup.find(dipDupItem => dipDupItem.token_id.toString() === tzKtItem.tokenId),
+		}));
+		return data.map(item =>
+			new TokenTreeItem(item.tzKtItem.tokenId, item.dipDupItem?.metadata?.name, item.dipDupItem?.metadata ?? undefined)
+		);
+	}
+
+	private async getTokensFromTzKt(element: SmartContractChildrenTreeItem): Promise<TokenInfo_TzKt[]> {
+		const tzktBaseUrl = this.sandboxStates[element.parent.sandboxName]?.getTzKtBaseUrl(element.parent.sandboxName);
+		if (!tzktBaseUrl) {
+			return [];
+		}
+		const contractAddress = element.parent.address;
+		const response = await fetch(`${tzktBaseUrl}/v1/tokens?contract.eq=${contractAddress}`);
+		const data = await response.json();
+		return data as TokenInfo_TzKt[];
+	}
+
+	private async getTokensFromDipdup(element: SmartContractChildrenTreeItem): Promise<TokenInfo_Dipdup[]> {
+		const connectionString = this.sandboxStates[element.parent.sandboxName]?.getDipupConnectionString(
+			element.parent.sandboxName,
+		);
+		if (!connectionString) {
+			return [];
+		}
+		const pool = this.pools.getPool(connectionString);
+		const contractAddress = element.parent.address;
+		const tokens = await pool.query(`select * from token_metadata where contract='${contractAddress}'`);
+		return tokens.rows.map(row => ({
+			id: row.id,
+			token_id: row.token_id,
+			link: row.link,
+			metadata: row.metadata,
+			image_processed: row.image_processed,
+		}));
+	}
 }

taqueria-vscode-extension/src/lib/helpers.ts

@@ -25,6 +25,7 @@ import {
 	SandboxTreeItem,
 	SandboxTreeItemBase,
 	SmartContractEntrypointTreeItem,
+	TokenTreeItem,
 } from './gui/SandboxTreeItemTypes';
 import { ScaffoldsDataProvider, ScaffoldTreeItem } from './gui/ScaffoldsDataProvider';
 import { SystemCheckDataProvider, SystemCheckTreeItem } from './gui/SystemCheckDataProvider';
@@ -241,6 +242,7 @@ export class VsCodeHelper {
 		this.exposeRefreshSandBoxDataCommand();
 		this.exposeShowEntrypointParametersCommand();
 		this.exposeShowOperationDetailsCommand();
+		this.exposeShowTokenMetadataCommand();
 		this.exposeInvokeEntrypointCommand();
 
 		await this.registerDataProviders();
@@ -1729,6 +1731,13 @@ export class VsCodeHelper {
 		});
 	}
 
+	exposeShowTokenMetadataCommand() {
+		this.registerCommand('show_token_metadata', async (item: TokenTreeItem) => {
+			const metadata = item.metadata;
+			this.logHelper.showOutput(JSON.stringify(metadata, null, 2));
+		});
+	}
+
 	exposeInvokeEntrypointCommand() {
 		this.registerCommand(
 			'invoke_entrypoint',