feat(redshift-driver): introspection for external schemas/tables (e.g…

…. Spectrum) (#8849) * use native Array.reduce instead of Ramda * define types for TablesSchema query results * extend redshift schema queries with external tables metadata * fix types in databricks driver * fix types in bigQuery driver * fix types in Quest driver * fix types in Materialize driver * remove unneeded ramda * fix typos in materialize * updated @types/node version * describe redshift fixtures config * fix getSchemas in redshift * add drivers-tests for redshift * add yarn cmds for redshift testing * implement missed getTablesForSpecificSchemas() * fix redshift export bucket test * add redshift fixtures description * add redshift test snapshots * add redshift to test matrix * add externalSchemaTests * remove unused
cube-js · Oct 24, 2024 · fa4b3b8 · fa4b3b8
1 parent bc582e5
commit fa4b3b8
Show file tree

Hide file tree

Showing 36 changed files with 31,758 additions and 130 deletions.
diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml
@@ -196,6 +196,8 @@ jobs:
         databricks-jdbc
         databricks-jdbc-export-bucket-s3
         databricks-jdbc-export-bucket-azure
+        redshift
+        redshift-export-bucket-s3
         snowflake
         snowflake-export-bucket-s3
         snowflake-export-bucket-azure
@@ -221,6 +223,8 @@ jobs:
           - mssql
           - mysql
           - postgres
+          - redshift
+          - redshift-export-bucket-s3
           - snowflake
           - snowflake-export-bucket-s3
           - snowflake-export-bucket-azure
@@ -312,6 +316,11 @@ jobs:
           DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY }}
           DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET }}
 
+          # Redshift
+          DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_HOST: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_HOST }}
+          DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_USER }}
+          DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_PASS }}
+
           # Snowflake
           DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER }}
           DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS }}
@@ -324,3 +333,4 @@ jobs:
             cd ./packages/cubejs-testing-drivers
             export DEBUG=testcontainers
             yarn ${{ matrix.database }}-full
+
diff --git a/packages/cubejs-backend-maven/package.json b/packages/cubejs-backend-maven/package.json
@@ -38,7 +38,7 @@
   "devDependencies": {
     "@cubejs-backend/linter": "^1.0.0",
     "@types/jest": "^27",
-    "@types/node": "^14",
+    "@types/node": "^18",
     "jest": "^27",
     "typescript": "~5.2.2"
   },

diff --git a/packages/cubejs-backend-native/package.json b/packages/cubejs-backend-native/package.json
@@ -35,7 +35,7 @@
   "devDependencies": {
     "@cubejs-backend/linter": "^1.0.0",
     "@types/jest": "^27",
-    "@types/node": "^14",
+    "@types/node": "^18",
     "cargo-cp-artifact": "^0.1.9",
     "jest": "^27",
     "pg": "^8.11.3",

diff --git a/packages/cubejs-base-driver/package.json b/packages/cubejs-base-driver/package.json
@@ -34,14 +34,12 @@
     "@azure/identity": "^4.4.1",
     "@azure/storage-blob": "^12.9.0",
     "@cubejs-backend/shared": "1.0.3",
-    "@google-cloud/storage": "^7.13.0",
-    "ramda": "^0.27.0"
+    "@google-cloud/storage": "^7.13.0"
   },
   "devDependencies": {
     "@cubejs-backend/linter": "^1.0.0",
     "@types/jest": "^27",
-    "@types/node": "^14",
-    "@types/ramda": "^0.27.32",
+    "@types/node": "^18",
     "jest": "^27",
     "typescript": "~5.2.2"
   },

diff --git a/packages/cubejs-base-driver/src/BaseDriver.ts b/packages/cubejs-base-driver/src/BaseDriver.ts
@@ -14,7 +14,6 @@ import {
   isSslKey,
   isSslCert,
 } from '@cubejs-backend/shared';
-import { reduce } from 'ramda';
 import fs from 'fs';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import { S3, GetObjectCommand, S3ClientConfig } from '@aws-sdk/client-s3';
@@ -53,6 +52,7 @@ import {
   TableMemoryData,
   PrimaryKeysQueryResult,
   ForeignKeysQueryResult,
+  DatabaseStructure,
 } from './driver.interface';
 
 /**
@@ -402,28 +402,28 @@ export abstract class BaseDriver implements DriverInterface {
     return false;
   }
 
-  protected informationColumnsSchemaReducer(result: any, i: any) {
+  protected informationColumnsSchemaReducer(result: any, i: any): DatabaseStructure {
     let schema = (result[i.table_schema] || {});
-    const tables = (schema[i.table_name] || []);
+    const columns = (schema[i.table_name] || []);
 
-    tables.push({
+    columns.push({
       name: i.column_name,
       type: i.data_type,
       attributes: i.key_type ? ['primaryKey'] : []
     });
 
-    tables.sort();
-    schema[i.table_name] = tables;
+    columns.sort();
+    schema[i.table_name] = columns;
     schema = sortByKeys(schema);
     result[i.table_schema] = schema;
 
     return sortByKeys(result);
   }
 
-  public tablesSchema() {
+  public tablesSchema(): Promise<DatabaseStructure> {
     const query = this.informationSchemaQuery();
 
-    return this.query(query).then(data => reduce(this.informationColumnsSchemaReducer, {}, data));
+    return this.query(query, []).then(data => data.reduce<DatabaseStructure>(this.informationColumnsSchemaReducer, {}));
   }
 
   // Extended version of tablesSchema containing primary and foreign keys

diff --git a/packages/cubejs-base-driver/src/driver.interface.ts b/packages/cubejs-base-driver/src/driver.interface.ts
@@ -2,9 +2,7 @@
 export type GenericDataBaseType = string;
 
 export interface TableColumn {
-  // eslint-disable-next-line camelcase
   name: string;
-  // eslint-disable-next-line camelcase
   type: GenericDataBaseType;
   attributes?: string[]
 }

diff --git a/packages/cubejs-bigquery-driver/src/BigQueryDriver.ts b/packages/cubejs-bigquery-driver/src/BigQueryDriver.ts
@@ -21,6 +21,7 @@ import {
 import { Bucket, Storage } from '@google-cloud/storage';
 import {
   BaseDriver,
+  DatabaseStructure,
   DriverCapabilities,
   DriverInterface,
   QueryColumnsResult,
@@ -202,7 +203,7 @@ export class BigQueryDriver extends BaseDriver implements DriverInterface {
         );
       }
 
-      return [];
+      return {};
     } catch (e) {
       if ((<any>e).message.includes('Permission bigquery.tables.get denied on table')) {
         return {};
@@ -212,7 +213,7 @@ export class BigQueryDriver extends BaseDriver implements DriverInterface {
     }
   }
 
-  public async tablesSchema() {
+  public async tablesSchema(): Promise<DatabaseStructure> {
     const dataSets = await this.bigquery.getDatasets();
     const dataSetsColumns = await Promise.all(
       dataSets[0].map((dataSet) => this.loadTablesForDataset(dataSet))

diff --git a/packages/cubejs-cli/package.json b/packages/cubejs-cli/package.json
@@ -58,7 +58,7 @@
     "@types/inquirer": "^7.3.1",
     "@types/jest": "^27",
     "@types/jsonwebtoken": "^9.0.2",
-    "@types/node": "^14",
+    "@types/node": "^18",
     "@types/request-promise": "^4.1.46",
     "@types/semver": "^7.3.4",
     "husky": "^4.2.3",

diff --git a/packages/cubejs-cubestore-driver/package.json b/packages/cubejs-cubestore-driver/package.json
@@ -44,7 +44,7 @@
     "@cubejs-backend/linter": "^1.0.0",
     "@types/csv-write-stream": "^2.0.0",
     "@types/generic-pool": "^3.1.9",
-    "@types/node": "^14",
+    "@types/node": "^18",
     "@types/ws": "^7.4.0",
     "jest": "^27",
     "typescript": "~5.2.2"

diff --git a/packages/cubejs-databricks-jdbc-driver/package.json b/packages/cubejs-databricks-jdbc-driver/package.json
@@ -41,7 +41,7 @@
     "@cubejs-backend/linter": "^1.0.0",
     "@types/generic-pool": "^3.1.9",
     "@types/jest": "^27",
-    "@types/node": "^14",
+    "@types/node": "^18",
     "@types/ramda": "^0.27.34",
     "@types/uuid": "^8.3.4",
     "jest": "^27",

diff --git a/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts b/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts
@@ -15,7 +15,9 @@ import {
   QuerySchemasResult,
   QueryTablesResult,
   UnloadOptions,
-  GenericDataBaseType
+  GenericDataBaseType,
+  TableColumn,
+  DatabaseStructure,
 } from '@cubejs-backend/base-driver';
 import {
   JDBCDriver,
@@ -392,10 +394,10 @@ export class DatabricksDriver extends JDBCDriver {
   /**
    * Returns tables meta data object.
    */
-  public override async tablesSchema(): Promise<Record<string, Record<string, object>>> {
+  public override async tablesSchema(): Promise<DatabaseStructure> {
     const tables = await this.getTables();
 
-    const metadata: Record<string, Record<string, object>> = {};
+    const metadata: DatabaseStructure = {};
 
     await Promise.all(tables.map(async ({ database, tableName }) => {
       if (!(database in metadata)) {
@@ -499,7 +501,7 @@ export class DatabricksDriver extends JDBCDriver {
   /**
    * Returns table columns types.
    */
-  public override async tableColumnTypes(table: string): Promise<{ name: any; type: string; }[]> {
+  public override async tableColumnTypes(table: string): Promise<TableColumn[]> {
     let tableFullName = '';
     const tableArray = table.split('.');
 
@@ -538,6 +540,7 @@ export class DatabricksDriver extends JDBCDriver {
       result.push({
         name: column.col_name,
         type: this.toGenericType(column.data_type),
+        attributes: [],
       });
     }
 

diff --git a/packages/cubejs-materialize-driver/package.json b/packages/cubejs-materialize-driver/package.json
@@ -32,7 +32,6 @@
     "@cubejs-backend/shared": "1.0.3",
     "@types/pg": "^8.6.0",
     "pg": "^8.6.0",
-    "ramda": "0.27.2",
     "semver": "7.3.7"
   },
   "license": "Apache-2.0",

diff --git a/packages/cubejs-materialize-driver/src/MaterializeDriver.ts b/packages/cubejs-materialize-driver/src/MaterializeDriver.ts
@@ -5,9 +5,16 @@
  */
 
 import { PostgresDriver, PostgresDriverConfiguration } from '@cubejs-backend/postgres-driver';
-import { BaseDriver, DownloadTableMemoryData, IndexesSQL, StreamOptions, StreamTableDataWithTypes, TableStructure } from '@cubejs-backend/base-driver';
+import {
+  BaseDriver,
+  DatabaseStructure,
+  DownloadTableMemoryData,
+  IndexesSQL,
+  StreamOptions,
+  StreamTableDataWithTypes,
+  TableStructure
+} from '@cubejs-backend/base-driver';
 import { PoolClient, QueryResult } from 'pg';
-import { reduce } from 'ramda';
 import { Readable } from 'stream';
 import semver from 'semver';
 
@@ -18,16 +25,6 @@ export type ReadableStreamTableDataWithTypes = StreamTableDataWithTypes & {
   rowStream: Readable;
 };
 
-export type SchemaResponse = {
-  [schema: string]: {
-    [schemaObject: string]: {
-      name: string;
-      type: string;
-      attributes: any[];
-  }[];
-  }
-};
-
 /**
  * Materialize driver class.
  */
@@ -73,6 +70,7 @@ export class MaterializeDriver extends PostgresDriver {
       /**
        * Application name to set for the connection.
        */
+      // eslint-disable-next-line camelcase
       application_name?: string,
     } = {},
   ) {
@@ -130,7 +128,7 @@ export class MaterializeDriver extends PostgresDriver {
   }
 
   /**
-   * Materialize quereable schema
+   * Materialize queryable schema
    * Returns materialized sources, materialized views, and tables
    * @returns {string} schemaQuery
    */
@@ -171,15 +169,15 @@ export class MaterializeDriver extends PostgresDriver {
   public async getMaterializeVersion(): Promise<string> {
     const [{ version }] = await this.query<{version: string}>('SELECT mz_version() as version;', []);
 
-    // Materialzie returns the version as follows: 'v0.24.3-alpha.5 (65778f520)'
+    // Materialize returns the version as follows: 'v0.24.3-alpha.5 (65778f520)'
     return version.split(' ')[0];
   }
 
-  public async tablesSchema(): Promise<SchemaResponse> {
+  public async tablesSchema(): Promise<DatabaseStructure> {
     const version = await this.getMaterializeVersion();
     const query = this.informationSchemaQueryWithFilter(version);
 
-    return this.query(query, []).then(data => reduce(this.informationColumnsSchemaReducer, {}, data));
+    return this.query(query, []).then(data => data.reduce<DatabaseStructure>(this.informationColumnsSchemaReducer, {}));
   }
 
   protected async* asyncFetcher<R extends unknown>(conn: PoolClient, cursorId: string): AsyncGenerator<R> {

diff --git a/packages/cubejs-questdb-driver/src/QuestDriver.ts b/packages/cubejs-questdb-driver/src/QuestDriver.ts
@@ -17,6 +17,7 @@ import {
   BaseDriver, DownloadQueryResultsOptions,
   DownloadTableMemoryData, DriverInterface,
   IndexesSQL, TableStructure, QueryOptions,
+  DatabaseStructure,
 } from '@cubejs-backend/base-driver';
 import { QuestQuery } from './QuestQuery';
 
@@ -179,14 +180,14 @@ export class QuestDriver<Config extends QuestDriverConfiguration = QuestDriverCo
     // no-op as there are no schemas in QuestDB
   }
 
-  public async tablesSchema() {
+  public async tablesSchema(): Promise<DatabaseStructure> {
     const tables = await this.getTablesQuery('');
 
     // QuestDB doesn't have a notion of schema/logical database while the driver
     // has to return a `{ 'schema_name': { 'table1': {...} } }` object. So, we use
     // empty schema name ('') as a workaround to avoid the schema prefix
     // ('schema_name.') being used for table names in the generated queries.
-    const metadata: Record<string, Record<string, object>> = { '': {} };
+    const metadata: DatabaseStructure = { '': {} };
 
     // eslint-disable-next-line camelcase
     await Promise.all(tables.map(async ({ table_name: tableName }) => {
@@ -209,7 +210,7 @@ export class QuestDriver<Config extends QuestDriverConfiguration = QuestDriverCo
     return this.query('SHOW TABLES', []);
   }
 
-  public async tableColumnTypes(table: string) {
+  public async tableColumnTypes(table: string): Promise<TableStructure> {
     const response: any[] = await this.query(`SHOW COLUMNS FROM '${table}'`, []);
 
     return response.map((row) => ({ name: row.column, type: this.toGenericType(row.type) }));