[pull] master from cube-js:master

packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts

@@ -18,10 +18,11 @@ export class BaseMeasure {
 
   protected preparePatchedMeasure(sourceMeasure: string, newMeasureType: string | null, addFilters: Array<{sql: Function}>): MeasureDefinition {
     const source = this.query.cubeEvaluator.measureByPath(sourceMeasure);
+    const aggType = source.aggType ?? source.type;
 
-    let resultMeasureType = source.type;
+    let resultMeasureType = aggType;
     if (newMeasureType !== null) {
-      switch (source.type) {
+      switch (aggType) {
         case 'sum':
         case 'avg':
         case 'min':
@@ -32,29 +33,35 @@ export class BaseMeasure {
             case 'min':
             case 'max':
             case 'count_distinct':
+            case 'countDistinct':
             case 'count_distinct_approx':
+            case 'countDistinctApprox':
               // Can change from avg/... to count_distinct
               // Latter does not care what input value is
               // ok, do nothing
               break;
             default:
               throw new UserError(
-                `Unsupported measure type replacement for ${sourceMeasure}: ${source.type} => ${newMeasureType}`
+                `Unsupported measure type replacement for ${sourceMeasure}: ${aggType} => ${newMeasureType}`
               );
           }
           break;
         case 'count_distinct':
+        case 'countDistinct':
         case 'count_distinct_approx':
+        case 'countDistinctApprox':
           switch (newMeasureType) {
             case 'count_distinct':
+            case 'countDistinct':
             case 'count_distinct_approx':
+            case 'countDistinctApprox':
               // ok, do nothing
               break;
             default:
               // Can not change from count_distinct to avg/...
               // Latter do care what input value is, and original measure can be defined on strings
               throw new UserError(
-                `Unsupported measure type replacement for ${sourceMeasure}: ${source.type} => ${newMeasureType}`
+                `Unsupported measure type replacement for ${sourceMeasure}: ${aggType} => ${newMeasureType}`
               );
           }
           break;
@@ -64,7 +71,7 @@ export class BaseMeasure {
           // Can not change from count
           // There's no SQL at all
           throw new UserError(
-            `Unsupported measure type replacement for ${sourceMeasure}: ${source.type} => ${newMeasureType}`
+            `Unsupported measure type replacement for ${sourceMeasure}: ${aggType} => ${newMeasureType}`
           );
       }
 
@@ -81,14 +88,16 @@ export class BaseMeasure {
         case 'max':
         case 'count':
         case 'count_distinct':
+        case 'countDistinct':
         case 'count_distinct_approx':
+        case 'countDistinctApprox':
           // ok, do nothing
           break;
         default:
           // Can not add filters to string, time, boolean, number
           // Aggregation is already included in SQL, it's hard to patch that
           throw new UserError(
-            `Unsupported additional filters for measure ${sourceMeasure} type ${source.type}`
+            `Unsupported additional filters for measure ${sourceMeasure} type ${aggType}`
           );
       }
 
@@ -97,9 +106,16 @@ export class BaseMeasure {
 
     const patchedFrom = this.query.cubeEvaluator.parsePath('measures', sourceMeasure);
 
+    // For view measures, `type` is `number` (aggregation is embedded in SQL)
+    // while `aggType` carries the real aggregation kind. We must preserve that
+    // distinction to avoid double-wrapping (e.g. SUM(SUM(...))).
+    const typeFields = source.aggType != null
+      ? { type: source.type, aggType: resultMeasureType }
+      : { type: resultMeasureType };
+
     return {
       ...source,
-      type: resultMeasureType,
+      ...typeFields,
       filters: resultFilters,
       patchedFrom: {
         cubeName: patchedFrom[0],

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -3319,13 +3319,28 @@ export class BaseQuery {
         const primaryKeys = this.cubeEvaluator.primaryKeys[cubeName];
         const orderBySql = (symbol.orderBy || []).map(o => ({ sql: this.evaluateSql(cubeName, o.sql), dir: o.dir }));
         let sql;
+        let patchedSymbol = symbol;
         if (symbol.type !== 'rank') {
-          sql = symbol.sql && this.evaluateSql(cubeName, symbol.sql) ||
+          const evaluateSql = () => symbol.sql && this.evaluateSql(cubeName, symbol.sql) ||
             primaryKeys.length && (
               primaryKeys.length > 1 ?
                 this.concatStringsSql(primaryKeys.map((pk) => this.castToString(this.primaryKeySql(pk, cubeName))))
                 : this.primaryKeySql(primaryKeys[0], cubeName)
             ) || '*';
+          // For patched view measures (aggType is set), the view's sql resolves to
+          // already-aggregated SQL (e.g. SUM(col)). Filters must be applied inside
+          // that aggregation, not outside. We pre-evaluate the filter SQL at the
+          // view level, push it down via context, and skip filters at this level.
+          const isPatchedViewMeasure = symbol.aggType && symbol.patchedFrom && symbol.filters?.length;
+          if (isPatchedViewMeasure) {
+            const pushDownFilterSql = this.evaluateFiltersArray(symbol.filters, cubeName);
+            sql = this.evaluateSymbolSqlWithContext(evaluateSql, {
+              patchMeasurePushDownFilterSql: pushDownFilterSql,
+            });
+            patchedSymbol = { ...symbol, filters: [] };
+          } else {
+            sql = evaluateSql();
+          }
         }
         const result = this.renderSqlMeasure(
           name,
@@ -3335,7 +3350,7 @@ export class BaseQuery {
               sql,
               isMemberExpr,
             ),
-            symbol,
+            patchedSymbol,
             cubeName
           ),
           symbol,
@@ -3836,11 +3851,21 @@ export class BaseQuery {
   }
 
   applyMeasureFilters(evaluateSql, symbol, cubeName) {
-    if (!symbol.filters || !symbol.filters.length) {
+    const pushDownFilterSql = this.safeEvaluateSymbolContext().patchMeasurePushDownFilterSql;
+    const hasOwnFilters = symbol.filters && symbol.filters.length;
+
+    if (!hasOwnFilters && !pushDownFilterSql) {
       return evaluateSql;
     }
 
-    const where = this.evaluateMeasureFilters(symbol, cubeName);
+    const parts = [];
+    if (hasOwnFilters) {
+      parts.push(this.evaluateMeasureFilters(symbol, cubeName));
+    }
+    if (pushDownFilterSql) {
+      parts.push(pushDownFilterSql);
+    }
+    const where = parts.join(' AND ');
 
     return `CASE WHEN ${where} THEN ${evaluateSql === '*' ? '1' : evaluateSql} END`;
   }

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -60,6 +60,7 @@ export type TimeShiftDefinitionReference = {
 
 export type MeasureDefinition = {
   type: string;
+  aggType?: string,
   sql(): string;
   ownedByCube: boolean;
   rollingWindow?: any

packages/cubejs-testing/birdbox-fixtures/postgresql/schema/Orders.js

@@ -24,6 +24,10 @@ cube(`Orders`, {
       type: `count_distinct`,
       sql: `CASE WHEN ${Orders.status} = 'shipped' THEN ${CUBE}.id END`
     },
+    approxOrderCount: {
+      type: `count_distinct_approx`,
+      sql: `CASE WHEN ${Orders.status} = 'shipped' THEN ${CUBE}.id END`
+    },
     netCollectionCompleted: {
       type: `sum`,
       sql: `CASE WHEN ${Orders.status} = 'shipped' THEN ${CUBE}.amount END`
@@ -50,6 +54,24 @@ cube(`Orders`, {
       format: `currency`,
       currency: `usd`,
     },
+    avgAmount: {
+      sql: `amount`,
+      type: `avg`,
+      format: `currency`,
+      currency: `usd`,
+    },
+    minAmount: {
+      sql: `amount`,
+      type: `min`,
+      format: `currency`,
+      currency: `usd`,
+    },
+    maxAmount: {
+      sql: `amount`,
+      type: `max`,
+      format: `currency`,
+      currency: `usd`,
+    },
     toRemove: {
       type: `count`,
     },

packages/cubejs-testing/test/__snapshots__/smoke-cubesql.test.ts.snap

@@ -149,8 +149,8 @@ Object {
 exports[`SQL API Cube SQL over HTTP sql4sql regular query with missing column 1`] = `
 Object {
   "body": Object {
-    "error": "Error: SQLCompilationError: Internal: Initial planning error: Error during planning: Invalid identifier '#foobar' for schema fields:[Orders.count, Orders.orderCount, Orders.netCollectionCompleted, Orders.arpu, Orders.refundRate, Orders.refundOrdersCount, Orders.overallOrders, Orders.totalAmount, Orders.toRemove, Orders.numberTotal, Orders.amountRank, Orders.amountReducedByStatus, Orders.statusPercentageOfTotal, Orders.amountRankView, Orders.amountRankDateMax, Orders.amountRankDate, Orders.countAndTotalAmount, Orders.createdAtMax, Orders.createdAtMaxProxy, Orders.id, Orders.status, Orders.createdAt, Orders.updatedAt, Orders.__user, Orders.__cubeJoinField], metadata:{}",
-    "stack": "Error: SQLCompilationError: Internal: Initial planning error: Error during planning: Invalid identifier '#foobar' for schema fields:[Orders.count, Orders.orderCount, Orders.netCollectionCompleted, Orders.arpu, Orders.refundRate, Orders.refundOrdersCount, Orders.overallOrders, Orders.totalAmount, Orders.toRemove, Orders.numberTotal, Orders.amountRank, Orders.amountReducedByStatus, Orders.statusPercentageOfTotal, Orders.amountRankView, Orders.amountRankDateMax, Orders.amountRankDate, Orders.countAndTotalAmount, Orders.createdAtMax, Orders.createdAtMaxProxy, Orders.id, Orders.status, Orders.createdAt, Orders.updatedAt, Orders.__user, Orders.__cubeJoinField], metadata:{}",
+    "error": "Error: SQLCompilationError: Internal: Initial planning error: Error during planning: Invalid identifier '#foobar' for schema fields:[Orders.count, Orders.orderCount, Orders.approxOrderCount, Orders.netCollectionCompleted, Orders.arpu, Orders.refundRate, Orders.refundOrdersCount, Orders.overallOrders, Orders.totalAmount, Orders.avgAmount, Orders.minAmount, Orders.maxAmount, Orders.toRemove, Orders.numberTotal, Orders.amountRank, Orders.amountReducedByStatus, Orders.statusPercentageOfTotal, Orders.amountRankView, Orders.amountRankDateMax, Orders.amountRankDate, Orders.countAndTotalAmount, Orders.createdAtMax, Orders.createdAtMaxProxy, Orders.id, Orders.status, Orders.createdAt, Orders.updatedAt, Orders.__user, Orders.__cubeJoinField], metadata:{}",
+    "stack": "Error: SQLCompilationError: Internal: Initial planning error: Error during planning: Invalid identifier '#foobar' for schema fields:[Orders.count, Orders.orderCount, Orders.approxOrderCount, Orders.netCollectionCompleted, Orders.arpu, Orders.refundRate, Orders.refundOrdersCount, Orders.overallOrders, Orders.totalAmount, Orders.avgAmount, Orders.minAmount, Orders.maxAmount, Orders.toRemove, Orders.numberTotal, Orders.amountRank, Orders.amountReducedByStatus, Orders.statusPercentageOfTotal, Orders.amountRankView, Orders.amountRankDateMax, Orders.amountRankDate, Orders.countAndTotalAmount, Orders.createdAtMax, Orders.createdAtMaxProxy, Orders.id, Orders.status, Orders.createdAt, Orders.updatedAt, Orders.__user, Orders.__cubeJoinField], metadata:{}",
   },
   "headers": Headers {
     Symbol(map): Object {
@@ -161,7 +161,7 @@ Object {
         "keep-alive",
       ],
       "content-length": Array [
-        "1431",
+        "1589",
       ],
       "content-type": Array [
         "application/json; charset=utf-8",
@@ -557,6 +557,18 @@ Array [
 ]
 `;
 
+exports[`SQL API Postgres (Data) measure in view with ad-hoc filter: measure-in-view-with-ad-hoc-filters 1`] = `
+Array [
+  Object {
+    "new_amount": 800,
+    "new_avg_amount": 400,
+    "new_count_distinct": "1",
+    "new_max_amount": 500,
+    "new_min_amount": 300,
+  },
+]
+`;
+
 exports[`SQL API Postgres (Data) measure with ad-hoc filter and original measure: measure-with-ad-hoc-filters-and-original-measure 1`] = `
 Array [
   Object {

packages/cubejs-testing/test/smoke-cubesql.test.ts

@@ -980,6 +980,38 @@ filter_subq AS (
       expect(res.rows).toMatchSnapshot('measure-with-ad-hoc-filters-and-original-measure');
     });
 
+    test('measure in view with ad-hoc filter', async () => {
+      const query = `
+      SELECT
+        SUM(CASE
+          WHEN status = 'processed' THEN totalAmount
+        END) AS new_amount,
+        AVG(CASE
+          WHEN status = 'processed' THEN avgAmount
+        END) AS new_avg_amount,
+        MIN(CASE
+          WHEN status = 'processed' THEN minAmount
+        END) AS new_min_amount,
+        MAX(CASE
+          WHEN status = 'processed' THEN maxAmount
+        END) AS new_max_amount,
+        COUNT(DISTINCT CASE
+          WHEN status = 'shipped' THEN orderCount
+        END) AS new_count_distinct
+
+        /* Works but testing Postgres does not include "hll_hash_any" function
+        APPROX_DISTINCT(CASE
+          WHEN status = 'shipped' THEN approxOrderCount
+        END) AS new_approx_distinct
+        */
+      FROM
+        OrdersView
+      `;
+
+      const res = await connection.query(query);
+      expect(res.rows).toMatchSnapshot('measure-in-view-with-ad-hoc-filters');
+    });
+
     /// Query references `updatedAt` in three places: in outer projection, in grouping key and in window
     /// Incoming query is consistent: all three references same column
     /// This tests that generated SQL for pushdown remains consistent: