Fix JSON operators

.changeset/olive-games-destroy.md

@@ -0,0 +1,6 @@
+---
+'@powersync/service-sync-rules': minor
+'@powersync/service-image': minor
+---
+
+Add the `fixed_json_extract` compatibility option. When enabled, JSON-extracting operators are updated to match SQLite more closely.

packages/sync-rules/src/SqlBucketDescriptor.ts

@@ -65,7 +65,7 @@ export class SqlBucketDescriptor implements BucketSource {
     if (this.bucketParameters == null) {
       throw new Error('Bucket parameters must be defined');
     }
-    const dataRows = SqlDataQuery.fromSql(this.name, this.bucketParameters, sql, options);
+    const dataRows = SqlDataQuery.fromSql(this.name, this.bucketParameters, sql, options, this.compatibility);
 
     this.dataQueries.push(dataRows);
 

packages/sync-rules/src/SqlDataQuery.ts

@@ -5,20 +5,26 @@ import { SqlRuleError } from './errors.js';
 import { ExpressionType } from './ExpressionType.js';
 import { SourceTableInterface } from './SourceTableInterface.js';
 import { SqlTools } from './sql_filters.js';
-import { castAsText } from './sql_functions.js';
 import { checkUnsupportedFeatures, isClauseError } from './sql_support.js';
 import { SyncRulesOptions } from './SqlSyncRules.js';
 import { TablePattern } from './TablePattern.js';
 import { TableQuerySchema } from './TableQuerySchema.js';
 import { BucketIdTransformer, EvaluationResult, ParameterMatchClause, QuerySchema, SqliteRow } from './types.js';
 import { getBucketId, isSelectStatement } from './utils.js';
+import { CompatibilityContext } from './compatibility.js';
 
 export interface SqlDataQueryOptions extends BaseSqlDataQueryOptions {
   filter: ParameterMatchClause;
 }
 
 export class SqlDataQuery extends BaseSqlDataQuery {
-  static fromSql(descriptorName: string, bucketParameters: string[], sql: string, options: SyncRulesOptions) {
+  static fromSql(
+    descriptorName: string,
+    bucketParameters: string[],
+    sql: string,
+    options: SyncRulesOptions,
+    compatibility: CompatibilityContext
+  ) {
     const parsed = parse(sql, { locationTracking: true });
     const schema = options.schema;
 
@@ -67,6 +73,7 @@ export class SqlDataQuery extends BaseSqlDataQuery {
       table: alias,
       parameterTables: ['bucket'],
       valueTables: [alias],
+      compatibilityContext: compatibility,
       sql,
       schema: querySchema
     });

packages/sync-rules/src/SqlParameterQuery.ts

@@ -123,6 +123,7 @@ export class SqlParameterQuery {
       sql,
       supportsExpandingParameters: true,
       supportsParameterExpressions: true,
+      compatibilityContext: options.compatibility,
       schema: querySchema
     });
     tools.checkSpecificNameCase(tableRef);

packages/sync-rules/src/StaticSqlParameterQuery.ts

@@ -45,6 +45,7 @@ export class StaticSqlParameterQuery {
       table: undefined,
       parameterTables: ['token_parameters', 'user_parameters'],
       supportsParameterExpressions: true,
+      compatibilityContext: options.compatibility,
       sql
     });
     const where = q.where;

packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts

@@ -1,8 +1,8 @@
-import { FromCall, SelectedColumn, SelectFromStatement } from 'pgsql-ast-parser';
+import { FromCall, SelectFromStatement } from 'pgsql-ast-parser';
 import { SqlRuleError } from './errors.js';
 import { SqlTools } from './sql_filters.js';
 import { checkUnsupportedFeatures, isClauseError, isParameterValueClause, sqliteBool } from './sql_support.js';
-import { TABLE_VALUED_FUNCTIONS, TableValuedFunction } from './TableValuedFunctions.js';
+import { generateTableValuedFunctions, TableValuedFunction } from './TableValuedFunctions.js';
 import {
   BucketIdTransformer,
   ParameterValueClause,
@@ -49,11 +49,13 @@ export class TableValuedFunctionSqlParameterQuery {
     options: QueryParseOptions,
     queryId: string
   ): TableValuedFunctionSqlParameterQuery {
+    const compatibility = options.compatibility;
     let errors: SqlRuleError[] = [];
 
     errors.push(...checkUnsupportedFeatures(sql, q));
 
-    if (!(call.function.name in TABLE_VALUED_FUNCTIONS)) {
+    const tableValuedFunctions = generateTableValuedFunctions(compatibility);
+    if (!(call.function.name in tableValuedFunctions)) {
       throw new SqlRuleError(`Table-valued function ${call.function.name} is not defined.`, sql, call);
     }
 
@@ -64,6 +66,7 @@ export class TableValuedFunctionSqlParameterQuery {
       table: callTable,
       parameterTables: ['token_parameters', 'user_parameters', callTable],
       supportsParameterExpressions: true,
+      compatibilityContext: compatibility,
       sql
     });
     const where = q.where;
@@ -73,7 +76,7 @@ export class TableValuedFunctionSqlParameterQuery {
     const columns = q.columns ?? [];
     const bucketParameters = columns.map((column) => tools.getOutputName(column));
 
-    const functionImpl = TABLE_VALUED_FUNCTIONS[call.function.name]!;
+    const functionImpl = tableValuedFunctions[call.function.name]!;
     let priority = options.priority;
     let parameterExtractors: Record<string, ParameterValueClause> = {};
 

packages/sync-rules/src/TableValuedFunctions.ts

@@ -1,3 +1,4 @@
+import { CompatibilityContext, CompatibilityOption } from './compatibility.js';
 import { SqliteJsonValue, SqliteRow, SqliteValue } from './types.js';
 import { jsonValueToSqlite } from './utils.js';
 
@@ -8,38 +9,40 @@ export interface TableValuedFunction {
   documentation: string;
 }
 
-export const JSON_EACH: TableValuedFunction = {
-  name: 'json_each',
-  call(args: SqliteValue[]) {
-    if (args.length != 1) {
-      throw new Error(`json_each expects 1 argument, got ${args.length}`);
-    }
-    const valueString = args[0];
-    if (valueString === null) {
-      return [];
-    } else if (typeof valueString !== 'string') {
-      throw new Error(`Expected json_each to be called with a string, got ${valueString}`);
-    }
-    let values: SqliteJsonValue[] = [];
-    try {
-      values = JSON.parse(valueString);
-    } catch (e) {
-      throw new Error('Expected JSON string');
-    }
-    if (!Array.isArray(values)) {
-      throw new Error(`Expected an array, got ${valueString}`);
-    }
+function jsonEachImplementation(fixedJsonBehavior: boolean): TableValuedFunction {
+  return {
+    name: 'json_each',
+    call(args: SqliteValue[]) {
+      if (args.length != 1) {
+        throw new Error(`json_each expects 1 argument, got ${args.length}`);
+      }
+      const valueString = args[0];
+      if (valueString === null) {
+        return [];
+      } else if (typeof valueString !== 'string') {
+        throw new Error(`Expected json_each to be called with a string, got ${valueString}`);
+      }
+      let values: SqliteJsonValue[] = [];
+      try {
+        values = JSON.parse(valueString);
+      } catch (e) {
+        throw new Error('Expected JSON string');
+      }
+      if (!Array.isArray(values)) {
+        throw new Error(`Expected an array, got ${valueString}`);
+      }
 
-    return values.map((v) => {
-      return {
-        value: jsonValueToSqlite(v)
-      };
-    });
-  },
-  detail: 'Each element of a JSON array',
-  documentation: 'Returns each element of a JSON array as a separate row.'
-};
+      return values.map((v) => {
+        return {
+          value: jsonValueToSqlite(fixedJsonBehavior, v)
+        };
+      });
+    },
+    detail: 'Each element of a JSON array',
+    documentation: 'Returns each element of a JSON array as a separate row.'
+  };
+}
 
-export const TABLE_VALUED_FUNCTIONS: Record<string, TableValuedFunction> = {
-  json_each: JSON_EACH
-};
+export function generateTableValuedFunctions(compatibility: CompatibilityContext): Record<string, TableValuedFunction> {
+  return { json_each: jsonEachImplementation(compatibility.isEnabled(CompatibilityOption.fixedJsonExtract)) };
+}

packages/sync-rules/src/compatibility.ts

@@ -28,9 +28,16 @@ export class CompatibilityOption {
     CompatibilityEdition.SYNC_STREAMS
   );
 
+  static fixedJsonExtract = new CompatibilityOption(
+    'fixed_json_extract',
+    "Old versions of the sync service used to treat `->> 'foo.bar'` as a two-element JSON path. With this compatibility option enabled, it follows modern SQLite and treats it as a single key. The `$.` prefix would be required to split on `.`.",
+    CompatibilityEdition.SYNC_STREAMS
+  );
+
   static byName: Record<string, CompatibilityOption> = Object.freeze({
     timestamps_iso8601: this.timestampsIso8601,
-    versioned_bucket_ids: this.versionedBucketIds
+    versioned_bucket_ids: this.versionedBucketIds,
+    fixed_json_extract: this.fixedJsonExtract
   });
 }
 

packages/sync-rules/src/events/SqlEventDescriptor.ts

@@ -23,7 +23,7 @@ export class SqlEventDescriptor {
   }
 
   addSourceQuery(sql: string, options: SyncRulesOptions): QueryParseResult {
-    const source = SqlEventSourceQuery.fromSql(this.name, sql, options);
+    const source = SqlEventSourceQuery.fromSql(this.name, sql, options, this.compatibility);
 
     // Each source query should be for a unique table
     const existingSourceQuery = this.sourceQueries.find((q) => q.table == source.table);

packages/sync-rules/src/events/SqlEventSourceQuery.ts

@@ -10,6 +10,7 @@ import { TablePattern } from '../TablePattern.js';
 import { TableQuerySchema } from '../TableQuerySchema.js';
 import { EvaluationError, QuerySchema, SqliteJsonRow, SqliteRow } from '../types.js';
 import { isSelectStatement } from '../utils.js';
+import { CompatibilityContext } from '../compatibility.js';
 
 export type EvaluatedEventSourceRow = {
   data: SqliteJsonRow;
@@ -24,7 +25,7 @@ export type EvaluatedEventRowWithErrors = {
  * Defines how a Replicated Row is mapped to source parameters for events.
  */
 export class SqlEventSourceQuery extends BaseSqlDataQuery {
-  static fromSql(descriptor_name: string, sql: string, options: SyncRulesOptions) {
+  static fromSql(descriptor_name: string, sql: string, options: SyncRulesOptions, compatibility: CompatibilityContext) {
     const parsed = parse(sql, { locationTracking: true });
     const schema = options.schema;
 
@@ -73,7 +74,8 @@ export class SqlEventSourceQuery extends BaseSqlDataQuery {
       parameterTables: [],
       valueTables: [alias],
       sql,
-      schema: querySchema
+      schema: querySchema,
+      compatibilityContext: compatibility
     });
 
     let extractors: RowValueExtractor[] = [];

packages/sync-rules/src/request_functions.ts

@@ -1,5 +1,6 @@
 import { ExpressionType } from './ExpressionType.js';
-import { jsonExtractFromRecord } from './sql_functions.js';
+import { CompatibilityContext, CompatibilityEdition, CompatibilityOption } from './compatibility.js';
+import { generateSqlFunctions } from './sql_functions.js';
 import { ParameterValueSet, SqliteValue } from './types.js';
 
 export interface SqlParameterFunction {
@@ -15,6 +16,9 @@ export interface SqlParameterFunction {
   documentation: string;
 }
 
+const jsonExtractFromRecord = generateSqlFunctions(
+  new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS)
+).jsonExtractFromRecord;
 /**
  * Defines a `parameters` function and a `parameter` function.
  *
@@ -50,12 +54,11 @@ export function parameterFunctions(options: {
     parameterCount: 1,
     call(parameters: ParameterValueSet, path) {
       const parsed = options.extractJsonParsed(parameters);
+      // jsonExtractFromRecord uses the correct behavior of only splitting the path if it starts with $.
+      // This particular JSON extract function always had that behavior, so we don't need to take backwards
+      // compatibility into account.
       if (typeof path == 'string') {
-        if (path.startsWith('$.')) {
-          return jsonExtractFromRecord(parsed, path, '->>');
-        } else {
-          return parsed[path];
-        }
+        return jsonExtractFromRecord(parsed, path, '->>');
       }
 
       return null;

packages/sync-rules/src/sql_filters.ts

@@ -10,13 +10,11 @@ import {
   OPERATOR_IN,
   OPERATOR_IS_NOT_NULL,
   OPERATOR_IS_NULL,
-  OPERATOR_JSON_EXTRACT_JSON,
-  OPERATOR_JSON_EXTRACT_SQL,
   OPERATOR_NOT,
   OPERATOR_OVERLAP,
-  SQL_FUNCTIONS,
   SqlFunction,
   castOperator,
+  generateSqlFunctions,
   getOperatorFunction,
   sqliteTypeOf
 } from './sql_functions.js';
@@ -47,7 +45,7 @@ import {
   TrueIfParametersMatch
 } from './types.js';
 import { isJsonValue } from './utils.js';
-import { STREAM_FUNCTIONS } from './streams/functions.js';
+import { CompatibilityContext } from './compatibility.js';
 
 export const MATCH_CONST_FALSE: TrueIfParametersMatch = [];
 export const MATCH_CONST_TRUE: TrueIfParametersMatch = [{}];
@@ -105,6 +103,11 @@ export interface SqlToolsOptions {
    * Schema for validations.
    */
   schema?: QuerySchema;
+
+  /**
+   * Context controling how functions should behave if we've made backwards-incompatible change to them.
+   */
+  compatibilityContext: CompatibilityContext;
 }
 
 export class SqlTools {
@@ -121,6 +124,8 @@ export class SqlTools {
   readonly supportsExpandingParameters: boolean;
   readonly supportsParameterExpressions: boolean;
   readonly parameterFunctions: Record<string, Record<string, SqlParameterFunction>>;
+  readonly compatibilityContext: CompatibilityContext;
+  readonly functions: ReturnType<typeof generateSqlFunctions>;
 
   schema?: QuerySchema;
 
@@ -140,6 +145,9 @@ export class SqlTools {
     this.supportsExpandingParameters = options.supportsExpandingParameters ?? false;
     this.supportsParameterExpressions = options.supportsParameterExpressions ?? false;
     this.parameterFunctions = options.parameterFunctions ?? { request: REQUEST_FUNCTIONS };
+    this.compatibilityContext = options.compatibilityContext;
+
+    this.functions = generateSqlFunctions(this.compatibilityContext);
   }
 
   error(message: string, expr: NodeLocation | Expr | undefined): ClauseError {
@@ -315,7 +323,7 @@ export class SqlTools {
 
       if (schema == null) {
         // Just fn()
-        const fnImpl = SQL_FUNCTIONS[fn];
+        const fnImpl = this.functions.named[fn];
         if (fnImpl == null) {
           return this.error(`Function '${fn}' is not defined`, expr);
         }
@@ -377,9 +385,9 @@ export class SqlTools {
       const debugArgs: Expr[] = [expr.operand, expr];
       const args: CompiledClause[] = [operand, staticValueClause(expr.member)];
       if (expr.op == '->') {
-        return this.composeFunction(OPERATOR_JSON_EXTRACT_JSON, args, debugArgs);
+        return this.composeFunction(this.functions.operatorJsonExtractJson, args, debugArgs);
       } else {
-        return this.composeFunction(OPERATOR_JSON_EXTRACT_SQL, args, debugArgs);
+        return this.composeFunction(this.functions.operatorJsonExtractSql, args, debugArgs);
       }
     } else if (expr.type == 'cast') {
       const operand = this.compileClause(expr.operand);