From fbc2cbff466ff05f657676d0468c2c901a5e1ea2 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 28 Jul 2026 23:20:12 -0700 Subject: [PATCH 01/18] feat: ddl create and alter statements --- docs/configuration/index.md | 1 + docs/development/extensions-core/catalog.md | 77 ++- .../catalog/CatalogDdlAndIngestTest.java | 254 +++++++++ .../catalog/guice/CatalogClientModule.java | 8 + .../druid/catalog/http/CatalogResource.java | 4 + .../druid/catalog/sync/CatalogClient.java | 55 +- .../catalog/sync/CatalogSqlTableWriter.java | 184 +++++++ .../http/catalog/CatalogResourceTest.java | 81 +-- .../druid/catalog/model/ColumnSpec.java | 9 +- .../apache/druid/catalog/model/Columns.java | 16 +- .../model/table/DatasourceTableTest.java | 14 +- sql/src/main/codegen/config.fmpp | 18 + sql/src/main/codegen/includes/ddl.ftl | 174 ++++++ .../druid/sql/calcite/external/Externals.java | 66 +-- .../calcite/parser/DruidSqlAlterTable.java | 305 +++++++++++ .../parser/DruidSqlColumnDeclaration.java | 132 +++++ .../calcite/parser/DruidSqlCreateTable.java | 181 +++++++ .../sql/calcite/parser/DruidSqlParser.java | 13 +- .../parser/DruidSqlPropertyAssignment.java | 103 ++++ .../calcite/planner/CatalogColumnTypes.java | 145 +++++ .../calcite/planner/CatalogDdlHandler.java | 505 ++++++++++++++++++ .../calcite/planner/CatalogTableWriter.java | 123 +++++ .../sql/calcite/planner/DruidPlanner.java | 35 ++ .../sql/calcite/planner/PlannerConfig.java | 27 +- .../sql/calcite/planner/PlannerFactory.java | 39 ++ .../sql/calcite/planner/PlannerToolbox.java | 45 ++ .../org/apache/druid/sql/guice/SqlModule.java | 4 + .../sql/calcite/CalciteCatalogDdlTest.java | 453 ++++++++++++++++ .../sql/calcite/SqlTestFrameworkConfig.java | 1 + .../calcite/parser/DruidSqlDdlParserTest.java | 307 +++++++++++ .../planner/CalcitePlannerModuleTest.java | 1 + .../sql/calcite/util/SqlTestFramework.java | 19 + 32 files changed, 3288 insertions(+), 111 deletions(-) create mode 100644 embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java create mode 100644 extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogSqlTableWriter.java create mode 100644 sql/src/main/codegen/includes/ddl.ftl create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlAlterTable.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlColumnDeclaration.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlPropertyAssignment.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogColumnTypes.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java create mode 100644 sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java create mode 100644 sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java diff --git a/docs/configuration/index.md b/docs/configuration/index.md index f4c40ba15b77..556643bdd973 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -1896,6 +1896,7 @@ The Druid SQL server is configured through the following properties on the Broke |`druid.sql.planner.metadataSegmentCacheEnable`|Whether to keep a cache of published segments on Broker that can be used to serve queries against `sys.segments`. If true, broker polls coordinator in background to get segments from metadata store and maintains a local cache. If false, coordinator's REST API will be invoked when broker needs published segments info.|true| |`druid.sql.planner.metadataSegmentPollPeriod`|How often to poll coordinator for published segments list if `druid.sql.planner.metadataSegmentCacheEnable` is set to true. Poll period is in milliseconds. |60000| |`druid.sql.planner.authorizeSystemTablesDirectly`|If true, Druid authorizes queries against any of the system schema tables (`sys` in SQL) as `SYSTEM_TABLE` resources which require `READ` access, in addition to permissions based content filtering.|false| +|`druid.sql.planner.enableCatalogDdl`|If true, `CREATE TABLE` and `ALTER TABLE` statements may be used to define [catalog](../development/extensions-core/catalog.md) tables. These statements require `WRITE` permission on the datasource, the same permission the catalog API requires, so enabling this lets anyone who can ingest into a datasource also change its catalog definition. Requires the `druid-catalog` extension. Cannot be overridden per query.|false| |`druid.sql.planner.useNativeQueryExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan as a JSON representation of equivalent native query(s), else it will return the original version of explain plan generated by Calcite. It can be overridden per query with `useNativeQueryExplain` context key.|true| |`druid.sql.planner.maxNumericInFilters`|Max limit for the amount of numeric values that can be compared for a string type dimension when the entire SQL WHERE clause of a query translates to an [OR](../querying/filters.md#or) of [Bound filter](../querying/filters.md#bound-filter). By default, Druid does not restrict the amount of numeric Bound Filters on String columns, although this situation may block other queries from running. Set this property to a smaller value to prevent Druid from running queries that have prohibitively long segment processing times. The optimal limit requires some trial and error; we recommend starting with 100. Users who submit a query that exceeds the limit of `maxNumericInFilters` should instead rewrite their queries to use strings in the `WHERE` clause instead of numbers. For example, `WHERE someString IN (‘123’, ‘456’)`. If this value is disabled, `maxNumericInFilters` set through query context is ignored.|`-1` (disabled)| |`druid.sql.approxCountDistinct.function`|Implementation to use for the [`APPROX_COUNT_DISTINCT` function](../querying/sql-aggregations.md). Without extensions loaded, the only valid value is `APPROX_COUNT_DISTINCT_BUILTIN` (a HyperLogLog, or HLL, based implementation). If the [DataSketches extension](../development/extensions-core/datasketches-extension.md) is loaded, this can also be `APPROX_COUNT_DISTINCT_DS_HLL` (alternative HLL implementation) or `APPROX_COUNT_DISTINCT_DS_THETA`.

Theta sketches use significantly more memory than HLL sketches, so you should prefer one of the two HLL implementations.|`APPROX_COUNT_DISTINCT_BUILTIN`| diff --git a/docs/development/extensions-core/catalog.md b/docs/development/extensions-core/catalog.md index 1825d56ef706..81ed7f9a2115 100644 --- a/docs/development/extensions-core/catalog.md +++ b/docs/development/extensions-core/catalog.md @@ -43,6 +43,74 @@ allowing queries to be more concise, and simpler to write. This also allows the written into a defined column of the table is consistent with that columns definition, minimizing errors where unexpected data is written into a particular column of the table. +### SQL DDL + +Tables can be defined with SQL instead of by posting a table specification. `CREATE TABLE` and `ALTER TABLE` are +submitted to the Broker like any other SQL statement, and write the same catalog metadata the REST API does. They +return no rows. + +These statements change catalog metadata only. They never create, modify, or delete segments: defining a table does +not ingest anything, and altering a column does not rewrite existing data. Column changes take effect for subsequent +ingestion. + +These statements are disabled by default. Set `druid.sql.planner.enableCatalogDdl` to `true` on the Broker to enable +them. They require `WRITE` permission on the datasource, the same permission the catalog API requires, so enabling +them lets anyone who can ingest into a datasource also change its catalog definition; leave them disabled if you +manage catalog entries with your own tooling. The setting cannot be overridden per query. + +The `druid-catalog` extension must be loaded on both the Broker and the Coordinator; without it, these statements +report that the extension is not available. + +```sql +CREATE [OR REPLACE] TABLE [IF NOT EXISTS] + [ ( [, ...] ) ] + [ PARTITIONED BY ] + [ CLUSTERED BY [, ...] ] +``` + +`OR REPLACE` replaces the specification of an existing table; `IF NOT EXISTS` leaves an existing table unchanged. +The two cannot be combined. `PARTITIONED BY` sets [`segmentGranularity`](#table-properties) and `CLUSTERED BY` sets +`clusterKeys`, both of which a later `INSERT` or `REPLACE` inherits unless it states its own. + +Column types are written as SQL types, such as `VARCHAR`, `BIGINT`, `DOUBLE`, or `VARCHAR ARRAY`. The `__time` column +is written as `TIMESTAMP`. Types that have no SQL spelling, such as complex types, use `TYPE('...')` with the Druid +native type string: + +```sql +CREATE TABLE "druid"."visits" ( + __time TIMESTAMP, + user_id VARCHAR, + pages_visited BIGINT, + sketch TYPE('COMPLEX') +) +PARTITIONED BY DAY +CLUSTERED BY user_id +``` + +`ALTER TABLE` supports one change per statement, so that each statement is a single atomic catalog operation: + +```sql +ALTER TABLE
ADD COLUMN +ALTER TABLE
DROP COLUMN +ALTER TABLE
ALTER COLUMN SET DATA TYPE +ALTER TABLE
SET PROPERTIES ( = [, ...] ) +``` + +`SET PROPERTIES` merges the given [table properties](#table-properties) into the table. A value of `NULL` removes a +property. Values must be literals: + +```sql +ALTER TABLE "druid"."visits" SET PROPERTIES (targetSegmentRows = 3000000, sealed = TRUE) +ALTER TABLE "druid"."visits" SET PROPERTIES (sealed = NULL) +``` + +Both statements require the `WRITE` permission on the datasource, the same permission the REST API checks. Table +names may be unqualified or qualified with the `druid` schema; other schemas are rejected. Names are case-sensitive. + +There is no `DROP TABLE`. Deleting a table's catalog entry without deleting its data would be a surprising meaning +for the statement, so removing a specification is left to the [delete API](#delete-a-table) until the semantics are +settled. + ### API Objects #### TableSpec @@ -90,10 +158,11 @@ The endpoint supports a set of optional query parameters to enforce optimistic l is meant to update a table rather than create a new one. In the default case, with no query parameters set, this request will return an error if a table of the same name already exists in the schema specified. -| Parameter | Type | Description | -|-------------|---------|-------------------------------------------------------------------------------------------------------------------------------| -| `version` | Long | the expected version of an existing table. The version must match. If not (or if the table does not exist), returns an error. | -| `overwrite` | boolean | if true, then overwrites any existing table. Otherwise, the operation fails if the table already exists. | +| Parameter | Type | Description | +|---------------|---------|-------------------------------------------------------------------------------------------------------------------------------| +| `version` | Long | the expected version of an existing table. The version must match. If not (or if the table does not exist), returns an error. | +| `overwrite` | boolean | if true, then overwrites any existing table. Otherwise, the operation fails if the table already exists. | +| `ifNotExists` | boolean | if true, then leaves an existing table unchanged and reports a version of 0. Otherwise, the operation fails if the table already exists. | ##### Responses diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java new file mode 100644 index 000000000000..060dda567cfb --- /dev/null +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testing.embedded.catalog; + +import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.TableId; +import org.apache.druid.catalog.model.TableMetadata; +import org.apache.druid.catalog.model.table.ClusterKeySpec; +import org.apache.druid.catalog.model.table.DatasourceDefn; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.query.http.SqlTaskStatus; +import org.apache.druid.segment.TestHelper; +import org.apache.druid.testing.embedded.EmbeddedDruidCluster; +import org.apache.druid.testing.embedded.msq.EmbeddedMSQApis; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end coverage of catalog DDL: define a table with SQL, ingest into it, and query it. This is what proves the + * Broker-to-Coordinator write path and the cache refresh actually work against a running cluster, rather than against + * a recording stub. + */ +public class CatalogDdlAndIngestTest extends CatalogTestBase +{ + private TestCatalogClient client; + private EmbeddedMSQApis msqApis; + + @Override + protected EmbeddedDruidCluster createCluster() + { + // Catalog DDL is opt-in. + broker.addProperty("druid.sql.planner.enableCatalogDdl", "true"); + return super.createCluster(); + } + + @BeforeAll + public void initializeClient() + { + client = new TestCatalogClient(cluster); + msqApis = new EmbeddedMSQApis(cluster, overlord); + } + + @Test + public void testCreateTableThenIngestAndQuery() + { + final String tableName = dataSource; + + cluster.callApi().runSql( + "CREATE TABLE \"%s\" (\n" + + " __time TIMESTAMP,\n" + + " varchar_col1 VARCHAR,\n" + + " double_col1 DOUBLE\n" + + ")\n" + + "PARTITIONED BY DAY\n" + + "CLUSTERED BY varchar_col1", + tableName + ); + + // The spec reached the Coordinator, in the declared column order. + final TableMetadata table = client.readTable(TableId.datasource(tableName)); + assertEquals( + List.of("__time", "varchar_col1", "double_col1"), + columnNames(table) + ); + assertEquals( + List.of("TIMESTAMP", "VARCHAR", "DOUBLE"), + table.spec().columns().stream().map(ColumnSpec::dataType).collect(Collectors.toList()) + ); + assertEquals("P1D", table.spec().properties().get(DatasourceDefn.SEGMENT_GRANULARITY_PROPERTY)); + + // An INSERT that omits PARTITIONED BY picks it up from the catalog entry the DDL just wrote, which only works + // if this Broker's catalog cache saw the write. + ingest( + "INSERT INTO \"%s\"\n" + + "SELECT\n" + + " TIME_PARSE(a) AS __time,\n" + + " b AS varchar_col1,\n" + + " c AS double_col1\n" + + "FROM TABLE(\n" + + " EXTERN(\n" + + " '{\"type\":\"inline\",\"data\":\"2022-12-26T12:34:56,foo,10\\n2022-12-26T12:34:56,bar,20\"}',\n" + + " '{\"type\":\"csv\",\"findColumnsFromHeader\":false,\"columns\":[\"a\",\"b\",\"c\"]}'\n" + + " )\n" + + ") EXTEND (a VARCHAR, b VARCHAR, c BIGINT)\n", + tableName + ); + + // The BIGINT source column is coerced to the DOUBLE declared by the DDL, and rows come back in clustered order. + cluster.callApi().verifySqlQuery( + "SELECT * FROM %s", + tableName, + "2022-12-26T12:34:56.000Z,bar,20.0\n" + + "2022-12-26T12:34:56.000Z,foo,10.0" + ); + } + + @Test + public void testAlterTableAddColumnThenIngest() + { + final String tableName = dataSource; + + cluster.callApi().runSql("CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR) PARTITIONED BY DAY", tableName); + cluster.callApi().runSql("ALTER TABLE \"%s\" ADD COLUMN b BIGINT", tableName); + + final TableMetadata table = client.readTable(TableId.datasource(tableName)); + assertEquals(List.of("__time", "a", "b"), columnNames(table)); + + ingest( + "INSERT INTO \"%s\"\n" + + "SELECT TIME_PARSE(x) AS __time, y AS a, z AS b\n" + + "FROM TABLE(\n" + + " EXTERN(\n" + + " '{\"type\":\"inline\",\"data\":\"2022-12-26T12:34:56,hello,7\"}',\n" + + " '{\"type\":\"csv\",\"findColumnsFromHeader\":false,\"columns\":[\"x\",\"y\",\"z\"]}'\n" + + " )\n" + + ") EXTEND (x VARCHAR, y VARCHAR, z BIGINT)\n", + tableName + ); + + cluster.callApi().verifySqlQuery("SELECT * FROM %s", tableName, "2022-12-26T12:34:56.000Z,hello,7"); + } + + @Test + public void testAlterTableColumnAndProperties() + { + final String tableName = dataSource; + + cluster.callApi().runSql("CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR, b VARCHAR)", tableName); + + cluster.callApi().runSql("ALTER TABLE \"%s\" ALTER COLUMN b SET DATA TYPE BIGINT", tableName); + cluster.callApi().runSql("ALTER TABLE \"%s\" DROP COLUMN a", tableName); + cluster.callApi().runSql( + "ALTER TABLE \"%s\" SET PROPERTIES (segmentGranularity = 'P1D', sealed = TRUE)", + tableName + ); + + final TableMetadata table = client.readTable(TableId.datasource(tableName)); + assertEquals(List.of("__time", "b"), columnNames(table)); + assertEquals("BIGINT", table.spec().columns().get(1).dataType()); + assertEquals("P1D", table.spec().properties().get(DatasourceDefn.SEGMENT_GRANULARITY_PROPERTY)); + assertEquals(true, table.spec().properties().get(DatasourceDefn.SEALED_PROPERTY)); + + // A null value removes a property. + cluster.callApi().runSql("ALTER TABLE \"%s\" SET PROPERTIES (sealed = NULL)", tableName); + assertNull( + client.readTable(TableId.datasource(tableName)).spec().properties().get(DatasourceDefn.SEALED_PROPERTY) + ); + } + + @Test + public void testCreateOrReplaceAndIfNotExists() + { + final String tableName = dataSource; + + cluster.callApi().runSql("CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR) CLUSTERED BY a", tableName); + + // IF NOT EXISTS leaves the original definition alone. + cluster.callApi().runSql("CREATE TABLE IF NOT EXISTS \"%s\" (__time TIMESTAMP, zzz VARCHAR)", tableName); + assertEquals(List.of("__time", "a"), columnNames(client.readTable(TableId.datasource(tableName)))); + + // OR REPLACE swaps the whole spec, including dropping the clustering the first statement set. + cluster.callApi().runSql("CREATE OR REPLACE TABLE \"%s\" (__time TIMESTAMP, b BIGINT)", tableName); + final TableMetadata replaced = client.readTable(TableId.datasource(tableName)); + assertEquals(List.of("__time", "b"), columnNames(replaced)); + assertNull(replaced.spec().properties().get(DatasourceDefn.CLUSTER_KEYS_PROPERTY)); + } + + @Test + public void testCreateTableClusterKeys() + { + final String tableName = dataSource; + cluster.callApi().runSql("CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR, b BIGINT) CLUSTERED BY a, b", tableName); + + // Catalog properties round trip as untyped JSON, so decode before asserting. + final TableMetadata table = client.readTable(TableId.datasource(tableName)); + final List keys = TestHelper.JSON_MAPPER.convertValue( + table.spec().properties().get(DatasourceDefn.CLUSTER_KEYS_PROPERTY), + ClusterKeySpec.CLUSTER_KEY_LIST_TYPE_REF + ); + assertEquals(List.of("a", "b"), keys.stream().map(ClusterKeySpec::expr).collect(Collectors.toList())); + assertTrue(keys.stream().noneMatch(ClusterKeySpec::desc)); + } + + /** + * A rejection from the Coordinator's own validation must reach the SQL user as the Coordinator worded it, not as + * a generic remote-call failure. Segment granularity is validated only on the Coordinator, so it exercises the + * whole round trip. + */ + @Test + public void testCoordinatorValidationErrorSurfacesToSqlUser() + { + final String tableName = dataSource; + cluster.callApi().runSql("CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR)", tableName); + + final Exception e = assertThrows( + Exception.class, + () -> cluster.callApi().runSql( + "ALTER TABLE \"%s\" SET PROPERTIES (segmentGranularity = 'not_a_granularity')", + tableName + ) + ); + assertTrue(e.getMessage().contains("granularity"), e.getMessage()); + } + + @Test + public void testCreateTableAlreadyExistsFails() + { + final String tableName = dataSource; + cluster.callApi().runSql("CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR)", tableName); + + final Exception e = assertThrows( + Exception.class, + () -> cluster.callApi().runSql("CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR)", tableName) + ); + assertTrue(e.getMessage().contains("duplicate table"), e.getMessage()); + } + + private void ingest(String sqlPattern, String tableName) + { + final SqlTaskStatus status = msqApis.submitTaskSql(StringUtils.format(sqlPattern, tableName)); + cluster.callApi().waitForTaskToSucceed(status.getTaskId(), overlord); + cluster.callApi().waitForAllSegmentsToBeAvailable(tableName, coordinator, broker); + } + + private static List columnNames(TableMetadata table) + { + return table.spec().columns().stream().map(ColumnSpec::name).collect(Collectors.toList()); + } +} diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/guice/CatalogClientModule.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/guice/CatalogClientModule.java index a9ca664b2be0..8c44c87a487e 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/guice/CatalogClientModule.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/guice/CatalogClientModule.java @@ -29,6 +29,7 @@ import org.apache.druid.catalog.sync.CatalogClient; import org.apache.druid.catalog.sync.CatalogClientConfig; import org.apache.druid.catalog.sync.CatalogSource; +import org.apache.druid.catalog.sync.CatalogSqlTableWriter; import org.apache.druid.catalog.sync.CatalogUpdateListener; import org.apache.druid.catalog.sync.CatalogUpdateReceiver; import org.apache.druid.discovery.NodeRole; @@ -41,6 +42,7 @@ import org.apache.druid.guice.annotations.LoadScope; import org.apache.druid.initialization.DruidModule; import org.apache.druid.sql.calcite.planner.CatalogResolver; +import org.apache.druid.sql.calcite.planner.CatalogTableWriter; /** * Configures the metadata catalog on the Broker and Overlord to use a cache @@ -101,6 +103,12 @@ public void configure(Binder binder) .to(LiveCatalogResolver.class) .in(LazySingleton.class); + // Catalog writer for DDL statements. This will override the base binding. + binder + .bind(CatalogTableWriter.class) + .to(CatalogSqlTableWriter.class) + .in(LazySingleton.class); + // The listener resource sends to the catalog listener (the cached catalog.) Jerseys.addResource(binder, CatalogListenerResource.class); } diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/CatalogResource.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/CatalogResource.java index f96f042e7223..3d51cc179ebb 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/CatalogResource.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/CatalogResource.java @@ -129,6 +129,7 @@ public Response postTable( TableSpec spec, @QueryParam("version") long version, @QueryParam("overwrite") boolean overwrite, + @QueryParam("ifNotExists") boolean ifNotExists, @Context final HttpServletRequest req ) { @@ -159,6 +160,9 @@ public Response postTable( if (overwrite) { // User wants to overwrite, so do so. newVersion = catalog.tables().replace(table); + } else if (ifNotExists) { + // User asked to leave an existing table alone. Report no change, as the edit operations do. + newVersion = 0; } else { throw e; } diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogClient.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogClient.java index 1c3a2be487b7..7a0bf20ee783 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogClient.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogClient.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.ListenableFuture; import org.apache.druid.catalog.http.CatalogResource; +import org.apache.druid.catalog.http.TableEditRequest; import org.apache.druid.catalog.model.ResolvedTable; import org.apache.druid.catalog.model.TableDefnRegistry; import org.apache.druid.catalog.model.TableId; @@ -59,6 +60,7 @@ public class CatalogClient implements CatalogSource public static final String SCHEMA_SYNC_PATH = CatalogResource.ROOT_PATH + CatalogResource.SCHEMA_SYNC; public static final String TABLE_SYNC_PATH = CatalogResource.ROOT_PATH + CatalogResource.TABLE_SYNC; private static final String TABLE_CREATE_PATH = CatalogResource.ROOT_PATH + "/schemas/{schema}/tables/{name}"; + private static final String TABLE_EDIT_PATH = TABLE_CREATE_PATH + "/edit"; private final ServiceClient serviceClient; private final ObjectMapper jsonMapper; @@ -105,12 +107,38 @@ public ResolvedTable resolveTable(TableId id) /** * Creates a table for the given {@link TableId} and {@link TableSpec}. * If a table already exists for this id, it is overwritten. - *

- * This method is currently used only in tests. */ public void createTable(TableId tableId, TableSpec tableSpec) { - getResult(postCreateTable(tableId, tableSpec)); + getResult(postCreateTable(tableId, tableSpec, false, true)); + } + + /** + * Creates a table for the given {@link TableId} and {@link TableSpec}. + * + * @param ifNotExists leave an existing table alone rather than failing + * @param overwrite replace the spec of an existing table + */ + public void createTable(TableId tableId, TableSpec tableSpec, boolean ifNotExists, boolean overwrite) + { + FutureUtils.getUnchecked(postCreateTable(tableId, tableSpec, ifNotExists, overwrite), true); + } + + /** + * Applies an edit to an existing table's catalog entry. + *

+ * API: {@code POST /druid/coordinator/v1/catalog/schemas/{schema}/tables/{name}/edit} + */ + public void editTable(TableId tableId, TableEditRequest editRequest) + { + String path = tablePath(TABLE_EDIT_PATH, tableId); + FutureUtils.getUnchecked( + serviceClient.asyncRequest( + new RequestBuilder(HttpMethod.POST, path).jsonContent(jsonMapper, editRequest), + IgnoreHttpResponseHandler.INSTANCE + ), + true + ); } /** @@ -118,10 +146,19 @@ public void createTable(TableId tableId, TableSpec tableSpec) *

* API: {@code POST /druid/coordinator/v1/catalog/schemas/{schema}/tables/{name}} */ - private ListenableFuture postCreateTable(TableId tableId, TableSpec tableSpec) + private ListenableFuture postCreateTable( + TableId tableId, + TableSpec tableSpec, + boolean ifNotExists, + boolean overwrite + ) { - String path = StringUtils.replace(TABLE_CREATE_PATH, "{schema}", StringUtils.urlEncode(tableId.schema())); - path = StringUtils.replace(path, "{name}", StringUtils.urlEncode(tableId.name())); + final String path = StringUtils.format( + "%s?overwrite=%s&ifNotExists=%s", + tablePath(TABLE_CREATE_PATH, tableId), + overwrite, + ifNotExists + ); return serviceClient.asyncRequest( new RequestBuilder(HttpMethod.POST, path) @@ -130,6 +167,12 @@ private ListenableFuture postCreateTable(TableId tableId, TableSpec tableS ); } + private static String tablePath(String template, TableId tableId) + { + String path = StringUtils.replace(template, "{schema}", StringUtils.urlEncode(tableId.schema())); + return StringUtils.replace(path, "{name}", StringUtils.urlEncode(tableId.name())); + } + /** * Fetches the metadata of a table from the Coordinator. *

diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogSqlTableWriter.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogSqlTableWriter.java new file mode 100644 index 000000000000..c8813384d9b2 --- /dev/null +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogSqlTableWriter.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.catalog.sync; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Throwables; +import org.apache.druid.catalog.CatalogException; +import org.apache.druid.catalog.http.TableEditRequest; +import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.TableId; +import org.apache.druid.catalog.model.TableMetadata; +import org.apache.druid.catalog.model.TableSpec; +import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.rpc.HttpResponseException; +import org.apache.druid.sql.calcite.planner.CatalogTableWriter; + +import javax.annotation.Nullable; +import javax.inject.Inject; +import java.util.List; +import java.util.Map; + +/** + * Applies catalog DDL by calling the Coordinator, which owns catalog metadata. + *

+ * The Coordinator is authoritative for validation, so the errors it reports are unwrapped and presented as the + * statement's own error rather than as a failed HTTP call. + */ +public class CatalogSqlTableWriter implements CatalogTableWriter +{ + private static final Logger LOG = new Logger(CatalogSqlTableWriter.class); + + private final CatalogClient client; + private final CachedMetadataCatalog cache; + private final ObjectMapper jsonMapper; + + @Inject + public CatalogSqlTableWriter( + final CatalogClient client, + final CachedMetadataCatalog cache, + final ObjectMapper jsonMapper + ) + { + this.client = client; + this.cache = cache; + this.jsonMapper = jsonMapper; + } + + @Override + public void createTable(TableId tableId, TableSpec spec, boolean ifNotExists, boolean replace) + { + execute(tableId, () -> client.createTable(tableId, spec, ifNotExists, replace)); + } + + @Override + public void updateColumns(TableId tableId, List columns) + { + execute(tableId, () -> client.editTable(tableId, new TableEditRequest.UpdateColumns(columns))); + } + + @Override + public void dropColumns(TableId tableId, List columns) + { + execute(tableId, () -> client.editTable(tableId, new TableEditRequest.DropColumns(columns))); + } + + @Override + public void updateProperties(TableId tableId, Map properties) + { + execute(tableId, () -> client.editTable(tableId, new TableEditRequest.UpdateProperties(properties))); + } + + @Nullable + @Override + public TableMetadata readTable(TableId tableId) + { + return client.table(tableId); + } + + /** + * Run a catalog write, translate any failure into a statement error, then refresh this Broker's cache. + *

+ * The refresh matters because the Coordinator's update notification is asynchronous: without it, a CREATE TABLE + * followed immediately by an INSERT on the same connection could plan against the pre-DDL schema. Other Brokers + * still converge through the normal notification and polling path. + */ + private void execute(TableId tableId, Runnable operation) + { + try { + operation.run(); + } + catch (Exception e) { + throw translateError(tableId, e); + } + refreshCache(tableId); + } + + private void refreshCache(TableId tableId) + { + try { + final TableMetadata table = client.table(tableId); + cache.updated( + new UpdateEvent( + table == null ? UpdateEvent.EventType.DELETE : UpdateEvent.EventType.UPDATE, + table == null ? TableMetadata.empty(tableId) : table + ) + ); + } + catch (Exception e) { + // The write succeeded, so failing the statement here would be misleading. The cache converges on the next + // notification or poll; the only cost is that this Broker may briefly plan against the older spec. + LOG.warn(e, "Could not refresh the catalog cache for table[%s] after a DDL statement", tableId); + } + } + + /** + * Surface the Coordinator's own error message. Catalog validation lives there, so its wording is what explains + * why the statement was rejected; wrapping it in "server error" would bury it. + */ + private DruidException translateError(TableId tableId, Exception e) + { + final HttpResponseException httpError = findHttpResponseException(e); + if (httpError != null) { + final String message = errorMessage(httpError); + if (message != null) { + return DruidException.forPersona(DruidException.Persona.USER) + .ofCategory(DruidException.Category.INVALID_INPUT) + .build("%s", message); + } + } + if (e instanceof DruidException) { + return (DruidException) e; + } + return DruidException.forPersona(DruidException.Persona.USER) + .ofCategory(DruidException.Category.RUNTIME_FAILURE) + .build(e, "Could not update the catalog entry for table[%s]", tableId.name()); + } + + @Nullable + private String errorMessage(HttpResponseException e) + { + final String content = e.getResponse().getContent(); + if (content == null || content.isEmpty()) { + return null; + } + try { + final Map payload = jsonMapper.readValue(content, Map.class); + final Object message = payload.get(CatalogException.ERR_MSG_KEY); + return message == null ? null : message.toString(); + } + catch (Exception ignored) { + // Not a catalog error payload; fall back to the generic message. + return null; + } + } + + @Nullable + private static HttpResponseException findHttpResponseException(Throwable t) + { + for (Throwable cause : Throwables.getCausalChain(t)) { + if (cause instanceof HttpResponseException) { + return (HttpResponseException) cause; + } + } + return null; + } +} diff --git a/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/CatalogResourceTest.java b/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/CatalogResourceTest.java index 8f99eec43f56..623e53caeebd 100644 --- a/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/CatalogResourceTest.java +++ b/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/CatalogResourceTest.java @@ -102,48 +102,48 @@ public void testCreate() TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D").buildSpec(); // Blank schema name: infer the schema. - Response resp = resource.postTable("", tableName, dsSpec, 0, false, postBy(CatalogTests.SUPER_USER)); + Response resp = resource.postTable("", tableName, dsSpec, 0, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); // Missing table name - resp = resource.postTable(TableId.DRUID_SCHEMA, "", dsSpec, 0, false, postBy(CatalogTests.SUPER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, "", dsSpec, 0, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); // Invalid table name - resp = resource.postTable(TableId.DRUID_SCHEMA, " bogus ", dsSpec, 0, false, postBy(CatalogTests.SUPER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, " bogus ", dsSpec, 0, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); // Unknown schema - resp = resource.postTable("bogus", tableName, dsSpec, 0, false, postBy(CatalogTests.SUPER_USER)); + resp = resource.postTable("bogus", tableName, dsSpec, 0, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp.getStatus()); // Immutable schema - resp = resource.postTable(TableId.CATALOG_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.SUPER_USER)); + resp = resource.postTable(TableId.CATALOG_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); // Wrong definition type. - resp = resource.postTable(TableId.EXTERNAL_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.SUPER_USER)); + resp = resource.postTable(TableId.EXTERNAL_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); // No permissions assertThrows( ForbiddenException.class, - () -> resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.DENY_USER)) + () -> resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.DENY_USER)) ); // Read permission assertThrows( ForbiddenException.class, - () -> resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.READER_USER)) + () -> resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.READER_USER)) ); // Write permission - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); assertTrue(getVersion(resp) > 0); // Duplicate - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); // Invalid column type: table-level validation failures (which raise DruidException rather than IAE) must also @@ -151,7 +151,7 @@ public void testCreate() TableSpec badTypeSpec = TableBuilder.datasource("badType", "P1D") .column("foo", "FOO") .buildSpec(); - resp = resource.postTable(TableId.DRUID_SCHEMA, "badType", badTypeSpec, 0, false, postBy(CatalogTests.SUPER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, "badType", badTypeSpec, 0, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); // Inline input source @@ -162,11 +162,11 @@ public void testCreate() .column("b", Columns.STRING) .column("c", Columns.LONG) .buildSpec(); - resp = resource.postTable(TableId.EXTERNAL_SCHEMA, "inline", inputSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.EXTERNAL_SCHEMA, "inline", inputSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); // Wrong spec type - resp = resource.postTable(TableId.DRUID_SCHEMA, "invalid", inputSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, "invalid", inputSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); } @@ -187,36 +187,57 @@ public void testUpdate() TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D").buildSpec(); // Does not exist - Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 10, false, postBy(CatalogTests.SUPER_USER)); + Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 10, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp.getStatus()); // Create the table - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); long version = getVersion(resp); // No update permission assertThrows( ForbiddenException.class, - () -> resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.READER_USER)) + () -> resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.READER_USER)) ); // Out-of-date version - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 10, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 10, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp.getStatus()); // Valid version - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, version, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, version, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); assertTrue(getVersion(resp) > version); version = getVersion(resp); // Overwrite - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, true, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, true, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); assertTrue(getVersion(resp) > version); } + @Test + public void testCreateIfNotExists() + { + final String tableName = "ifNotExists"; + TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D").buildSpec(); + + Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, true, postBy(CatalogTests.WRITER_USER)); + assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + final long version = getVersion(resp); + assertTrue(version > 0); + + // Creating again reports no change rather than failing, and leaves the existing spec in place. + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, true, postBy(CatalogTests.WRITER_USER)); + assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + assertEquals(0, getVersion(resp)); + + // Without the flag, a duplicate is still an error. + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); + assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); + } + @Test public void testRead() { @@ -240,7 +261,7 @@ public void testRead() assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp.getStatus()); // Create the table - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); long version = getVersion(resp); @@ -323,7 +344,7 @@ public void testGetSchemas() // Create a table final String tableName = "list"; TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D").buildSpec(); - resp = resource.postTable(TableId.DRUID_SCHEMA, "list", dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, "list", dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); // Table paths - no read access @@ -370,7 +391,7 @@ public void testGetSchemaTables() // Create a table final String tableName = "list"; TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D").buildSpec(); - resp = resource.postTable(TableId.DRUID_SCHEMA, "list", dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, "list", dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); // No read access - name @@ -416,7 +437,7 @@ public void testSync() { final String tableName = "sync"; TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D").buildSpec(); - Response resp = resource.postTable(TableId.DRUID_SCHEMA, "list", dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + Response resp = resource.postTable(TableId.DRUID_SCHEMA, "list", dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); // Internal sync schema API @@ -456,7 +477,7 @@ public void testDelete() // Create the table TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D").buildSpec(); - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); // No write permission @@ -479,7 +500,7 @@ public void testLifecycle() // Operations for one table - create String table1Name = "lifecycle1"; TableSpec dsSpec = TableBuilder.datasource(table1Name, "P1D").buildSpec(); - Response resp = resource.postTable(TableId.DRUID_SCHEMA, table1Name, dsSpec, 0, true, postBy(CatalogTests.WRITER_USER)); + Response resp = resource.postTable(TableId.DRUID_SCHEMA, table1Name, dsSpec, 0, true, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); long version = getVersion(resp); @@ -507,7 +528,7 @@ public void testLifecycle() // update TableSpec table2Spec = TableBuilder.datasource(table1Name, "PT1H").buildSpec(); - resp = resource.postTable(TableId.DRUID_SCHEMA, table1Name, table2Spec, version, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, table1Name, table2Spec, version, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); assertTrue(getVersion(resp) > version); version = getVersion(resp); @@ -522,7 +543,7 @@ public void testLifecycle() // add second table String table2Name = "lifecycle2"; - resp = resource.postTable(TableId.DRUID_SCHEMA, table2Name, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, table2Name, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); TableId id2 = TableId.of(TableId.DRUID_SCHEMA, table2Name); @@ -568,7 +589,7 @@ public void testMoveColumn() .column("b", "BIGINT") .column("c", "FLOAT") .buildSpec(); - Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); long version = getVersion(resp); @@ -612,7 +633,7 @@ public void testHideColumns() String tableName = "hide"; TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D") .buildSpec(); - Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); long version = getVersion(resp); @@ -673,7 +694,7 @@ public void testDropColumns() .column("c", "FLOAT") .buildSpec(); - Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); long version = getVersion(resp); diff --git a/server/src/main/java/org/apache/druid/catalog/model/ColumnSpec.java b/server/src/main/java/org/apache/druid/catalog/model/ColumnSpec.java index cfc9fc55cf43..23f333b658da 100644 --- a/server/src/main/java/org/apache/druid/catalog/model/ColumnSpec.java +++ b/server/src/main/java/org/apache/druid/catalog/model/ColumnSpec.java @@ -27,6 +27,7 @@ import org.apache.druid.catalog.model.ModelProperties.PropertyDefn; import org.apache.druid.guice.annotations.UnstableApi; import org.apache.druid.java.util.common.IAE; +import org.apache.druid.segment.column.ColumnType; import javax.annotation.Nullable; @@ -104,11 +105,15 @@ public void validate() throw new IAE("Column name is required"); } if (Columns.isTimeColumn(name)) { - if (dataType != null && !Columns.LONG.equalsIgnoreCase(dataType)) { + // Any spelling that resolves to a LONG is fine: the time column is written as TIMESTAMP in SQL and as LONG + // natively, and both mean the same stored type. + if (dataType != null && !ColumnType.LONG.equals(Columns.druidTypeFromString(dataType))) { throw new IAE( - "[%s] column must have type [%s] or no type. Found [%s]", + "[%s] column must have a type that resolves to [%s], such as [%s] or [%s], or no type. Found [%s]", name, Columns.LONG, + Columns.SQL_TIMESTAMP, + Columns.SQL_BIGINT, dataType ); } diff --git a/server/src/main/java/org/apache/druid/catalog/model/Columns.java b/server/src/main/java/org/apache/druid/catalog/model/Columns.java index 9d64a4ed22e9..9be0d4ab3122 100644 --- a/server/src/main/java/org/apache/druid/catalog/model/Columns.java +++ b/server/src/main/java/org/apache/druid/catalog/model/Columns.java @@ -25,6 +25,7 @@ import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.ValueType; +import javax.annotation.Nullable; import java.util.List; import java.util.Map; @@ -76,12 +77,25 @@ private Columns() { } + /** + * The Druid type of a column, which for {@code __time} is always {@link ColumnType#LONG} whatever was declared. + * Use {@link #druidTypeFromString} to resolve a declared type on its own terms. + */ public static ColumnType druidType(ColumnSpec spec) { if (isTimeColumn(spec.name())) { return ColumnType.LONG; } - String dataType = spec.dataType(); + return druidTypeFromString(spec.dataType()); + } + + /** + * Resolve a declared type string to its Druid type: a SQL type name such as {@code BIGINT} or {@code VARCHAR ARRAY}, + * or a native type string such as {@code COMPLEX}. Returns null if the string is null or does not name a type. + */ + @Nullable + public static ColumnType druidTypeFromString(@Nullable String dataType) + { if (dataType == null) { return null; } diff --git a/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java b/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java index ba32f61de7fc..66c956876536 100644 --- a/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java +++ b/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java @@ -521,9 +521,19 @@ public void testTimeColumn() assertSame(ColumnType.LONG, col.druidType()); } - { + // Any spelling that resolves to a LONG is accepted: TIMESTAMP is how SQL names the time column, BIGINT is its + // SQL storage type, and LONG is the native name. + for (String timeType : new String[]{Columns.SQL_TIMESTAMP, Columns.SQL_BIGINT, Columns.LONG, "long"}) { + TableSpec spec = builder.copy() + .column(Columns.TIME_COLUMN, timeType) + .buildSpec(); + registry.resolve(spec).validate(); + } + + // Types that do not resolve to a LONG are rejected, as are types that do not resolve at all. + for (String badType : new String[]{Columns.STRING, Columns.SQL_VARCHAR, Columns.SQL_DOUBLE, "NOT_A_TYPE"}) { TableSpec spec = builder.copy() - .column(Columns.TIME_COLUMN, Columns.STRING) + .column(Columns.TIME_COLUMN, badType) .buildSpec(); expectValidationFails(spec); } diff --git a/sql/src/main/codegen/config.fmpp b/sql/src/main/codegen/config.fmpp index 837c04f42a63..714d8af28f0d 100644 --- a/sql/src/main/codegen/config.fmpp +++ b/sql/src/main/codegen/config.fmpp @@ -50,14 +50,19 @@ data: { imports: [ "java.util.List" "org.apache.calcite.sql.SqlNode" + "org.apache.calcite.sql.SqlCreate" "org.apache.calcite.sql.SqlInsert" "org.apache.calcite.sql.SqlNodeList" "org.apache.calcite.sql.SqlBasicCall" "org.apache.druid.java.util.common.granularity.Granularity" "org.apache.druid.java.util.common.granularity.GranularityType" "org.apache.druid.java.util.common.granularity.Granularities" + "org.apache.druid.sql.calcite.parser.DruidSqlAlterTable" + "org.apache.druid.sql.calcite.parser.DruidSqlColumnDeclaration" + "org.apache.druid.sql.calcite.parser.DruidSqlCreateTable" "org.apache.druid.sql.calcite.parser.DruidSqlInsert" "org.apache.druid.sql.calcite.parser.DruidSqlParserUtils" + "org.apache.druid.sql.calcite.parser.DruidSqlPropertyAssignment" "org.apache.druid.sql.calcite.external.ExtendOperator" "org.apache.druid.sql.calcite.external.ParameterizeOperator" "org.apache.druid.sql.calcite.parser.ExternalDestinationSqlIdentifier" @@ -71,11 +76,15 @@ data: { "OVERWRITE" "PARTITIONED" "EXTERN" + "IF" + "PROPERTIES" ] nonReservedKeywordsToAdd: [ "OVERWRITE" "EXTERN" + "IF" + "PROPERTIES" ] # List of methods for parsing custom SQL statements. @@ -85,6 +94,14 @@ data: { "DruidSqlInsertEof()" "DruidSqlExplain()" "DruidSqlReplaceEof()" + "DruidSqlAlterTable()" + ] + + # CREATE reaches DruidSqlCreateTable through Calcite's stock SqlCreate() production, which supplies the + # (Span, boolean replace) arguments. DROP TABLE is deliberately absent: catalog-only deletion would be a + # surprising meaning for the statement, so it is left unclaimed until the semantics are settled. + createStatementParserMethods: [ + "DruidSqlCreateTable" ] # List of methods for parsing custom data types. @@ -100,6 +117,7 @@ data: { # "dataTypeParserMethods". implementationFiles: [ "common.ftl" + "ddl.ftl" "explain.ftl" "replace.ftl" ] diff --git a/sql/src/main/codegen/includes/ddl.ftl b/sql/src/main/codegen/includes/ddl.ftl new file mode 100644 index 000000000000..46fa5e3d0059 --- /dev/null +++ b/sql/src/main/codegen/includes/ddl.ftl @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Druid catalog DDL. These statements write catalog metadata only; they do not create or delete data. +// +// CREATE TABLE is reached through the standard Calcite SqlCreate() production, which has already consumed +// CREATE [OR REPLACE], so this production must not consume : the enclosing SqlStmtList() handles statement +// separators. ALTER TABLE is a top-level statement production instead, because Calcite's stock SqlAlter() mandates +// a SYSTEM or SESSION scope that does not apply here. + +SqlCreate DruidSqlCreateTable(Span s, boolean replace) : +{ + boolean ifNotExists = false; + final SqlIdentifier id; + SqlNodeList columnList = SqlNodeList.EMPTY; + SqlGranularityLiteral partitionedBy = null; + SqlNodeList clusteredBy = null; +} +{ +

+ [ { ifNotExists = true; } ] + id = CompoundTableIdentifier() + [ columnList = DruidColumnDeclarationList() ] + [ + + partitionedBy = PartitionGranularity() + ] + [ + clusteredBy = ClusteredBy() + ] + { + return new DruidSqlCreateTable(s.end(this), replace, ifNotExists, id, columnList, partitionedBy, clusteredBy); + } +} + +SqlNodeList DruidColumnDeclarationList() : +{ + final Span s; + final List list = new ArrayList(); +} +{ + { s = span(); } + AddDruidColumnDeclaration(list) + ( + AddDruidColumnDeclaration(list) + )* + + { + return new SqlNodeList(list, s.end(this)); + } +} + +void AddDruidColumnDeclaration(List list) : +{ + final DruidSqlColumnDeclaration column; +} +{ + column = DruidColumnDeclaration() + { + list.add(column); + } +} + +DruidSqlColumnDeclaration DruidColumnDeclaration() : +{ + final SqlIdentifier name; + final SqlDataTypeSpec dataType; +} +{ + name = SimpleIdentifier() + dataType = DataType() + // Nullability is accepted but ignored: all Druid columns are nullable, matching the EXTEND clause of INSERT. + [ | ] + { + return new DruidSqlColumnDeclaration( + name.getParserPosition().plus(dataType.getParserPosition()), + name, + dataType + ); + } +} + +SqlNode DruidSqlAlterTable() : +{ + final Span s; + final SqlIdentifier id; + final SqlIdentifier columnName; + final SqlDataTypeSpec dataType; + final DruidSqlColumnDeclaration column; + final SqlNodeList properties; +} +{ + { s = span(); }
id = CompoundTableIdentifier() + ( + column = DruidColumnDeclaration() + { + return new DruidSqlAlterTable.AddColumn(s.end(this), id, column); + } + | + columnName = SimpleIdentifier() + { + return new DruidSqlAlterTable.DropColumn(s.end(this), id, columnName); + } + | + columnName = SimpleIdentifier() dataType = DataType() + { + return new DruidSqlAlterTable.AlterColumn( + s.end(this), + id, + new DruidSqlColumnDeclaration( + columnName.getParserPosition().plus(dataType.getParserPosition()), + columnName, + dataType + ) + ); + } + | + properties = DruidPropertyList() + { + return new DruidSqlAlterTable.SetProperties(s.end(this), id, properties); + } + ) +} + +SqlNodeList DruidPropertyList() : +{ + final Span s; + final List list = new ArrayList(); +} +{ + { s = span(); } + AddDruidPropertyAssignment(list) + ( + AddDruidPropertyAssignment(list) + )* + + { + return new SqlNodeList(list, s.end(this)); + } +} + +void AddDruidPropertyAssignment(List list) : +{ + final SqlIdentifier key; + final SqlNode value; +} +{ + key = SimpleIdentifier() value = Literal() + { + list.add( + new DruidSqlPropertyAssignment( + key.getParserPosition().plus(value.getParserPosition()), + key, + value + ) + ); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/external/Externals.java b/sql/src/main/java/org/apache/druid/sql/calcite/external/Externals.java index 53908dbc4705..dd29e5e0d376 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/external/Externals.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/external/Externals.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; -import org.apache.calcite.avatica.SqlType; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.schema.FunctionParameter; @@ -32,10 +31,8 @@ import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlOperandCountRange; import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlTypeNameSpec; import org.apache.calcite.sql.type.SqlOperandCountRanges; import org.apache.calcite.sql.type.SqlOperandMetadata; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.catalog.model.ColumnSpec; import org.apache.druid.catalog.model.table.ExternalTableSpec; import org.apache.druid.catalog.model.table.TableFunction; @@ -48,6 +45,7 @@ import org.apache.druid.server.security.Resource; import org.apache.druid.server.security.ResourceAction; import org.apache.druid.server.security.ResourceType; +import org.apache.druid.sql.calcite.planner.CatalogColumnTypes; import org.apache.druid.sql.calcite.planner.DruidTypeSystem; import org.apache.druid.sql.calcite.table.ExternalTable; @@ -232,7 +230,7 @@ public static List convertColumns(SqlNodeList schema) final List columns = new ArrayList<>(); for (int i = 0; i < schema.size(); i += 2) { final String name = convertName((SqlIdentifier) schema.get(i)); - final String sqlType = convertType(name, (SqlDataTypeSpec) schema.get(i + 1)); + final String sqlType = CatalogColumnTypes.forExternalColumn(name, (SqlDataTypeSpec) schema.get(i + 1)); columns.add(new ColumnSpec(name, sqlType, null)); } return columns; @@ -254,66 +252,6 @@ private static String convertName(SqlIdentifier ident) return ident.getSimple(); } - /** - * Define the SQL input column type from a type provided in the - * EXTEND clause. Calcite allows any form of type. But, Druid - * requires only the Druid supported types (and their aliases.) - *

- * Druid has its own rules for nullability. We ignore any nullability - * clause in the EXTEND list. - */ - private static String convertType(String name, SqlDataTypeSpec dataType) - { - SqlTypeNameSpec spec = dataType.getTypeNameSpec(); - if (spec == null) { - throw unsupportedType(name, dataType); - } - SqlIdentifier typeNameIdentifier = spec.getTypeName(); - if (typeNameIdentifier == null || !typeNameIdentifier.isSimple()) { - throw unsupportedType(name, dataType); - } - String simpleName = typeNameIdentifier.getSimple(); - if (StringUtils.toLowerCase(simpleName).startsWith("complex<")) { - // Parse and validate rather than passing the raw string downstream, where a malformed type string would - // silently resolve to a different type; return the canonical form. - final ColumnType complexType = ColumnType.fromString(simpleName); - if (complexType == null) { - throw unsupportedType(name, dataType); - } - return complexType.asTypeString(); - } - SqlTypeName type = SqlTypeName.get(simpleName); - if (type == null) { - throw unsupportedType(name, dataType); - } - if (SqlTypeName.CHAR_TYPES.contains(type)) { - return SqlTypeName.VARCHAR.name(); - } - if (SqlTypeName.INT_TYPES.contains(type)) { - return SqlTypeName.BIGINT.name(); - } - switch (type) { - case DOUBLE: - return SqlType.DOUBLE.name(); - case FLOAT: - case REAL: - return SqlType.FLOAT.name(); - case ARRAY: - return convertType(name, dataType.getComponentTypeSpec()) + " " + SqlType.ARRAY.name(); - default: - throw unsupportedType(name, dataType); - } - } - - private static RuntimeException unsupportedType(String name, SqlDataTypeSpec dataType) - { - return new IAE(StringUtils.format( - "Column [%s] has an unsupported type: [%s]", - name, - dataType - )); - } - /** * Create an MSQ ExternalTable given an external table spec. Enforces type restructions * (which should be revisited.) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlAlterTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlAlterTable.java new file mode 100644 index 000000000000..bf6c20f4a604 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlAlterTable.java @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.parser; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.util.ImmutableNullableList; + +import javax.annotation.Nonnull; +import java.util.List; + +/** + * {@code ALTER TABLE }, which edits the catalog metadata of an existing table. + *

+ * Each concrete subclass corresponds to exactly one catalog edit operation, so that a single statement is always a + * single atomic change on the Coordinator. See {@link org.apache.druid.sql.calcite.planner.CatalogDdlHandler}. + */ +public abstract class DruidSqlAlterTable extends SqlCall +{ + private final SqlIdentifier name; + + protected DruidSqlAlterTable(SqlParserPos pos, SqlIdentifier name) + { + super(pos); + this.name = name; + } + + public SqlIdentifier getName() + { + return name; + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("ALTER TABLE"); + name.unparse(writer, leftPrec, rightPrec); + unparseOperation(writer, leftPrec, rightPrec); + } + + /** + * Unparse the portion of the statement following the table name. + */ + protected abstract void unparseOperation(SqlWriter writer, int leftPrec, int rightPrec); + + /** + * {@code ALTER TABLE ADD COLUMN }. + */ + public static class AddColumn extends DruidSqlAlterTable + { + public static final SqlOperator OPERATOR = new Operator("ALTER TABLE ADD COLUMN"); + + private final DruidSqlColumnDeclaration column; + + public AddColumn(SqlParserPos pos, SqlIdentifier name, DruidSqlColumnDeclaration column) + { + super(pos, name); + this.column = column; + } + + public DruidSqlColumnDeclaration getColumn() + { + return column; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of(getName(), column); + } + + @Override + protected void unparseOperation(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("ADD COLUMN"); + column.unparse(writer, 0, 0); + } + + private static class Operator extends SqlSpecialOperator + { + Operator(String name) + { + super(name, SqlKind.ALTER_TABLE); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new AddColumn(pos, (SqlIdentifier) operands[0], (DruidSqlColumnDeclaration) operands[1]); + } + } + } + + /** + * {@code ALTER TABLE DROP COLUMN }. Removes the column from the catalog spec; existing segments are + * unaffected. + */ + public static class DropColumn extends DruidSqlAlterTable + { + public static final SqlOperator OPERATOR = new Operator("ALTER TABLE DROP COLUMN"); + + private final SqlIdentifier column; + + public DropColumn(SqlParserPos pos, SqlIdentifier name, SqlIdentifier column) + { + super(pos, name); + this.column = column; + } + + public SqlIdentifier getColumn() + { + return column; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of(getName(), column); + } + + @Override + protected void unparseOperation(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("DROP COLUMN"); + column.unparse(writer, 0, 0); + } + + private static class Operator extends SqlSpecialOperator + { + Operator(String name) + { + super(name, SqlKind.ALTER_TABLE); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new DropColumn(pos, (SqlIdentifier) operands[0], (SqlIdentifier) operands[1]); + } + } + } + + /** + * {@code ALTER TABLE ALTER COLUMN SET DATA TYPE }. + */ + public static class AlterColumn extends DruidSqlAlterTable + { + public static final SqlOperator OPERATOR = new Operator("ALTER TABLE ALTER COLUMN"); + + private final DruidSqlColumnDeclaration column; + + public AlterColumn(SqlParserPos pos, SqlIdentifier name, DruidSqlColumnDeclaration column) + { + super(pos, name); + this.column = column; + } + + public DruidSqlColumnDeclaration getColumn() + { + return column; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of(getName(), column); + } + + @Override + protected void unparseOperation(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("ALTER COLUMN"); + column.getName().unparse(writer, 0, 0); + writer.keyword("SET DATA TYPE"); + DruidSqlColumnDeclaration.unparseDataType(writer, column.getDataType()); + } + + private static class Operator extends SqlSpecialOperator + { + Operator(String name) + { + super(name, SqlKind.ALTER_TABLE); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new AlterColumn(pos, (SqlIdentifier) operands[0], (DruidSqlColumnDeclaration) operands[1]); + } + } + } + + /** + * {@code ALTER TABLE SET PROPERTIES ( = , ...)}. A {@code NULL} value removes the property. + */ + public static class SetProperties extends DruidSqlAlterTable + { + public static final SqlOperator OPERATOR = new Operator("ALTER TABLE SET PROPERTIES"); + + private final SqlNodeList properties; + + public SetProperties(SqlParserPos pos, SqlIdentifier name, SqlNodeList properties) + { + super(pos, name); + this.properties = properties; + } + + /** + * The property assignments, each a {@link DruidSqlPropertyAssignment}. + */ + public SqlNodeList getProperties() + { + return properties; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of(getName(), properties); + } + + @Override + protected void unparseOperation(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("SET PROPERTIES"); + final SqlWriter.Frame frame = writer.startList("(", ")"); + for (SqlNode property : properties) { + writer.sep(","); + property.unparse(writer, 0, 0); + } + writer.endList(frame); + } + + private static class Operator extends SqlSpecialOperator + { + Operator(String name) + { + super(name, SqlKind.ALTER_TABLE); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new SetProperties(pos, (SqlIdentifier) operands[0], (SqlNodeList) operands[1]); + } + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlColumnDeclaration.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlColumnDeclaration.java new file mode 100644 index 000000000000..de700e8d66e3 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlColumnDeclaration.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.parser; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.util.ImmutableNullableList; + +import javax.annotation.Nonnull; +import java.util.List; + +/** + * A single {@code } column declaration within a Druid DDL statement, such as the column list of + * {@code CREATE TABLE} or the target of {@code ALTER TABLE ... ADD COLUMN}. + */ +public class DruidSqlColumnDeclaration extends SqlCall +{ + public static final SqlOperator OPERATOR = new DruidSqlColumnDeclarationOperator(); + + /** + * The {@code TYPE('...')} escape hatch used to name Druid native types that have no SQL spelling, defined by the + * {@code DruidType()} production in {@code common.ftl}. + */ + public static final String COMPLEX_TYPE_FUNCTION = "TYPE"; + + private final SqlIdentifier name; + private final SqlDataTypeSpec dataType; + + public DruidSqlColumnDeclaration( + SqlParserPos pos, + SqlIdentifier name, + SqlDataTypeSpec dataType + ) + { + super(pos); + this.name = name; + this.dataType = dataType; + } + + public SqlIdentifier getName() + { + return name; + } + + public SqlDataTypeSpec getDataType() + { + return dataType; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of(name, dataType); + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) + { + name.unparse(writer, 0, 0); + unparseDataType(writer, dataType); + } + + /** + * Unparse a column type. Types named by {@link #COMPLEX_TYPE_FUNCTION} (the {@code TYPE('...')} escape hatch used + * for Druid native types such as {@code COMPLEX}) are written back in that form, since + * {@link SqlUserDefinedTypeNameSpec} would otherwise emit the bare name, which does not parse. + */ + public static void unparseDataType(SqlWriter writer, SqlDataTypeSpec dataType) + { + if (dataType.getTypeNameSpec() instanceof SqlUserDefinedTypeNameSpec) { + writer.keyword(COMPLEX_TYPE_FUNCTION); + final SqlWriter.Frame frame = writer.startList(SqlWriter.FrameTypeEnum.FUN_CALL, "(", ")"); + SqlLiteral.createCharString(dataType.getTypeName().toString(), dataType.getParserPosition()) + .unparse(writer, 0, 0); + writer.endList(frame); + } else { + dataType.unparse(writer, 0, 0); + } + } + + private static class DruidSqlColumnDeclarationOperator extends SqlSpecialOperator + { + public DruidSqlColumnDeclarationOperator() + { + super("COLUMN_DECL", SqlKind.COLUMN_DECL); + } + + @Override + public SqlCall createCall( + SqlLiteral functionQualifier, + SqlParserPos pos, + SqlNode... operands + ) + { + return new DruidSqlColumnDeclaration(pos, (SqlIdentifier) operands[0], (SqlDataTypeSpec) operands[1]); + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java new file mode 100644 index 000000000000..91e3ebfd2ae3 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.parser; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlCreate; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.util.ImmutableNullableList; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; + +/** + * {@code CREATE [OR REPLACE] TABLE [IF NOT EXISTS] () [PARTITIONED BY ] + * [CLUSTERED BY ]}, which defines a table in the Druid catalog. + *

+ * This statement writes catalog metadata only; it neither creates segments nor otherwise touches data. See + * {@link org.apache.druid.sql.calcite.planner.CatalogDdlHandler} for the execution side. + */ +public class DruidSqlCreateTable extends SqlCreate +{ + public static final SqlOperator OPERATOR = new DruidSqlCreateTableOperator(); + + private final SqlIdentifier name; + private final SqlNodeList columnList; + @Nullable + private final SqlGranularityLiteral partitionedBy; + @Nullable + private final SqlNodeList clusteredBy; + + public DruidSqlCreateTable( + SqlParserPos pos, + boolean replace, + boolean ifNotExists, + SqlIdentifier name, + SqlNodeList columnList, + @Nullable SqlGranularityLiteral partitionedBy, + @Nullable SqlNodeList clusteredBy + ) + { + super(OPERATOR, pos, replace, ifNotExists); + this.name = name; + this.columnList = columnList; + this.partitionedBy = partitionedBy; + this.clusteredBy = clusteredBy; + } + + public SqlIdentifier getName() + { + return name; + } + + /** + * The declared columns, each a {@link DruidSqlColumnDeclaration}. Order is significant: it is the order columns are + * recorded in the catalog table spec. + */ + public SqlNodeList getColumnList() + { + return columnList; + } + + @Nullable + public SqlGranularityLiteral getPartitionedBy() + { + return partitionedBy; + } + + @Nullable + public SqlNodeList getClusteredBy() + { + return clusteredBy; + } + + public boolean isIfNotExists() + { + return ifNotExists; + } + + @Nonnull + @Override + public List getOperandList() + { + // The replace and ifNotExists flags travel as operands so that createCall() can rebuild an equivalent node. + return ImmutableNullableList.of( + name, + columnList, + partitionedBy, + clusteredBy, + SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), + SqlLiteral.createBoolean(ifNotExists, SqlParserPos.ZERO) + ); + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("CREATE"); + if (getReplace()) { + writer.keyword("OR REPLACE"); + } + writer.keyword("TABLE"); + if (ifNotExists) { + writer.keyword("IF NOT EXISTS"); + } + name.unparse(writer, leftPrec, rightPrec); + + final SqlWriter.Frame frame = writer.startList("(", ")"); + for (SqlNode column : columnList) { + writer.sep(","); + column.unparse(writer, 0, 0); + } + writer.endList(frame); + + if (partitionedBy != null) { + writer.keyword("PARTITIONED BY"); + partitionedBy.unparse(writer, 0, 0); + } + + if (clusteredBy != null) { + writer.keyword("CLUSTERED BY"); + final SqlWriter.Frame clusterFrame = writer.startList("", ""); + for (SqlNode clusterByOpts : clusteredBy.getList()) { + writer.sep(","); + clusterByOpts.unparse(writer, leftPrec, rightPrec); + } + writer.endList(clusterFrame); + } + } + + private static class DruidSqlCreateTableOperator extends SqlSpecialOperator + { + public DruidSqlCreateTableOperator() + { + super("CREATE TABLE", SqlKind.CREATE_TABLE); + } + + @Override + public SqlCall createCall( + SqlLiteral functionQualifier, + SqlParserPos pos, + SqlNode... operands + ) + { + return new DruidSqlCreateTable( + pos, + ((SqlLiteral) operands[4]).booleanValue(), + ((SqlLiteral) operands[5]).booleanValue(), + (SqlIdentifier) operands[0], + (SqlNodeList) operands[1], + (SqlGranularityLiteral) operands[2], + (SqlNodeList) operands[3] + ); + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlParser.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlParser.java index ab9e9e1878b9..da2fccb62d41 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlParser.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlParser.java @@ -149,6 +149,17 @@ private static StatementAndSetContext processStatementList( */ @Nullable static Object sqlLiteralToContextValue(final SqlLiteral literal) + { + return sqlLiteralToJavaValue(literal, "SET"); + } + + /** + * Coerces a SQL literal to a plain Java value, as used for query context entries and catalog table properties. + * + * @param what the clause the literal came from, named in the error message if its type has no Java equivalent + */ + @Nullable + public static Object sqlLiteralToJavaValue(final SqlLiteral literal, final String what) { if (SqlUtil.isNullLiteral(literal, false)) { return null; @@ -172,7 +183,7 @@ static Object sqlLiteralToContextValue(final SqlLiteral literal) } else if (literal.getTypeName() == SqlTypeName.TIMESTAMP) { return Calcites.CALCITE_TIMESTAMP_PARSER.parse(literal.getValue().toString()).toString(); } else { - throw InvalidSqlInput.exception("Unsupported type for SET[%s]", literal.getTypeName()); + throw InvalidSqlInput.exception("Unsupported type for %s[%s]", what, literal.getTypeName()); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlPropertyAssignment.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlPropertyAssignment.java new file mode 100644 index 000000000000..e24f069a2904 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlPropertyAssignment.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.parser; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.util.ImmutableNullableList; + +import javax.annotation.Nonnull; +import java.util.List; + +/** + * A single {@code = } pair within {@code ALTER TABLE ... SET PROPERTIES (...)}. The value is a literal; + * a {@code NULL} literal means "remove this property". + */ +public class DruidSqlPropertyAssignment extends SqlCall +{ + public static final SqlOperator OPERATOR = new DruidSqlPropertyAssignmentOperator(); + + private final SqlIdentifier key; + private final SqlNode value; + + public DruidSqlPropertyAssignment(SqlParserPos pos, SqlIdentifier key, SqlNode value) + { + super(pos); + this.key = key; + this.value = value; + } + + public SqlIdentifier getKey() + { + return key; + } + + /** + * The assigned value. Normally a {@link SqlLiteral}, but a multi-line string literal parses as a concatenation + * call, so callers must handle a non-literal node rather than assume the cast succeeds. + */ + public SqlNode getValue() + { + return value; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of(key, value); + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) + { + key.unparse(writer, 0, 0); + writer.keyword("="); + value.unparse(writer, 0, 0); + } + + private static class DruidSqlPropertyAssignmentOperator extends SqlSpecialOperator + { + public DruidSqlPropertyAssignmentOperator() + { + super("PROPERTY_ASSIGNMENT", SqlKind.OTHER); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new DruidSqlPropertyAssignment(pos, (SqlIdentifier) operands[0], operands[1]); + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogColumnTypes.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogColumnTypes.java new file mode 100644 index 000000000000..e7f3fa3e4bcd --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogColumnTypes.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.planner; + +import org.apache.calcite.avatica.SqlType; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlTypeNameSpec; +import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.druid.catalog.model.Columns; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.segment.column.ColumnType; + +/** + * Converts a SQL type as written in a statement into the type string stored in the catalog. Calcite accepts any + * type spelling; Druid accepts only its own supported types and their aliases, plus the {@code TYPE('...')} escape + * hatch for native type strings that have no SQL spelling, such as {@code COMPLEX}. + *

+ * Druid has its own rules for nullability, so any nullability clause is ignored. + */ +public class CatalogColumnTypes +{ + /** + * Which statement the type was written in. The two differ in what they accept, so they are named rather than + * flagged. + */ + private enum Target + { + /** + * The {@code EXTEND} clause, describing columns read from an external input source. Those are read as their + * underlying storage type, so {@code TIMESTAMP} is not among them, and the escape hatch is limited to complex + * types. + */ + EXTERNAL, + + /** + * A catalog DDL statement, which additionally accepts {@code TIMESTAMP} (how {@code __time} is spelled in SQL) + * and any native type string through the escape hatch. + */ + CATALOG + } + + private CatalogColumnTypes() + { + // No instantiation. + } + + public static String forExternalColumn(String name, SqlDataTypeSpec dataType) + { + return convert(name, dataType, Target.EXTERNAL); + } + + public static String forCatalogColumn(String name, SqlDataTypeSpec dataType) + { + final String typeString = convert(name, dataType, Target.CATALOG); + // the catalog rejects unparseable types at write time, but catching it here attributes the error to the statement + // rather than to a Coordinator round trip. + if (Columns.druidTypeFromString(typeString) == null) { + throw unsupportedType(name, dataType); + } + return typeString; + } + + private static String convert(String name, SqlDataTypeSpec dataType, Target target) + { + final SqlTypeNameSpec spec = dataType.getTypeNameSpec(); + if (spec == null) { + throw unsupportedType(name, dataType); + } + final SqlIdentifier typeNameIdentifier = spec.getTypeName(); + if (typeNameIdentifier == null || !typeNameIdentifier.isSimple()) { + throw unsupportedType(name, dataType); + } + final String simpleName = typeNameIdentifier.getSimple(); + + if (spec instanceof SqlUserDefinedTypeNameSpec) { + // The TYPE('...') escape hatch names a Druid native type. Parse and validate rather than passing the raw + // string downstream, where a malformed type string would silently resolve to a different type, and return the + // canonical form. + if (target == Target.EXTERNAL && !StringUtils.toLowerCase(simpleName).startsWith("complex<")) { + throw unsupportedType(name, dataType); + } + final ColumnType nativeType = ColumnType.fromString(simpleName); + if (nativeType == null) { + throw unsupportedType(name, dataType); + } + return nativeType.asTypeString(); + } + + final SqlTypeName type = SqlTypeName.get(simpleName); + if (type == null) { + throw unsupportedType(name, dataType); + } + if (SqlTypeName.CHAR_TYPES.contains(type)) { + return SqlTypeName.VARCHAR.name(); + } + if (SqlTypeName.INT_TYPES.contains(type)) { + return SqlTypeName.BIGINT.name(); + } + switch (type) { + case DOUBLE: + return SqlType.DOUBLE.name(); + case FLOAT: + case REAL: + return SqlType.FLOAT.name(); + case ARRAY: + return convert(name, dataType.getComponentTypeSpec(), target) + " " + SqlType.ARRAY.name(); + case TIMESTAMP: + if (target == Target.CATALOG) { + return Columns.SQL_TIMESTAMP; + } + throw unsupportedType(name, dataType); + default: + throw unsupportedType(name, dataType); + } + } + + private static RuntimeException unsupportedType(String name, SqlDataTypeSpec dataType) + { + return new IAE(StringUtils.format( + "Column [%s] has an unsupported type: [%s]", + name, + dataType + )); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java new file mode 100644 index 000000000000..bc3a36f11507 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java @@ -0,0 +1,505 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.planner; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.Columns; +import org.apache.druid.catalog.model.TableId; +import org.apache.druid.catalog.model.TableMetadata; +import org.apache.druid.catalog.model.TableSpec; +import org.apache.druid.catalog.model.table.ClusterKeySpec; +import org.apache.druid.catalog.model.table.DatasourceDefn; +import org.apache.druid.common.utils.IdUtils; +import org.apache.druid.error.DruidException; +import org.apache.druid.error.InvalidSqlInput; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.granularity.Granularity; +import org.apache.druid.java.util.common.granularity.PeriodGranularity; +import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.query.explain.ExplainAttributes; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.server.QueryResponse; +import org.apache.druid.server.security.Action; +import org.apache.druid.server.security.Resource; +import org.apache.druid.server.security.ResourceAction; +import org.apache.druid.server.security.ResourceType; +import org.apache.druid.sql.calcite.parser.DruidSqlAlterTable; +import org.apache.druid.sql.calcite.parser.DruidSqlColumnDeclaration; +import org.apache.druid.sql.calcite.parser.DruidSqlCreateTable; +import org.apache.druid.sql.calcite.parser.DruidSqlParser; +import org.apache.druid.sql.calcite.parser.DruidSqlPropertyAssignment; +import org.apache.druid.sql.calcite.parser.SqlGranularityLiteral; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Handles the catalog DDL statements: {@code CREATE TABLE} and {@code ALTER TABLE}. + *

+ * These statements are metadata operations, not queries. They are validated here, converted to a catalog + * {@link TableSpec} or column/property edit, and applied through {@link CatalogTableWriter}, which forwards them to + * the Coordinator. No Calcite validation or query planning takes place, and no rows are returned. + *

+ * Validation is deliberately split. This class checks what it can attribute to a position in the statement (type + * spellings, duplicate columns, granularity, clustering) so that the error names the offending SQL. The Coordinator + * remains authoritative: {@code DatasourceDefn.validate} runs on write, and its message is surfaced verbatim. + */ +public abstract class CatalogDdlHandler extends SqlStatementHandler.BaseStatementHandler +{ + /** + * DDL produces no rows. A single-column type still has to be declared, because JDBC clients ask for a result set + * signature when preparing the statement. + */ + private static final RelDataType RESULT_TYPE = resultType(); + + protected final SqlIdentifier tableIdentifier; + protected TableId tableId; + + protected CatalogDdlHandler(SqlStatementHandler.HandlerContext handlerContext, SqlIdentifier tableIdentifier) + { + super(handlerContext); + this.tableIdentifier = tableIdentifier; + } + + /** + * The runtime property that gates these statements. Read from {@link PlannerConfig} rather than the query context + * so that a user cannot turn the feature on for their own statement. + */ + public static final String ENABLE_CATALOG_DDL_PROPERTY = "druid.sql.planner.enableCatalogDdl"; + + @Override + public void validate() + { + if (!handlerContext.plannerContext().getPlannerConfig().isEnableCatalogDdl()) { + throw DruidException.forPersona(DruidException.Persona.ADMIN) + .ofCategory(DruidException.Category.UNSUPPORTED) + .build( + "Catalog DDL statements are disabled. Set [%s] to true on the Broker to enable [%s].", + ENABLE_CATALOG_DDL_PROPERTY, + operationName() + ); + } + if (!handlerContext.plannerContext().getParameters().isEmpty()) { + throw InvalidSqlInput.exception("Dynamic parameters are not supported for [%s]", operationName()); + } + tableId = TableId.datasource(resolveTableName()); + resourceActions = Collections.singleton( + new ResourceAction(new Resource(tableId.name(), ResourceType.DATASOURCE), Action.WRITE) + ); + validateStatement(); + } + + /** + * Statement-specific validation, which also prepares whatever {@link #execute} will apply. + */ + protected abstract void validateStatement(); + + protected abstract void execute(CatalogTableWriter writer); + + protected abstract String operationName(); + + @Override + public void prepare() + { + // Nothing to prepare: there is no query to plan. + } + + @Override + public PrepareResult prepareResult() + { + return new PrepareResult(RESULT_TYPE, RESULT_TYPE, DruidTypeSystem.TYPE_FACTORY.createStructType( + Collections.emptyList(), + Collections.emptyList() + )); + } + + @Override + public PlannerResult plan() + { + execute(handlerContext.plannerContext().getPlannerToolbox().catalogTableWriter()); + final Supplier> resultsSupplier = Suppliers.ofInstance( + QueryResponse.withEmptyContext(Sequences.empty()) + ); + return new PlannerResult(resultsSupplier, RESULT_TYPE); + } + + @Override + public ExplainAttributes explainAttributes() + { + throw InvalidSqlInput.exception("EXPLAIN is not supported for [%s]", operationName()); + } + + /** + * Resolve the table name, which may be unqualified or qualified by the Druid schema. Other schemas are rejected: + * only datasources have catalog specs that DDL can write. + */ + private String resolveTableName() + { + final String tableName; + if (tableIdentifier.names.size() == 1) { + tableName = tableIdentifier.names.get(0); + } else if (tableIdentifier.names.size() == 2) { + final String defaultSchemaName = + Iterables.getOnlyElement(CalciteSchema.from(handlerContext.defaultSchema()).path(null)); + if (!defaultSchemaName.equals(tableIdentifier.names.get(0))) { + throw InvalidSqlInput.exception( + "Table [%s] does not support operation [%s] because it is not a Druid datasource", + tableIdentifier, + operationName() + ); + } + tableName = tableIdentifier.names.get(1); + } else { + throw InvalidSqlInput.exception( + "Table name [%s] is not valid for operation [%s]", + tableIdentifier, + operationName() + ); + } + IdUtils.validateId("table", tableName); + return tableName; + } + + /** + * Convert a parsed column declaration into its catalog form, checking that the type is one Druid can store. + */ + protected static ColumnSpec toColumnSpec(DruidSqlColumnDeclaration declaration) + { + final String name = simpleName(declaration.getName(), "Column"); + final String type; + try { + type = CatalogColumnTypes.forCatalogColumn(name, declaration.getDataType()); + } + catch (IAE e) { + throw InvalidSqlInput.exception(e, "%s", e.getMessage()); + } + if (Columns.isTimeColumn(name) && !ColumnType.LONG.equals(Columns.druidTypeFromString(type))) { + throw InvalidSqlInput.exception( + "Column [%s] must have type [%s] or [%s], but was [%s]", + Columns.TIME_COLUMN, + Columns.SQL_TIMESTAMP, + Columns.SQL_BIGINT, + type + ); + } + return new ColumnSpec(name, type, null); + } + + protected static String simpleName(SqlIdentifier identifier, String what) + { + if (!identifier.isSimple()) { + throw InvalidSqlInput.exception("%s name [%s] must be a simple name", what, identifier); + } + return identifier.getSimple(); + } + + /** + * The catalog stores a segment granularity as either {@code ALL} or an ISO period string. + */ + protected static String toGranularityString(SqlGranularityLiteral partitionedBy) + { + final Granularity granularity = partitionedBy.getGranularity(); + if (Granularities.ALL.equals(granularity)) { + return DatasourceDefn.ALL_GRANULARITY; + } + if (granularity instanceof PeriodGranularity) { + return ((PeriodGranularity) granularity).getPeriod().toString(); + } + throw InvalidSqlInput.exception("Granularity [%s] is not supported by the catalog", partitionedBy); + } + + /** + * The catalog's clustering keys are plain ascending column references. Expressions, ordinals and DESC have no + * catalog representation, so they are rejected here rather than silently dropped. + */ + protected static List toClusterKeys(SqlNodeList clusteredBy) + { + final List keys = new ArrayList<>(clusteredBy.size()); + for (SqlNode node : clusteredBy) { + if (!(node instanceof SqlIdentifier) || !((SqlIdentifier) node).isSimple()) { + throw InvalidSqlInput.exception( + "CLUSTERED BY column [%s] must be a column name; expressions, ordinals and DESC are not supported when" + + " defining a table", + node + ); + } + keys.add(new ClusterKeySpec(((SqlIdentifier) node).getSimple(), false)); + } + return keys; + } + + private static RelDataType resultType() + { + final RelDataTypeFactory typeFactory = DruidTypeSystem.TYPE_FACTORY; + return typeFactory.createStructType( + ImmutableList.of(Calcites.createSqlType(typeFactory, SqlTypeName.VARCHAR)), + ImmutableList.of("RESULT") + ); + } + + /** + * {@code CREATE [OR REPLACE] TABLE [IF NOT EXISTS] ...}. + */ + public static class CreateTableHandler extends CatalogDdlHandler + { + private final DruidSqlCreateTable createTable; + private TableSpec tableSpec; + + public CreateTableHandler(SqlStatementHandler.HandlerContext handlerContext, DruidSqlCreateTable createTable) + { + super(handlerContext, createTable.getName()); + this.createTable = createTable; + } + + @Override + protected void validateStatement() + { + if (createTable.getReplace() && createTable.isIfNotExists()) { + throw InvalidSqlInput.exception("Cannot specify both OR REPLACE and IF NOT EXISTS"); + } + + final List columns = new ArrayList<>(createTable.getColumnList().size()); + final Set seen = new HashSet<>(); + for (SqlNode node : createTable.getColumnList()) { + final ColumnSpec column = toColumnSpec((DruidSqlColumnDeclaration) node); + if (!seen.add(column.name())) { + throw InvalidSqlInput.exception("Column [%s] is declared more than once", column.name()); + } + columns.add(column); + } + + final Map properties = new LinkedHashMap<>(); + if (createTable.getPartitionedBy() != null) { + properties.put( + DatasourceDefn.SEGMENT_GRANULARITY_PROPERTY, + toGranularityString(createTable.getPartitionedBy()) + ); + } + if (createTable.getClusteredBy() != null) { + properties.put(DatasourceDefn.CLUSTER_KEYS_PROPERTY, toClusterKeys(createTable.getClusteredBy())); + } + + tableSpec = new TableSpec(DatasourceDefn.TABLE_TYPE, properties, columns); + } + + @Override + protected void execute(CatalogTableWriter writer) + { + writer.createTable(tableId, tableSpec, createTable.isIfNotExists(), createTable.getReplace()); + } + + @Override + protected String operationName() + { + return "CREATE TABLE"; + } + } + + /** + * {@code ALTER TABLE ... ADD COLUMN}. The Coordinator merges columns by name, so an existing column would be + * silently updated; this checks first so that {@code ADD} means add. + */ + public static class AddColumnHandler extends CatalogDdlHandler + { + private final DruidSqlAlterTable.AddColumn alterTable; + private ColumnSpec column; + + public AddColumnHandler(SqlStatementHandler.HandlerContext handlerContext, DruidSqlAlterTable.AddColumn alterTable) + { + super(handlerContext, alterTable.getName()); + this.alterTable = alterTable; + } + + @Override + protected void validateStatement() + { + column = toColumnSpec(alterTable.getColumn()); + } + + @Override + protected void execute(CatalogTableWriter writer) + { + final TableMetadata existing = writer.readTable(tableId); + if (existing == null) { + throw InvalidSqlInput.exception("Table [%s] does not have a catalog entry", tableId.name()); + } + if (existing.spec().columns() != null + && existing.spec().columns().stream().anyMatch(c -> column.name().equals(c.name()))) { + throw InvalidSqlInput.exception( + "Column [%s] already exists in table [%s]; use ALTER COLUMN to change its type", + column.name(), + tableId.name() + ); + } + writer.updateColumns(tableId, Collections.singletonList(column)); + } + + @Override + protected String operationName() + { + return "ALTER TABLE ADD COLUMN"; + } + } + + /** + * {@code ALTER TABLE ... DROP COLUMN}. + */ + public static class DropColumnHandler extends CatalogDdlHandler + { + private final DruidSqlAlterTable.DropColumn alterTable; + private String column; + + public DropColumnHandler( + SqlStatementHandler.HandlerContext handlerContext, + DruidSqlAlterTable.DropColumn alterTable + ) + { + super(handlerContext, alterTable.getName()); + this.alterTable = alterTable; + } + + @Override + protected void validateStatement() + { + column = simpleName(alterTable.getColumn(), "Column"); + } + + @Override + protected void execute(CatalogTableWriter writer) + { + writer.dropColumns(tableId, Collections.singletonList(column)); + } + + @Override + protected String operationName() + { + return "ALTER TABLE DROP COLUMN"; + } + } + + /** + * {@code ALTER TABLE ... ALTER COLUMN ... SET DATA TYPE}. Merging a column by name is exactly what changing its + * type requires, so this reuses the same update as ADD COLUMN without the existence check. + */ + public static class AlterColumnHandler extends CatalogDdlHandler + { + private final DruidSqlAlterTable.AlterColumn alterTable; + private ColumnSpec column; + + public AlterColumnHandler( + SqlStatementHandler.HandlerContext handlerContext, + DruidSqlAlterTable.AlterColumn alterTable + ) + { + super(handlerContext, alterTable.getName()); + this.alterTable = alterTable; + } + + @Override + protected void validateStatement() + { + column = toColumnSpec(alterTable.getColumn()); + } + + @Override + protected void execute(CatalogTableWriter writer) + { + writer.updateColumns(tableId, Collections.singletonList(column)); + } + + @Override + protected String operationName() + { + return "ALTER TABLE ALTER COLUMN"; + } + } + + /** + * {@code ALTER TABLE ... SET PROPERTIES}. A NULL value removes the property. The set of legal keys is not checked + * here: the Coordinator's table definition registry is what knows them. + */ + public static class SetPropertiesHandler extends CatalogDdlHandler + { + private final DruidSqlAlterTable.SetProperties alterTable; + private Map properties; + + public SetPropertiesHandler( + SqlStatementHandler.HandlerContext handlerContext, + DruidSqlAlterTable.SetProperties alterTable + ) + { + super(handlerContext, alterTable.getName()); + this.alterTable = alterTable; + } + + @Override + protected void validateStatement() + { + properties = new LinkedHashMap<>(); + for (SqlNode node : alterTable.getProperties()) { + final DruidSqlPropertyAssignment assignment = (DruidSqlPropertyAssignment) node; + final String key = simpleName(assignment.getKey(), "Property"); + if (properties.containsKey(key)) { + throw InvalidSqlInput.exception("Property [%s] is assigned more than once", key); + } + properties.put(key, propertyValue(key, assignment.getValue())); + } + } + + private static Object propertyValue(String key, SqlNode value) + { + if (!(value instanceof SqlLiteral)) { + throw InvalidSqlInput.exception("Value for property [%s] must be a literal", key); + } + // A NULL literal coerces to null, which the catalog treats as "remove this property". + return DruidSqlParser.sqlLiteralToJavaValue((SqlLiteral) value, "property " + key); + } + + @Override + protected void execute(CatalogTableWriter writer) + { + writer.updateProperties(tableId, properties); + } + + @Override + protected String operationName() + { + return "ALTER TABLE SET PROPERTIES"; + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java new file mode 100644 index 000000000000..1a6bd2aefb45 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.planner; + +import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.TableId; +import org.apache.druid.catalog.model.TableMetadata; +import org.apache.druid.catalog.model.TableSpec; +import org.apache.druid.error.DruidException; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Map; + +/** + * Write side of the catalog, the counterpart to the read-only {@link CatalogResolver}. Catalog metadata is owned by + * the Coordinator, so an implementation of this interface makes a remote call; it is bound by the {@code druid-catalog} + * extension and defaults to {@link #NOT_AVAILABLE} when that extension is absent. + *

+ * The methods are deliberately semantic rather than a generic "apply this edit request", so that the extension's edit + * request types need not be visible here. Each corresponds to exactly one atomic Coordinator operation, which is why + * every DDL statement maps to a single call: the catalog stores properties and columns as separate blobs with separate + * update paths, so a statement that changed both would not be atomic. + */ +public interface CatalogTableWriter +{ + CatalogTableWriter NOT_AVAILABLE = new UnavailableCatalogTableWriter(); + + /** + * Create a table. + * + * @param ifNotExists if the table already exists, do nothing rather than failing + * @param replace overwrite any existing spec for this table + */ + void createTable(TableId tableId, TableSpec spec, boolean ifNotExists, boolean replace); + + /** + * Merge the given columns into the table's column list, matching on name. Columns that do not yet exist are + * appended; existing columns are updated in place. + */ + void updateColumns(TableId tableId, List columns); + + /** + * Remove the named columns from the table's column list. Segments are unaffected. + */ + void dropColumns(TableId tableId, List columns); + + /** + * Merge the given properties into the table's properties. A null value removes the property. + */ + void updateProperties(TableId tableId, Map properties); + + /** + * Read a table's current metadata directly from the Coordinator, bypassing any local cache, or null if the table + * has no catalog entry. Used for pre-checks that must not race against a stale cache. + */ + @Nullable + TableMetadata readTable(TableId tableId); + + /** + * Stand-in used when the {@code druid-catalog} extension is not loaded. Every operation fails with an explanation + * rather than silently doing nothing. + */ + class UnavailableCatalogTableWriter implements CatalogTableWriter + { + @Override + public void createTable(TableId tableId, TableSpec spec, boolean ifNotExists, boolean replace) + { + throw notAvailable(); + } + + @Override + public void updateColumns(TableId tableId, List columns) + { + throw notAvailable(); + } + + @Override + public void dropColumns(TableId tableId, List columns) + { + throw notAvailable(); + } + + @Override + public void updateProperties(TableId tableId, Map properties) + { + throw notAvailable(); + } + + @Nullable + @Override + public TableMetadata readTable(TableId tableId) + { + throw notAvailable(); + } + + private static DruidException notAvailable() + { + return DruidException.forPersona(DruidException.Persona.USER) + .ofCategory(DruidException.Category.UNSUPPORTED) + .build( + "Catalog DDL statements require the [druid-catalog] extension, which is not loaded" + ); + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java index f361e43797b4..7c76ad8babb3 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java @@ -35,6 +35,8 @@ import org.apache.druid.server.security.AuthorizationResult; import org.apache.druid.server.security.Resource; import org.apache.druid.server.security.ResourceAction; +import org.apache.druid.sql.calcite.parser.DruidSqlAlterTable; +import org.apache.druid.sql.calcite.parser.DruidSqlCreateTable; import org.apache.druid.sql.calcite.parser.DruidSqlInsert; import org.apache.druid.sql.calcite.parser.DruidSqlReplace; import org.apache.druid.sql.calcite.run.SqlEngine; @@ -147,6 +149,16 @@ private SqlStatementHandler createHandler(final SqlNode node) } SqlStatementHandler.HandlerContext handlerContext = new HandlerContextImpl(); + + if (query instanceof DruidSqlCreateTable || query instanceof DruidSqlAlterTable) { + // The grammar does not admit EXPLAIN of a DDL statement; this guards the case anyway, since a DDL statement + // has no query to explain. + if (explain != null) { + throw InvalidSqlInput.exception("EXPLAIN is not supported for [%s]", query.getKind()); + } + return createDdlHandler(handlerContext, query); + } + if (query.getKind() == SqlKind.INSERT) { if (query instanceof DruidSqlInsert) { return new IngestHandler.InsertHandler(handlerContext, (DruidSqlInsert) query, explain); @@ -161,6 +173,29 @@ private SqlStatementHandler createHandler(final SqlNode node) throw InvalidSqlInput.exception("Unsupported SQL statement [%s]", node.getKind()); } + private static SqlStatementHandler createDdlHandler( + final SqlStatementHandler.HandlerContext handlerContext, + final SqlNode query + ) + { + if (query instanceof DruidSqlCreateTable) { + return new CatalogDdlHandler.CreateTableHandler(handlerContext, (DruidSqlCreateTable) query); + } + if (query instanceof DruidSqlAlterTable.AddColumn) { + return new CatalogDdlHandler.AddColumnHandler(handlerContext, (DruidSqlAlterTable.AddColumn) query); + } + if (query instanceof DruidSqlAlterTable.DropColumn) { + return new CatalogDdlHandler.DropColumnHandler(handlerContext, (DruidSqlAlterTable.DropColumn) query); + } + if (query instanceof DruidSqlAlterTable.AlterColumn) { + return new CatalogDdlHandler.AlterColumnHandler(handlerContext, (DruidSqlAlterTable.AlterColumn) query); + } + if (query instanceof DruidSqlAlterTable.SetProperties) { + return new CatalogDdlHandler.SetPropertiesHandler(handlerContext, (DruidSqlAlterTable.SetProperties) query); + } + throw DruidException.defensive("Unhandled catalog DDL statement [%s]", query.getClass().getSimpleName()); + } + /** * Uses {@link SqlParameterizerShuttle} to rewrite {@link SqlNode} to swap out any * {@link org.apache.calcite.sql.SqlDynamicParam} early for their {@link org.apache.calcite.sql.SqlLiteral} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java index d8c77c695894..f66928f967f0 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java @@ -83,6 +83,9 @@ public class PlannerConfig @JsonProperty private boolean enableSysQueriesTable = false; + @JsonProperty + private boolean enableCatalogDdl = false; + public int getMaxNumericInFilters() { return maxNumericInFilters; @@ -160,6 +163,16 @@ public boolean isEnableSysQueriesTable() return enableSysQueriesTable; } + /** + * Whether catalog DDL statements (CREATE TABLE, ALTER TABLE) may be executed. Off by default: the Broker's SQL + * endpoint is typically reachable by far more people than the Coordinator's catalog API, so enabling this widens + * what an existing datasource WRITE permission allows. Deliberately not overridable from the query context. + */ + public boolean isEnableCatalogDdl() + { + return enableCatalogDdl; + } + public PlannerConfig withOverrides(final Map queryContext) { if (queryContext.isEmpty()) { @@ -189,6 +202,7 @@ public boolean equals(Object o) && forceExpressionVirtualColumns == that.forceExpressionVirtualColumns && maxNumericInFilters == that.maxNumericInFilters && enableSysQueriesTable == that.enableSysQueriesTable + && enableCatalogDdl == that.enableCatalogDdl && Objects.equals(sqlTimeZone, that.sqlTimeZone) && Objects.equals(nativeQuerySqlPlanningMode, that.nativeQuerySqlPlanningMode); } @@ -210,7 +224,8 @@ public int hashCode() forceExpressionVirtualColumns, maxNumericInFilters, nativeQuerySqlPlanningMode, - enableSysQueriesTable + enableSysQueriesTable, + enableCatalogDdl ); } @@ -227,6 +242,7 @@ public String toString() ", useNativeQueryExplain=" + useNativeQueryExplain + ", nativeQuerySqlPlanningMode=" + nativeQuerySqlPlanningMode + ", enableSysQueriesTable=" + enableSysQueriesTable + + ", enableCatalogDdl=" + enableCatalogDdl + '}'; } @@ -262,6 +278,7 @@ public static class Builder private int maxNumericInFilters; private String nativeQuerySqlPlanningMode; private boolean enableSysQueriesTable; + private boolean enableCatalogDdl; public Builder(PlannerConfig base) { @@ -282,6 +299,7 @@ public Builder(PlannerConfig base) maxNumericInFilters = base.getMaxNumericInFilters(); nativeQuerySqlPlanningMode = base.getNativeQuerySqlPlanningMode(); enableSysQueriesTable = base.isEnableSysQueriesTable(); + enableCatalogDdl = base.isEnableCatalogDdl(); } public Builder requireTimeCondition(boolean option) @@ -362,6 +380,12 @@ public Builder enableSysQueriesTable(boolean option) return this; } + public Builder enableCatalogDdl(boolean option) + { + this.enableCatalogDdl = option; + return this; + } + public Builder withOverrides(final Map queryContext) { useApproximateCountDistinct = QueryContexts.parseBoolean( @@ -459,6 +483,7 @@ public PlannerConfig build() config.forceExpressionVirtualColumns = forceExpressionVirtualColumns; config.nativeQuerySqlPlanningMode = nativeQuerySqlPlanningMode; config.enableSysQueriesTable = enableSysQueriesTable; + config.enableCatalogDdl = enableCatalogDdl; return config; } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java index 32ca488e923a..2071415ae587 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java @@ -59,6 +59,43 @@ public class PlannerFactory extends PlannerToolbox { + /** + * Convenience for callers that never execute catalog DDL, such as tests and benchmarks. + */ + public PlannerFactory( + final DruidSchemaCatalog rootSchema, + final DruidOperatorTable operatorTable, + final ExprMacroTable macroTable, + final PlannerConfig plannerConfig, + final AuthorizerMapper authorizerMapper, + final @Json ObjectMapper jsonMapper, + final @DruidSchemaName String druidSchemaName, + final CalciteRulesManager calciteRuleManager, + final JoinableFactoryWrapper joinableFactoryWrapper, + final CatalogResolver catalog, + final AuthConfig authConfig, + final PolicyEnforcer policyEnforcer, + final DruidHookDispatcher hookDispatcher + ) + { + this( + rootSchema, + operatorTable, + macroTable, + plannerConfig, + authorizerMapper, + jsonMapper, + druidSchemaName, + calciteRuleManager, + joinableFactoryWrapper, + catalog, + CatalogTableWriter.NOT_AVAILABLE, + authConfig, + policyEnforcer, + hookDispatcher + ); + } + @Inject public PlannerFactory( final DruidSchemaCatalog rootSchema, @@ -71,6 +108,7 @@ public PlannerFactory( final CalciteRulesManager calciteRuleManager, final JoinableFactoryWrapper joinableFactoryWrapper, final CatalogResolver catalog, + final CatalogTableWriter catalogTableWriter, final AuthConfig authConfig, final PolicyEnforcer policyEnforcer, final DruidHookDispatcher hookDispatcher @@ -84,6 +122,7 @@ public PlannerFactory( rootSchema, joinableFactoryWrapper, catalog, + catalogTableWriter, druidSchemaName, calciteRuleManager, authorizerMapper, diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerToolbox.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerToolbox.java index 17887afd06a5..6fc4ffdf2663 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerToolbox.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerToolbox.java @@ -38,6 +38,7 @@ public class PlannerToolbox protected final PlannerConfig plannerConfig; protected final DruidSchemaCatalog rootSchema; protected final CatalogResolver catalog; + protected final CatalogTableWriter catalogTableWriter; protected final String druidSchemaName; protected final CalciteRulesManager calciteRuleManager; protected final AuthorizerMapper authorizerMapper; @@ -45,6 +46,9 @@ public class PlannerToolbox protected final PolicyEnforcer policyEnforcer; protected final DruidHookDispatcher hookDispatcher; + /** + * Convenience for callers that never execute catalog DDL, such as tests and benchmarks. + */ public PlannerToolbox( final DruidOperatorTable operatorTable, final ExprMacroTable macroTable, @@ -60,6 +64,41 @@ public PlannerToolbox( final PolicyEnforcer policyEnforcer, final DruidHookDispatcher hookDispatcher ) + { + this( + operatorTable, + macroTable, + jsonMapper, + plannerConfig, + rootSchema, + joinableFactoryWrapper, + catalog, + CatalogTableWriter.NOT_AVAILABLE, + druidSchemaName, + calciteRuleManager, + authorizerMapper, + authConfig, + policyEnforcer, + hookDispatcher + ); + } + + public PlannerToolbox( + final DruidOperatorTable operatorTable, + final ExprMacroTable macroTable, + final ObjectMapper jsonMapper, + final PlannerConfig plannerConfig, + final DruidSchemaCatalog rootSchema, + final JoinableFactoryWrapper joinableFactoryWrapper, + final CatalogResolver catalog, + final CatalogTableWriter catalogTableWriter, + final String druidSchemaName, + final CalciteRulesManager calciteRuleManager, + final AuthorizerMapper authorizerMapper, + final AuthConfig authConfig, + final PolicyEnforcer policyEnforcer, + final DruidHookDispatcher hookDispatcher + ) { this.operatorTable = operatorTable; this.macroTable = macroTable; @@ -68,6 +107,7 @@ public PlannerToolbox( this.rootSchema = rootSchema; this.joinableFactoryWrapper = joinableFactoryWrapper; this.catalog = catalog; + this.catalogTableWriter = catalogTableWriter; this.druidSchemaName = druidSchemaName; this.calciteRuleManager = calciteRuleManager; this.authorizerMapper = authorizerMapper; @@ -106,6 +146,11 @@ public CatalogResolver catalogResolver() return catalog; } + public CatalogTableWriter catalogTableWriter() + { + return catalogTableWriter; + } + public String druidSchemaName() { return druidSchemaName; diff --git a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java index 1565527f601b..8fee6e2a2f98 100644 --- a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java +++ b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java @@ -41,6 +41,7 @@ import org.apache.druid.sql.calcite.expression.builtin.QueryLookupOperatorConversion; import org.apache.druid.sql.calcite.planner.CalcitePlannerModule; import org.apache.druid.sql.calcite.planner.CatalogResolver; +import org.apache.druid.sql.calcite.planner.CatalogTableWriter; import org.apache.druid.sql.calcite.planner.PlannerFactory; import org.apache.druid.sql.calcite.run.NativeSqlEngine; import org.apache.druid.sql.calcite.run.SqlEngine; @@ -125,6 +126,9 @@ public void configure(Binder binder) // Default do-nothing catalog resolver binder.bind(CatalogResolver.class).toInstance(CatalogResolver.NULL_RESOLVER); + // Default catalog writer, which reports that catalog DDL needs the druid-catalog extension + binder.bind(CatalogTableWriter.class).toInstance(CatalogTableWriter.NOT_AVAILABLE); + // Bind the engine Multibinder.newSetBinder(binder, SqlEngine.class) .addBinding() diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java new file mode 100644 index 000000000000..7e9c71a3699a --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java @@ -0,0 +1,453 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.TableId; +import org.apache.druid.catalog.model.TableMetadata; +import org.apache.druid.catalog.model.TableSpec; +import org.apache.druid.catalog.model.table.ClusterKeySpec; +import org.apache.druid.catalog.model.table.DatasourceDefn; +import org.apache.druid.error.DruidException; +import org.apache.druid.server.security.Action; +import org.apache.druid.server.security.AuthConfig; +import org.apache.druid.server.security.Resource; +import org.apache.druid.server.security.ResourceAction; +import org.apache.druid.server.security.ResourceType; +import org.apache.druid.sql.DirectStatement; +import org.apache.druid.sql.SqlQueryPlus; +import org.apache.druid.sql.calcite.CalciteCatalogDdlTest.CatalogDdlComponentSupplier; +import org.apache.druid.sql.calcite.planner.CatalogTableWriter; +import org.apache.druid.sql.calcite.planner.PlannerConfig; +import org.apache.druid.sql.calcite.util.CalciteTests; +import org.apache.druid.sql.calcite.util.SqlTestFramework.StandardComponentSupplier; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests that catalog DDL statements plan into the catalog operations they claim to, using a writer that records + * calls instead of contacting a Coordinator. + */ +@SqlTestFrameworkConfig.ComponentSupplier(CatalogDdlComponentSupplier.class) +public class CalciteCatalogDdlTest extends BaseCalciteQueryTest +{ + private static final RecordingCatalogTableWriter WRITER = new RecordingCatalogTableWriter(); + + public static class CatalogDdlComponentSupplier extends StandardComponentSupplier + { + public CatalogDdlComponentSupplier(TempDirProducer tempFolderProducer) + { + super(tempFolderProducer); + } + + @Override + public CatalogTableWriter createCatalogTableWriter() + { + return WRITER; + } + } + + @BeforeEach + public void resetWriter() + { + WRITER.reset(); + } + + @Test + public void testCreateTable() + { + execute("CREATE TABLE tbl (__time TIMESTAMP, page VARCHAR, cnt BIGINT)"); + + assertEquals(1, WRITER.calls.size()); + final RecordingCatalogTableWriter.Call call = WRITER.calls.get(0); + assertEquals("createTable", call.operation); + assertEquals(TableId.datasource("tbl"), call.tableId); + assertEquals(DatasourceDefn.TABLE_TYPE, call.spec.type()); + assertEquals(ImmutableMap.of(), call.spec.properties()); + assertEquals( + ImmutableList.of( + new ColumnSpec("__time", "TIMESTAMP", null), + new ColumnSpec("page", "VARCHAR", null), + new ColumnSpec("cnt", "BIGINT", null) + ), + call.spec.columns() + ); + assertFalse(call.ifNotExists); + assertFalse(call.replace); + } + + @Test + public void testCreateTableWithPartitioningAndClustering() + { + execute("CREATE TABLE tbl (page VARCHAR, cnt BIGINT) PARTITIONED BY DAY CLUSTERED BY page, cnt"); + + final TableSpec spec = WRITER.calls.get(0).spec; + assertEquals("P1D", spec.properties().get(DatasourceDefn.SEGMENT_GRANULARITY_PROPERTY)); + assertEquals( + ImmutableList.of(new ClusterKeySpec("page", false), new ClusterKeySpec("cnt", false)), + spec.properties().get(DatasourceDefn.CLUSTER_KEYS_PROPERTY) + ); + } + + @Test + public void testCreateTablePartitionedByAll() + { + execute("CREATE TABLE tbl (page VARCHAR) PARTITIONED BY ALL TIME"); + assertEquals("ALL", WRITER.calls.get(0).spec.properties().get(DatasourceDefn.SEGMENT_GRANULARITY_PROPERTY)); + } + + @Test + public void testCreateTableTypeCanonicalization() + { + execute( + "CREATE TABLE tbl (a CHAR, b INTEGER, c REAL, d DOUBLE, e VARCHAR ARRAY, f TYPE('complex'))" + ); + assertEquals( + ImmutableList.of( + new ColumnSpec("a", "VARCHAR", null), + new ColumnSpec("b", "BIGINT", null), + new ColumnSpec("c", "FLOAT", null), + new ColumnSpec("d", "DOUBLE", null), + new ColumnSpec("e", "VARCHAR ARRAY", null), + new ColumnSpec("f", "COMPLEX", null) + ), + WRITER.calls.get(0).spec.columns() + ); + } + + @Test + public void testCreateTableFlags() + { + execute("CREATE OR REPLACE TABLE tbl (a VARCHAR)"); + assertTrue(WRITER.calls.get(0).replace); + + WRITER.reset(); + execute("CREATE TABLE IF NOT EXISTS tbl (a VARCHAR)"); + assertTrue(WRITER.calls.get(0).ifNotExists); + } + + @Test + public void testCreateTableInDruidSchema() + { + execute("CREATE TABLE druid.tbl (a VARCHAR)"); + assertEquals(TableId.datasource("tbl"), WRITER.calls.get(0).tableId); + } + + @Test + public void testResourceActionIsDatasourceWrite() + { + final DirectStatement stmt = statement("CREATE TABLE tbl (a VARCHAR)"); + stmt.execute(); + assertEquals( + Collections.singleton(new ResourceAction(new Resource("tbl", ResourceType.DATASOURCE), Action.WRITE)), + stmt.resources() + ); + } + + @Test + public void testDdlReturnsNoRows() + { + final DirectStatement stmt = statement("CREATE TABLE tbl (a VARCHAR)"); + final List results = stmt.execute().getResults().toList(); + assertEquals(ImmutableList.of(), results); + } + + @Test + public void testAlterTableAddColumn() + { + WRITER.existing.put(TableId.datasource("tbl"), tableWithColumns("a")); + execute("ALTER TABLE tbl ADD COLUMN b BIGINT"); + + final RecordingCatalogTableWriter.Call call = WRITER.lastCall("updateColumns"); + assertEquals(ImmutableList.of(new ColumnSpec("b", "BIGINT", null)), call.columns); + } + + @Test + public void testAlterTableAddExistingColumnFails() + { + WRITER.existing.put(TableId.datasource("tbl"), tableWithColumns("a")); + final DruidException e = assertThrows( + DruidException.class, + () -> execute("ALTER TABLE tbl ADD COLUMN a BIGINT") + ); + assertTrue(e.getMessage().contains("Column [a] already exists")); + } + + @Test + public void testAlterTableAddColumnToMissingTableFails() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("ALTER TABLE tbl ADD COLUMN a BIGINT") + ); + assertTrue(e.getMessage().contains("does not have a catalog entry")); + } + + @Test + public void testAlterTableDropColumn() + { + execute("ALTER TABLE tbl DROP COLUMN gone"); + assertEquals(ImmutableList.of("gone"), WRITER.lastCall("dropColumns").droppedColumns); + } + + @Test + public void testAlterTableAlterColumn() + { + // Unlike ADD COLUMN, changing a type does not require the column to be absent, so no read is needed. + execute("ALTER TABLE tbl ALTER COLUMN cnt SET DATA TYPE DOUBLE"); + assertEquals( + ImmutableList.of(new ColumnSpec("cnt", "DOUBLE", null)), + WRITER.lastCall("updateColumns").columns + ); + } + + @Test + public void testAlterTableSetProperties() + { + execute("ALTER TABLE tbl SET PROPERTIES (targetSegmentRows = 3000000, sealed = TRUE, description = 'hi')"); + + final Map properties = WRITER.lastCall("updateProperties").properties; + assertEquals(3000000L, properties.get("targetSegmentRows")); + assertEquals(true, properties.get("sealed")); + assertEquals("hi", properties.get("description")); + } + + @Test + public void testAlterTableSetPropertyToNullRemovesIt() + { + execute("ALTER TABLE tbl SET PROPERTIES (description = NULL)"); + final Map properties = WRITER.lastCall("updateProperties").properties; + assertTrue(properties.containsKey("description")); + assertNull(properties.get("description")); + } + + @Test + public void testCreateTableRejectsDuplicateColumn() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a VARCHAR, a BIGINT)") + ); + assertTrue(e.getMessage().contains("Column [a] is declared more than once")); + } + + @Test + public void testCreateTableRejectsUnsupportedType() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a TYPE('NOT_A_TYPE'))") + ); + assertTrue(e.getMessage().contains("unsupported type")); + } + + /** + * Any spelling that resolves to a LONG is accepted for the time column, and is stored as written. + */ + @Test + public void testCreateTableTimeColumnSpellings() + { + execute("CREATE TABLE tbl (__time BIGINT)"); + assertEquals(ImmutableList.of(new ColumnSpec("__time", "BIGINT", null)), WRITER.calls.get(0).spec.columns()); + + WRITER.reset(); + execute("CREATE TABLE tbl (__time TYPE('LONG'))"); + assertEquals(ImmutableList.of(new ColumnSpec("__time", "LONG", null)), WRITER.calls.get(0).spec.columns()); + } + + @Test + public void testCreateTableRejectsNonLongTimeColumn() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (__time VARCHAR)") + ); + assertTrue(e.getMessage().contains("Column [__time] must have type")); + } + + @Test + public void testCreateTableRejectsNonDruidSchema() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE lookup.tbl (a VARCHAR)") + ); + assertTrue(e.getMessage().contains("is not a Druid datasource")); + } + + @Test + public void testCreateTableRejectsBothReplaceAndIfNotExists() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE OR REPLACE TABLE IF NOT EXISTS tbl (a VARCHAR)") + ); + assertTrue(e.getMessage().contains("Cannot specify both OR REPLACE and IF NOT EXISTS")); + } + + @Test + public void testCreateTableRejectsClusteringExpression() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a VARCHAR) CLUSTERED BY a DESC") + ); + assertTrue(e.getMessage().contains("must be a column name")); + } + + /** + * The feature is off unless an operator turns it on, so that upgrading a cluster does not silently widen what a + * datasource WRITE permission allows. + */ + @Test + public void testDdlIsDisabledByDefault() + { + final DirectStatement stmt = getSqlStatementFactory(PlannerConfig.builder().build(), new AuthConfig()) + .directStatement( + SqlQueryPlus.builder("CREATE TABLE tbl (a VARCHAR)") + .auth(CalciteTests.SUPER_USER_AUTH_RESULT) + .build() + ); + final DruidException e = assertThrows(DruidException.class, stmt::execute); + assertTrue(e.getMessage().contains("druid.sql.planner.enableCatalogDdl"), e.getMessage()); + assertEquals(ImmutableList.of(), WRITER.calls); + } + + private void execute(String sql) + { + statement(sql).execute(); + } + + private DirectStatement statement(String sql) + { + return getSqlStatementFactory(PlannerConfig.builder().enableCatalogDdl(true).build(), new AuthConfig()) + .directStatement( + SqlQueryPlus.builder(sql).auth(CalciteTests.SUPER_USER_AUTH_RESULT).build() + ); + } + + private static TableMetadata tableWithColumns(String... names) + { + final List columns = new ArrayList<>(); + for (String name : names) { + columns.add(new ColumnSpec(name, "VARCHAR", null)); + } + return TableMetadata.newTable( + TableId.datasource("tbl"), + new TableSpec(DatasourceDefn.TABLE_TYPE, ImmutableMap.of(), columns) + ); + } + + /** + * Records what a DDL statement asked the catalog to do, so tests can assert on the resulting operation rather + * than on a Coordinator round trip. + */ + private static class RecordingCatalogTableWriter implements CatalogTableWriter + { + static class Call + { + String operation; + TableId tableId; + TableSpec spec; + boolean ifNotExists; + boolean replace; + List columns; + List droppedColumns; + Map properties; + } + + final List calls = new ArrayList<>(); + final Map existing = new HashMap<>(); + + void reset() + { + calls.clear(); + existing.clear(); + } + + Call lastCall(String operation) + { + for (int i = calls.size() - 1; i >= 0; i--) { + if (operation.equals(calls.get(i).operation)) { + return calls.get(i); + } + } + throw new AssertionError("No call to [" + operation + "] in " + calls); + } + + @Override + public void createTable(TableId tableId, TableSpec spec, boolean ifNotExists, boolean replace) + { + final Call call = record("createTable", tableId); + call.spec = spec; + call.ifNotExists = ifNotExists; + call.replace = replace; + } + + @Override + public void updateColumns(TableId tableId, List columns) + { + record("updateColumns", tableId).columns = columns; + } + + @Override + public void dropColumns(TableId tableId, List columns) + { + record("dropColumns", tableId).droppedColumns = columns; + } + + @Override + public void updateProperties(TableId tableId, Map properties) + { + record("updateProperties", tableId).properties = properties; + } + + @Nullable + @Override + public TableMetadata readTable(TableId tableId) + { + return existing.get(tableId); + } + + private Call record(String operation, TableId tableId) + { + final Call call = new Call(); + call.operation = operation; + call.tableId = tableId; + calls.add(call); + return call; + } + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java b/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java index f950f933a7bd..3fdb4b313234 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java @@ -385,6 +385,7 @@ public static class ConfigurationInstance SqlTestFramework.Builder builder = new SqlTestFramework.Builder(testHost) .withConfig(config) .catalogResolver(testHost.createCatalogResolver()) + .catalogTableWriter(testHost.createCatalogTableWriter()) .mergeBufferCount(config.numMergeBuffers) .withOverrideModule(config.resultCache.makeModule()); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java new file mode 100644 index 000000000000..1778888308af --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.parser; + +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlSetOption; +import org.apache.calcite.sql.dialect.CalciteSqlDialect; +import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.junit.jupiter.api.Test; + +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Parser coverage for the catalog DDL statements: {@code CREATE TABLE} and {@code ALTER TABLE}. + */ +public class DruidSqlDdlParserTest +{ + @Test + public void testCreateTableMinimal() + { + final DruidSqlCreateTable create = parseCreate("CREATE TABLE tbl (a VARCHAR, b BIGINT)"); + + assertEquals("tbl", create.getName().toString()); + assertFalse(create.getReplace()); + assertFalse(create.isIfNotExists()); + assertNull(create.getPartitionedBy()); + assertNull(create.getClusteredBy()); + assertEquals("a VARCHAR, b BIGINT", columnsOf(create)); + } + + @Test + public void testCreateTableWithSchemaQualifiedName() + { + final DruidSqlCreateTable create = parseCreate("CREATE TABLE \"druid\".tbl (a VARCHAR)"); + assertEquals("druid.tbl", create.getName().toString()); + } + + @Test + public void testCreateTableWithoutColumns() + { + // A catalog entry that only carries properties is legal. + final DruidSqlCreateTable create = parseCreate("CREATE TABLE tbl PARTITIONED BY DAY"); + assertEquals(0, create.getColumnList().size()); + assertEquals(Granularities.DAY, create.getPartitionedBy().getGranularity()); + } + + @Test + public void testCreateTableAllClauses() + { + final DruidSqlCreateTable create = parseCreate( + "CREATE OR REPLACE TABLE \"druid\".sales (\n" + + " __time TIMESTAMP,\n" + + " page VARCHAR NOT NULL,\n" + + " cnt BIGINT,\n" + + " vals DOUBLE ARRAY,\n" + + " usr TYPE('COMPLEX')\n" + + ")\n" + + "PARTITIONED BY HOUR\n" + + "CLUSTERED BY page, cnt" + ); + + assertTrue(create.getReplace()); + assertFalse(create.isIfNotExists()); + // Calcite renders a user-defined type name as a quoted identifier; TYPE('...') round-trips correctly through + // unparse, which is what actually matters (see testUnparseRoundTrip). + assertEquals( + "__time TIMESTAMP, page VARCHAR, cnt BIGINT, vals DOUBLE ARRAY, usr `COMPLEX`", + columnsOf(create) + ); + assertEquals(Granularities.HOUR, create.getPartitionedBy().getGranularity()); + assertEquals("`page`, `cnt`", create.getClusteredBy().toString()); + } + + @Test + public void testCreateTableIfNotExists() + { + final DruidSqlCreateTable create = parseCreate("CREATE TABLE IF NOT EXISTS tbl (a VARCHAR)"); + assertTrue(create.isIfNotExists()); + assertFalse(create.getReplace()); + } + + @Test + public void testCreateTableExpressionGranularity() + { + final DruidSqlCreateTable create = parseCreate("CREATE TABLE tbl (a VARCHAR) PARTITIONED BY FLOOR(__time TO HOUR)"); + assertEquals(Granularities.HOUR, create.getPartitionedBy().getGranularity()); + } + + @Test + public void testAlterTableAddColumn() + { + final DruidSqlAlterTable.AddColumn alter = parseAlter( + "ALTER TABLE tbl ADD COLUMN added DOUBLE", + DruidSqlAlterTable.AddColumn.class + ); + assertEquals("tbl", alter.getName().toString()); + assertEquals("added", alter.getColumn().getName().toString()); + assertEquals("DOUBLE", alter.getColumn().getDataType().toString()); + } + + @Test + public void testAlterTableDropColumn() + { + final DruidSqlAlterTable.DropColumn alter = parseAlter( + "ALTER TABLE tbl DROP COLUMN gone", + DruidSqlAlterTable.DropColumn.class + ); + assertEquals("gone", alter.getColumn().toString()); + } + + @Test + public void testAlterTableAlterColumn() + { + final DruidSqlAlterTable.AlterColumn alter = parseAlter( + "ALTER TABLE tbl ALTER COLUMN cnt SET DATA TYPE DOUBLE", + DruidSqlAlterTable.AlterColumn.class + ); + assertEquals("cnt", alter.getColumn().getName().toString()); + assertEquals("DOUBLE", alter.getColumn().getDataType().toString()); + } + + @Test + public void testAlterTableAlterColumnToComplexType() + { + final DruidSqlAlterTable.AlterColumn alter = parseAlter( + "ALTER TABLE tbl ALTER COLUMN payload SET DATA TYPE TYPE('COMPLEX')", + DruidSqlAlterTable.AlterColumn.class + ); + assertEquals("COMPLEX", alter.getColumn().getDataType().getTypeName().toString()); + } + + @Test + public void testAlterTableSetProperties() + { + final DruidSqlAlterTable.SetProperties alter = parseAlter( + "ALTER TABLE tbl SET PROPERTIES (targetSegmentRows = 3000000, sealed = TRUE, description = NULL)", + DruidSqlAlterTable.SetProperties.class + ); + assertEquals(3, alter.getProperties().size()); + assertEquals( + "targetSegmentRows = 3000000, sealed = TRUE, description = NULL", + alter.getProperties() + .stream() + .map(p -> { + final DruidSqlPropertyAssignment assignment = (DruidSqlPropertyAssignment) p; + return assignment.getKey() + " = " + assignment.getValue(); + }) + .collect(Collectors.joining(", ")) + ); + } + + /** + * DDL nodes must round-trip through {@link SqlNode#unparse}, which is what makes them safe to log and re-print. + */ + @Test + public void testUnparseRoundTrip() + { + assertUnparseRoundTrips("CREATE TABLE \"tbl\" (\"a\" VARCHAR, \"b\" BIGINT)"); + assertUnparseRoundTrips("CREATE OR REPLACE TABLE \"tbl\" (\"a\" VARCHAR)"); + assertUnparseRoundTrips("CREATE TABLE IF NOT EXISTS \"tbl\" (\"a\" VARCHAR)"); + assertUnparseRoundTrips("CREATE TABLE \"tbl\" (\"a\" VARCHAR) PARTITIONED BY DAY"); + assertUnparseRoundTrips("CREATE TABLE \"tbl\" (\"a\" VARCHAR) PARTITIONED BY DAY CLUSTERED BY \"a\""); + assertUnparseRoundTrips("CREATE TABLE \"tbl\" (\"p\" TYPE('COMPLEX'))"); + assertUnparseRoundTrips("ALTER TABLE \"tbl\" ADD COLUMN \"a\" DOUBLE"); + assertUnparseRoundTrips("ALTER TABLE \"tbl\" DROP COLUMN \"a\""); + assertUnparseRoundTrips("ALTER TABLE \"tbl\" ALTER COLUMN \"a\" SET DATA TYPE BIGINT"); + assertUnparseRoundTrips("ALTER TABLE \"tbl\" SET PROPERTIES (\"sealed\" = TRUE)"); + } + + @Test + public void testDdlAfterSetStatement() + { + final SqlNode node = parse("SET sqlQueryId = 'abc'; CREATE TABLE tbl (a VARCHAR)"); + assertInstanceOf(DruidSqlCreateTable.class, node); + } + + @Test + public void testDdlWithTrailingSemicolon() + { + assertInstanceOf(DruidSqlCreateTable.class, parse("CREATE TABLE tbl (a VARCHAR);")); + assertInstanceOf(DruidSqlAlterTable.AddColumn.class, parse("ALTER TABLE tbl ADD COLUMN a VARCHAR;")); + } + + @Test + public void testDdlBeforeAnotherStatementIsRejected() + { + final DruidException e = assertThrows( + DruidException.class, + () -> parse("CREATE TABLE tbl (a VARCHAR); SELECT 1") + ); + assertTrue(e.getMessage().contains("Only SET statements can appear before the final statement")); + } + + /** + * {@code ALTER SYSTEM}/{@code ALTER SESSION} must keep working: {@code ALTER TABLE} is dispatched by a two-token + * lookahead ahead of Calcite's stock {@code SqlAlter()} production. + */ + @Test + public void testAlterSystemStillParses() throws SqlParseException + { + // Parsed directly rather than through DruidSqlParser.parse, which folds SET options into the query context and + // then requires a non-SET statement to execute. + assertInstanceOf(SqlSetOption.class, parseStatementList("ALTER SYSTEM SET \"a\" = 1").get(0)); + assertInstanceOf(SqlSetOption.class, parseStatementList("ALTER SESSION SET \"a\" = 1").get(0)); + } + + /** + * {@code IF} and {@code PROPERTIES} are added as non-reserved keywords, so they must remain usable as identifiers. + */ + @Test + public void testNewKeywordsRemainUsableAsIdentifiers() + { + final DruidSqlCreateTable create = parseCreate("CREATE TABLE properties (if VARCHAR, properties BIGINT)"); + assertEquals("properties", create.getName().toString()); + assertEquals("if VARCHAR, properties BIGINT", columnsOf(create)); + } + + @Test + public void testExplainOfDdlIsRejected() + { + assertThrows(DruidException.class, () -> parse("EXPLAIN PLAN FOR CREATE TABLE tbl (a VARCHAR)")); + } + + @Test + public void testCreateTableWithoutTypeIsRejected() + { + assertThrows(DruidException.class, () -> parse("CREATE TABLE tbl (a)")); + } + + @Test + public void testAlterTableWithoutOperationIsRejected() + { + assertThrows(DruidException.class, () -> parse("ALTER TABLE tbl")); + } + + @Test + public void testDropTableIsNotSupported() + { + // DROP TABLE is deliberately unclaimed; it must not silently parse as something else. + assertThrows(DruidException.class, () -> parse("DROP TABLE tbl")); + } + + private static void assertUnparseRoundTrips(String sql) + { + final SqlNode node = parse(sql); + assertEquals(sql, node.toSqlString(CalciteSqlDialect.DEFAULT).getSql().replace("\n", " ")); + } + + private static String columnsOf(DruidSqlCreateTable create) + { + return create.getColumnList() + .stream() + .map(c -> { + final DruidSqlColumnDeclaration column = (DruidSqlColumnDeclaration) c; + return column.getName() + " " + column.getDataType(); + }) + .collect(Collectors.joining(", ")); + } + + private static SqlNode parse(String sql) + { + return DruidSqlParser.parse(sql, true).getMainStatement(); + } + + private static SqlNodeList parseStatementList(String sql) throws SqlParseException + { + return (SqlNodeList) SqlParser.create(sql, DruidSqlParser.PARSER_CONFIG).parseStmtList(); + } + + private static DruidSqlCreateTable parseCreate(String sql) + { + return assertInstanceOf(DruidSqlCreateTable.class, parse(sql)); + } + + private static T parseAlter(String sql, Class clazz) + { + return assertInstanceOf(clazz, parse(sql)); + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModuleTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModuleTest.java index c283a364d1af..0df3eb6aa533 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModuleTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModuleTest.java @@ -140,6 +140,7 @@ public void onMatch(RelOptRuleCall call) binder.bind(DruidSchemaCatalog.class).toInstance(rootSchema); binder.bind(JoinableFactoryWrapper.class).toInstance(joinableFactoryWrapper); binder.bind(CatalogResolver.class).toInstance(CatalogResolver.NULL_RESOLVER); + binder.bind(CatalogTableWriter.class).toInstance(CatalogTableWriter.NOT_AVAILABLE); }, target, binder -> { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 487c50570f3b..70b5b3662f99 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -104,6 +104,7 @@ import org.apache.druid.sql.calcite.TempDirProducer; import org.apache.druid.sql.calcite.planner.CalciteRulesManager; import org.apache.druid.sql.calcite.planner.CatalogResolver; +import org.apache.druid.sql.calcite.planner.CatalogTableWriter; import org.apache.druid.sql.calcite.planner.DruidOperatorTable; import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerFactory; @@ -220,6 +221,11 @@ default CatalogResolver createCatalogResolver() return CatalogResolver.NULL_RESOLVER; } + default CatalogTableWriter createCatalogTableWriter() + { + return CatalogTableWriter.NOT_AVAILABLE; + } + /** * Configure the JSON mapper. */ @@ -751,6 +757,7 @@ public static class Builder private final QueryComponentSupplier componentSupplier; private int mergeBufferCount; private CatalogResolver catalogResolver = CatalogResolver.NULL_RESOLVER; + private CatalogTableWriter catalogTableWriter = CatalogTableWriter.NOT_AVAILABLE; private List overrideModules = new ArrayList<>(); private SqlTestFrameworkConfig config; private Closer resourceCloser = Closer.create(); @@ -772,6 +779,12 @@ public Builder catalogResolver(CatalogResolver catalogResolver) return this; } + public Builder catalogTableWriter(CatalogTableWriter catalogTableWriter) + { + this.catalogTableWriter = catalogTableWriter; + return this; + } + public Builder withOverrideModule(Module m) { this.overrideModules.add(m); @@ -799,6 +812,11 @@ public CatalogResolver getCatalogResolver() return catalogResolver; } + public CatalogTableWriter getCatalogTableWriter() + { + return catalogTableWriter; + } + public Closer getResourceCloser() { return resourceCloser; @@ -848,6 +866,7 @@ public PlannerFixture( new CalciteRulesManager(componentSupplier.extensionCalciteRules()), framework.injector.getInstance(JoinableFactoryWrapper.class), framework.builder.catalogResolver, + framework.builder.catalogTableWriter, authConfig != null ? authConfig : new AuthConfig(), NoopPolicyEnforcer.instance(), new DruidHookDispatcher() From ae9c072408fdc3eb5298182fbc431b8bbbcc6ec6 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 29 Jul 2026 14:30:22 -0700 Subject: [PATCH 02/18] feat: ddl + projections --- docs/development/extensions-core/catalog.md | 49 ++- .../catalog/CatalogDdlAndIngestTest.java | 101 +++++ .../druid/catalog/http/TableEditRequest.java | 49 ++- .../druid/catalog/http/TableEditor.java | 97 ++++- .../catalog/sync/CatalogSqlTableWriter.java | 13 + .../druid/server/http/catalog/EditorTest.java | 58 +++ .../catalog/model/table/DatasourceDefn.java | 78 ++++ .../model/table/DatasourceTableTest.java | 72 ++++ sql/src/main/codegen/config.fmpp | 6 + sql/src/main/codegen/includes/ddl.ftl | 134 ++++-- .../calcite/parser/DruidSqlAlterTable.java | 161 +++++++ .../calcite/parser/DruidSqlCreateTable.java | 23 +- .../sql/calcite/parser/SqlProjectionSpec.java | 107 +++++ .../calcite/planner/CatalogDdlHandler.java | 140 ++++++ .../calcite/planner/CatalogTableWriter.java | 27 ++ .../sql/calcite/planner/DruidPlanner.java | 17 +- .../sql/calcite/planner/PlannerFactory.java | 69 ++- .../planner/ProjectionSpecTranslator.java | 397 ++++++++++++++++++ .../calcite/planner/ProjectionSqlEngine.java | 156 +++++++ .../calcite/planner/SqlStatementHandler.java | 5 + .../sql/calcite/CalciteCatalogDdlTest.java | 224 ++++++++++ .../calcite/parser/DruidSqlDdlParserTest.java | 134 ++++++ 22 files changed, 2083 insertions(+), 34 deletions(-) create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlProjectionSpec.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSqlEngine.java diff --git a/docs/development/extensions-core/catalog.md b/docs/development/extensions-core/catalog.md index 81ed7f9a2115..351d13aa2631 100644 --- a/docs/development/extensions-core/catalog.md +++ b/docs/development/extensions-core/catalog.md @@ -63,7 +63,7 @@ report that the extension is not available. ```sql CREATE [OR REPLACE] TABLE [IF NOT EXISTS]

- [ ( [, ...] ) ] + [ ( { | PROJECTION AS (
ADD COLUMN ALTER TABLE
DROP COLUMN ALTER TABLE
ALTER COLUMN SET DATA TYPE +ALTER TABLE
ADD [IF NOT EXISTS] PROJECTION AS (
DROP PROJECTION [IF EXISTS] ALTER TABLE
SET PROPERTIES ( = [, ...] ) ``` +#### Projections + +A table may declare [projections](../../querying/projections.md), which are pre-aggregated views stored inside each +segment. A projection is written as a `SELECT` over the table's own columns, with no `FROM` clause: + +```sql +CREATE TABLE "druid"."visits" ( + __time TIMESTAMP, + user_id VARCHAR, + user_agent VARCHAR, + pages_visited BIGINT, + PROJECTION daily_by_agent AS ( + SELECT TIME_FLOOR(__time, 'P1D'), user_agent, SUM(pages_visited) AS total_pages + WHERE user_agent IS NOT NULL + GROUP BY 1, 2 + ) +) +PARTITIONED BY DAY +``` + +The body is planned exactly as the equivalent query would be, so a projection matches the queries it was written to +serve. Every aggregate needs an alias, which becomes the name of the stored column. Time granularity is expressed +with `TIME_FLOOR`, as it would be in a query. + +A projection body accepts a select list, an optional `WHERE` and an optional `GROUP BY`. It cannot use `ORDER BY`, +`LIMIT` or `HAVING`: a projection's ordering follows its grouping columns and is not something you choose. It also +cannot use joins, subqueries, or expressions computed over aggregates. Store the aggregates instead: `SUM(x)` and +`COUNT(x)` rather than `AVG(x)`. + +Projections may also be added to and removed from an existing table: + +```sql +ALTER TABLE "druid"."visits" ADD [IF NOT EXISTS] PROJECTION by_agent AS ( + SELECT user_agent, SUM(pages_visited) AS total_pages GROUP BY user_agent +) +ALTER TABLE "druid"."visits" DROP PROJECTION [IF EXISTS] by_agent +``` + +Both take effect for subsequent ingestion. Segments already built keep whatever projections they were built with, so +dropping a projection does not rewrite data. + +The name `__base` is reserved for the projection describing the table's own physical layout, and is not accepted yet. + +#### Setting table properties + `SET PROPERTIES` merges the given [table properties](#table-properties) into the table. A value of `NULL` removes a property. Values must be literals: diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java index 060dda567cfb..5513dfa14dee 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java @@ -19,7 +19,9 @@ package org.apache.druid.testing.embedded.catalog; +import com.fasterxml.jackson.core.type.TypeReference; import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import org.apache.druid.catalog.model.TableId; import org.apache.druid.catalog.model.TableMetadata; import org.apache.druid.catalog.model.table.ClusterKeySpec; @@ -27,6 +29,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.http.SqlTaskStatus; import org.apache.druid.segment.TestHelper; +import org.apache.druid.server.metrics.LatchableEmitter; import org.apache.druid.testing.embedded.EmbeddedDruidCluster; import org.apache.druid.testing.embedded.msq.EmbeddedMSQApis; import org.junit.jupiter.api.BeforeAll; @@ -240,6 +243,104 @@ public void testCreateTableAlreadyExistsFails() assertTrue(e.getMessage().contains("duplicate table"), e.getMessage()); } + /** + * A projection defined in SQL must actually be used at query time, which is the whole point of storing one: the + * specification the translator produces has to match what the planner generates for the equivalent query. + */ + @Test + public void testCreateTableWithProjectionThenQuery() + { + final String tableName = dataSource; + + cluster.callApi().runSql( + "CREATE TABLE \"%s\" (\n" + + " __time TIMESTAMP,\n" + + " varchar_col1 VARCHAR,\n" + + " bigint_col1 BIGINT,\n" + + " PROJECTION by_varchar AS (\n" + + " SELECT varchar_col1, SUM(bigint_col1) AS sum_bigint_col1\n" + + " GROUP BY varchar_col1\n" + + " )\n" + + ")\n" + + "PARTITIONED BY DAY", + tableName + ); + + ingest( + "INSERT INTO \"%s\"\n" + + "SELECT TIME_PARSE(a) AS __time, b AS varchar_col1, c AS bigint_col1\n" + + "FROM TABLE(\n" + + " EXTERN(\n" + + " '{\"type\":\"inline\",\"data\":\"2022-12-26T12:34:56,foo,10\\n2022-12-26T12:34:56,foo,9" + + "\\n2022-12-26T12:34:56,bar,8\"}',\n" + + " '{\"type\":\"csv\",\"findColumnsFromHeader\":false,\"columns\":[\"a\",\"b\",\"c\"]}'\n" + + " )\n" + + ") EXTEND (a VARCHAR, b VARCHAR, c BIGINT)\n", + tableName + ); + + final LatchableEmitter emitter = historical.latchableEmitter(); + emitter.flush(); + + cluster.callApi().verifySqlQuery( + "SELECT varchar_col1, SUM(bigint_col1) FROM %s GROUP BY 1 ORDER BY 1", + tableName, + "bar,8\nfoo,19" + ); + + // The segment-scan metrics name the projection that served the query. + emitter.waitForEvent( + event -> event.hasMetricName("query/segment/time").hasDimension("projection", "by_varchar") + ); + } + + @Test + public void testAlterTableAddAndDropProjection() + { + final String tableName = dataSource; + + cluster.callApi().runSql( + "CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR, b BIGINT) PARTITIONED BY DAY", + tableName + ); + cluster.callApi().runSql( + "ALTER TABLE \"%s\" ADD PROJECTION p AS (SELECT a, SUM(b) AS sum_b GROUP BY a)", + tableName + ); + assertEquals(1, projectionsOf(tableName).size()); + assertEquals("p", projectionsOf(tableName).get(0).getSpec().getName()); + + // Adding it again is an error, but IF NOT EXISTS tolerates it. + assertThrows( + Exception.class, + () -> cluster.callApi().runSql( + "ALTER TABLE \"%s\" ADD PROJECTION p AS (SELECT a, SUM(b) AS sum_b GROUP BY a)", + tableName + ) + ); + cluster.callApi().runSql( + "ALTER TABLE \"%s\" ADD IF NOT EXISTS PROJECTION p AS (SELECT a, SUM(b) AS sum_b GROUP BY a)", + tableName + ); + assertEquals(1, projectionsOf(tableName).size()); + + cluster.callApi().runSql("ALTER TABLE \"%s\" DROP PROJECTION p", tableName); + assertNull( + client.readTable(TableId.datasource(tableName)) + .spec().properties().get(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY) + ); + cluster.callApi().runSql("ALTER TABLE \"%s\" DROP PROJECTION IF EXISTS p", tableName); + } + + private List projectionsOf(String tableName) + { + return TestHelper.JSON_MAPPER.convertValue( + client.readTable(TableId.datasource(tableName)) + .spec().properties().get(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY), + new TypeReference>() {} + ); + } + private void ingest(String sqlPattern, String tableName) { final SqlTaskStatus status = msqApis.submitTaskSql(StringUtils.format(sqlPattern, tableName)); diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java index 79136d4a0595..428f982c7848 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java @@ -25,9 +25,9 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import javax.annotation.Nullable; - import java.util.List; import java.util.Map; @@ -51,6 +51,8 @@ @Type(name = "updateProperties", value = TableEditRequest.UpdateProperties.class), @Type(name = "updateColumns", value = TableEditRequest.UpdateColumns.class), @Type(name = "moveColumn", value = TableEditRequest.MoveColumn.class), + @Type(name = "addProjection", value = TableEditRequest.AddProjection.class), + @Type(name = "dropProjection", value = TableEditRequest.DropProjection.class), }) public class TableEditRequest { @@ -169,4 +171,49 @@ public MoveColumn( this.anchor = anchor; } } + + /** + * Append a projection to the list of projections. Fails if a projection of the same name already exists, unless + * {@code ifNotExists} is set, in which case the existing projection is left alone. + */ + public static class AddProjection extends TableEditRequest + { + @JsonProperty("projection") + public final DatasourceProjectionMetadata projection; + @JsonProperty("ifNotExists") + public final boolean ifNotExists; + + @JsonCreator + public AddProjection( + @JsonProperty("projection") DatasourceProjectionMetadata projection, + @JsonProperty("ifNotExists") boolean ifNotExists + ) + { + this.projection = projection; + this.ifNotExists = ifNotExists; + } + } + + /** + * Remove the named projection from the list of projections. Fails if no such projection exists, unless + * {@code ifExists} is set. Segments already built keep whatever projections they were built with. + */ + public static class DropProjection extends TableEditRequest + { + @JsonProperty("projection") + public final String projection; + @JsonProperty("ifExists") + public final boolean ifExists; + + @JsonCreator + public DropProjection( + @JsonProperty("projection") String projection, + @JsonProperty("ifExists") boolean ifExists + ) + { + this.projection = projection; + this.ifExists = ifExists; + } + } + } diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java index b3a762b05686..f9419691a340 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java @@ -21,7 +21,9 @@ import com.google.common.base.Strings; import org.apache.druid.catalog.CatalogException; +import org.apache.druid.catalog.http.TableEditRequest.AddProjection; import org.apache.druid.catalog.http.TableEditRequest.DropColumns; +import org.apache.druid.catalog.http.TableEditRequest.DropProjection; import org.apache.druid.catalog.http.TableEditRequest.HideColumns; import org.apache.druid.catalog.http.TableEditRequest.MoveColumn; import org.apache.druid.catalog.http.TableEditRequest.MoveColumn.Position; @@ -30,6 +32,8 @@ import org.apache.druid.catalog.http.TableEditRequest.UpdateProperties; import org.apache.druid.catalog.model.CatalogUtils; import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; +import org.apache.druid.catalog.model.ResolvedTable; import org.apache.druid.catalog.model.TableDefn; import org.apache.druid.catalog.model.TableId; import org.apache.druid.catalog.model.TableMetadata; @@ -41,7 +45,6 @@ import org.apache.druid.utils.CollectionUtils; import javax.ws.rs.core.Response; - import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -99,6 +102,12 @@ public long go() throws CatalogException return updateProperties(((UpdateProperties) editRequest).properties); } else if (editRequest instanceof UpdateColumns) { return updateColumns(((UpdateColumns) editRequest).columns); + } else if (editRequest instanceof AddProjection) { + final AddProjection addProjection = (AddProjection) editRequest; + return addProjection(addProjection.projection, addProjection.ifNotExists); + } else if (editRequest instanceof DropProjection) { + final DropProjection dropProjection = (DropProjection) editRequest; + return dropProjection(dropProjection.projection, dropProjection.ifExists); } else if (editRequest instanceof MoveColumn) { return moveColumn(((MoveColumn) editRequest)); } else { @@ -379,4 +388,90 @@ private TableSpec applyMoveColumn( return existingSpec.withColumns(revised); } + + /** + * Projections are stored as a table property, so adding one is a read-modify-write of that property performed + * inside the update transaction rather than by the caller. + */ + private long addProjection(DatasourceProjectionMetadata projection, boolean ifNotExists) throws CatalogException + { + if (projection == null || projection.getSpec() == null) { + throw CatalogException.badRequest("A projection is required"); + } + final String name = projection.getSpec().getName(); + return catalog.tables().updateProperties(id, table -> { + final List existing = projectionsOf(table); + for (DatasourceProjectionMetadata current : existing) { + if (name.equals(current.getSpec().getName())) { + if (ifNotExists) { + return null; + } + throw CatalogException.badRequest( + "Projection [%s] already exists in table %s", + name, + id.sqlName() + ); + } + } + final List revised = new ArrayList<>(existing); + revised.add(projection); + return withProjections(table, revised); + }); + } + + private long dropProjection(String name, boolean ifExists) throws CatalogException + { + if (Strings.isNullOrEmpty(name)) { + throw CatalogException.badRequest("A projection name is required"); + } + return catalog.tables().updateProperties(id, table -> { + final List existing = projectionsOf(table); + final List revised = new ArrayList<>(existing.size()); + for (DatasourceProjectionMetadata current : existing) { + if (!name.equals(current.getSpec().getName())) { + revised.add(current); + } + } + if (revised.size() == existing.size()) { + if (ifExists) { + return null; + } + throw CatalogException.badRequest( + "Projection [%s] does not exist in table %s", + name, + id.sqlName() + ); + } + return withProjections(table, revised); + }); + } + + private List projectionsOf(TableMetadata table) throws CatalogException + { + final ResolvedTable resolved = catalog.tableRegistry().resolve(table.spec()); + final List projections = + resolved.decodeProperty(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY); + return projections == null ? Collections.emptyList() : projections; + } + + private TableSpec withProjections(TableMetadata table, List projections) + throws CatalogException + { + final TableSpec existingSpec = table.spec(); + final Map revised = new HashMap<>(existingSpec.properties()); + if (projections.isEmpty()) { + revised.remove(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY); + } else { + revised.put(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, projections); + } + final TableSpec revisedSpec = existingSpec.withProperties(revised); + try { + catalog.tableRegistry().resolve(revisedSpec).validate(); + } + catch (IAE | DruidException e) { + throw CatalogException.badRequest(e.getMessage()); + } + return revisedSpec; + } + } diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogSqlTableWriter.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogSqlTableWriter.java index c8813384d9b2..f75b2bd27eec 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogSqlTableWriter.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogSqlTableWriter.java @@ -24,6 +24,7 @@ import org.apache.druid.catalog.CatalogException; import org.apache.druid.catalog.http.TableEditRequest; import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import org.apache.druid.catalog.model.TableId; import org.apache.druid.catalog.model.TableMetadata; import org.apache.druid.catalog.model.TableSpec; @@ -87,6 +88,18 @@ public void updateProperties(TableId tableId, Map properties) execute(tableId, () -> client.editTable(tableId, new TableEditRequest.UpdateProperties(properties))); } + @Override + public void addProjection(TableId tableId, DatasourceProjectionMetadata projection, boolean ifNotExists) + { + execute(tableId, () -> client.editTable(tableId, new TableEditRequest.AddProjection(projection, ifNotExists))); + } + + @Override + public void dropProjection(TableId tableId, String projectionName, boolean ifExists) + { + execute(tableId, () -> client.editTable(tableId, new TableEditRequest.DropProjection(projectionName, ifExists))); + } + @Nullable @Override public TableMetadata readTable(TableId tableId) diff --git a/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java b/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java index 107cedd565a9..18491161803d 100644 --- a/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java +++ b/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java @@ -24,7 +24,9 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.catalog.CatalogException; import org.apache.druid.catalog.http.TableEditRequest; +import org.apache.druid.catalog.http.TableEditRequest.AddProjection; import org.apache.druid.catalog.http.TableEditRequest.DropColumns; +import org.apache.druid.catalog.http.TableEditRequest.DropProjection; import org.apache.druid.catalog.http.TableEditRequest.HideColumns; import org.apache.druid.catalog.http.TableEditRequest.MoveColumn; import org.apache.druid.catalog.http.TableEditRequest.UnhideColumns; @@ -34,6 +36,7 @@ import org.apache.druid.catalog.model.CatalogUtils; import org.apache.druid.catalog.model.ColumnSpec; import org.apache.druid.catalog.model.Columns; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import org.apache.druid.catalog.model.TableId; import org.apache.druid.catalog.model.TableMetadata; import org.apache.druid.catalog.model.table.ClusterKeySpec; @@ -41,8 +44,11 @@ import org.apache.druid.catalog.model.table.TableBuilder; import org.apache.druid.catalog.storage.CatalogStorage; import org.apache.druid.catalog.storage.CatalogTests; +import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.metadata.TestDerbyConnector; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -57,6 +63,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; public class EditorTest { @@ -486,4 +493,55 @@ public void testUpdateColumns() throws CatalogException CatalogUtils.columnNames(revised.spec().columns()) ); } + + @Test + public void testAddAndDropProjection() throws CatalogException + { + final String tableName = "projections"; + final TableMetadata table = TableBuilder.datasource(tableName, "P1D") + .timeColumn() + .column("dim", "VARCHAR") + .column("met", "BIGINT") + .build(); + catalog.tables().create(table); + + final DatasourceProjectionMetadata daily = new DatasourceProjectionMetadata( + AggregateProjectionSpec.builder("daily") + .groupingColumns(new StringDimensionSchema("dim")) + .aggregators(new LongSumAggregatorFactory("sum_met", "met")) + .build() + ); + + assertTrue(new TableEditor(catalog, table.id(), new AddProjection(daily, false)).go() > 0); + assertEquals(List.of(daily), projectionsOf(tableName)); + + // Adding the same name again is an error, unless the caller said to leave it alone. + assertThrows( + CatalogException.class, + () -> new TableEditor(catalog, table.id(), new AddProjection(daily, false)).go() + ); + assertEquals(0, new TableEditor(catalog, table.id(), new AddProjection(daily, true)).go()); + assertEquals(List.of(daily), projectionsOf(tableName)); + + // Dropping a projection that is not there is likewise an error unless tolerated. + assertThrows( + CatalogException.class, + () -> new TableEditor(catalog, table.id(), new DropProjection("nope", false)).go() + ); + assertEquals(0, new TableEditor(catalog, table.id(), new DropProjection("nope", true)).go()); + + assertTrue(new TableEditor(catalog, table.id(), new DropProjection("daily", false)).go() > 0); + assertNull( + catalog.tables().read(TableId.datasource(tableName)) + .spec().properties().get(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY) + ); + } + + private List projectionsOf(String tableName) throws CatalogException + { + return catalog.tableRegistry() + .resolve(catalog.tables().read(TableId.datasource(tableName)).spec()) + .decodeProperty(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY); + } + } diff --git a/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java b/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java index 89982a94883f..ae1b7a75f6e6 100644 --- a/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java +++ b/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.catalog.model.CatalogUtils; import org.apache.druid.catalog.model.Columns; import org.apache.druid.catalog.model.DatasourceBaseTableMetadata; import org.apache.druid.catalog.model.DatasourceProjectionMetadata; @@ -30,12 +31,20 @@ import org.apache.druid.catalog.model.ResolvedTable; import org.apache.druid.catalog.model.TableDefn; import org.apache.druid.catalog.model.TableSpec; +import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.data.input.impl.DimensionSchema; import org.apache.druid.error.InvalidInput; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.segment.VirtualColumn; +import org.apache.druid.segment.indexing.DataSchema; +import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; public class DatasourceDefn extends TableDefn { @@ -123,6 +132,75 @@ public void validate(ResolvedTable table) // fail fast instead of surfacing layout problems at ingest time. baseTable.createSpec(table.spec().columns()); } + validateProjections(table); + } + + /** + * Cross-validate the declared projections. Names must be unique, and a projection must not be coarser than the + * segments it lives in. For a sealed table the declared columns are the whole schema, so a projection that reads a + * column the table does not declare can never be built and is rejected; for a non-sealed table ingestion may add + * columns the catalog has not seen, so only the projections' internal consistency is checked. + */ + private void validateProjections(ResolvedTable table) + { + final List projections = table.decodeProperty(PROJECTIONS_KEYS_PROPERTY); + if (projections == null || projections.isEmpty()) { + return; + } + + final List specs = new ArrayList<>(projections.size()); + for (DatasourceProjectionMetadata projection : projections) { + if (projection == null || projection.getSpec() == null) { + throw InvalidInput.exception("Projections must each have a [spec]"); + } + specs.add(projection.getSpec()); + } + + final String granularity = table.stringProperty(SEGMENT_GRANULARITY_PROPERTY); + DataSchema.validateProjections( + specs, + granularity == null ? null : CatalogUtils.asDruidGranularity(granularity) + ); + + if (!table.booleanProperty(SEALED_PROPERTY) || table.spec().columns() == null) { + return; + } + final Set declared = new HashSet<>(CatalogUtils.columnNames(table.spec().columns())); + declared.add(Columns.TIME_COLUMN); + for (AggregateProjectionSpec spec : specs) { + final Set available = new HashSet<>(declared); + for (VirtualColumn virtualColumn : spec.getVirtualColumns().getVirtualColumns()) { + available.add(virtualColumn.getOutputName()); + } + for (String required : requiredColumns(spec)) { + if (!available.contains(required)) { + throw InvalidInput.exception( + "Projection [%s] references column [%s], which table [%s] does not declare", + spec.getName(), + required, + table.spec().type() + ); + } + } + } + } + + private static Set requiredColumns(AggregateProjectionSpec spec) + { + final Set required = new HashSet<>(); + for (VirtualColumn virtualColumn : spec.getVirtualColumns().getVirtualColumns()) { + required.addAll(virtualColumn.requiredColumns()); + } + for (DimensionSchema groupingColumn : spec.getGroupingColumns()) { + required.add(groupingColumn.getName()); + } + for (AggregatorFactory aggregator : spec.getAggregators()) { + required.addAll(aggregator.requiredFields()); + } + if (spec.getFilter() != null) { + required.addAll(spec.getFilter().getRequiredColumns()); + } + return required; } /** diff --git a/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java b/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java index 66c956876536..3d409723785a 100644 --- a/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java +++ b/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java @@ -29,17 +29,22 @@ import org.apache.druid.catalog.model.ColumnSpec; import org.apache.druid.catalog.model.Columns; import org.apache.druid.catalog.model.DatasourceBaseTableMetadata; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import org.apache.druid.catalog.model.ResolvedTable; import org.apache.druid.catalog.model.TableDefn; import org.apache.druid.catalog.model.TableDefnRegistry; import org.apache.druid.catalog.model.TableSpec; import org.apache.druid.catalog.model.facade.DatasourceFacade; import org.apache.druid.catalog.model.facade.DatasourceFacade.ColumnFacade; +import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.error.DruidException; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; @@ -483,6 +488,73 @@ public void testColumns() } } + /** + * Projections hold polymorphic aggregators and dimension schemas, so validating them needs a mapper that knows + * those subtypes; the shared mapper in this class does not register them. + */ + @Test + public void testProjectionValidation() + { + final TableDefnRegistry projectionRegistry = new TableDefnRegistry(TestHelper.makeJsonMapper()); + final TableBuilder builder = TableBuilder.datasource("foo", "P1D") + .timeColumn() + .column("dim", Columns.SQL_VARCHAR) + .column("met", Columns.SQL_BIGINT); + + final AggregateProjectionSpec good = AggregateProjectionSpec + .builder("daily") + .groupingColumns(new StringDimensionSchema("dim")) + .aggregators(new LongSumAggregatorFactory("sum_met", "met")) + .build(); + + projectionRegistry.resolve( + builder.copy() + .property(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, List.of(new DatasourceProjectionMetadata(good))) + .buildSpec() + ).validate(); + + // Two projections cannot share a name. + final TableSpec duplicateNames = + builder.copy() + .property( + DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, + List.of(new DatasourceProjectionMetadata(good), new DatasourceProjectionMetadata(good)) + ) + .buildSpec(); + assertThrows(DruidException.class, () -> projectionRegistry.resolve(duplicateNames).validate()); + + // A sealed table declares its whole schema, so a projection over an undeclared column can never be built. + final AggregateProjectionSpec undeclared = AggregateProjectionSpec + .builder("bad") + .groupingColumns(new StringDimensionSchema("nope")) + .aggregators(new LongSumAggregatorFactory("sum_met", "met")) + .build(); + + final TableSpec sealedWithUndeclared = + builder.copy() + .sealed(true) + .property( + DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, + List.of(new DatasourceProjectionMetadata(undeclared)) + ) + .buildSpec(); + final DruidException e = assertThrows( + DruidException.class, + () -> projectionRegistry.resolve(sealedWithUndeclared).validate() + ); + assertTrue(e.getMessage().contains("references column [nope]")); + + // Ingestion may add columns to a table that is not sealed, so the same projection is allowed there. + projectionRegistry.resolve( + builder.copy() + .property( + DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, + List.of(new DatasourceProjectionMetadata(undeclared)) + ) + .buildSpec() + ).validate(); + } + @Test public void testTimeColumn() { diff --git a/sql/src/main/codegen/config.fmpp b/sql/src/main/codegen/config.fmpp index 714d8af28f0d..2eb61970328a 100644 --- a/sql/src/main/codegen/config.fmpp +++ b/sql/src/main/codegen/config.fmpp @@ -53,6 +53,9 @@ data: { "org.apache.calcite.sql.SqlCreate" "org.apache.calcite.sql.SqlInsert" "org.apache.calcite.sql.SqlNodeList" + "org.apache.calcite.sql.SqlSelect" + "org.apache.calcite.sql.SqlLiteral" + "org.apache.calcite.sql.parser.SqlParserPos" "org.apache.calcite.sql.SqlBasicCall" "org.apache.druid.java.util.common.granularity.Granularity" "org.apache.druid.java.util.common.granularity.GranularityType" @@ -65,6 +68,7 @@ data: { "org.apache.druid.sql.calcite.parser.DruidSqlPropertyAssignment" "org.apache.druid.sql.calcite.external.ExtendOperator" "org.apache.druid.sql.calcite.external.ParameterizeOperator" + "org.apache.druid.sql.calcite.parser.SqlProjectionSpec" "org.apache.druid.sql.calcite.parser.ExternalDestinationSqlIdentifier" "java.util.HashMap" ] @@ -78,6 +82,7 @@ data: { "EXTERN" "IF" "PROPERTIES" + "PROJECTION" ] nonReservedKeywordsToAdd: [ @@ -85,6 +90,7 @@ data: { "EXTERN" "IF" "PROPERTIES" + "PROJECTION" ] # List of methods for parsing custom SQL statements. diff --git a/sql/src/main/codegen/includes/ddl.ftl b/sql/src/main/codegen/includes/ddl.ftl index 46fa5e3d0059..cfb30c1f55a2 100644 --- a/sql/src/main/codegen/includes/ddl.ftl +++ b/sql/src/main/codegen/includes/ddl.ftl @@ -28,7 +28,9 @@ SqlCreate DruidSqlCreateTable(Span s, boolean replace) : { boolean ifNotExists = false; final SqlIdentifier id; - SqlNodeList columnList = SqlNodeList.EMPTY; + final List columns = new ArrayList(); + final List projections = new ArrayList(); + Span elementSpan = null; SqlGranularityLiteral partitionedBy = null; SqlNodeList clusteredBy = null; } @@ -36,7 +38,14 @@ SqlCreate DruidSqlCreateTable(Span s, boolean replace) :
[ { ifNotExists = true; } ] id = CompoundTableIdentifier() - [ columnList = DruidColumnDeclarationList() ] + [ + { elementSpan = span(); } + AddDruidTableElement(columns, projections) + ( + AddDruidTableElement(columns, projections) + )* + + ] [ partitionedBy = PartitionGranularity() @@ -45,35 +54,90 @@ SqlCreate DruidSqlCreateTable(Span s, boolean replace) : clusteredBy = ClusteredBy() ] { - return new DruidSqlCreateTable(s.end(this), replace, ifNotExists, id, columnList, partitionedBy, clusteredBy); + final SqlParserPos elementPos = elementSpan == null ? s.pos() : elementSpan.end(this); + return new DruidSqlCreateTable( + s.end(this), + replace, + ifNotExists, + id, + new SqlNodeList(columns, elementPos), + new SqlNodeList(projections, elementPos), + partitionedBy, + clusteredBy + ); } } -SqlNodeList DruidColumnDeclarationList() : +// A table element is either a column declaration or a projection definition. A column may legitimately be named +// "projection" (the keyword is non-reserved) and may have a bare-identifier type, so two tokens are not enough to +// tell the two apart: a projection definition is distinguished by its third token, which is always '(' or AS. +void AddDruidTableElement(List columns, List projections) : { - final Span s; - final List list = new ArrayList(); + final DruidSqlColumnDeclaration column; + final SqlProjectionSpec projection; } { - { s = span(); } - AddDruidColumnDeclaration(list) - ( - AddDruidColumnDeclaration(list) - )* - + LOOKAHEAD(3) + projection = DruidProjectionDefinition() { - return new SqlNodeList(list, s.end(this)); + projections.add(projection); + } +| + column = DruidColumnDeclaration() + { + columns.add(column); } } -void AddDruidColumnDeclaration(List list) : +// The body is a SELECT with no FROM: the table the projection belongs to is implicit. Only a select list, WHERE and +// GROUP BY are admitted; a projection has no way to express ordering, limits or having. +SqlProjectionSpec DruidProjectionDefinition() : { - final DruidSqlColumnDeclaration column; + final Span s; + final Span bodySpan; + final SqlIdentifier name; + final List keywords = new ArrayList(); + final SqlNodeList keywordList; + final List selectList = new ArrayList(); + SqlLiteral keyword = null; + final SqlNode where; + final SqlNodeList groupBy; } { - column = DruidColumnDeclaration() + { s = span(); } + name = SimpleIdentifier() + [ ] + +
id = CompoundTableIdentifier() ( - column = DruidColumnDeclaration() - { - return new DruidSqlAlterTable.AddColumn(s.end(this), id, column); - } + + ( + column = DruidColumnDeclaration() + { + return new DruidSqlAlterTable.AddColumn(s.end(this), id, column); + } + | + [ { ifNotExists = true; } ] projection = DruidProjectionDefinition() + { + return new DruidSqlAlterTable.AddProjection(s.end(this), id, projection, ifNotExists); + } + ) | - columnName = SimpleIdentifier() - { - return new DruidSqlAlterTable.DropColumn(s.end(this), id, columnName); - } + + ( + columnName = SimpleIdentifier() + { + return new DruidSqlAlterTable.DropColumn(s.end(this), id, columnName); + } + | + [ { ifExists = true; } ] projectionName = SimpleIdentifier() + { + return new DruidSqlAlterTable.DropProjection(s.end(this), id, projectionName, ifExists); + } + ) | columnName = SimpleIdentifier() dataType = DataType() { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlAlterTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlAlterTable.java index bf6c20f4a604..d9cadd2d4c12 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlAlterTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlAlterTable.java @@ -239,6 +239,167 @@ public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNod } } + /** + * {@code ALTER TABLE ADD [IF NOT EXISTS] PROJECTION AS ( ... )}. + */ + public static class AddProjection extends DruidSqlAlterTable + { + public static final SqlOperator OPERATOR = new Operator("ALTER TABLE ADD PROJECTION"); + + private final SqlProjectionSpec projection; + private final boolean ifNotExists; + + public AddProjection( + SqlParserPos pos, + SqlIdentifier name, + SqlProjectionSpec projection, + boolean ifNotExists + ) + { + super(pos, name); + this.projection = projection; + this.ifNotExists = ifNotExists; + } + + public SqlProjectionSpec getProjection() + { + return projection; + } + + public boolean isIfNotExists() + { + return ifNotExists; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of( + getName(), + projection, + SqlLiteral.createBoolean(ifNotExists, SqlParserPos.ZERO) + ); + } + + @Override + protected void unparseOperation(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("ADD"); + if (ifNotExists) { + writer.keyword("IF NOT EXISTS"); + } + projection.unparse(writer, 0, 0); + } + + private static class Operator extends SqlSpecialOperator + { + Operator(String name) + { + super(name, SqlKind.ALTER_TABLE); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new AddProjection( + pos, + (SqlIdentifier) operands[0], + (SqlProjectionSpec) operands[1], + ((SqlLiteral) operands[2]).booleanValue() + ); + } + } + } + + /** + * {@code ALTER TABLE DROP PROJECTION [IF EXISTS] }. Existing segments keep whatever projections they + * were built with; this only stops future ingestion from building it. + */ + public static class DropProjection extends DruidSqlAlterTable + { + public static final SqlOperator OPERATOR = new Operator("ALTER TABLE DROP PROJECTION"); + + private final SqlIdentifier projectionName; + private final boolean ifExists; + + public DropProjection( + SqlParserPos pos, + SqlIdentifier name, + SqlIdentifier projectionName, + boolean ifExists + ) + { + super(pos, name); + this.projectionName = projectionName; + this.ifExists = ifExists; + } + + public SqlIdentifier getProjectionName() + { + return projectionName; + } + + public boolean isIfExists() + { + return ifExists; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of( + getName(), + projectionName, + SqlLiteral.createBoolean(ifExists, SqlParserPos.ZERO) + ); + } + + @Override + protected void unparseOperation(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("DROP PROJECTION"); + if (ifExists) { + writer.keyword("IF EXISTS"); + } + projectionName.unparse(writer, 0, 0); + } + + private static class Operator extends SqlSpecialOperator + { + Operator(String name) + { + super(name, SqlKind.ALTER_TABLE); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new DropProjection( + pos, + (SqlIdentifier) operands[0], + (SqlIdentifier) operands[1], + ((SqlLiteral) operands[2]).booleanValue() + ); + } + } + } + /** * {@code ALTER TABLE SET PROPERTIES ( = , ...)}. A {@code NULL} value removes the property. */ diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java index 91e3ebfd2ae3..325ec9388ec5 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java @@ -49,6 +49,7 @@ public class DruidSqlCreateTable extends SqlCreate private final SqlIdentifier name; private final SqlNodeList columnList; + private final SqlNodeList projectionList; @Nullable private final SqlGranularityLiteral partitionedBy; @Nullable @@ -60,6 +61,7 @@ public DruidSqlCreateTable( boolean ifNotExists, SqlIdentifier name, SqlNodeList columnList, + SqlNodeList projectionList, @Nullable SqlGranularityLiteral partitionedBy, @Nullable SqlNodeList clusteredBy ) @@ -67,6 +69,7 @@ public DruidSqlCreateTable( super(OPERATOR, pos, replace, ifNotExists); this.name = name; this.columnList = columnList; + this.projectionList = projectionList; this.partitionedBy = partitionedBy; this.clusteredBy = clusteredBy; } @@ -85,6 +88,14 @@ public SqlNodeList getColumnList() return columnList; } + /** + * The declared projections, each a {@link SqlProjectionSpec}. + */ + public SqlNodeList getProjectionList() + { + return projectionList; + } + @Nullable public SqlGranularityLiteral getPartitionedBy() { @@ -110,6 +121,7 @@ public List getOperandList() return ImmutableNullableList.of( name, columnList, + projectionList, partitionedBy, clusteredBy, SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), @@ -135,6 +147,10 @@ public void unparse(SqlWriter writer, int leftPrec, int rightPrec) writer.sep(","); column.unparse(writer, 0, 0); } + for (SqlNode projection : projectionList) { + writer.sep(","); + projection.unparse(writer, 0, 0); + } writer.endList(frame); if (partitionedBy != null) { @@ -169,12 +185,13 @@ public SqlCall createCall( { return new DruidSqlCreateTable( pos, - ((SqlLiteral) operands[4]).booleanValue(), ((SqlLiteral) operands[5]).booleanValue(), + ((SqlLiteral) operands[6]).booleanValue(), (SqlIdentifier) operands[0], (SqlNodeList) operands[1], - (SqlGranularityLiteral) operands[2], - (SqlNodeList) operands[3] + (SqlNodeList) operands[2], + (SqlGranularityLiteral) operands[3], + (SqlNodeList) operands[4] ); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlProjectionSpec.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlProjectionSpec.java new file mode 100644 index 000000000000..4e8d75ab5973 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlProjectionSpec.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.parser; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.util.ImmutableNullableList; + +import javax.annotation.Nonnull; +import java.util.List; + +/** + * A {@code PROJECTION AS ( SELECT ... )} clause, which defines a projection of the table it appears in. + *

+ * The body is a {@link SqlSelect} with no FROM clause: the table is implicit, and the grammar admits only a select + * list, an optional WHERE and an optional GROUP BY. ORDER BY, LIMIT, HAVING, joins and set operations are + * structurally excluded rather than validated away, because a projection cannot express them: its ordering is + * derived from its grouping columns. + */ +public class SqlProjectionSpec extends SqlCall +{ + public static final SqlOperator OPERATOR = new SqlProjectionSpecOperator(); + + private final SqlIdentifier name; + private final SqlSelect body; + + public SqlProjectionSpec(SqlParserPos pos, SqlIdentifier name, SqlSelect body) + { + super(pos); + this.name = name; + this.body = body; + } + + public SqlIdentifier getName() + { + return name; + } + + public SqlSelect getBody() + { + return body; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of(name, body); + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("PROJECTION"); + name.unparse(writer, 0, 0); + writer.keyword("AS"); + final SqlWriter.Frame frame = writer.startList("(", ")"); + body.unparse(writer, 0, 0); + writer.endList(frame); + } + + private static class SqlProjectionSpecOperator extends SqlSpecialOperator + { + public SqlProjectionSpecOperator() + { + super("PROJECTION", SqlKind.OTHER); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new SqlProjectionSpec(pos, (SqlIdentifier) operands[0], (SqlSelect) operands[1]); + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java index bc3a36f11507..872d1f7f1223 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java @@ -33,6 +33,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.catalog.model.ColumnSpec; import org.apache.druid.catalog.model.Columns; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import org.apache.druid.catalog.model.TableId; import org.apache.druid.catalog.model.TableMetadata; import org.apache.druid.catalog.model.TableSpec; @@ -48,6 +49,7 @@ import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.query.explain.ExplainAttributes; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.projections.Projections; import org.apache.druid.server.QueryResponse; import org.apache.druid.server.security.Action; import org.apache.druid.server.security.Resource; @@ -59,6 +61,7 @@ import org.apache.druid.sql.calcite.parser.DruidSqlParser; import org.apache.druid.sql.calcite.parser.DruidSqlPropertyAssignment; import org.apache.druid.sql.calcite.parser.SqlGranularityLiteral; +import org.apache.druid.sql.calcite.parser.SqlProjectionSpec; import java.util.ArrayList; import java.util.Collections; @@ -102,6 +105,12 @@ protected CatalogDdlHandler(SqlStatementHandler.HandlerContext handlerContext, S */ public static final String ENABLE_CATALOG_DDL_PROPERTY = "druid.sql.planner.enableCatalogDdl"; + /** + * The reserved name of the base-table projection, which describes the physical layout of the table itself. Handled + * as a separate catalog property, not as one of the aggregate projections. + */ + public static final String BASE_PROJECTION_NAME = "__base"; + @Override public void validate() { @@ -220,6 +229,39 @@ protected static ColumnSpec toColumnSpec(DruidSqlColumnDeclaration declaration) return new ColumnSpec(name, type, null); } + /** + * Translate one projection definition into the catalog form. + *

+ * {@code __base} is reserved for the base-table projection, which is a different catalog entity: it describes the + * physical layout of the table itself rather than an additional aggregate. It is rejected here rather than being + * translated as an ordinary projection. + */ + protected static DatasourceProjectionMetadata translateProjection( + final SqlStatementHandler.HandlerContext handlerContext, + final String tableName, + final List columns, + final SqlProjectionSpec projection + ) + { + final String name = simpleName(projection.getName(), "Projection"); + if (BASE_PROJECTION_NAME.equals(name)) { + throw InvalidSqlInput.exception( + "Projection [%s] names the base table layout, which is not supported yet", + BASE_PROJECTION_NAME + ); + } + try { + Projections.validateProjectionName(name); + } + catch (DruidException e) { + throw InvalidSqlInput.exception(e, "%s", e.getMessage()); + } + return new DatasourceProjectionMetadata( + new ProjectionSpecTranslator(handlerContext.plannerFactory()) + .translate(tableName, columns, name, projection.getBody()) + ); + } + protected static String simpleName(SqlIdentifier identifier, String what) { if (!identifier.isSimple()) { @@ -313,6 +355,20 @@ protected void validateStatement() if (createTable.getClusteredBy() != null) { properties.put(DatasourceDefn.CLUSTER_KEYS_PROPERTY, toClusterKeys(createTable.getClusteredBy())); } + if (!createTable.getProjectionList().isEmpty()) { + final List projections = + new ArrayList<>(createTable.getProjectionList().size()); + final Set seenProjections = new HashSet<>(); + for (SqlNode node : createTable.getProjectionList()) { + final SqlProjectionSpec projection = (SqlProjectionSpec) node; + final String name = simpleName(projection.getName(), "Projection"); + if (!seenProjections.add(name)) { + throw InvalidSqlInput.exception("Projection [%s] is declared more than once", name); + } + projections.add(translateProjection(handlerContext, tableId.name(), columns, projection)); + } + properties.put(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, projections); + } tableSpec = new TableSpec(DatasourceDefn.TABLE_TYPE, properties, columns); } @@ -449,6 +505,90 @@ protected String operationName() } } + /** + * {@code ALTER TABLE ... ADD PROJECTION}. The body is translated against the table's current declared columns, so + * the table must already have a catalog entry. + */ + public static class AddProjectionHandler extends CatalogDdlHandler + { + private final DruidSqlAlterTable.AddProjection alterTable; + private String projectionName; + + public AddProjectionHandler( + SqlStatementHandler.HandlerContext handlerContext, + DruidSqlAlterTable.AddProjection alterTable + ) + { + super(handlerContext, alterTable.getName()); + this.alterTable = alterTable; + } + + @Override + protected void validateStatement() + { + projectionName = simpleName(alterTable.getProjection().getName(), "Projection"); + } + + @Override + protected void execute(CatalogTableWriter writer) + { + final TableMetadata existing = writer.readTable(tableId); + if (existing == null) { + throw InvalidSqlInput.exception("Table [%s] does not have a catalog entry", tableId.name()); + } + final List columns = + existing.spec().columns() == null ? Collections.emptyList() : existing.spec().columns(); + writer.addProjection( + tableId, + translateProjection(handlerContext, tableId.name(), columns, alterTable.getProjection()), + alterTable.isIfNotExists() + ); + } + + @Override + protected String operationName() + { + return "ALTER TABLE ADD PROJECTION"; + } + } + + /** + * {@code ALTER TABLE ... DROP PROJECTION}. Segments already built keep whatever projections they were built with; + * this only stops future ingestion from building it. + */ + public static class DropProjectionHandler extends CatalogDdlHandler + { + private final DruidSqlAlterTable.DropProjection alterTable; + private String projectionName; + + public DropProjectionHandler( + SqlStatementHandler.HandlerContext handlerContext, + DruidSqlAlterTable.DropProjection alterTable + ) + { + super(handlerContext, alterTable.getName()); + this.alterTable = alterTable; + } + + @Override + protected void validateStatement() + { + projectionName = simpleName(alterTable.getProjectionName(), "Projection"); + } + + @Override + protected void execute(CatalogTableWriter writer) + { + writer.dropProjection(tableId, projectionName, alterTable.isIfExists()); + } + + @Override + protected String operationName() + { + return "ALTER TABLE DROP PROJECTION"; + } + } + /** * {@code ALTER TABLE ... SET PROPERTIES}. A NULL value removes the property. The set of legal keys is not checked * here: the Coordinator's table definition registry is what knows them. diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java index 1a6bd2aefb45..20fab1cbf41d 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java @@ -20,6 +20,7 @@ package org.apache.druid.sql.calcite.planner; import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import org.apache.druid.catalog.model.TableId; import org.apache.druid.catalog.model.TableMetadata; import org.apache.druid.catalog.model.TableSpec; @@ -67,6 +68,20 @@ public interface CatalogTableWriter */ void updateProperties(TableId tableId, Map properties); + /** + * Append a projection to the table's projections. + * + * @param ifNotExists if a projection of the same name exists, do nothing rather than failing + */ + void addProjection(TableId tableId, DatasourceProjectionMetadata projection, boolean ifNotExists); + + /** + * Remove the named projection from the table's projections. + * + * @param ifExists if no projection of that name exists, do nothing rather than failing + */ + void dropProjection(TableId tableId, String projectionName, boolean ifExists); + /** * Read a table's current metadata directly from the Coordinator, bypassing any local cache, or null if the table * has no catalog entry. Used for pre-checks that must not race against a stale cache. @@ -104,6 +119,18 @@ public void updateProperties(TableId tableId, Map properties) throw notAvailable(); } + @Override + public void addProjection(TableId tableId, DatasourceProjectionMetadata projection, boolean ifNotExists) + { + throw notAvailable(); + } + + @Override + public void dropProjection(TableId tableId, String projectionName, boolean ifExists) + { + throw notAvailable(); + } + @Nullable @Override public TableMetadata readTable(TableId tableId) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java index 7c76ad8babb3..614764d66881 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java @@ -99,6 +99,7 @@ public AuthResult( private final PlannerContext plannerContext; private final SqlEngine engine; private final PlannerHook hook; + private final PlannerFactory plannerFactory; private State state = State.START; private SqlStatementHandler handler; private boolean authorized; @@ -107,9 +108,11 @@ public AuthResult( final FrameworkConfig frameworkConfig, final PlannerContext plannerContext, final SqlEngine engine, - final PlannerHook hook + final PlannerHook hook, + final PlannerFactory plannerFactory ) { + this.plannerFactory = plannerFactory; this.frameworkConfig = frameworkConfig; this.planner = new CalcitePlanner(frameworkConfig); this.plannerContext = plannerContext; @@ -190,6 +193,12 @@ private static SqlStatementHandler createDdlHandler( if (query instanceof DruidSqlAlterTable.AlterColumn) { return new CatalogDdlHandler.AlterColumnHandler(handlerContext, (DruidSqlAlterTable.AlterColumn) query); } + if (query instanceof DruidSqlAlterTable.AddProjection) { + return new CatalogDdlHandler.AddProjectionHandler(handlerContext, (DruidSqlAlterTable.AddProjection) query); + } + if (query instanceof DruidSqlAlterTable.DropProjection) { + return new CatalogDdlHandler.DropProjectionHandler(handlerContext, (DruidSqlAlterTable.DropProjection) query); + } if (query instanceof DruidSqlAlterTable.SetProperties) { return new CatalogDdlHandler.SetPropertiesHandler(handlerContext, (DruidSqlAlterTable.SetProperties) query); } @@ -354,6 +363,12 @@ public PlannerHook hook() { return hook; } + + @Override + public PlannerFactory plannerFactory() + { + return plannerFactory; + } } public static DruidException translateException(Exception e) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java index 2071415ae587..0bcd1fbf36f0 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java @@ -21,15 +21,20 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import org.apache.calcite.config.CalciteConnectionConfig; import org.apache.calcite.config.CalciteConnectionConfigImpl; import org.apache.calcite.config.CalciteConnectionProperty; +import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.plan.Context; import org.apache.calcite.plan.ConventionTraitDef; import org.apache.calcite.plan.volcano.DruidVolcanoCost; import org.apache.calcite.rel.RelCollationTraitDef; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.Table; +import org.apache.calcite.schema.impl.AbstractSchema; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.validate.SqlConformance; import org.apache.calcite.sql2rel.SqlToRelConverter; @@ -50,9 +55,11 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.schema.DruidSchemaName; +import org.apache.druid.sql.calcite.table.DruidTable; import org.apache.druid.sql.hook.DruidHook; import org.apache.druid.sql.hook.DruidHookDispatcher; +import java.util.Collections; import java.util.Map; import java.util.Properties; import java.util.Set; @@ -166,7 +173,7 @@ public DruidPlanner createPlanner( ); context.dispatchHook(DruidHook.SQL, sql); - return new DruidPlanner(buildFrameworkConfig(context), context, engine, hook); + return new DruidPlanner(buildFrameworkConfig(context), context, engine, hook, this); } /** @@ -198,12 +205,70 @@ public DruidPlanner createPlannerForTesting( return thePlanner; } + /** + * Create a planner for a statement that refers to a table which may not be in the schema, or which should be seen + * with a different definition than the one the schema holds. Used to plan the body of a projection definition + * against the columns its {@code CREATE TABLE} or {@code ALTER TABLE} statement declares, before those columns + * have been written to the catalog. + *

+ * The schema holds only that table. Resource typing still consults the real schema, so authorization is + * unaffected; callers are expected to have authorized the enclosing statement already. + */ + public DruidPlanner createPlannerForTable( + final SqlEngine engine, + final String sql, + final SqlNode sqlNode, + final Map queryContext, + final String tableName, + final DruidTable table + ) + { + final PlannerContext context = PlannerContext.create( + this, + sql, + sqlNode, + engine, + Collections.emptySet(), + queryContext, + null + ); + final SchemaPlus defaultSchema = CalciteSchema.createRootSchema(false, false) + .plus() + .add(druidSchemaName, new SingleTableSchema(tableName, table)); + return new DruidPlanner(buildFrameworkConfig(context, defaultSchema), context, engine, null, this); + } + + /** + * A schema holding exactly the table being defined. The body of a projection has no FROM clause, so the table it + * belongs to is the only one it can name; anything else is a mistake worth reporting as an unknown table. + */ + private static class SingleTableSchema extends AbstractSchema + { + private final Map tables; + + SingleTableSchema(String tableName, Table table) + { + this.tables = ImmutableMap.of(tableName, table); + } + + @Override + protected Map getTableMap() + { + return tables; + } + } + public AuthorizerMapper getAuthorizerMapper() { return authorizerMapper; } private FrameworkConfig buildFrameworkConfig(PlannerContext plannerContext) + { + return buildFrameworkConfig(plannerContext, rootSchema.getSubSchema(druidSchemaName)); + } + + private FrameworkConfig buildFrameworkConfig(PlannerContext plannerContext, SchemaPlus defaultSchema) { final SqlToRelConverter.Config sqlToRelConverterConfig = SqlToRelConverter .config() @@ -223,7 +288,7 @@ private FrameworkConfig buildFrameworkConfig(PlannerContext plannerContext) .programs(calciteRuleManager.programs(plannerContext)) .executor(new DruidRexExecutor(plannerContext)) .typeSystem(DruidTypeSystem.INSTANCE) - .defaultSchema(rootSchema.getSubSchema(druidSchemaName)) + .defaultSchema(defaultSchema) .sqlToRelConverterConfig(sqlToRelConverterConfig) .context(new Context() { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java new file mode 100644 index 000000000000..5c50a91da949 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java @@ -0,0 +1,397 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.planner; + +import com.google.common.collect.ImmutableMap; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.util.SqlBasicVisitor; +import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.Columns; +import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.data.input.impl.DimensionSchema; +import org.apache.druid.error.DruidException; +import org.apache.druid.error.InvalidSqlInput; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.query.DataSource; +import org.apache.druid.query.Query; +import org.apache.druid.query.QueryContexts; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.dimension.DefaultDimensionSpec; +import org.apache.druid.query.dimension.DimensionSpec; +import org.apache.druid.query.filter.AndDimFilter; +import org.apache.druid.query.filter.DimFilter; +import org.apache.druid.query.filter.RangeFilter; +import org.apache.druid.query.groupby.GroupByQuery; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.segment.VirtualColumns; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.server.security.AuthorizationResult; +import org.apache.druid.server.security.NoopEscalator; +import org.apache.druid.sql.calcite.rel.DruidQuery; +import org.apache.druid.sql.calcite.rel.Grouping; +import org.apache.druid.sql.calcite.table.DatasourceTable; +import org.apache.druid.sql.calcite.table.DatasourceTable.PhysicalDatasourceMetadata; +import org.apache.druid.sql.calcite.table.DruidTable; +import org.joda.time.Interval; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Turns the SQL body of a projection definition into the {@link AggregateProjectionSpec} the catalog stores. + *

+ * The body is planned through the normal pipeline against the columns the enclosing statement declares, and the + * specification is lifted out of the resulting native query. Going through the planner is the point: a projection is + * only useful if it matches the queries the planner generates at query time, and that agreement is guaranteed when + * the same machinery produces both. It also means aggregators contributed by extensions work without a second + * registry. + */ +public class ProjectionSpecTranslator +{ + /** + * Planning is deterministic in the shapes the lift understands: no timeseries or topN rewrite to hide the grouping + * columns, and no approximation choices that depend on unrelated configuration. + */ + private static final Map CONTEXT = ImmutableMap.of( + PlannerContext.CTX_SQL_USE_GRANULARITY, false, + QueryContexts.TIME_BOUNDARY_PLANNING_KEY, false, + PlannerConfig.CTX_KEY_USE_APPROXIMATE_TOPN, false + ); + + private final PlannerFactory plannerFactory; + + public ProjectionSpecTranslator(PlannerFactory plannerFactory) + { + this.plannerFactory = plannerFactory; + } + + /** + * Translate one projection definition. + * + * @param tableName the table the projection belongs to + * @param columns the table's declared columns, which are the only ones the body may reference + */ + public AggregateProjectionSpec translate( + final String tableName, + final List columns, + final String projectionName, + final SqlSelect body + ) + { + rejectSubqueries(projectionName, body); + + final DruidQuery druidQuery = planBody(tableName, columns, projectionName, body); + return lift(projectionName, tableName, druidQuery); + } + + /** + * Plan the body against a table built from the declared columns. The table is synthesized rather than looked up + * because for {@code CREATE TABLE} it does not exist yet, and for {@code ALTER TABLE} the statement's own columns + * are what the projection must agree with, not whatever a possibly stale cache holds. + */ + private DruidQuery planBody( + final String tableName, + final List columns, + final String projectionName, + final SqlSelect body + ) + { + final SqlSelect query = (SqlSelect) body.clone(body.getParserPosition()); + query.setFrom(new SqlIdentifier(tableName, SqlParserPos.ZERO)); + + final ProjectionSqlEngine engine = new ProjectionSqlEngine(); + final String sql = query.toString(); + try (DruidPlanner planner = plannerFactory.createPlannerForTable( + engine, + sql, + query, + CONTEXT, + tableName, + tableFor(tableName, columns) + )) { + planner.getPlannerContext() + .setAuthenticationResult(NoopEscalator.getInstance().createEscalatedAuthenticationResult()); + planner.validate(); + planner.authorize(ra -> AuthorizationResult.ALLOW_NO_RESTRICTION, Collections.emptySet()); + planner.plan().run(); + } + catch (DruidException e) { + throw contextualize(projectionName, e); + } + return engine.captured(); + } + + /** + * Build the table the body is planned against. Mirrors how a catalog-only table is presented to the planner: + * declared columns in declared order, with {@code __time} supplied if the statement did not declare it. + */ + private static DruidTable tableFor(final String tableName, final List columns) + { + RowSignature.Builder builder = RowSignature.builder(); + boolean hasTime = false; + for (ColumnSpec column : columns) { + ColumnType type = Columns.druidType(column); + if (type == null) { + type = ColumnType.STRING; + } + if (Columns.isTimeColumn(column.name())) { + hasTime = true; + } + builder.add(column.name(), type); + } + if (!hasTime) { + builder = RowSignature.builder() + .add(Columns.TIME_COLUMN, ColumnType.LONG) + .addAll(builder.build()); + } + final RowSignature signature = builder.build(); + return new DatasourceTable( + signature, + new PhysicalDatasourceMetadata(new TableDataSource(tableName), signature, false, false), + DatasourceTable.EffectiveMetadata.of(signature) + ); + } + + /** + * Lift the projection specification out of the planned query. + */ + private static AggregateProjectionSpec lift( + final String projectionName, + final String tableName, + final DruidQuery druidQuery + ) + { + final DataSource dataSource = druidQuery.getDataSource(); + if (!(dataSource instanceof TableDataSource) || !tableName.equals(((TableDataSource) dataSource).getName())) { + throw invalid( + projectionName, + "its body requires more than one pass over the data. Rewrite it as a single aggregation, for example by" + + " using APPROX_COUNT_DISTINCT instead of COUNT(DISTINCT ...)" + ); + } + + final Grouping grouping = druidQuery.getGrouping(); + if (grouping == null) { + throw invalid(projectionName, "its body does not aggregate. Add a GROUP BY clause, or use SELECT DISTINCT"); + } + if (!grouping.getPostAggregators().isEmpty()) { + throw invalid( + projectionName, + "its body computes an expression over aggregates, which a projection cannot store. Store the aggregates" + + " themselves instead, for example SUM(x) and COUNT(x) rather than AVG(x)" + ); + } + if (grouping.getHavingFilter() != null) { + throw invalid(projectionName, "its body has a HAVING clause, which a projection cannot store"); + } + + final Query query = druidQuery.getQuery(); + final List dimensions; + final List aggregators; + final VirtualColumnsAndFilter virtualColumnsAndFilter; + if (query instanceof GroupByQuery) { + final GroupByQuery groupBy = (GroupByQuery) query; + dimensions = groupBy.getDimensions(); + aggregators = groupBy.getAggregatorSpecs(); + virtualColumnsAndFilter = new VirtualColumnsAndFilter( + groupBy.getVirtualColumns(), + groupBy.getDimFilter(), + groupBy.getIntervals() + ); + } else if (query instanceof TimeseriesQuery) { + // GROUP BY () plans to a timeseries over all time; it has no grouping columns. + final TimeseriesQuery timeseries = (TimeseriesQuery) query; + dimensions = Collections.emptyList(); + aggregators = List.of(timeseries.getAggregatorSpecs().toArray(new AggregatorFactory[0])); + virtualColumnsAndFilter = new VirtualColumnsAndFilter( + timeseries.getVirtualColumns(), + timeseries.getFilter(), + timeseries.getIntervals() + ); + } else { + throw invalid( + projectionName, + "its body did not plan to an aggregation. Add a GROUP BY clause, or use SELECT DISTINCT" + ); + } + + return AggregateProjectionSpec + .builder(projectionName) + .virtualColumns(virtualColumnsAndFilter.virtualColumns) + .filter(virtualColumnsAndFilter.filter(projectionName)) + .groupingColumns(groupingColumns(projectionName, dimensions)) + .aggregators(renameToOutputNames(projectionName, druidQuery, aggregators)) + .build(); + } + + private static List groupingColumns( + final String projectionName, + final List dimensions + ) + { + final List groupingColumns = new ArrayList<>(dimensions.size()); + for (DimensionSpec dimension : dimensions) { + if (!(dimension instanceof DefaultDimensionSpec)) { + throw invalid( + projectionName, + "its grouping column [" + dimension.getOutputName() + "] is not a plain column reference" + ); + } + final ColumnType type = dimension.getOutputType(); + if (type == null || (!type.isPrimitive() && !type.isArray())) { + throw invalid( + projectionName, + "its grouping column [" + dimension.getDimension() + "] has type [" + type + + "], which a projection cannot group on" + ); + } + // The stored name is the physical column or virtual column the planner grouped on, not the SELECT alias: + // projections are matched structurally against a query's grouping columns, not by output name. + groupingColumns.add(DimensionSchema.getDefaultSchemaForBuiltInType(dimension.getDimension(), type)); + } + return groupingColumns; + } + + /** + * Aggregators are stored under the name the projection's own column will have, which is the SELECT alias. The + * planner names them {@code a0}, {@code a1}, ... internally, so an explicit alias is required. + */ + private static AggregatorFactory[] renameToOutputNames( + final String projectionName, + final DruidQuery druidQuery, + final List aggregators + ) + { + final RowSignature internal = druidQuery.getOutputRowSignature(); + final List outputNames = druidQuery.getOutputRowType().getFieldNames(); + final AggregatorFactory[] renamed = new AggregatorFactory[aggregators.size()]; + for (int i = 0; i < aggregators.size(); i++) { + final AggregatorFactory aggregator = aggregators.get(i); + final int position = internal.indexOf(aggregator.getName()); + final String outputName = position < 0 || position >= outputNames.size() + ? aggregator.getName() + : outputNames.get(position); + if (isPlannerGeneratedName(outputName)) { + throw invalid( + projectionName, + "one of its aggregate expressions has no name. Give every aggregate an alias, for example" + + " SUM(x) AS sum_x" + ); + } + renamed[i] = aggregator.withName(outputName); + } + return renamed; + } + + private static boolean isPlannerGeneratedName(String name) + { + return name.startsWith("EXPR$"); + } + + /** + * The virtual columns, filter and intervals of the planned query. Kept together because a time filter written in + * the body is moved out of the filter and into the query's intervals during planning, and has to be put back: + * a projection has nowhere to store an interval. + */ + private static class VirtualColumnsAndFilter + { + private final VirtualColumns virtualColumns; + @Nullable + private final DimFilter dimFilter; + private final List intervals; + + VirtualColumnsAndFilter( + VirtualColumns virtualColumns, + @Nullable DimFilter dimFilter, + List intervals + ) + { + this.virtualColumns = virtualColumns; + this.dimFilter = dimFilter; + this.intervals = intervals; + } + + @Nullable + DimFilter filter(String projectionName) + { + if (intervals.size() == 1 && Intervals.ETERNITY.equals(intervals.get(0))) { + return dimFilter; + } + if (intervals.size() != 1) { + throw invalid( + projectionName, + "its WHERE clause selects more than one time range, which a projection cannot store" + ); + } + final Interval interval = intervals.get(0); + final DimFilter timeFilter = new RangeFilter( + Columns.TIME_COLUMN, + ColumnType.LONG, + interval.getStartMillis(), + interval.getEndMillis(), + false, + true, + null + ); + return dimFilter == null ? timeFilter : new AndDimFilter(timeFilter, dimFilter); + } + } + + /** + * Subqueries would be planned as a second pass over the data, which a projection cannot represent. The grammar + * cannot exclude them, because they appear inside expressions. + */ + private static void rejectSubqueries(final String projectionName, final SqlSelect body) + { + body.accept(new SqlBasicVisitor() + { + @Override + public Void visit(SqlCall call) + { + if (call instanceof SqlSelect && call != body) { + throw invalid(projectionName, "its body contains a subquery, which a projection cannot store"); + } + return super.visit(call); + } + }); + } + + private static DruidException contextualize(final String projectionName, final DruidException e) + { + if (e.getTargetPersona() == DruidException.Persona.USER) { + return InvalidSqlInput.exception(e, "Cannot define projection [%s]: %s", projectionName, e.getMessage()); + } + return e; + } + + private static DruidException invalid(final String projectionName, final String reason) + { + return InvalidSqlInput.exception("Cannot define projection [%s] because %s", projectionName, reason); + } + +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSqlEngine.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSqlEngine.java new file mode 100644 index 000000000000..e0fc7db7e1df --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSqlEngine.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.planner; + +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.server.QueryResponse; +import org.apache.druid.sql.SqlStatementFactory; +import org.apache.druid.sql.calcite.rel.DruidQuery; +import org.apache.druid.sql.calcite.run.EngineFeature; +import org.apache.druid.sql.calcite.run.QueryMaker; +import org.apache.druid.sql.calcite.run.SqlEngine; +import org.apache.druid.sql.calcite.run.SqlEngines; +import org.apache.druid.sql.destination.IngestDestination; + +import java.util.Map; + +/** + * Engine used to plan the body of a projection definition without running it. The planned {@link DruidQuery} is + * captured so the projection specification can be lifted out of it. + *

+ * A projection is defined by SQL but stored as a native specification, and it is only useful if it matches the + * queries the planner generates at query time. Planning the body through the normal pipeline is what makes the two + * agree: the same aggregator factories, virtual column expressions and filters come out either way. + *

+ * Not a singleton, unlike {@link org.apache.druid.sql.calcite.view.ViewSqlEngine}: each instance captures one query. + */ +public class ProjectionSqlEngine implements SqlEngine +{ + private static final String NAME = "projection"; + + private DruidQuery captured; + + /** + * The query planned for the projection body, available after the statement has been planned and run. + */ + public DruidQuery captured() + { + if (captured == null) { + throw DruidException.defensive("Projection body was not planned into a native query"); + } + return captured; + } + + @Override + public String name() + { + return NAME; + } + + @Override + public boolean featureAvailable(EngineFeature feature) + { + switch (feature) { + case CAN_SELECT: + case GROUPING_SETS: + return true; + + // A projection stores grouping columns and aggregators, so the body must plan to a group-by rather than to a + // specialized query shape that would hide them. + case TIMESERIES_QUERY: + case TOPN_QUERY: + case TIME_BOUNDARY_QUERY: + case GROUPBY_IMPLICITLY_SORTS: + case ALLOW_BINDABLE_PLAN: + return false; + + // The body has no FROM clause, so it can only read the table it belongs to. + case READ_EXTERNAL_DATA: + case WRITE_EXTERNAL_DATA: + case CAN_INSERT: + case CAN_REPLACE: + case SCAN_ORDER_BY_NON_TIME: + case WINDOW_FUNCTIONS: + case WINDOW_LEAF_OPERATOR: + case UNNEST: + case ALLOW_BROADCAST_RIGHTY_JOIN: + case ALLOW_TOP_LEVEL_UNION_ALL: + return false; + + default: + throw SqlEngines.generateUnrecognizedFeatureException(ProjectionSqlEngine.class.getSimpleName(), feature); + } + } + + @Override + public void validateContext(Map queryContext) + { + // The context is supplied by the translator, not by the user. + } + + @Override + public RelDataType resultTypeForSelect( + RelDataTypeFactory typeFactory, + RelDataType validatedRowType, + Map queryContext + ) + { + return validatedRowType; + } + + @Override + public RelDataType resultTypeForInsert( + RelDataTypeFactory typeFactory, + RelDataType validatedRowType, + Map queryContext + ) + { + throw new UnsupportedOperationException(); + } + + @Override + public QueryMaker buildQueryMakerForSelect(RelRoot relRoot, PlannerContext plannerContext) + { + return druidQuery -> { + captured = druidQuery; + return QueryResponse.withEmptyContext(Sequences.empty()); + }; + } + + @Override + public QueryMaker buildQueryMakerForInsert( + IngestDestination destination, + RelRoot relRoot, + PlannerContext plannerContext + ) + { + throw new UnsupportedOperationException(); + } + + @Override + public SqlStatementFactory getSqlStatementFactory() + { + throw new UnsupportedOperationException(); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/SqlStatementHandler.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/SqlStatementHandler.java index 393363587f7d..7d490c85b975 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/SqlStatementHandler.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/SqlStatementHandler.java @@ -57,6 +57,11 @@ interface HandlerContext ObjectMapper jsonMapper(); DateTimeZone timeZone(); PlannerHook hook(); + + /** + * The factory that created the enclosing planner, for statements that need to plan a nested query of their own. + */ + PlannerFactory plannerFactory(); } abstract class BaseStatementHandler implements SqlStatementHandler diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java index 7e9c71a3699a..ebc00343e419 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java @@ -22,12 +22,15 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import org.apache.druid.catalog.model.TableId; import org.apache.druid.catalog.model.TableMetadata; import org.apache.druid.catalog.model.TableSpec; import org.apache.druid.catalog.model.table.ClusterKeySpec; import org.apache.druid.catalog.model.table.DatasourceDefn; +import org.apache.druid.data.input.impl.AggregateProjectionSpec; import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.server.security.Action; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.Resource; @@ -52,6 +55,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -345,6 +349,207 @@ public void testDdlIsDisabledByDefault() assertEquals(ImmutableList.of(), WRITER.calls); } + /** + * The stored specification must be the one the planner would produce for the equivalent query, since that is what + * makes a projection match at query time. Pinned as JSON so a change in planner output is visible here. + */ + @Test + public void testCreateTableWithProjection() throws Exception + { + execute( + "CREATE TABLE tbl (__time TIMESTAMP, page VARCHAR, cnt BIGINT," + + " PROJECTION daily AS (SELECT TIME_FLOOR(__time, 'P1D'), page, SUM(cnt) AS total GROUP BY 1, 2))" + ); + + assertEquals( + "[{\"spec\":{\"type\":\"aggregate\",\"name\":\"daily\"," + + "\"virtualColumns\":[{\"type\":\"expression\",\"name\":\"v0\"," + + "\"expression\":\"timestamp_floor(\\\"__time\\\",'P1D',null,'UTC')\",\"outputType\":\"LONG\"}]," + + "\"groupingColumns\":[{\"type\":\"long\",\"name\":\"v0\",\"multiValueHandling\":\"SORTED_ARRAY\"," + + "\"createBitmapIndex\":false},{\"type\":\"string\",\"name\":\"page\"," + + "\"multiValueHandling\":\"SORTED_ARRAY\",\"createBitmapIndex\":true}]," + + "\"aggregators\":[{\"type\":\"longSum\",\"name\":\"total\",\"fieldName\":\"cnt\"}]," + + "\"ordering\":[{\"columnName\":\"v0\",\"order\":\"ascending\"}," + + "{\"columnName\":\"page\",\"order\":\"ascending\"}]}}]", + projectionsJson() + ); + } + + /** + * A projection defined with TIME_FLOOR must carry a granularity the segment layer can recover, which is how the + * projection gets matched to time-grouped queries. + */ + @Test + public void testProjectionGranularityIsRecoverable() + { + execute( + "CREATE TABLE tbl (__time TIMESTAMP, page VARCHAR, cnt BIGINT," + + " PROJECTION hourly AS (SELECT TIME_FLOOR(__time, 'PT1H'), page, SUM(cnt) AS total GROUP BY 1, 2))" + ); + + final AggregateProjectionSpec spec = projection(0).getSpec(); + final String timeColumn = spec.toMetadataSchema().getTimeColumnName(); + assertEquals("v0", timeColumn); + assertEquals( + Granularities.HOUR, + Granularities.fromVirtualColumn(spec.getVirtualColumns().getVirtualColumn(timeColumn)) + ); + } + + @Test + public void testProjectionWithFilter() + { + execute( + "CREATE TABLE tbl (__time TIMESTAMP, page VARCHAR, cnt BIGINT," + + " PROJECTION filtered AS (SELECT page, SUM(cnt) AS total WHERE page <> 'skip' GROUP BY page))" + ); + assertEquals("!page = skip", projection(0).getSpec().getFilter().toString()); + } + + /** + * A time bound written in the body is moved into the query's intervals during planning, and has to be put back: + * a projection stores a filter, not an interval. + */ + @Test + public void testProjectionWithTimeFilter() + { + execute( + "CREATE TABLE tbl (__time TIMESTAMP, page VARCHAR, cnt BIGINT," + + " PROJECTION recent AS (SELECT page, SUM(cnt) AS total" + + " WHERE __time >= TIMESTAMP '2020-01-01 00:00:00' GROUP BY page))" + ); + assertNotNull(projection(0).getSpec().getFilter(), "time filter must survive as a filter"); + assertTrue(projection(0).getSpec().getFilter().getRequiredColumns().contains("__time")); + } + + @Test + public void testProjectionSelectDistinct() + { + execute("CREATE TABLE tbl (a VARCHAR, PROJECTION d AS (SELECT DISTINCT a))"); + final AggregateProjectionSpec spec = projection(0).getSpec(); + assertEquals(1, spec.getGroupingColumns().size()); + assertEquals("a", spec.getGroupingColumns().get(0).getName()); + assertEquals(0, spec.getAggregators().length); + } + + @Test + public void testMultipleProjections() + { + execute( + "CREATE TABLE tbl (a VARCHAR, b BIGINT," + + " PROJECTION p1 AS (SELECT a, SUM(b) AS s GROUP BY a)," + + " PROJECTION p2 AS (SELECT b, COUNT(*) AS c GROUP BY b))" + ); + assertEquals(List.of("p1", "p2"), List.of(projection(0).getSpec().getName(), projection(1).getSpec().getName())); + } + + @Test + public void testAlterTableAddProjection() + { + WRITER.existing.put(TableId.datasource("tbl"), tableWithColumns("a")); + execute("ALTER TABLE tbl ADD PROJECTION p AS (SELECT a, COUNT(*) AS c GROUP BY a)"); + + final RecordingCatalogTableWriter.Call call = WRITER.lastCall("addProjection"); + assertEquals("p", call.projection.getSpec().getName()); + assertFalse(call.ifNotExists); + } + + @Test + public void testAlterTableAddProjectionIfNotExists() + { + WRITER.existing.put(TableId.datasource("tbl"), tableWithColumns("a")); + execute("ALTER TABLE tbl ADD IF NOT EXISTS PROJECTION p AS (SELECT a GROUP BY a)"); + assertTrue(WRITER.lastCall("addProjection").ifNotExists); + } + + @Test + public void testAlterTableDropProjection() + { + execute("ALTER TABLE tbl DROP PROJECTION p"); + final RecordingCatalogTableWriter.Call call = WRITER.lastCall("dropProjection"); + assertEquals("p", call.projectionName); + assertFalse(call.ifExists); + + WRITER.reset(); + execute("ALTER TABLE tbl DROP PROJECTION IF EXISTS p"); + assertTrue(WRITER.lastCall("dropProjection").ifExists); + } + + @Test + public void testProjectionRejectsUnaliasedAggregate() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a VARCHAR, b BIGINT, PROJECTION p AS (SELECT a, SUM(b) GROUP BY a))") + ); + assertTrue(e.getMessage().contains("no name"), e.getMessage()); + } + + @Test + public void testProjectionRejectsPostAggregation() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a VARCHAR, b BIGINT, PROJECTION p AS (SELECT a, AVG(b) AS m GROUP BY a))") + ); + assertTrue(e.getMessage().contains("expression over aggregates"), e.getMessage()); + } + + @Test + public void testProjectionRejectsUnknownColumn() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a VARCHAR, PROJECTION p AS (SELECT nope GROUP BY nope))") + ); + assertTrue(e.getMessage().contains("nope"), e.getMessage()); + } + + @Test + public void testProjectionRejectsNonAggregatingBody() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a VARCHAR, PROJECTION p AS (SELECT a))") + ); + assertTrue(e.getMessage().contains("does not aggregate"), e.getMessage()); + } + + @Test + public void testProjectionRejectsReservedBaseName() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a VARCHAR, PROJECTION __base AS (SELECT a GROUP BY a))") + ); + assertTrue(e.getMessage().contains("base table layout"), e.getMessage()); + } + + @Test + public void testProjectionRejectsDuplicateName() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute( + "CREATE TABLE tbl (a VARCHAR, PROJECTION p AS (SELECT a GROUP BY a)," + + " PROJECTION p AS (SELECT a GROUP BY a))" + ) + ); + assertTrue(e.getMessage().contains("declared more than once"), e.getMessage()); + } + + @SuppressWarnings("unchecked") + private DatasourceProjectionMetadata projection(int index) + { + return ((List) WRITER.calls.get(0).spec.properties().get("projections")).get(index); + } + + private String projectionsJson() throws Exception + { + return queryFramework().queryJsonMapper() + .writeValueAsString(WRITER.calls.get(0).spec.properties().get("projections")); + } + private void execute(String sql) { statement(sql).execute(); @@ -386,6 +591,9 @@ static class Call List columns; List droppedColumns; Map properties; + DatasourceProjectionMetadata projection; + String projectionName; + boolean ifExists; } final List calls = new ArrayList<>(); @@ -434,6 +642,22 @@ public void updateProperties(TableId tableId, Map properties) record("updateProperties", tableId).properties = properties; } + @Override + public void addProjection(TableId tableId, DatasourceProjectionMetadata projection, boolean ifNotExists) + { + final Call call = record("addProjection", tableId); + call.projection = projection; + call.ifNotExists = ifNotExists; + } + + @Override + public void dropProjection(TableId tableId, String projectionName, boolean ifExists) + { + final Call call = record("dropProjection", tableId); + call.projectionName = projectionName; + call.ifExists = ifExists; + } + @Nullable @Override public TableMetadata readTable(TableId tableId) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java index 1778888308af..26a47b9d2411 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java @@ -34,6 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -177,6 +178,139 @@ public void testAlterTableSetProperties() ); } + @Test + public void testCreateTableWithProjection() + { + final DruidSqlCreateTable create = parseCreate( + "CREATE TABLE events (\n" + + " __time TIMESTAMP,\n" + + " user_id VARCHAR,\n" + + " pages_visited BIGINT,\n" + + " PROJECTION daily_visits AS (\n" + + " SELECT TIME_FLOOR(__time, 'P1D'), user_id, SUM(pages_visited) AS total\n" + + " WHERE user_id IS NOT NULL\n" + + " GROUP BY 1, 2\n" + + " )\n" + + ")" + ); + + assertEquals("__time TIMESTAMP, user_id VARCHAR, pages_visited BIGINT", columnsOf(create)); + assertEquals(1, create.getProjectionList().size()); + + final SqlProjectionSpec projection = (SqlProjectionSpec) create.getProjectionList().get(0); + assertEquals("daily_visits", projection.getName().toString()); + assertNull(projection.getBody().getFrom(), "projection body must have no FROM clause"); + assertEquals(3, projection.getBody().getSelectList().size()); + assertNotNull(projection.getBody().getWhere()); + assertEquals(2, projection.getBody().getGroup().size()); + } + + @Test + public void testCreateTableProjectionWithoutAs() + { + // ClickHouse spells this without AS; both are accepted. + final DruidSqlCreateTable create = parseCreate( + "CREATE TABLE t (a VARCHAR, PROJECTION p (SELECT a, COUNT(*) AS c GROUP BY a))" + ); + assertEquals(1, create.getProjectionList().size()); + assertEquals("a VARCHAR", columnsOf(create)); + } + + @Test + public void testCreateTableMultipleProjections() + { + final DruidSqlCreateTable create = parseCreate( + "CREATE TABLE t (a VARCHAR, b BIGINT," + + " PROJECTION p1 AS (SELECT a, SUM(b) AS s GROUP BY a)," + + " PROJECTION p2 AS (SELECT b, COUNT(*) AS c GROUP BY b))" + ); + assertEquals(2, create.getProjectionList().size()); + assertEquals("a VARCHAR, b BIGINT", columnsOf(create)); + } + + /** + * A column may be named "projection": the keyword is non-reserved, and a projection definition is told apart by + * its third token, which is always '(' or AS. + */ + @Test + public void testColumnNamedProjection() + { + assertEquals("projection VARCHAR", columnsOf(parseCreate("CREATE TABLE t (projection VARCHAR)"))); + // A bare-identifier type is the case two tokens of lookahead could not resolve. Calcite renders such a type as + // a quoted identifier. + assertEquals("projection `LONG`", columnsOf(parseCreate("CREATE TABLE t (projection LONG)"))); + assertEquals( + "a VARCHAR, projection `LONG`", + columnsOf(parseCreate("CREATE TABLE t (a VARCHAR, projection LONG)")) + ); + + final DruidSqlCreateTable both = parseCreate( + "CREATE TABLE t (projection LONG, PROJECTION projection AS (SELECT projection GROUP BY projection))" + ); + assertEquals("projection `LONG`", columnsOf(both)); + assertEquals(1, both.getProjectionList().size()); + } + + @Test + public void testAlterTableAddProjection() + { + final DruidSqlAlterTable.AddProjection alter = parseAlter( + "ALTER TABLE t ADD PROJECTION p AS (SELECT a, SUM(b) AS s GROUP BY a)", + DruidSqlAlterTable.AddProjection.class + ); + assertEquals("t", alter.getName().toString()); + assertEquals("p", alter.getProjection().getName().toString()); + assertFalse(alter.isIfNotExists()); + } + + @Test + public void testAlterTableAddProjectionIfNotExists() + { + final DruidSqlAlterTable.AddProjection alter = parseAlter( + "ALTER TABLE t ADD IF NOT EXISTS PROJECTION p AS (SELECT a GROUP BY a)", + DruidSqlAlterTable.AddProjection.class + ); + assertTrue(alter.isIfNotExists()); + } + + @Test + public void testAlterTableDropProjection() + { + final DruidSqlAlterTable.DropProjection alter = parseAlter( + "ALTER TABLE t DROP PROJECTION p", + DruidSqlAlterTable.DropProjection.class + ); + assertEquals("p", alter.getProjectionName().toString()); + assertFalse(alter.isIfExists()); + + assertTrue( + parseAlter("ALTER TABLE t DROP PROJECTION IF EXISTS p", DruidSqlAlterTable.DropProjection.class) + .isIfExists() + ); + } + + /** + * A projection has no way to express ordering or limits, so the grammar excludes them rather than validating them + * away later. + */ + @Test + public void testProjectionBodyRejectsUnsupportedClauses() + { + for (String body : new String[]{ + "SELECT a GROUP BY a ORDER BY a", + "SELECT a GROUP BY a LIMIT 10", + "SELECT a GROUP BY a HAVING COUNT(*) > 1", + "SELECT a FROM other GROUP BY a", + "SELECT a GROUP BY a UNION ALL SELECT b GROUP BY b" + }) { + assertThrows( + DruidException.class, + () -> parse("CREATE TABLE t (a VARCHAR, PROJECTION p AS (" + body + "))"), + body + ); + } + } + /** * DDL nodes must round-trip through {@link SqlNode#unparse}, which is what makes them safe to log and re-print. */ From b0617ff15ecf17bcc6fdc9341e055f651c21bb1f Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 30 Jul 2026 00:39:22 -0700 Subject: [PATCH 03/18] feat: ddl + base table projection --- docs/development/extensions-core/catalog.md | 48 +++- .../catalog/CatalogDdlAndIngestTest.java | 110 +++++++++ sql/src/main/codegen/config.fmpp | 2 + sql/src/main/codegen/includes/ddl.ftl | 11 +- .../calcite/parser/DruidSqlCreateTable.java | 23 +- .../sql/calcite/parser/SqlProjectionSpec.java | 40 +++- .../calcite/planner/CatalogDdlHandler.java | 107 ++++++++- .../planner/ProjectionSpecTranslator.java | 149 ++++++++++++ .../sql/calcite/CalciteCatalogDdlTest.java | 217 +++++++++++++++++- .../calcite/parser/DruidSqlDdlParserTest.java | 53 +++++ 10 files changed, 740 insertions(+), 20 deletions(-) diff --git a/docs/development/extensions-core/catalog.md b/docs/development/extensions-core/catalog.md index 351d13aa2631..aef6f2cc9556 100644 --- a/docs/development/extensions-core/catalog.md +++ b/docs/development/extensions-core/catalog.md @@ -66,11 +66,16 @@ CREATE [OR REPLACE] TABLE [IF NOT EXISTS]

[ ( { | PROJECTION AS (
@@ -53,6 +54,9 @@ SqlCreate DruidSqlCreateTable(Span s, boolean replace) : [ clusteredBy = ClusteredBy() ] + [ + { sealed = true; } + ] { final SqlParserPos elementPos = elementSpan == null ? s.pos() : elementSpan.end(this); return new DruidSqlCreateTable( @@ -63,7 +67,8 @@ SqlCreate DruidSqlCreateTable(Span s, boolean replace) : new SqlNodeList(columns, elementPos), new SqlNodeList(projections, elementPos), partitionedBy, - clusteredBy + clusteredBy, + sealed ); } } @@ -102,6 +107,7 @@ SqlProjectionSpec DruidProjectionDefinition() : SqlLiteral keyword = null; final SqlNode where; final SqlNodeList groupBy; + SqlNodeList clusteredBy = null; } { { s = span(); } @@ -117,11 +123,14 @@ SqlProjectionSpec DruidProjectionDefinition() : )* ( where = Where() | { where = null; } ) ( groupBy = GroupBy() | { groupBy = null; } ) + // Only meaningful for the reserved __base projection, where it names the columns segments are clustered on. + [ clusteredBy = ClusteredBy() ] { return new SqlProjectionSpec( s.end(this), name, + clusteredBy, new SqlSelect( bodySpan.end(this), keywordList, diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java index 325ec9388ec5..a9c9e6c77c80 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java @@ -54,6 +54,7 @@ public class DruidSqlCreateTable extends SqlCreate private final SqlGranularityLiteral partitionedBy; @Nullable private final SqlNodeList clusteredBy; + private final boolean sealed; public DruidSqlCreateTable( SqlParserPos pos, @@ -63,10 +64,12 @@ public DruidSqlCreateTable( SqlNodeList columnList, SqlNodeList projectionList, @Nullable SqlGranularityLiteral partitionedBy, - @Nullable SqlNodeList clusteredBy + @Nullable SqlNodeList clusteredBy, + boolean sealed ) { super(OPERATOR, pos, replace, ifNotExists); + this.sealed = sealed; this.name = name; this.columnList = columnList; this.projectionList = projectionList; @@ -113,6 +116,14 @@ public boolean isIfNotExists() return ifNotExists; } + /** + * Whether the statement declared SEALED, which requires every ingested column to be declared. + */ + public boolean isSealed() + { + return sealed; + } + @Nonnull @Override public List getOperandList() @@ -125,7 +136,8 @@ public List getOperandList() partitionedBy, clusteredBy, SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), - SqlLiteral.createBoolean(ifNotExists, SqlParserPos.ZERO) + SqlLiteral.createBoolean(ifNotExists, SqlParserPos.ZERO), + SqlLiteral.createBoolean(sealed, SqlParserPos.ZERO) ); } @@ -167,6 +179,10 @@ public void unparse(SqlWriter writer, int leftPrec, int rightPrec) } writer.endList(clusterFrame); } + + if (sealed) { + writer.keyword("SEALED"); + } } private static class DruidSqlCreateTableOperator extends SqlSpecialOperator @@ -191,7 +207,8 @@ public SqlCall createCall( (SqlNodeList) operands[1], (SqlNodeList) operands[2], (SqlGranularityLiteral) operands[3], - (SqlNodeList) operands[4] + (SqlNodeList) operands[4], + ((SqlLiteral) operands[7]).booleanValue() ); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlProjectionSpec.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlProjectionSpec.java index 4e8d75ab5973..8cfa53cc733b 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlProjectionSpec.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlProjectionSpec.java @@ -24,6 +24,7 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlSpecialOperator; @@ -32,6 +33,7 @@ import org.apache.calcite.util.ImmutableNullableList; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.util.List; /** @@ -47,15 +49,33 @@ public class SqlProjectionSpec extends SqlCall public static final SqlOperator OPERATOR = new SqlProjectionSpecOperator(); private final SqlIdentifier name; + @Nullable + private final SqlNodeList clusteredBy; private final SqlSelect body; - public SqlProjectionSpec(SqlParserPos pos, SqlIdentifier name, SqlSelect body) + public SqlProjectionSpec( + SqlParserPos pos, + SqlIdentifier name, + @Nullable SqlNodeList clusteredBy, + SqlSelect body + ) { super(pos); this.name = name; + this.clusteredBy = clusteredBy; this.body = body; } + /** + * The columns segments are clustered on, meaningful only for the reserved base-table projection: an aggregate + * projection is ordered by its grouping columns and has nothing to choose. + */ + @Nullable + public SqlNodeList getClusteredBy() + { + return clusteredBy; + } + public SqlIdentifier getName() { return name; @@ -77,7 +97,7 @@ public SqlOperator getOperator() @Override public List getOperandList() { - return ImmutableNullableList.of(name, body); + return ImmutableNullableList.of(name, clusteredBy, body); } @Override @@ -88,6 +108,15 @@ public void unparse(SqlWriter writer, int leftPrec, int rightPrec) writer.keyword("AS"); final SqlWriter.Frame frame = writer.startList("(", ")"); body.unparse(writer, 0, 0); + if (clusteredBy != null) { + writer.keyword("CLUSTERED BY"); + final SqlWriter.Frame clusterFrame = writer.startList("", ""); + for (SqlNode column : clusteredBy) { + writer.sep(","); + column.unparse(writer, 0, 0); + } + writer.endList(clusterFrame); + } writer.endList(frame); } @@ -101,7 +130,12 @@ public SqlProjectionSpecOperator() @Override public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) { - return new SqlProjectionSpec(pos, (SqlIdentifier) operands[0], (SqlSelect) operands[1]); + return new SqlProjectionSpec( + pos, + (SqlIdentifier) operands[0], + (SqlNodeList) operands[1], + (SqlSelect) operands[2] + ); } } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java index 872d1f7f1223..c12cc826ae40 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java @@ -31,6 +31,7 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.druid.catalog.model.ClusteredValueGroupsBaseTableMetadata; import org.apache.druid.catalog.model.ColumnSpec; import org.apache.druid.catalog.model.Columns; import org.apache.druid.catalog.model.DatasourceProjectionMetadata; @@ -244,24 +245,56 @@ protected static DatasourceProjectionMetadata translateProjection( ) { final String name = simpleName(projection.getName(), "Projection"); - if (BASE_PROJECTION_NAME.equals(name)) { - throw InvalidSqlInput.exception( - "Projection [%s] names the base table layout, which is not supported yet", - BASE_PROJECTION_NAME - ); - } try { Projections.validateProjectionName(name); } catch (DruidException e) { throw InvalidSqlInput.exception(e, "%s", e.getMessage()); } + if (projection.getClusteredBy() != null) { + throw InvalidSqlInput.exception( + "Projection [%s] cannot use CLUSTERED BY: an aggregate projection is ordered by its grouping columns." + + " Only the [%s] projection, which describes the table's own layout, chooses a clustering", + name, + BASE_PROJECTION_NAME + ); + } return new DatasourceProjectionMetadata( new ProjectionSpecTranslator(handlerContext.plannerFactory()) .translate(tableName, columns, name, projection.getBody()) ); } + /** + * Translate the reserved {@code __base} projection, which describes the physical layout of the table rather than an + * additional aggregate, and so becomes the {@code baseTable} property instead of one of the projections. + */ + protected static ClusteredValueGroupsBaseTableMetadata translateBaseTable( + final SqlStatementHandler.HandlerContext handlerContext, + final String tableName, + final List columns, + final SqlProjectionSpec projection + ) + { + return new ProjectionSpecTranslator(handlerContext.plannerFactory()) + .translateBaseTable(tableName, columns, projection.getBody(), projection.getClusteredBy()); + } + + /** + * A base table layout derives the physical segment schema from the declared columns, so a column that is not + * declared cannot be stored. The catalog enforces this too, but saying it here names the clause that is missing. + */ + protected static void requireSealed(boolean sealed) + { + if (!sealed) { + throw InvalidSqlInput.exception( + "A table with a [%s] projection must be declared SEALED: its columns define the physical segment schema," + + " so columns that are not declared cannot be ingested", + BASE_PROJECTION_NAME + ); + } + } + protected static String simpleName(SqlIdentifier identifier, String what) { if (!identifier.isSimple()) { @@ -355,6 +388,9 @@ protected void validateStatement() if (createTable.getClusteredBy() != null) { properties.put(DatasourceDefn.CLUSTER_KEYS_PROPERTY, toClusterKeys(createTable.getClusteredBy())); } + if (createTable.isSealed()) { + properties.put(DatasourceDefn.SEALED_PROPERTY, true); + } if (!createTable.getProjectionList().isEmpty()) { final List projections = new ArrayList<>(createTable.getProjectionList().size()); @@ -365,9 +401,19 @@ protected void validateStatement() if (!seenProjections.add(name)) { throw InvalidSqlInput.exception("Projection [%s] is declared more than once", name); } - projections.add(translateProjection(handlerContext, tableId.name(), columns, projection)); + if (BASE_PROJECTION_NAME.equals(name)) { + requireSealed(createTable.isSealed()); + properties.put( + DatasourceDefn.BASE_TABLE_PROPERTY, + translateBaseTable(handlerContext, tableId.name(), columns, projection) + ); + } else { + projections.add(translateProjection(handlerContext, tableId.name(), columns, projection)); + } + } + if (!projections.isEmpty()) { + properties.put(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, projections); } - properties.put(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, projections); } tableSpec = new TableSpec(DatasourceDefn.TABLE_TYPE, properties, columns); @@ -538,6 +584,30 @@ protected void execute(CatalogTableWriter writer) } final List columns = existing.spec().columns() == null ? Collections.emptyList() : existing.spec().columns(); + + if (BASE_PROJECTION_NAME.equals(projectionName)) { + // The base table is a property of the table, not one of its projections, so it is set rather than appended. + if (existing.spec().properties().get(DatasourceDefn.BASE_TABLE_PROPERTY) != null) { + if (alterTable.isIfNotExists()) { + return; + } + throw InvalidSqlInput.exception( + "Table [%s] already has a [%s] projection; drop it before defining another", + tableId.name(), + BASE_PROJECTION_NAME + ); + } + requireSealed(Boolean.TRUE.equals(existing.spec().properties().get(DatasourceDefn.SEALED_PROPERTY))); + writer.updateProperties( + tableId, + Collections.singletonMap( + DatasourceDefn.BASE_TABLE_PROPERTY, + translateBaseTable(handlerContext, tableId.name(), columns, alterTable.getProjection()) + ) + ); + return; + } + writer.addProjection( tableId, translateProjection(handlerContext, tableId.name(), columns, alterTable.getProjection()), @@ -579,6 +649,27 @@ protected void validateStatement() @Override protected void execute(CatalogTableWriter writer) { + if (BASE_PROJECTION_NAME.equals(projectionName)) { + // Removing the layout leaves the declared columns alone; only future segments are affected. + final TableMetadata existing = writer.readTable(tableId); + final boolean present = existing != null + && existing.spec().properties().get(DatasourceDefn.BASE_TABLE_PROPERTY) != null; + if (!present) { + if (alterTable.isIfExists()) { + return; + } + throw InvalidSqlInput.exception( + "Table [%s] does not have a [%s] projection", + tableId.name(), + BASE_PROJECTION_NAME + ); + } + writer.updateProperties( + tableId, + Collections.singletonMap(DatasourceDefn.BASE_TABLE_PROPERTY, null) + ); + return; + } writer.dropProjection(tableId, projectionName, alterTable.isIfExists()); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java index 5c50a91da949..1efe8e0c2504 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java @@ -22,9 +22,12 @@ import com.google.common.collect.ImmutableMap; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.util.SqlBasicVisitor; +import org.apache.druid.catalog.model.ClusteredValueGroupsBaseTableMetadata; import org.apache.druid.catalog.model.ColumnSpec; import org.apache.druid.catalog.model.Columns; import org.apache.druid.data.input.impl.AggregateProjectionSpec; @@ -32,6 +35,8 @@ import org.apache.druid.error.DruidException; import org.apache.druid.error.InvalidSqlInput; import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.DataSource; import org.apache.druid.query.Query; import org.apache.druid.query.QueryContexts; @@ -43,10 +48,13 @@ import org.apache.druid.query.filter.DimFilter; import org.apache.druid.query.filter.RangeFilter; import org.apache.druid.query.groupby.GroupByQuery; +import org.apache.druid.query.scan.ScanQuery; import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.server.security.AuthorizationResult; import org.apache.druid.server.security.NoopEscalator; import org.apache.druid.sql.calcite.rel.DruidQuery; @@ -73,6 +81,11 @@ */ public class ProjectionSpecTranslator { + /** + * The reserved projection name that describes the table's own physical layout. + */ + public static final String BASE_PROJECTION_NAME = "__base"; + /** * Planning is deterministic in the shapes the lift understands: no timeseries or topN rewrite to hide the grouping * columns, and no approximation choices that depend on unrelated configuration. @@ -109,6 +122,142 @@ public AggregateProjectionSpec translate( return lift(projectionName, tableName, druidQuery); } + /** + * Translate the reserved base-table projection, which describes the physical layout of the table itself rather + * than an additional aggregate. + *

+ * The body enumerates the table's columns in the order segments store them, so it must name every declared column, + * in declared order. An item written as {@code AS } makes that column computed at ingest time: the + * expression becomes a virtual column materializing the declared column, which is why the declared type has to + * match what the expression produces. + * + * @param clusteredBy the columns segments are clustered on, which must be the leading prefix of the column list + */ + public ClusteredValueGroupsBaseTableMetadata translateBaseTable( + final String tableName, + final List columns, + final SqlSelect body, + @Nullable final SqlNodeList clusteredBy + ) + { + if (body.getWhere() != null || body.getGroup() != null) { + throw invalid( + BASE_PROJECTION_NAME, + "its body filters or groups. The base table stores every ingested row, so it can do neither" + ); + } + rejectSubqueries(BASE_PROJECTION_NAME, body); + + final DruidQuery druidQuery = planBody(tableName, columns, BASE_PROJECTION_NAME, body); + final ClusteredValueGroupsBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata( + clusteringColumns(clusteredBy), + liftComputedColumns(columns, druidQuery), + null + ); + + // Derive the physical spec now. The catalog does this too when the write lands, but doing it here attributes + // layout problems to the statement that caused them rather than to a Coordinator round trip. + try { + metadata.createSpec(columns); + } + catch (DruidException e) { + throw contextualize(BASE_PROJECTION_NAME, e); + } + return metadata; + } + + private static List clusteringColumns(@Nullable final SqlNodeList clusteredBy) + { + if (clusteredBy == null) { + return Collections.emptyList(); + } + final List names = new ArrayList<>(clusteredBy.size()); + for (SqlNode node : clusteredBy) { + if (!(node instanceof SqlIdentifier) || !((SqlIdentifier) node).isSimple()) { + throw invalid( + BASE_PROJECTION_NAME, + "its CLUSTERED BY names [" + node + "], which is not a column. Segments are clustered on stored columns;" + + " to cluster on a computed value, declare it as a column of the table" + ); + } + names.add(((SqlIdentifier) node).getSimple()); + } + return names; + } + + /** + * Pair the planned output with the declared columns and lift the virtual columns behind the computed ones. + *

+ * The planner names its virtual columns {@code v0}, {@code v1}, ...; each is renamed to the declared column it + * fills, which is what makes it a materialized column rather than an anonymous intermediate. + */ + private static VirtualColumns liftComputedColumns( + final List columns, + final DruidQuery druidQuery + ) + { + final Query query = druidQuery.getQuery(); + if (!(query instanceof ScanQuery)) { + throw invalid( + BASE_PROJECTION_NAME, + "its body does not select rows directly. The base table stores every ingested row as it arrives" + ); + } + final List selected = ((ScanQuery) query).getColumns(); + final List outputNames = druidQuery.getOutputRowType().getFieldNames(); + + if (outputNames.size() != columns.size()) { + throw invalid( + BASE_PROJECTION_NAME, + StringUtils.format( + "it selects %d column(s) but the table declares %d. The body lists the columns in the order segments" + + " store them, so it must name every declared column", + outputNames.size(), + columns.size() + ) + ); + } + + final VirtualColumns planned = ((ScanQuery) query).getVirtualColumns(); + final List materialized = new ArrayList<>(); + for (int i = 0; i < columns.size(); i++) { + final String declared = columns.get(i).name(); + if (!declared.equals(outputNames.get(i))) { + throw invalid( + BASE_PROJECTION_NAME, + StringUtils.format( + "its column %d is [%s] but the table declares [%s] there. The body lists the columns in the order" + + " segments store them", + i + 1, + outputNames.get(i), + declared + ) + ); + } + final VirtualColumn virtualColumn = planned.getVirtualColumn(selected.get(i)); + if (virtualColumn == null) { + // A plain reference: the column is ingested as it arrives. + continue; + } + if (!(virtualColumn instanceof ExpressionVirtualColumn)) { + throw invalid( + BASE_PROJECTION_NAME, + "column [" + declared + "] is computed by an expression the base table cannot store" + ); + } + final ExpressionVirtualColumn expression = (ExpressionVirtualColumn) virtualColumn; + materialized.add( + new ExpressionVirtualColumn( + declared, + expression.getExpression(), + expression.getOutputType(), + ExprMacroTable.nil() + ) + ); + } + return VirtualColumns.create(materialized); + } + /** * Plan the body against a table built from the declared columns. The table is synthesized rather than looked up * because for {@code CREATE TABLE} it does not exist yet, and for {@code ALTER TABLE} the statement's own columns diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java index ebc00343e419..240af0d5c40a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.apache.druid.catalog.model.ClusteredValueGroupsBaseTableMetadata; import org.apache.druid.catalog.model.ColumnSpec; import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import org.apache.druid.catalog.model.TableId; @@ -515,14 +516,18 @@ public void testProjectionRejectsNonAggregatingBody() assertTrue(e.getMessage().contains("does not aggregate"), e.getMessage()); } + /** + * {@code __base} names the table's own layout and is handled separately; every other name beginning with the + * reserved prefix stays unavailable. + */ @Test - public void testProjectionRejectsReservedBaseName() + public void testProjectionRejectsOtherReservedNames() { final DruidException e = assertThrows( DruidException.class, - () -> execute("CREATE TABLE tbl (a VARCHAR, PROJECTION __base AS (SELECT a GROUP BY a))") + () -> execute("CREATE TABLE tbl (a VARCHAR, PROJECTION __other AS (SELECT a GROUP BY a))") ); - assertTrue(e.getMessage().contains("base table layout"), e.getMessage()); + assertTrue(e.getMessage().contains("reserved name"), e.getMessage()); } @Test @@ -550,6 +555,212 @@ private String projectionsJson() throws Exception .writeValueAsString(WRITER.calls.get(0).spec.properties().get("projections")); } + /** + * The reserved {@code __base} projection describes the table's own layout, so it becomes the baseTable property + * rather than one of the projections. A computed column becomes a virtual column materializing the declared column + * it fills. + */ + @Test + public void testCreateTableWithBaseProjection() throws Exception + { + execute( + "CREATE TABLE tbl (" + + " tenant VARCHAR," + + " bucket BIGINT," + + " __time TIMESTAMP," + + " user_id BIGINT," + + " PROJECTION __base AS (" + + " SELECT tenant, ABS(user_id) AS bucket, __time, user_id" + + " CLUSTERED BY tenant, bucket" + + " )" + + ") PARTITIONED BY DAY SEALED" + ); + + final TableSpec spec = WRITER.calls.get(0).spec; + assertEquals(true, spec.properties().get(DatasourceDefn.SEALED_PROPERTY)); + assertNull(spec.properties().get(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY)); + assertEquals( + "{\"clusteringColumns\":[\"tenant\",\"bucket\"]," + + "\"virtualColumns\":[{\"type\":\"expression\",\"name\":\"bucket\"," + + "\"expression\":\"abs(\\\"user_id\\\")\",\"outputType\":\"LONG\"}]," + + "\"type\":\"clusteredValueGroups\"}", + queryFramework().queryJsonMapper() + .writeValueAsString(spec.properties().get(DatasourceDefn.BASE_TABLE_PROPERTY)) + ); + } + + @Test + public void testBaseProjectionWithoutComputedColumns() + { + execute( + "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + + " PROJECTION __base AS (SELECT tenant, __time, v CLUSTERED BY tenant)) SEALED" + ); + final ClusteredValueGroupsBaseTableMetadata baseTable = baseTable(); + assertEquals(List.of("tenant"), baseTable.getClusteringColumns()); + assertEquals(0, baseTable.getVirtualColumns().getVirtualColumns().length); + } + + /** + * A base table and aggregate projections are different catalog entities and may coexist. + */ + @Test + public void testBaseProjectionAlongsideAggregateProjection() + { + execute( + "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + + " PROJECTION __base AS (SELECT tenant, __time, v CLUSTERED BY tenant)," + + " PROJECTION by_tenant AS (SELECT tenant, SUM(v) AS sum_v GROUP BY tenant)) SEALED" + ); + final TableSpec spec = WRITER.calls.get(0).spec; + assertNotNull(spec.properties().get(DatasourceDefn.BASE_TABLE_PROPERTY)); + assertEquals(1, ((List) spec.properties().get(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY)).size()); + } + + @Test + public void testBaseProjectionRequiresSealed() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute( + "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP," + + " PROJECTION __base AS (SELECT tenant, __time CLUSTERED BY tenant))" + ) + ); + assertTrue(e.getMessage().contains("must be declared SEALED"), e.getMessage()); + } + + /** + * The body lists the columns in the order segments store them, so it must match the declaration exactly. + */ + @Test + public void testBaseProjectionColumnOrderMustMatch() + { + final DruidException wrongOrder = assertThrows( + DruidException.class, + () -> execute( + "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + + " PROJECTION __base AS (SELECT __time, tenant, v CLUSTERED BY tenant)) SEALED" + ) + ); + assertTrue(wrongOrder.getMessage().contains("the table declares"), wrongOrder.getMessage()); + + final DruidException missing = assertThrows( + DruidException.class, + () -> execute( + "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + + " PROJECTION __base AS (SELECT tenant, __time CLUSTERED BY tenant)) SEALED" + ) + ); + assertTrue(missing.getMessage().contains("must name every declared column"), missing.getMessage()); + } + + /** + * Clustering columns must lead the declared column list, since the declared order is the physical order. The + * catalog enforces this on write; catching it here names the statement that caused it. + */ + @Test + public void testBaseProjectionClusteringMustBeLeadingPrefix() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute( + "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + + " PROJECTION __base AS (SELECT tenant, __time, v CLUSTERED BY v)) SEALED" + ) + ); + assertTrue(e.getMessage().contains("__base"), e.getMessage()); + } + + @Test + public void testBaseProjectionRejectsFilterOrGrouping() + { + for (String body : new String[]{ + "SELECT tenant, __time WHERE tenant <> 'x'", + "SELECT tenant, __time GROUP BY tenant, __time" + }) { + final DruidException e = assertThrows( + DruidException.class, + () -> execute( + "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, PROJECTION __base AS (" + body + ")) SEALED" + ), + body + ); + assertTrue(e.getMessage().contains("filters or groups"), e.getMessage()); + } + } + + /** + * Only the base projection chooses a clustering; an aggregate projection is ordered by its grouping columns. + */ + @Test + public void testAggregateProjectionRejectsClusteredBy() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute( + "CREATE TABLE tbl (a VARCHAR, b BIGINT," + + " PROJECTION p AS (SELECT a, SUM(b) AS s GROUP BY a CLUSTERED BY a))" + ) + ); + assertTrue(e.getMessage().contains("cannot use CLUSTERED BY"), e.getMessage()); + } + + @Test + public void testAlterTableAddBaseProjection() + { + WRITER.existing.put( + TableId.datasource("tbl"), + TableMetadata.newTable( + TableId.datasource("tbl"), + new TableSpec( + DatasourceDefn.TABLE_TYPE, + ImmutableMap.of(DatasourceDefn.SEALED_PROPERTY, true), + List.of( + new ColumnSpec("tenant", "VARCHAR", null), + new ColumnSpec("__time", "TIMESTAMP", null) + ) + ) + ) + ); + execute("ALTER TABLE tbl ADD PROJECTION __base AS (SELECT tenant, __time CLUSTERED BY tenant)"); + + final RecordingCatalogTableWriter.Call call = WRITER.lastCall("updateProperties"); + assertNotNull(call.properties.get(DatasourceDefn.BASE_TABLE_PROPERTY)); + } + + @Test + public void testAlterTableDropBaseProjection() + { + WRITER.existing.put( + TableId.datasource("tbl"), + TableMetadata.newTable( + TableId.datasource("tbl"), + new TableSpec( + DatasourceDefn.TABLE_TYPE, + ImmutableMap.of(DatasourceDefn.BASE_TABLE_PROPERTY, ImmutableMap.of("type", "clusteredValueGroups")), + List.of(new ColumnSpec("tenant", "VARCHAR", null)) + ) + ) + ); + execute("ALTER TABLE tbl DROP PROJECTION __base"); + + final RecordingCatalogTableWriter.Call call = WRITER.lastCall("updateProperties"); + assertTrue(call.properties.containsKey(DatasourceDefn.BASE_TABLE_PROPERTY)); + assertNull(call.properties.get(DatasourceDefn.BASE_TABLE_PROPERTY)); + + // Dropping one that is not there is an error unless tolerated. + WRITER.reset(); + assertThrows(DruidException.class, () -> execute("ALTER TABLE tbl DROP PROJECTION __base")); + execute("ALTER TABLE tbl DROP PROJECTION IF EXISTS __base"); + } + + private ClusteredValueGroupsBaseTableMetadata baseTable() + { + return (ClusteredValueGroupsBaseTableMetadata) + WRITER.calls.get(0).spec.properties().get(DatasourceDefn.BASE_TABLE_PROPERTY); + } + private void execute(String sql) { statement(sql).execute(); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java index 26a47b9d2411..d1bb79b11998 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java @@ -311,6 +311,58 @@ public void testProjectionBodyRejectsUnsupportedClauses() } } + @Test + public void testCreateTableWithBaseProjectionAndSealed() + { + final DruidSqlCreateTable create = parseCreate( + "CREATE TABLE t (\n" + + " tenant VARCHAR,\n" + + " bucket BIGINT,\n" + + " __time TIMESTAMP,\n" + + " PROJECTION __base AS (\n" + + " SELECT tenant, ABS(user_id) AS bucket, __time\n" + + " CLUSTERED BY tenant, bucket\n" + + " )\n" + + ") PARTITIONED BY DAY SEALED" + ); + + assertTrue(create.isSealed()); + assertEquals(1, create.getProjectionList().size()); + + final SqlProjectionSpec base = (SqlProjectionSpec) create.getProjectionList().get(0); + assertEquals("__base", base.getName().toString()); + assertEquals("`tenant`, `bucket`", base.getClusteredBy().toString()); + assertNull(base.getBody().getGroup()); + } + + @Test + public void testSealedWithoutProjection() + { + assertTrue(parseCreate("CREATE TABLE t (a VARCHAR) SEALED").isSealed()); + assertFalse(parseCreate("CREATE TABLE t (a VARCHAR)").isSealed()); + } + + /** + * SEALED is a non-reserved keyword, so it remains usable as an identifier. + */ + @Test + public void testSealedUsableAsIdentifier() + { + assertEquals("sealed VARCHAR", columnsOf(parseCreate("CREATE TABLE t (sealed VARCHAR)"))); + assertTrue(parseCreate("CREATE TABLE sealed (a VARCHAR) SEALED").isSealed()); + } + + @Test + public void testAlterTableAddBaseProjection() + { + final DruidSqlAlterTable.AddProjection alter = parseAlter( + "ALTER TABLE t ADD PROJECTION __base AS (SELECT a, __time CLUSTERED BY a)", + DruidSqlAlterTable.AddProjection.class + ); + assertEquals("__base", alter.getProjection().getName().toString()); + assertEquals("`a`", alter.getProjection().getClusteredBy().toString()); + } + /** * DDL nodes must round-trip through {@link SqlNode#unparse}, which is what makes them safe to log and re-print. */ @@ -327,6 +379,7 @@ public void testUnparseRoundTrip() assertUnparseRoundTrips("ALTER TABLE \"tbl\" DROP COLUMN \"a\""); assertUnparseRoundTrips("ALTER TABLE \"tbl\" ALTER COLUMN \"a\" SET DATA TYPE BIGINT"); assertUnparseRoundTrips("ALTER TABLE \"tbl\" SET PROPERTIES (\"sealed\" = TRUE)"); + assertUnparseRoundTrips("CREATE TABLE \"tbl\" (\"a\" VARCHAR) SEALED"); } @Test From c71e15f3b3416c2f1756e6a91a96c45f0756a055 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 30 Jul 2026 11:14:03 -0700 Subject: [PATCH 04/18] fix --- .../apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java index d1bb79b11998..fc56d62b7db2 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java @@ -26,6 +26,7 @@ import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; import org.junit.jupiter.api.Test; @@ -458,7 +459,7 @@ public void testDropTableIsNotSupported() private static void assertUnparseRoundTrips(String sql) { final SqlNode node = parse(sql); - assertEquals(sql, node.toSqlString(CalciteSqlDialect.DEFAULT).getSql().replace("\n", " ")); + assertEquals(sql, StringUtils.replace(node.toSqlString(CalciteSqlDialect.DEFAULT).getSql(), "\n", " ")); } private static String columnsOf(DruidSqlCreateTable create) From 00ccd65da3bdcc733110f72d6d095520d531bf70 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 30 Jul 2026 16:40:13 -0700 Subject: [PATCH 05/18] fixes --- .../druid/catalog/http/TableEditor.java | 2 +- .../sql/avatica/DruidAvaticaHandlerTest.java | 2 ++ .../druid/sql/calcite/NotYetSupported.java | 4 ++- .../calcite/parser/DruidSqlDdlParserTest.java | 34 +++++++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java index f9419691a340..b69cb48acf6b 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java @@ -446,7 +446,7 @@ private long dropProjection(String name, boolean ifExists) throws CatalogExcepti }); } - private List projectionsOf(TableMetadata table) throws CatalogException + private List projectionsOf(TableMetadata table) { final ResolvedTable resolved = catalog.tableRegistry().resolve(table.spec()); final List projections = diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java index 20c46f3a1082..7be5e70a288b 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java @@ -84,6 +84,7 @@ import org.apache.druid.sql.calcite.planner.CalciteRulesManager; import org.apache.druid.sql.calcite.planner.Calcites; import org.apache.druid.sql.calcite.planner.CatalogResolver; +import org.apache.druid.sql.calcite.planner.CatalogTableWriter; import org.apache.druid.sql.calcite.planner.DruidOperatorTable; import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerFactory; @@ -318,6 +319,7 @@ public void setUp() throws Exception binder.bind(CalciteRulesManager.class).toInstance(new CalciteRulesManager(ImmutableSet.of())); binder.bind(JoinableFactoryWrapper.class).toInstance(CalciteTests.createJoinableFactoryWrapper()); binder.bind(CatalogResolver.class).toInstance(CatalogResolver.NULL_RESOLVER); + binder.bind(CatalogTableWriter.class).toInstance(CatalogTableWriter.NOT_AVAILABLE); } ) .build(); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java b/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java index 25972ae556ca..4a65bcecefa7 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java @@ -94,7 +94,9 @@ enum Modes AGGREGATION_NOT_SUPPORT_TYPE(Scope.WINDOWING, DruidException.class, "Aggregation \\[(MIN|MAX)\\] does not support type \\[STRING\\]"), ALLDATA_CSV(Scope.WINDOWING, DruidException.class, "allData.csv"), BIGINT_TIME_COMPARE(Scope.WINDOWING, DruidException.class, "Cannot apply '.' to arguments of type"), - VIEWS_NOT_SUPPORTED(Scope.WINDOWING, DruidException.class, "Incorrect syntax near the keyword 'CREATE'"), + // CREATE starts a catalog DDL statement, so the parser now gets as far as the object being created before + // failing, rather than rejecting the CREATE keyword outright. + VIEWS_NOT_SUPPORTED(Scope.WINDOWING, DruidException.class, "Received an unexpected token \\[VIEW\\]"), RESULT_MISMATCH(Scope.WINDOWING, AssertionError.class, "(assertResulEquals|AssertionError: column content mismatch)"), LONG_CASTING(Scope.WINDOWING, AssertionError.class, "expected: java.lang.Long"), UNSUPPORTED_NULL_ORDERING(Scope.WINDOWING, DruidException.class, "(A|DE)SCENDING ordering with NULLS (LAST|FIRST)"), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java index fc56d62b7db2..f88440deea30 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java @@ -383,6 +383,40 @@ public void testUnparseRoundTrip() assertUnparseRoundTrips("CREATE TABLE \"tbl\" (\"a\" VARCHAR) SEALED"); } + /** + * Calcite clones a node by asking its operator to rebuild it from its operand list, which is how shuttles rewrite + * a statement. The operand order is hand-written per node, so a round trip is what proves it is right. + */ + @Test + public void testCloneRoundTrip() + { + for (String sql : new String[]{ + "CREATE TABLE \"tbl\" (\"a\" VARCHAR, \"b\" BIGINT)", + "CREATE OR REPLACE TABLE \"tbl\" (\"a\" VARCHAR)", + "CREATE TABLE IF NOT EXISTS \"tbl\" (\"a\" VARCHAR)", + "CREATE TABLE \"tbl\" (\"a\" VARCHAR) PARTITIONED BY DAY CLUSTERED BY \"a\"", + "CREATE TABLE \"tbl\" (\"a\" VARCHAR) SEALED", + "CREATE TABLE \"tbl\" (\"a\" VARCHAR, PROJECTION \"p\" AS (SELECT \"a\" GROUP BY \"a\"))", + "CREATE TABLE \"tbl\" (\"a\" VARCHAR, PROJECTION \"__base\" AS (SELECT \"a\" CLUSTERED BY \"a\")) SEALED", + "ALTER TABLE \"tbl\" ADD COLUMN \"a\" DOUBLE", + "ALTER TABLE \"tbl\" DROP COLUMN \"a\"", + "ALTER TABLE \"tbl\" ALTER COLUMN \"a\" SET DATA TYPE BIGINT", + "ALTER TABLE \"tbl\" ADD PROJECTION \"p\" AS (SELECT \"a\" GROUP BY \"a\")", + "ALTER TABLE \"tbl\" ADD IF NOT EXISTS PROJECTION \"p\" AS (SELECT \"a\" GROUP BY \"a\")", + "ALTER TABLE \"tbl\" DROP PROJECTION \"p\"", + "ALTER TABLE \"tbl\" DROP PROJECTION IF EXISTS \"p\"", + "ALTER TABLE \"tbl\" SET PROPERTIES (\"sealed\" = TRUE)" + }) { + final SqlNode node = parse(sql); + final SqlNode clone = node.clone(node.getParserPosition()); + assertEquals( + node.toSqlString(CalciteSqlDialect.DEFAULT).getSql(), + clone.toSqlString(CalciteSqlDialect.DEFAULT).getSql(), + sql + ); + } + } + @Test public void testDdlAfterSetStatement() { From aac737acac02401b777393b2d8eed0db54c5c330 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 6 Aug 2026 13:42:20 -0700 Subject: [PATCH 06/18] fixes --- docs/development/extensions-core/catalog.md | 12 ++ .../druid/catalog/http/TableEditRequest.java | 36 ++++ .../druid/catalog/http/TableEditor.java | 112 +++++++++--- .../storage/sql/SQLCatalogManager.java | 13 +- .../catalog/sync/CatalogSqlTableWriter.java | 10 +- .../druid/catalog/storage/CatalogTests.java | 3 +- .../druid/server/http/catalog/EditorTest.java | 161 ++++++++++++++++++ .../calcite/parser/DruidSqlCreateTable.java | 22 ++- .../sql/calcite/parser/SqlProjectionSpec.java | 5 +- .../calcite/planner/CatalogDdlHandler.java | 30 ++-- .../calcite/planner/CatalogTableWriter.java | 20 ++- .../planner/ProjectionSpecTranslator.java | 40 ++++- .../sql/calcite/CalciteCatalogDdlTest.java | 102 ++++++++--- .../calcite/parser/DruidSqlDdlParserTest.java | 12 ++ 14 files changed, 475 insertions(+), 103 deletions(-) diff --git a/docs/development/extensions-core/catalog.md b/docs/development/extensions-core/catalog.md index aef6f2cc9556..819a5eecacc3 100644 --- a/docs/development/extensions-core/catalog.md +++ b/docs/development/extensions-core/catalog.md @@ -103,6 +103,11 @@ ALTER TABLE

DROP PROJECTION [IF EXISTS] ALTER TABLE
SET PROPERTIES ( = [, ...] ) ``` +`ADD COLUMN` fails if the column already exists, and `ALTER COLUMN` fails if it does not, so a misspelled column name +is reported rather than quietly creating or replacing a column. Each statement is also checked against the rest of the +table definition, not only the part it changes: adding a column, changing a type, or setting a property is rejected if +the resulting table would be invalid, such as a segment granularity coarser than a projection the table declares. + #### Projections A table may declare [projections](../../querying/projections.md), which are pre-aggregated views stored inside each @@ -127,6 +132,13 @@ The body is planned exactly as the equivalent query would be, so a projection ma serve. Every aggregate needs an alias, which becomes the name of the stored column. Time granularity is expressed with `TIME_FLOOR`, as it would be in a query. +Because the body is planned like a query, it is planned under the statement's own query context, including any `SET` +clauses. Only context parameters that affect planning can change the stored definition; parameters that only affect +query execution have no effect, since the body is planned and stored rather than run. The context itself is not part +of the definition: what the catalog stores is the projection the body planned to, so nothing from the statement's +context is carried over to queries that later use it. Note also that a projection is matched to a query by its shape, +so a definition planned under a context that changes that shape only matches queries run under the same context. + A projection body accepts a select list, an optional `WHERE` and an optional `GROUP BY`. It cannot use `ORDER BY`, `LIMIT` or `HAVING`: a projection's ordering follows its grouping columns and is not something you choose. It also cannot use joins, subqueries, or expressions computed over aggregates. Store the aggregates instead: `SUM(x)` and diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java index 428f982c7848..fcf04757bb87 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java @@ -50,6 +50,8 @@ @Type(name = "dropColumns", value = TableEditRequest.DropColumns.class), @Type(name = "updateProperties", value = TableEditRequest.UpdateProperties.class), @Type(name = "updateColumns", value = TableEditRequest.UpdateColumns.class), + @Type(name = "addColumns", value = TableEditRequest.AddColumns.class), + @Type(name = "alterColumns", value = TableEditRequest.AlterColumns.class), @Type(name = "moveColumn", value = TableEditRequest.MoveColumn.class), @Type(name = "addProjection", value = TableEditRequest.AddProjection.class), @Type(name = "dropProjection", value = TableEditRequest.DropProjection.class), @@ -138,6 +140,40 @@ public UpdateColumns(@JsonProperty("columns") List columns) } } + /** + * Add columns that the table does not yet declare. Unlike {@link UpdateColumns}, which merges by name and so would + * quietly change an existing column, a column that already exists is an error. The check runs inside the update + * transaction, so two concurrent requests adding the same column cannot both succeed. + */ + public static class AddColumns extends TableEditRequest + { + @JsonProperty("columns") + public final List columns; + + @JsonCreator + public AddColumns(@JsonProperty("columns") List columns) + { + this.columns = columns; + } + } + + /** + * Change columns the table already declares. Unlike {@link UpdateColumns}, which appends a name it does not find and + * so would quietly create a column, a column that does not exist is an error. Checked inside the update transaction, + * as for {@link AddColumns}. + */ + public static class AlterColumns extends TableEditRequest + { + @JsonProperty("columns") + public final List columns; + + @JsonCreator + public AlterColumns(@JsonProperty("columns") List columns) + { + this.columns = columns; + } + } + /** * Move a column within the list of columns for a TableSpec. */ diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java index b69cb48acf6b..6784bb3e59c4 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java @@ -21,7 +21,9 @@ import com.google.common.base.Strings; import org.apache.druid.catalog.CatalogException; +import org.apache.druid.catalog.http.TableEditRequest.AddColumns; import org.apache.druid.catalog.http.TableEditRequest.AddProjection; +import org.apache.druid.catalog.http.TableEditRequest.AlterColumns; import org.apache.druid.catalog.http.TableEditRequest.DropColumns; import org.apache.druid.catalog.http.TableEditRequest.DropProjection; import org.apache.druid.catalog.http.TableEditRequest.HideColumns; @@ -44,6 +46,7 @@ import org.apache.druid.java.util.common.IAE; import org.apache.druid.utils.CollectionUtils; +import javax.annotation.Nullable; import javax.ws.rs.core.Response; import java.util.ArrayList; import java.util.Collections; @@ -102,6 +105,10 @@ public long go() throws CatalogException return updateProperties(((UpdateProperties) editRequest).properties); } else if (editRequest instanceof UpdateColumns) { return updateColumns(((UpdateColumns) editRequest).columns); + } else if (editRequest instanceof AddColumns) { + return addOrAlterColumns(((AddColumns) editRequest).columns, true); + } else if (editRequest instanceof AlterColumns) { + return addOrAlterColumns(((AlterColumns) editRequest).columns, false); } else if (editRequest instanceof AddProjection) { final AddProjection addProjection = (AddProjection) editRequest; return addProjection(addProjection.projection, addProjection.ifNotExists); @@ -120,6 +127,29 @@ public long go() throws CatalogException } } + /** + * Validate the revised spec as a whole before it is written back. Every edit is a read-modify-write of one part of + * the spec, but the rules that matter are cross-field: a projection is checked against the segment granularity, a + * base table layout against the declared columns. Validating only the part being edited would let an edit to either + * side leave the catalog holding a spec that later fails at ingest, so the update transactions load the whole spec + * (they still write back only the part they own) and each edit is checked against all of it. + * + * @param revised the revised spec, or null when the edit is a no-op and the transaction should roll back + */ + private TableSpec validated(@Nullable TableSpec revised) throws CatalogException + { + if (revised == null) { + return null; + } + try { + catalog.tableRegistry().resolve(revised).validate(); + } + catch (IAE | DruidException e) { + throw CatalogException.validationError(e); + } + return revised; + } + private long hideColumns(List columns) throws CatalogException { if (CollectionUtils.isNullOrEmpty(columns)) { @@ -127,7 +157,7 @@ private long hideColumns(List columns) throws CatalogException } return catalog.tables().updateProperties( id, - table -> applyHiddenColumns(table, columns) + table -> validated(applyHiddenColumns(table, columns)) ); } @@ -177,7 +207,7 @@ private long unHideColumns(List columns) throws CatalogException } return catalog.tables().updateProperties( id, - table -> applyUnhideColumns(table, columns) + table -> validated(applyUnhideColumns(table, columns)) ); } @@ -228,7 +258,7 @@ private long dropColumns(List columnsToDrop) throws CatalogException } return catalog.tables().updateColumns( id, - table -> applyDropColumns(table, columnsToDrop) + table -> validated(applyDropColumns(table, columnsToDrop)) ); } @@ -261,7 +291,7 @@ private long updateProperties(Map updates) throws CatalogExcepti } return catalog.tables().updateProperties( id, - table -> applyUpdateProperties(table, updates) + table -> validated(applyUpdateProperties(table, updates)) ); } @@ -276,12 +306,6 @@ private TableSpec applyUpdateProperties( existingSpec.properties(), updates ); - try { - defn.validate(revised, catalog.jsonMapper()); - } - catch (IAE e) { - throw CatalogException.badRequest(e.getMessage()); - } return existingSpec.withProperties(revised); } @@ -307,7 +331,7 @@ private long updateColumns(final List updates) throws CatalogExcepti } return catalog.tables().updateColumns( id, - table -> applyUpdateColumns(table, updates) + table -> validated(applyUpdateColumns(table, updates)) ); } @@ -319,15 +343,54 @@ private TableSpec applyUpdateColumns( final TableSpec existingSpec = table.spec(); final TableDefn defn = resolveDefn(existingSpec.type()); final List revised = defn.mergeColumns(existingSpec.columns(), updates); - try { - defn.validateColumns(revised); - } - catch (IAE | DruidException e) { - throw CatalogException.validationError(e); - } return existingSpec.withColumns(revised); } + /** + * Applies {@link AddColumns} and {@link AlterColumns}, which are {@link UpdateColumns} plus an existence rule. + * Merging by name is the right operation for both, but on its own it is silent about which columns already existed: + * a merge that finds the name changes that column, and one that does not appends a new one. So {@code ADD} would + * quietly alter and {@code ALTER} would quietly add. + *

+ * The rule is checked here, inside the Coordinator's update transaction, rather than by the caller beforehand. A + * caller's check would race: two requests adding the same column could both find it absent and the second merge + * would overwrite the first. + * + * @param mustBeAbsent true for {@code ADD} (no column may already exist), false for {@code ALTER} (all must exist) + */ + private long addOrAlterColumns(final List updates, final boolean mustBeAbsent) throws CatalogException + { + if (CollectionUtils.isNullOrEmpty(updates)) { + return 0; + } + return catalog.tables().updateColumns( + id, + table -> { + final TableSpec existingSpec = table.spec(); + final List existingColumns = + existingSpec.columns() == null ? Collections.emptyList() : existingSpec.columns(); + final Set existingNames = new HashSet<>(CatalogUtils.columnNames(existingColumns)); + for (ColumnSpec update : updates) { + if (mustBeAbsent && existingNames.contains(update.name())) { + throw CatalogException.badRequest( + "Column [%s] already exists in table %s; use ALTER COLUMN to change its type", + update.name(), + id.sqlName() + ); + } + if (!mustBeAbsent && !existingNames.contains(update.name())) { + throw CatalogException.badRequest( + "Column [%s] does not exist in table %s; use ADD COLUMN to define it", + update.name(), + id.sqlName() + ); + } + } + return validated(applyUpdateColumns(table, updates)); + } + ); + } + private long moveColumn(MoveColumn moveColumn) throws CatalogException { if (Strings.isNullOrEmpty(moveColumn.column)) { @@ -341,7 +404,7 @@ private long moveColumn(MoveColumn moveColumn) throws CatalogException } return catalog.tables().updateColumns( id, - table -> applyMoveColumn(table, moveColumn) + table -> validated(applyMoveColumn(table, moveColumn)) ); } @@ -415,7 +478,7 @@ private long addProjection(DatasourceProjectionMetadata projection, boolean ifNo } final List revised = new ArrayList<>(existing); revised.add(projection); - return withProjections(table, revised); + return validated(withProjections(table, revised)); }); } @@ -442,7 +505,7 @@ private long dropProjection(String name, boolean ifExists) throws CatalogExcepti id.sqlName() ); } - return withProjections(table, revised); + return validated(withProjections(table, revised)); }); } @@ -464,14 +527,7 @@ private TableSpec withProjections(TableMetadata table, List columns) + public void addColumns(TableId tableId, List columns) { - execute(tableId, () -> client.editTable(tableId, new TableEditRequest.UpdateColumns(columns))); + execute(tableId, () -> client.editTable(tableId, new TableEditRequest.AddColumns(columns))); + } + + @Override + public void alterColumns(TableId tableId, List columns) + { + execute(tableId, () -> client.editTable(tableId, new TableEditRequest.AlterColumns(columns))); } @Override diff --git a/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/CatalogTests.java b/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/CatalogTests.java index 0c30c073686f..2d1c47f5c017 100644 --- a/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/CatalogTests.java +++ b/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/CatalogTests.java @@ -27,6 +27,7 @@ import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.metadata.TestDerbyConnector.DerbyConnectorRule; +import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Action; import org.apache.druid.server.security.AuthenticationResult; @@ -73,7 +74,7 @@ public Access authorize( } public static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper().setInjectableValues( - new InjectableValues.Std().addValue(ExprMacroTable.class, ExprMacroTable.nil()) + new InjectableValues.Std().addValue(ExprMacroTable.class, TestExprMacroTable.INSTANCE) ); public static class DbFixture diff --git a/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java b/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java index 18491161803d..629543cbb75e 100644 --- a/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java +++ b/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java @@ -34,6 +34,7 @@ import org.apache.druid.catalog.http.TableEditRequest.UpdateProperties; import org.apache.druid.catalog.http.TableEditor; import org.apache.druid.catalog.model.CatalogUtils; +import org.apache.druid.catalog.model.ClusteredValueGroupsBaseTableMetadata; import org.apache.druid.catalog.model.ColumnSpec; import org.apache.druid.catalog.model.Columns; import org.apache.druid.catalog.model.DatasourceProjectionMetadata; @@ -45,8 +46,10 @@ import org.apache.druid.catalog.storage.CatalogStorage; import org.apache.druid.catalog.storage.CatalogTests; import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.data.input.impl.LongDimensionSchema; import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.metadata.TestDerbyConnector; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.junit.After; @@ -537,6 +540,164 @@ public void testAddAndDropProjection() throws CatalogException ); } + /** + * {@code AddColumns} means add: a column that already exists is an error rather than a silent in-place update, which + * is what plain {@code UpdateColumns} would do. + */ + @Test + public void testAddColumns() throws CatalogException + { + final String tableName = "addCols"; + final TableMetadata table = TableBuilder.datasource(tableName, "P1D") + .timeColumn() + .column("dim", "VARCHAR") + .build(); + catalog.tables().create(table); + + final TableMetadata revised = doEdit( + tableName, + new TableEditRequest.AddColumns(Collections.singletonList(new ColumnSpec("met", "BIGINT", null))) + ); + assertEquals( + Arrays.asList(Columns.TIME_COLUMN, "dim", "met"), + CatalogUtils.columnNames(revised.spec().columns()) + ); + + final CatalogException e = assertThrows( + CatalogException.class, + () -> doEdit( + tableName, + new TableEditRequest.AddColumns(Collections.singletonList(new ColumnSpec("dim", "BIGINT", null))) + ) + ); + assertTrue(e.getMessage(), e.getMessage().contains("Column [dim] already exists")); + // The rejected add did not change the existing column's type. + assertEquals("VARCHAR", columnType(tableName, "dim")); + } + + /** + * {@code AlterColumns} means alter: a column that does not exist is an error rather than being appended, so a + * misspelled target cannot quietly create a column. + */ + @Test + public void testAlterColumns() throws CatalogException + { + final String tableName = "alterCols"; + final TableMetadata table = TableBuilder.datasource(tableName, "P1D") + .timeColumn() + .column("dim", "VARCHAR") + .build(); + catalog.tables().create(table); + + doEdit( + tableName, + new TableEditRequest.AlterColumns(Collections.singletonList(new ColumnSpec("dim", "BIGINT", null))) + ); + assertEquals("BIGINT", columnType(tableName, "dim")); + + final CatalogException e = assertThrows( + CatalogException.class, + () -> doEdit( + tableName, + new TableEditRequest.AlterColumns(Collections.singletonList(new ColumnSpec("typo", "BIGINT", null))) + ) + ); + assertTrue(e.getMessage(), e.getMessage().contains("Column [typo] does not exist")); + // The misspelled target was not created. + assertEquals( + Arrays.asList(Columns.TIME_COLUMN, "dim"), + CatalogUtils.columnNames(catalog.tables().read(table.id()).spec().columns()) + ); + } + + private String columnType(String tableName, String columnName) throws CatalogException + { + return catalog.tables().read(TableId.datasource(tableName)).spec().columns().stream() + .filter(c -> columnName.equals(c.name())) + .findFirst() + .orElseThrow(() -> new AssertionError("No column [" + columnName + "]")) + .dataType(); + } + + /** + * A property edit is validated against the parts of the spec it does not touch. Segment granularity and the declared + * projections are only meaningful together, so coarsening the segments under a projection has to be caught here + * rather than at ingest time. + */ + @Test + public void testUpdatePropertiesValidatedAgainstProjections() throws CatalogException + { + final String tableName = "granularity"; + final TableMetadata table = TableBuilder.datasource(tableName, "P1D") + .timeColumn() + .column("dim", "VARCHAR") + .column("met", "BIGINT") + .build(); + catalog.tables().create(table); + + final DatasourceProjectionMetadata daily = new DatasourceProjectionMetadata( + AggregateProjectionSpec.builder("daily") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.DAY, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + new StringDimensionSchema("dim") + ) + .aggregators(new LongSumAggregatorFactory("sum_met", "met")) + .build() + ); + // A day-granularity projection is fine in day-granularity segments. + assertTrue(new TableEditor(catalog, table.id(), new AddProjection(daily, false)).go() > 0); + + assertThrows( + CatalogException.class, + () -> doEdit( + tableName, + new UpdateProperties(ImmutableMap.of(DatasourceDefn.SEGMENT_GRANULARITY_PROPERTY, "PT1H")) + ) + ); + // The rejected edit left the stored spec alone. + assertEquals( + "P1D", + catalog.tables().read(table.id()).spec().properties().get(DatasourceDefn.SEGMENT_GRANULARITY_PROPERTY) + ); + } + + /** + * The mirror case: a column edit is validated against the properties it does not touch. A base table layout names + * the columns it clusters on, so dropping one leaves a layout that can no longer be derived. + */ + @Test + public void testDropColumnsValidatedAgainstBaseTable() throws CatalogException + { + final String tableName = "clustered"; + final TableMetadata table = TableBuilder.datasource(tableName, "P1D") + // Declared order is the segment order, so the clustering column leads. + .column("tenant", "VARCHAR") + .timeColumn() + .column("met", "BIGINT") + .sealed(true) + .baseTable(new ClusteredValueGroupsBaseTableMetadata(ImmutableList.of("tenant"), null, null)) + .build(); + catalog.tables().create(table); + + assertThrows( + CatalogException.class, + () -> doEdit(tableName, new DropColumns(ImmutableList.of("tenant"))) + ); + assertEquals( + Arrays.asList("tenant", Columns.TIME_COLUMN, "met"), + CatalogUtils.columnNames(catalog.tables().read(table.id()).spec().columns()) + ); + + // Dropping a column the layout does not name is still allowed. + assertTrue(new TableEditor(catalog, table.id(), new DropColumns(ImmutableList.of("met"))).go() > 0); + } + private List projectionsOf(String tableName) throws CatalogException { return catalog.tableRegistry() diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java index a9c9e6c77c80..38d79ea8e2da 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java @@ -154,16 +154,20 @@ public void unparse(SqlWriter writer, int leftPrec, int rightPrec) } name.unparse(writer, leftPrec, rightPrec); - final SqlWriter.Frame frame = writer.startList("(", ")"); - for (SqlNode column : columnList) { - writer.sep(","); - column.unparse(writer, 0, 0); - } - for (SqlNode projection : projectionList) { - writer.sep(","); - projection.unparse(writer, 0, 0); + // The element list is optional and, when present, must hold at least one element, so a table declaring nothing + // prints no parentheses at all: "()" is not something the grammar can read back. + if (!columnList.isEmpty() || !projectionList.isEmpty()) { + final SqlWriter.Frame frame = writer.startList("(", ")"); + for (SqlNode column : columnList) { + writer.sep(","); + column.unparse(writer, 0, 0); + } + for (SqlNode projection : projectionList) { + writer.sep(","); + projection.unparse(writer, 0, 0); + } + writer.endList(frame); } - writer.endList(frame); if (partitionedBy != null) { writer.keyword("PARTITIONED BY"); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlProjectionSpec.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlProjectionSpec.java index 8cfa53cc733b..40beffc5e9cb 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlProjectionSpec.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlProjectionSpec.java @@ -107,7 +107,10 @@ public void unparse(SqlWriter writer, int leftPrec, int rightPrec) name.unparse(writer, 0, 0); writer.keyword("AS"); final SqlWriter.Frame frame = writer.startList("(", ")"); - body.unparse(writer, 0, 0); + // Unparse through the operator rather than the node: a SqlSelect in this position parenthesizes itself, which + // together with the frame above would print a second set the grammar cannot read back. The parentheses belong to + // the projection, not the body, because CLUSTERED BY goes inside them. + body.getOperator().unparse(writer, body, 0, 0); if (clusteredBy != null) { writer.keyword("CLUSTERED BY"); final SqlWriter.Frame clusterFrame = writer.startList("", ""); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java index c12cc826ae40..aa922054f41a 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java @@ -260,7 +260,7 @@ protected static DatasourceProjectionMetadata translateProjection( ); } return new DatasourceProjectionMetadata( - new ProjectionSpecTranslator(handlerContext.plannerFactory()) + new ProjectionSpecTranslator(handlerContext.plannerFactory(), handlerContext.queryContextMap()) .translate(tableName, columns, name, projection.getBody()) ); } @@ -276,7 +276,7 @@ protected static ClusteredValueGroupsBaseTableMetadata translateBaseTable( final SqlProjectionSpec projection ) { - return new ProjectionSpecTranslator(handlerContext.plannerFactory()) + return new ProjectionSpecTranslator(handlerContext.plannerFactory(), handlerContext.queryContextMap()) .translateBaseTable(tableName, columns, projection.getBody(), projection.getClusteredBy()); } @@ -433,8 +433,9 @@ protected String operationName() } /** - * {@code ALTER TABLE ... ADD COLUMN}. The Coordinator merges columns by name, so an existing column would be - * silently updated; this checks first so that {@code ADD} means add. + * {@code ALTER TABLE ... ADD COLUMN}. The Coordinator merges columns by name, which on its own would silently update + * an existing column, so the write requires the column to be absent; that check belongs to the Coordinator's update + * transaction rather than here, since checking first would race with a concurrent add of the same column. */ public static class AddColumnHandler extends CatalogDdlHandler { @@ -456,19 +457,7 @@ protected void validateStatement() @Override protected void execute(CatalogTableWriter writer) { - final TableMetadata existing = writer.readTable(tableId); - if (existing == null) { - throw InvalidSqlInput.exception("Table [%s] does not have a catalog entry", tableId.name()); - } - if (existing.spec().columns() != null - && existing.spec().columns().stream().anyMatch(c -> column.name().equals(c.name()))) { - throw InvalidSqlInput.exception( - "Column [%s] already exists in table [%s]; use ALTER COLUMN to change its type", - column.name(), - tableId.name() - ); - } - writer.updateColumns(tableId, Collections.singletonList(column)); + writer.addColumns(tableId, Collections.singletonList(column)); } @Override @@ -515,8 +504,9 @@ protected String operationName() } /** - * {@code ALTER TABLE ... ALTER COLUMN ... SET DATA TYPE}. Merging a column by name is exactly what changing its - * type requires, so this reuses the same update as ADD COLUMN without the existence check. + * {@code ALTER TABLE ... ALTER COLUMN ... SET DATA TYPE}. The mirror of {@link AddColumnHandler}: merging by name is + * what changing a type requires, but a name the Coordinator does not find is appended, so a misspelled target would + * create a column instead of failing. The write therefore requires the column to already exist. */ public static class AlterColumnHandler extends CatalogDdlHandler { @@ -541,7 +531,7 @@ protected void validateStatement() @Override protected void execute(CatalogTableWriter writer) { - writer.updateColumns(tableId, Collections.singletonList(column)); + writer.alterColumns(tableId, Collections.singletonList(column)); } @Override diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java index 20fab1cbf41d..b93516bb031a 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java @@ -53,10 +53,16 @@ public interface CatalogTableWriter void createTable(TableId tableId, TableSpec spec, boolean ifNotExists, boolean replace); /** - * Merge the given columns into the table's column list, matching on name. Columns that do not yet exist are - * appended; existing columns are updated in place. + * Append the given columns to the table's column list, failing if any of them already exists. Both this and + * {@link #alterColumns} match columns by name; they differ only in which outcome is an error, and both decide that + * inside the Coordinator's update transaction so that concurrent callers cannot both win. */ - void updateColumns(TableId tableId, List columns); + void addColumns(TableId tableId, List columns); + + /** + * Update the given columns in place, failing if any of them does not already exist. See {@link #addColumns}. + */ + void alterColumns(TableId tableId, List columns); /** * Remove the named columns from the table's column list. Segments are unaffected. @@ -102,7 +108,13 @@ public void createTable(TableId tableId, TableSpec spec, boolean ifNotExists, bo } @Override - public void updateColumns(TableId tableId, List columns) + public void addColumns(TableId tableId, List columns) + { + throw notAvailable(); + } + + @Override + public void alterColumns(TableId tableId, List columns) { throw notAvailable(); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java index 1efe8e0c2504..8f36d1086639 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java @@ -67,6 +67,7 @@ import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -87,8 +88,9 @@ public class ProjectionSpecTranslator public static final String BASE_PROJECTION_NAME = "__base"; /** - * Planning is deterministic in the shapes the lift understands: no timeseries or topN rewrite to hide the grouping - * columns, and no approximation choices that depend on unrelated configuration. + * Overrides applied on top of the statement's own context, so that planning is deterministic in the shapes the lift + * understands: no timeseries or topN rewrite to hide the grouping columns, and no approximation choices that depend + * on unrelated configuration. These win over anything the statement set. */ private static final Map CONTEXT = ImmutableMap.of( PlannerContext.CTX_SQL_USE_GRANULARITY, false, @@ -97,10 +99,20 @@ public class ProjectionSpecTranslator ); private final PlannerFactory plannerFactory; + private final Map queryContext; - public ProjectionSpecTranslator(PlannerFactory plannerFactory) + /** + * @param queryContext the context of the statement being planned, including anything its {@code SET} clauses set. + * A projection body is planned by the same machinery as the query it is meant to serve, which + * is what makes the two agree structurally, so it is planned under the same context too. Only + * parameters that affect planning can change the stored definition; execution-time parameters + * reach the nested planner but have nothing to act on, since the body is planned and lifted + * rather than run. + */ + public ProjectionSpecTranslator(PlannerFactory plannerFactory, Map queryContext) { this.plannerFactory = plannerFactory; + this.queryContext = queryContext; } /** @@ -203,7 +215,7 @@ private static VirtualColumns liftComputedColumns( "its body does not select rows directly. The base table stores every ingested row as it arrives" ); } - final List selected = ((ScanQuery) query).getColumns(); + final RowSignature sources = druidQuery.getOutputRowSignature(); final List outputNames = druidQuery.getOutputRowType().getFieldNames(); if (outputNames.size() != columns.size()) { @@ -234,8 +246,22 @@ private static VirtualColumns liftComputedColumns( ) ); } - final VirtualColumn virtualColumn = planned.getVirtualColumn(selected.get(i)); + final String source = sources.getColumnName(i); + final VirtualColumn virtualColumn = planned.getVirtualColumn(source); if (virtualColumn == null) { + if (!source.equals(declared)) { + throw invalid( + BASE_PROJECTION_NAME, + StringUtils.format( + "its column %d selects [%s] but declares it as [%s]. A base table column is either ingested under" + + " its own name or computed by an expression, so nothing would materialize [%s]", + i + 1, + source, + declared, + declared + ) + ); + } // A plain reference: the column is ingested as it arrives. continue; } @@ -275,11 +301,13 @@ private DruidQuery planBody( final ProjectionSqlEngine engine = new ProjectionSqlEngine(); final String sql = query.toString(); + final Map context = new HashMap<>(queryContext); + context.putAll(CONTEXT); try (DruidPlanner planner = plannerFactory.createPlannerForTable( engine, sql, query, - CONTEXT, + context, tableName, tableFor(tableName, columns) )) { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java index 240af0d5c40a..d3a00f2e903b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java @@ -192,34 +192,14 @@ public void testDdlReturnsNoRows() @Test public void testAlterTableAddColumn() { - WRITER.existing.put(TableId.datasource("tbl"), tableWithColumns("a")); + // ADD and ALTER differ only in which existence outcome is an error, and that rule is enforced inside the + // Coordinator's update transaction, so all the statement does is pick the verb. EditorTest covers the rules. execute("ALTER TABLE tbl ADD COLUMN b BIGINT"); - final RecordingCatalogTableWriter.Call call = WRITER.lastCall("updateColumns"); + final RecordingCatalogTableWriter.Call call = WRITER.lastCall("addColumns"); assertEquals(ImmutableList.of(new ColumnSpec("b", "BIGINT", null)), call.columns); } - @Test - public void testAlterTableAddExistingColumnFails() - { - WRITER.existing.put(TableId.datasource("tbl"), tableWithColumns("a")); - final DruidException e = assertThrows( - DruidException.class, - () -> execute("ALTER TABLE tbl ADD COLUMN a BIGINT") - ); - assertTrue(e.getMessage().contains("Column [a] already exists")); - } - - @Test - public void testAlterTableAddColumnToMissingTableFails() - { - final DruidException e = assertThrows( - DruidException.class, - () -> execute("ALTER TABLE tbl ADD COLUMN a BIGINT") - ); - assertTrue(e.getMessage().contains("does not have a catalog entry")); - } - @Test public void testAlterTableDropColumn() { @@ -230,11 +210,10 @@ public void testAlterTableDropColumn() @Test public void testAlterTableAlterColumn() { - // Unlike ADD COLUMN, changing a type does not require the column to be absent, so no read is needed. execute("ALTER TABLE tbl ALTER COLUMN cnt SET DATA TYPE DOUBLE"); assertEquals( ImmutableList.of(new ColumnSpec("cnt", "DOUBLE", null)), - WRITER.lastCall("updateColumns").columns + WRITER.lastCall("alterColumns").columns ); } @@ -376,6 +355,45 @@ public void testCreateTableWithProjection() throws Exception ); } + /** + * A projection body is planned under the statement's own context, so a SET clause that changes how the equivalent + * query would plan changes the stored definition the same way. Here the session time zone reaches the TIME_FLOOR. + */ + @Test + public void testProjectionBodyHonorsStatementContext() throws Exception + { + execute( + "SET sqlTimeZone = 'America/Los_Angeles';\n" + + "CREATE TABLE tbl (__time TIMESTAMP, page VARCHAR, cnt BIGINT," + + " PROJECTION daily AS (SELECT TIME_FLOOR(__time, 'P1D'), page, SUM(cnt) AS total GROUP BY 1, 2))" + ); + + assertTrue( + projectionsJson().contains("timestamp_floor(\\\"__time\\\",'P1D',null,'America/Los_Angeles')"), + projectionsJson() + ); + } + + /** + * The overrides the lift depends on are applied on top of the statement's context, so a SET clause cannot put the + * planner into a shape the lift does not understand. + */ + @Test + public void testProjectionBodyContextCannotOverrideDeterministicOverrides() throws Exception + { + execute( + "SET sqlUseGranularity = TRUE;\n" + + "CREATE TABLE tbl (__time TIMESTAMP, page VARCHAR, cnt BIGINT," + + " PROJECTION daily AS (SELECT TIME_FLOOR(__time, 'P1D'), page, SUM(cnt) AS total GROUP BY 1, 2))" + ); + + // Still lifted as an ordinary grouping column rather than a query granularity, exactly as without the SET. + assertTrue( + projectionsJson().contains("timestamp_floor(\\\"__time\\\",'P1D',null,'UTC')"), + projectionsJson() + ); + } + /** * A projection defined with TIME_FLOOR must carry a granularity the segment layer can recover, which is how the * projection gets matched to time-grouped queries. @@ -589,6 +607,30 @@ public void testCreateTableWithBaseProjection() throws Exception ); } + /** + * A base table column is stored under the name it declares, so a body item that renames another column is rejected: + * only a virtual column materializes a name the body did not select, and a bare reference produces none. + *

+ * Selecting the same column twice also makes the scan's deduplicated column list shorter than the select list, so + * the source of each item is taken from the query's output signature. Reading the scan's columns positionally used + * to run off the end of that list and fail as an internal error. + */ + @Test + public void testBaseProjectionRenamedColumnFails() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute( + "CREATE TABLE tbl (id BIGINT, copy BIGINT, __time TIMESTAMP," + + " PROJECTION __base AS (SELECT id, id AS copy, __time CLUSTERED BY id)) SEALED" + ) + ); + assertTrue( + e.getMessage().contains("its column 2 selects [id] but declares it as [copy]"), + e.getMessage() + ); + } + @Test public void testBaseProjectionWithoutComputedColumns() { @@ -836,9 +878,15 @@ public void createTable(TableId tableId, TableSpec spec, boolean ifNotExists, bo } @Override - public void updateColumns(TableId tableId, List columns) + public void addColumns(TableId tableId, List columns) + { + record("addColumns", tableId).columns = columns; + } + + @Override + public void alterColumns(TableId tableId, List columns) { - record("updateColumns", tableId).columns = columns; + record("alterColumns", tableId).columns = columns; } @Override diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java index f88440deea30..849627e077b7 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java @@ -371,6 +371,9 @@ public void testAlterTableAddBaseProjection() public void testUnparseRoundTrip() { assertUnparseRoundTrips("CREATE TABLE \"tbl\" (\"a\" VARCHAR, \"b\" BIGINT)"); + // A table may be declared with no columns at all; the element list is omitted rather than printed empty, since + // the grammar has no way to read "()" back. + assertUnparseRoundTrips("CREATE TABLE \"tbl\" PARTITIONED BY DAY"); assertUnparseRoundTrips("CREATE OR REPLACE TABLE \"tbl\" (\"a\" VARCHAR)"); assertUnparseRoundTrips("CREATE TABLE IF NOT EXISTS \"tbl\" (\"a\" VARCHAR)"); assertUnparseRoundTrips("CREATE TABLE \"tbl\" (\"a\" VARCHAR) PARTITIONED BY DAY"); @@ -381,6 +384,15 @@ public void testUnparseRoundTrip() assertUnparseRoundTrips("ALTER TABLE \"tbl\" ALTER COLUMN \"a\" SET DATA TYPE BIGINT"); assertUnparseRoundTrips("ALTER TABLE \"tbl\" SET PROPERTIES (\"sealed\" = TRUE)"); assertUnparseRoundTrips("CREATE TABLE \"tbl\" (\"a\" VARCHAR) SEALED"); + assertUnparseRoundTrips( + "CREATE TABLE \"tbl\" (\"a\" VARCHAR, \"c\" BIGINT," + + " PROJECTION \"p\" AS (SELECT \"a\", SUM(\"c\") AS \"total\" GROUP BY \"a\"))" + ); + assertUnparseRoundTrips( + "CREATE TABLE \"tbl\" (\"a\" VARCHAR, \"__time\" TIMESTAMP," + + " PROJECTION \"__base\" AS (SELECT \"a\", \"__time\" CLUSTERED BY \"a\"))" + ); + assertUnparseRoundTrips("ALTER TABLE \"tbl\" DROP PROJECTION \"p\""); } /** From 19fc09a894ef25cfc161acaea4bf6b8cab744a3f Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 6 Aug 2026 21:12:57 -0700 Subject: [PATCH 07/18] fix --- .../src/main/java/org/apache/druid/catalog/http/TableEditor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java index 6784bb3e59c4..5fb16102507b 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java @@ -518,7 +518,6 @@ private List projectionsOf(TableMetadata table) } private TableSpec withProjections(TableMetadata table, List projections) - throws CatalogException { final TableSpec existingSpec = table.spec(); final Map revised = new HashMap<>(existingSpec.properties()); From 957ad33b8f55b3c13a373e0508970fb9b11a0fb5 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 11 Aug 2026 16:27:34 -0700 Subject: [PATCH 08/18] fixes --- .../druid/catalog/http/TableEditRequest.java | 42 ++++++++++ .../druid/catalog/http/TableEditor.java | 60 ++++++++++++++ .../catalog/sync/CatalogSqlTableWriter.java | 13 +++ .../druid/server/http/catalog/EditorTest.java | 82 +++++++++++++++++++ .../catalog/model/table/DatasourceDefn.java | 52 +++++++++++- .../model/table/DatasourceTableTest.java | 19 +++++ .../calcite/planner/CatalogDdlHandler.java | 45 +++------- .../calcite/planner/CatalogTableWriter.java | 25 ++++++ .../sql/calcite/CalciteCatalogDdlTest.java | 36 ++++++-- 9 files changed, 328 insertions(+), 46 deletions(-) diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java index fcf04757bb87..1f50559e3e34 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceBaseTableMetadata; import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import javax.annotation.Nullable; @@ -55,6 +56,8 @@ @Type(name = "moveColumn", value = TableEditRequest.MoveColumn.class), @Type(name = "addProjection", value = TableEditRequest.AddProjection.class), @Type(name = "dropProjection", value = TableEditRequest.DropProjection.class), + @Type(name = "setBaseTable", value = TableEditRequest.SetBaseTable.class), + @Type(name = "dropBaseTable", value = TableEditRequest.DropBaseTable.class), }) public class TableEditRequest { @@ -252,4 +255,43 @@ public DropProjection( } } + /** + * Set the table's base table layout, which is a property rather than one of the projections. Like + * {@link AddProjection}, the layout must not already be defined; the check runs inside the update transaction so + * that two callers cannot both find it absent and have the later one overwrite the earlier. + */ + public static class SetBaseTable extends TableEditRequest + { + @JsonProperty("baseTable") + public final DatasourceBaseTableMetadata baseTable; + @JsonProperty("ifNotExists") + public final boolean ifNotExists; + + @JsonCreator + public SetBaseTable( + @JsonProperty("baseTable") DatasourceBaseTableMetadata baseTable, + @JsonProperty("ifNotExists") boolean ifNotExists + ) + { + this.baseTable = baseTable; + this.ifNotExists = ifNotExists; + } + } + + /** + * Remove the table's base table layout. The mirror of {@link SetBaseTable}: the layout must already be defined, and + * that is likewise decided inside the update transaction. + */ + public static class DropBaseTable extends TableEditRequest + { + @JsonProperty("ifExists") + public final boolean ifExists; + + @JsonCreator + public DropBaseTable(@JsonProperty("ifExists") boolean ifExists) + { + this.ifExists = ifExists; + } + } + } diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java index 5fb16102507b..bbc25b2e4ceb 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java @@ -24,16 +24,19 @@ import org.apache.druid.catalog.http.TableEditRequest.AddColumns; import org.apache.druid.catalog.http.TableEditRequest.AddProjection; import org.apache.druid.catalog.http.TableEditRequest.AlterColumns; +import org.apache.druid.catalog.http.TableEditRequest.DropBaseTable; import org.apache.druid.catalog.http.TableEditRequest.DropColumns; import org.apache.druid.catalog.http.TableEditRequest.DropProjection; import org.apache.druid.catalog.http.TableEditRequest.HideColumns; import org.apache.druid.catalog.http.TableEditRequest.MoveColumn; import org.apache.druid.catalog.http.TableEditRequest.MoveColumn.Position; +import org.apache.druid.catalog.http.TableEditRequest.SetBaseTable; import org.apache.druid.catalog.http.TableEditRequest.UnhideColumns; import org.apache.druid.catalog.http.TableEditRequest.UpdateColumns; import org.apache.druid.catalog.http.TableEditRequest.UpdateProperties; import org.apache.druid.catalog.model.CatalogUtils; import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceBaseTableMetadata; import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import org.apache.druid.catalog.model.ResolvedTable; import org.apache.druid.catalog.model.TableDefn; @@ -115,6 +118,11 @@ public long go() throws CatalogException } else if (editRequest instanceof DropProjection) { final DropProjection dropProjection = (DropProjection) editRequest; return dropProjection(dropProjection.projection, dropProjection.ifExists); + } else if (editRequest instanceof SetBaseTable) { + final SetBaseTable setBaseTable = (SetBaseTable) editRequest; + return setBaseTable(setBaseTable.baseTable, setBaseTable.ifNotExists); + } else if (editRequest instanceof DropBaseTable) { + return dropBaseTable(((DropBaseTable) editRequest).ifExists); } else if (editRequest instanceof MoveColumn) { return moveColumn(((MoveColumn) editRequest)); } else { @@ -509,6 +517,58 @@ private long dropProjection(String name, boolean ifExists) throws CatalogExcepti }); } + /** + * Sets the base table layout, which lives in a table property rather than in the projections list. Mirrors + * {@link #addProjection}: the layout must not already be defined, and that is decided here, inside the update + * transaction, so two callers cannot both find it absent and have the later write overwrite the earlier. + */ + private long setBaseTable(DatasourceBaseTableMetadata baseTable, boolean ifNotExists) throws CatalogException + { + if (baseTable == null) { + throw CatalogException.badRequest("A base table layout is required"); + } + return catalog.tables().updateProperties(id, table -> { + if (table.spec().properties().get(DatasourceDefn.BASE_TABLE_PROPERTY) != null) { + if (ifNotExists) { + return null; + } + throw CatalogException.badRequest( + "Table %s already has a base table layout; drop it before defining another", + id.sqlName() + ); + } + return validated(withBaseTable(table, baseTable)); + }); + } + + /** + * Removes the base table layout. The mirror of {@link #setBaseTable}, checked in the same transaction. + */ + private long dropBaseTable(boolean ifExists) throws CatalogException + { + return catalog.tables().updateProperties(id, table -> { + if (table.spec().properties().get(DatasourceDefn.BASE_TABLE_PROPERTY) == null) { + if (ifExists) { + return null; + } + throw CatalogException.badRequest("Table %s does not have a base table layout", id.sqlName()); + } + return validated(withBaseTable(table, null)); + }); + } + + private TableSpec withBaseTable(TableMetadata table, @Nullable DatasourceBaseTableMetadata baseTable) + { + final TableSpec existingSpec = table.spec(); + final Map revised = new HashMap<>(existingSpec.properties()); + if (baseTable == null) { + revised.remove(DatasourceDefn.BASE_TABLE_PROPERTY); + } else { + revised.put(DatasourceDefn.BASE_TABLE_PROPERTY, baseTable); + } + return existingSpec.withProperties(revised); + } + private List projectionsOf(TableMetadata table) { final ResolvedTable resolved = catalog.tableRegistry().resolve(table.spec()); diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogSqlTableWriter.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogSqlTableWriter.java index 5874bff16fdb..b68950379b30 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogSqlTableWriter.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogSqlTableWriter.java @@ -24,6 +24,7 @@ import org.apache.druid.catalog.CatalogException; import org.apache.druid.catalog.http.TableEditRequest; import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceBaseTableMetadata; import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import org.apache.druid.catalog.model.TableId; import org.apache.druid.catalog.model.TableMetadata; @@ -106,6 +107,18 @@ public void dropProjection(TableId tableId, String projectionName, boolean ifExi execute(tableId, () -> client.editTable(tableId, new TableEditRequest.DropProjection(projectionName, ifExists))); } + @Override + public void setBaseTable(TableId tableId, DatasourceBaseTableMetadata baseTable, boolean ifNotExists) + { + execute(tableId, () -> client.editTable(tableId, new TableEditRequest.SetBaseTable(baseTable, ifNotExists))); + } + + @Override + public void dropBaseTable(TableId tableId, boolean ifExists) + { + execute(tableId, () -> client.editTable(tableId, new TableEditRequest.DropBaseTable(ifExists))); + } + @Nullable @Override public TableMetadata readTable(TableId tableId) diff --git a/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java b/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java index 72bc77e7bf76..a6c60383d50b 100644 --- a/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java +++ b/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java @@ -667,6 +667,88 @@ public void testUpdatePropertiesValidatedAgainstProjections() throws CatalogExce ); } + /** + * The base table layout lives in a property rather than in the projections list, but it follows the same rules as + * {@code AddProjection} / {@code DropProjection}, decided inside the update transaction rather than by the caller. + */ + @Test + public void testSetAndDropBaseTable() throws CatalogException + { + final String tableName = "baseTable"; + final TableMetadata table = TableBuilder.datasource(tableName, "P1D") + .column("tenant", "VARCHAR") + .timeColumn() + .sealed(true) + .build(); + catalog.tables().create(table); + + final ClusteredValueGroupsBaseTableMetadata layout = + new ClusteredValueGroupsBaseTableMetadata(ImmutableList.of("tenant"), null, null); + + assertTrue(new TableEditor(catalog, table.id(), new TableEditRequest.SetBaseTable(layout, false)).go() > 0); + assertEquals( + layout, + catalog.tableRegistry() + .resolve(catalog.tables().read(table.id()).spec()) + .decodeProperty(DatasourceDefn.BASE_TABLE_PROPERTY) + ); + + // Defining a second layout is an error unless the caller said to leave the existing one alone. + final CatalogException e = assertThrows( + CatalogException.class, + () -> doEdit(tableName, new TableEditRequest.SetBaseTable(layout, false)) + ); + assertTrue(e.getMessage().contains("already has a base table layout"), e.getMessage()); + assertEquals(0, new TableEditor(catalog, table.id(), new TableEditRequest.SetBaseTable(layout, true)).go()); + + assertTrue(new TableEditor(catalog, table.id(), new TableEditRequest.DropBaseTable(false)).go() > 0); + assertNull( + catalog.tables().read(table.id()).spec().properties().get(DatasourceDefn.BASE_TABLE_PROPERTY) + ); + + // Dropping one that is not there is likewise an error unless tolerated. + assertThrows( + CatalogException.class, + () -> doEdit(tableName, new TableEditRequest.DropBaseTable(false)) + ); + assertEquals(0, new TableEditor(catalog, table.id(), new TableEditRequest.DropBaseTable(true)).go()); + } + + /** + * A projection records the type of each column it groups by, so retyping such a column would leave a projection the + * table can no longer build. The whole-spec validation catches it before the edit is committed. + */ + @Test + public void testAlterColumnsValidatedAgainstProjections() throws CatalogException + { + final String tableName = "retype"; + final TableMetadata table = TableBuilder.datasource(tableName, "P1D") + .timeColumn() + .column("dim", "VARCHAR") + .column("met", "BIGINT") + .build(); + catalog.tables().create(table); + + final DatasourceProjectionMetadata byDim = new DatasourceProjectionMetadata( + AggregateProjectionSpec.builder("by_dim") + .groupingColumns(new StringDimensionSchema("dim")) + .aggregators(new LongSumAggregatorFactory("sum_met", "met")) + .build() + ); + assertTrue(new TableEditor(catalog, table.id(), new AddProjection(byDim, false)).go() > 0); + + final CatalogException e = assertThrows( + CatalogException.class, + () -> doEdit( + tableName, + new TableEditRequest.AlterColumns(Collections.singletonList(new ColumnSpec("dim", "BIGINT", null))) + ) + ); + assertTrue(e.getMessage().contains("groups on column [dim]"), e.getMessage()); + // The rejected edit left the column as it was. + assertEquals("VARCHAR", columnType(tableName, "dim")); + } + /** * The mirror case: a column edit is validated against the properties it does not touch. A base table layout names * the columns it clusters on, so dropping one leaves a layout that can no longer be derived. diff --git a/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java b/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java index ae1b7a75f6e6..6e574d85a44b 100644 --- a/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java +++ b/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.druid.catalog.model.CatalogUtils; +import org.apache.druid.catalog.model.ColumnSpec; import org.apache.druid.catalog.model.Columns; import org.apache.druid.catalog.model.DatasourceBaseTableMetadata; import org.apache.druid.catalog.model.DatasourceProjectionMetadata; @@ -38,12 +39,15 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.VirtualColumn; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.indexing.DataSchema; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; public class DatasourceDefn extends TableDefn @@ -136,10 +140,11 @@ public void validate(ResolvedTable table) } /** - * Cross-validate the declared projections. Names must be unique, and a projection must not be coarser than the - * segments it lives in. For a sealed table the declared columns are the whole schema, so a projection that reads a - * column the table does not declare can never be built and is rejected; for a non-sealed table ingestion may add - * columns the catalog has not seen, so only the projections' internal consistency is checked. + * Cross-validate the declared projections. Names must be unique, a projection must not be coarser than the segments + * it lives in, and the types it groups by must agree with the types the table declares. For a sealed table the + * declared columns are the whole schema, so a projection that reads a column the table does not declare can never be + * built and is rejected; for a non-sealed table ingestion may add columns the catalog has not seen, so only the + * projections' internal consistency is checked. */ private void validateProjections(ResolvedTable table) { @@ -162,6 +167,8 @@ private void validateProjections(ResolvedTable table) granularity == null ? null : CatalogUtils.asDruidGranularity(granularity) ); + validateProjectionGroupingTypes(table, specs); + if (!table.booleanProperty(SEALED_PROPERTY) || table.spec().columns() == null) { return; } @@ -185,6 +192,43 @@ private void validateProjections(ResolvedTable table) } } + /** + * Reconcile the types a projection groups by against the types the table declares. Grouping columns are the only + * part of a projection that shares column components with the base table, such as dictionaries, so they are the only + * part whose types have to agree with it. A grouping column that disagrees was built against a definition the table + * no longer has. + */ + private static void validateProjectionGroupingTypes(ResolvedTable table, List specs) + { + final List columns = table.spec().columns(); + if (columns == null) { + return; + } + final Map declaredTypes = new HashMap<>(); + for (ColumnSpec column : columns) { + final ColumnType type = Columns.druidType(column); + if (type != null) { + declaredTypes.put(column.name(), type); + } + } + for (AggregateProjectionSpec spec : specs) { + for (DimensionSchema grouping : spec.getGroupingColumns()) { + final ColumnType declared = declaredTypes.get(grouping.getName()); + if (declared != null && !declared.equals(grouping.getColumnType())) { + throw InvalidInput.exception( + "Projection [%s] groups on column [%s] as type [%s], but the table declares it as type [%s]. A" + + " projection is built from the table's columns, so changing the type of a column requires redefining" + + " the projections that group on it", + spec.getName(), + grouping.getName(), + grouping.getColumnType(), + declared + ); + } + } + } + } + private static Set requiredColumns(AggregateProjectionSpec spec) { final Set required = new HashSet<>(); diff --git a/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java b/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java index 3d409723785a..23ee29ba39ae 100644 --- a/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java +++ b/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java @@ -553,6 +553,25 @@ public void testProjectionValidation() ) .buildSpec() ).validate(); + + // A projection groups by the type the column had when it was defined, so a table that now declares a different + // type for that column carries a projection it can no longer build. Not limited to sealed tables: an undeclared + // column has no type to disagree with, but a declared one does. + final TableSpec retypedGroupingColumn = + TableBuilder.datasource("foo", "P1D") + .timeColumn() + .column("dim", Columns.SQL_BIGINT) + .column("met", Columns.SQL_BIGINT) + .property(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, List.of(new DatasourceProjectionMetadata(good))) + .buildSpec(); + final DruidException retyped = assertThrows( + DruidException.class, + () -> projectionRegistry.resolve(retypedGroupingColumn).validate() + ); + assertTrue( + retyped.getMessage(), + retyped.getMessage().contains("groups on column [dim] as type [STRING], but the table declares it as type [LONG]") + ); } @Test diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java index aa922054f41a..115493f07a1e 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java @@ -577,23 +577,17 @@ protected void execute(CatalogTableWriter writer) if (BASE_PROJECTION_NAME.equals(projectionName)) { // The base table is a property of the table, not one of its projections, so it is set rather than appended. - if (existing.spec().properties().get(DatasourceDefn.BASE_TABLE_PROPERTY) != null) { - if (alterTable.isIfNotExists()) { - return; - } - throw InvalidSqlInput.exception( - "Table [%s] already has a [%s] projection; drop it before defining another", - tableId.name(), - BASE_PROJECTION_NAME - ); - } + // Whether one is already defined is decided by the Coordinator inside its update transaction; checking it + // from the read above would let two concurrent statements both find it absent. + // + // This read is still needed for the layout itself, which is derived from the declared columns. A concurrent + // column change would make it stale, but the Coordinator validates the resulting spec against the columns it + // commits against, so a stale layout is rejected rather than stored. requireSealed(Boolean.TRUE.equals(existing.spec().properties().get(DatasourceDefn.SEALED_PROPERTY))); - writer.updateProperties( + writer.setBaseTable( tableId, - Collections.singletonMap( - DatasourceDefn.BASE_TABLE_PROPERTY, - translateBaseTable(handlerContext, tableId.name(), columns, alterTable.getProjection()) - ) + translateBaseTable(handlerContext, tableId.name(), columns, alterTable.getProjection()), + alterTable.isIfNotExists() ); return; } @@ -640,24 +634,9 @@ protected void validateStatement() protected void execute(CatalogTableWriter writer) { if (BASE_PROJECTION_NAME.equals(projectionName)) { - // Removing the layout leaves the declared columns alone; only future segments are affected. - final TableMetadata existing = writer.readTable(tableId); - final boolean present = existing != null - && existing.spec().properties().get(DatasourceDefn.BASE_TABLE_PROPERTY) != null; - if (!present) { - if (alterTable.isIfExists()) { - return; - } - throw InvalidSqlInput.exception( - "Table [%s] does not have a [%s] projection", - tableId.name(), - BASE_PROJECTION_NAME - ); - } - writer.updateProperties( - tableId, - Collections.singletonMap(DatasourceDefn.BASE_TABLE_PROPERTY, null) - ); + // Removing the layout leaves the declared columns alone; only future segments are affected. Whether there is + // one to remove is decided inside the Coordinator's update transaction, as for adding it. + writer.dropBaseTable(tableId, alterTable.isIfExists()); return; } writer.dropProjection(tableId, projectionName, alterTable.isIfExists()); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java index b93516bb031a..0d10fc13a6af 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java @@ -20,6 +20,7 @@ package org.apache.druid.sql.calcite.planner; import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceBaseTableMetadata; import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import org.apache.druid.catalog.model.TableId; import org.apache.druid.catalog.model.TableMetadata; @@ -88,6 +89,18 @@ public interface CatalogTableWriter */ void dropProjection(TableId tableId, String projectionName, boolean ifExists); + /** + * Set the table's base table layout, which is a property rather than one of the projections. Fails if the table + * already declares one, unless {@code ifNotExists}. Decided inside the Coordinator's update transaction, as for + * {@link #addProjection}. + */ + void setBaseTable(TableId tableId, DatasourceBaseTableMetadata baseTable, boolean ifNotExists); + + /** + * Remove the table's base table layout, failing if it has none unless {@code ifExists}. See {@link #setBaseTable}. + */ + void dropBaseTable(TableId tableId, boolean ifExists); + /** * Read a table's current metadata directly from the Coordinator, bypassing any local cache, or null if the table * has no catalog entry. Used for pre-checks that must not race against a stale cache. @@ -143,6 +156,18 @@ public void dropProjection(TableId tableId, String projectionName, boolean ifExi throw notAvailable(); } + @Override + public void setBaseTable(TableId tableId, DatasourceBaseTableMetadata baseTable, boolean ifNotExists) + { + throw notAvailable(); + } + + @Override + public void dropBaseTable(TableId tableId, boolean ifExists) + { + throw notAvailable(); + } + @Nullable @Override public TableMetadata readTable(TableId tableId) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java index d3a00f2e903b..e72db24a8d48 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.catalog.model.ClusteredValueGroupsBaseTableMetadata; import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceBaseTableMetadata; import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import org.apache.druid.catalog.model.TableId; import org.apache.druid.catalog.model.TableMetadata; @@ -767,8 +768,14 @@ public void testAlterTableAddBaseProjection() ); execute("ALTER TABLE tbl ADD PROJECTION __base AS (SELECT tenant, __time CLUSTERED BY tenant)"); - final RecordingCatalogTableWriter.Call call = WRITER.lastCall("updateProperties"); - assertNotNull(call.properties.get(DatasourceDefn.BASE_TABLE_PROPERTY)); + // Whether a layout already exists is the Coordinator's call, inside its update transaction, so the statement only + // hands over the translated layout and the IF NOT EXISTS flag. EditorTest covers the rule itself. + final RecordingCatalogTableWriter.Call call = WRITER.lastCall("setBaseTable"); + assertNotNull(call.baseTable); + assertFalse(call.ifNotExists); + + execute("ALTER TABLE tbl ADD IF NOT EXISTS PROJECTION __base AS (SELECT tenant, __time CLUSTERED BY tenant)"); + assertTrue(WRITER.lastCall("setBaseTable").ifNotExists); } @Test @@ -786,15 +793,11 @@ public void testAlterTableDropBaseProjection() ) ); execute("ALTER TABLE tbl DROP PROJECTION __base"); + assertFalse(WRITER.lastCall("dropBaseTable").ifExists); - final RecordingCatalogTableWriter.Call call = WRITER.lastCall("updateProperties"); - assertTrue(call.properties.containsKey(DatasourceDefn.BASE_TABLE_PROPERTY)); - assertNull(call.properties.get(DatasourceDefn.BASE_TABLE_PROPERTY)); - - // Dropping one that is not there is an error unless tolerated. - WRITER.reset(); - assertThrows(DruidException.class, () -> execute("ALTER TABLE tbl DROP PROJECTION __base")); + // Whether there is one to drop is likewise the Coordinator's call; the statement only carries IF EXISTS. execute("ALTER TABLE tbl DROP PROJECTION IF EXISTS __base"); + assertTrue(WRITER.lastCall("dropBaseTable").ifExists); } private ClusteredValueGroupsBaseTableMetadata baseTable() @@ -847,6 +850,7 @@ static class Call DatasourceProjectionMetadata projection; String projectionName; boolean ifExists; + DatasourceBaseTableMetadata baseTable; } final List calls = new ArrayList<>(); @@ -917,6 +921,20 @@ public void dropProjection(TableId tableId, String projectionName, boolean ifExi call.ifExists = ifExists; } + @Override + public void setBaseTable(TableId tableId, DatasourceBaseTableMetadata baseTable, boolean ifNotExists) + { + final Call call = record("setBaseTable", tableId); + call.baseTable = baseTable; + call.ifNotExists = ifNotExists; + } + + @Override + public void dropBaseTable(TableId tableId, boolean ifExists) + { + record("dropBaseTable", tableId).ifExists = ifExists; + } + @Nullable @Override public TableMetadata readTable(TableId tableId) From 68d530ecb4a248dfa85e5311811bcd90ade45166 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 13 Aug 2026 02:44:23 -0700 Subject: [PATCH 09/18] fix --- .../storage/sql/SQLCatalogManager.java | 188 +++++++++--------- .../catalog/storage/TableManagerTest.java | 46 +++++ 2 files changed, 138 insertions(+), 96 deletions(-) diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/storage/sql/SQLCatalogManager.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/storage/sql/SQLCatalogManager.java index 78aa1c7f78d0..5864c10db774 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/storage/sql/SQLCatalogManager.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/storage/sql/SQLCatalogManager.java @@ -51,11 +51,13 @@ import org.skife.jdbi.v2.exceptions.UnableToExecuteStatementException; import org.skife.jdbi.v2.tweak.HandleCallback; +import javax.ws.rs.core.Response; import java.io.IOException; import java.util.Deque; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.function.Function; @ManageLifecycle public class SQLCatalogManager implements CatalogManager @@ -333,7 +335,7 @@ public TableMetadata withHandle(Handle handle) throws NotFoundException } private static final String SELECT_PROPERTIES_STMT = - "SELECT tableType, properties, columns\n" + + "SELECT tableType, properties, columns, updateTime\n" + "FROM %s\n" + "WHERE schemaName = :schemaName\n" + " AND name = :name\n" + @@ -344,7 +346,8 @@ public TableMetadata withHandle(Handle handle) throws NotFoundException " properties = :properties,\n" + " updateTime = :updateTime\n" + "WHERE schemaName = :schemaName\n" + - " AND name = :name\n"; + " AND name = :name\n" + + " AND updateTime = :oldVersion"; @Override public long updateProperties( @@ -352,83 +355,22 @@ public long updateProperties( final TableTransform transform ) throws CatalogException { - try { - final TableMetadata result = dbi.withHandle( - new HandleCallback<>() - { - @Override - public TableMetadata withHandle(Handle handle) throws CatalogException - { - handle.begin(); - try { - final Query> query = handle - .createQuery(statement(SELECT_PROPERTIES_STMT)) - .setFetchSize(connector.getStreamingFetchSize()) - .bind(SCHEMA_NAME_COL, id.schema()) - .bind(TABLE_NAME_COL, id.name()); - - final ResultIterator resultIterator = query - .map((index, r, ctx) -> - tableSpecFromBytes( - jsonMapper, - r.getString(1), - r.getBytes(2), - r.getBytes(3) - ) - ) - .iterator(); - final TableSpec tableSpec; - if (resultIterator.hasNext()) { - tableSpec = resultIterator.next(); - } else { - throw tableNotFound(id); - } - final TableSpec revised = transform.apply(TableMetadata.of(id, tableSpec)); - if (revised == null) { - handle.rollback(); - return null; - } - final long updateTime = System.currentTimeMillis(); - final int updateCount = handle - .createStatement(statement(UPDATE_PROPERTIES_STMT)) - .bind(SCHEMA_NAME_COL, id.schema()) - .bind(TABLE_NAME_COL, id.name()) - .bind(PROPERTIES_COL, JacksonUtils.toBytes(jsonMapper, revised.properties())) - .bind(UPDATE_TIME_COL, updateTime) - .execute(); - if (updateCount == 0) { - // Should never occur because we're holding a lock. - throw new ISE("Table %s: not found", id.sqlName()); - } - handle.commit(); - return TableMetadata.forUpdate(id, updateTime, revised); - } - catch (Exception e) { - handle.rollback(); - throw e; - } - } - } - ); - if (result == null) { - return 0; - } - sendUpdate(EventType.PROPERTY_UPDATE, result); - return result.updateTime(); - } - catch (CallbackFailedException e) { - if (e.getCause() instanceof CatalogException) { - throw (CatalogException) e.getCause(); - } - throw e; - } + return updateSpec( + id, + SELECT_PROPERTIES_STMT, + UPDATE_PROPERTIES_STMT, + PROPERTIES_COL, + TableSpec::properties, + EventType.PROPERTY_UPDATE, + transform + ); } /** * Loads the whole spec, not just the columns this transaction writes back. See {@link #SELECT_PROPERTIES_STMT}. */ private static final String SELECT_COLUMNS_STMT = - "SELECT tableType, properties, columns\n" + + "SELECT tableType, properties, columns, updateTime\n" + "FROM %s\n" + "WHERE schemaName = :schemaName\n" + " AND name = :name\n" + @@ -439,13 +381,51 @@ public TableMetadata withHandle(Handle handle) throws CatalogException " columns = :columns,\n" + " updateTime = :updateTime\n" + "WHERE schemaName = :schemaName\n" + - " AND name = :name\n"; + " AND name = :name\n" + + " AND updateTime = :oldVersion"; @Override public long updateColumns( final TableId id, final TableTransform transform ) throws CatalogException + { + return updateSpec( + id, + SELECT_COLUMNS_STMT, + UPDATE_COLUMNS_STMT, + COLUMNS_COL, + TableSpec::columns, + EventType.COLUMNS_UPDATE, + transform + ); + } + + /** + * Read-modify-write of one half of a table spec: the whole spec is read, the transform revises it, and only the blob + * this operation owns is written back, so that an edit to the properties cannot clobber a concurrent edit to the + * columns. + *

+ * The write carries the {@code updateTime} the read saw, so it applies only if nothing else has committed in + * between. That is what makes a transform's decisions binding: a check it makes against the spec it read, such as a + * projection not already existing, still holds at the moment the row changes. Opening a transaction is not enough + * on its own, since these run at the connection's default isolation, where two writers can read the same row and + * both commit. + *

+ * A write that does not apply fails rather than being retried here. Retrying would re-run the transform against the + * newer spec, but only the transform: an edit whose payload was derived from the spec the caller read, such as a + * projection planned against its columns, would be re-applied as it was. Reporting the conflict sends the caller + * back through the whole operation, which is what re-derives the edit from the state it will actually land on. + */ + private long updateSpec( + final TableId id, + final String selectStmt, + final String updateStmt, + final String blobColumn, + final Function blob, + final EventType eventType, + final TableTransform transform + ) throws CatalogException { try { final TableMetadata result = dbi.withHandle( @@ -456,44 +436,44 @@ public TableMetadata withHandle(Handle handle) throws CatalogException { handle.begin(); try { - final Query> query = handle - .createQuery(statement(SELECT_COLUMNS_STMT)) + final ResultIterator resultIterator = handle + .createQuery(statement(selectStmt)) .setFetchSize(connector.getStreamingFetchSize()) .bind(SCHEMA_NAME_COL, id.schema()) - .bind(TABLE_NAME_COL, id.name()); - - final ResultIterator resultIterator = query - .map((index, r, ctx) -> - tableSpecFromBytes( - jsonMapper, - r.getString(1), - r.getBytes(2), - r.getBytes(3) - ) - ) - .iterator(); - final TableSpec tableSpec; + .bind(TABLE_NAME_COL, id.name()) + .map((index, r, ctx) -> + TableMetadata.forUpdate( + id, + r.getLong(4), + tableSpecFromBytes(jsonMapper, r.getString(1), r.getBytes(2), r.getBytes(3)) + ) + ) + .iterator(); + final TableMetadata existing; if (resultIterator.hasNext()) { - tableSpec = resultIterator.next(); + existing = resultIterator.next(); } else { throw tableNotFound(id); } - final TableSpec revised = transform.apply(TableMetadata.of(id, tableSpec)); + final TableSpec revised = transform.apply(existing); if (revised == null) { handle.rollback(); return null; } final long updateTime = System.currentTimeMillis(); final int updateCount = handle - .createStatement(statement(UPDATE_COLUMNS_STMT)) + .createStatement(statement(updateStmt)) .bind(SCHEMA_NAME_COL, id.schema()) .bind(TABLE_NAME_COL, id.name()) - .bind(COLUMNS_COL, JacksonUtils.toBytes(jsonMapper, revised.columns())) + .bind(blobColumn, JacksonUtils.toBytes(jsonMapper, blob.apply(revised))) .bind(UPDATE_TIME_COL, updateTime) + .bind(OLD_VERSION_PARAM, existing.updateTime()) .execute(); if (updateCount == 0) { - // Should never occur because we're holding a lock. - throw new ISE("Table %s: not found", id.sqlName()); + // The row was updated or removed since it was read, so this revision was computed against a spec + // that is no longer current. Report the conflict rather than overwrite whatever landed in between. + handle.rollback(); + throw new StaleUpdateException(); } handle.commit(); return TableMetadata.forUpdate(id, updateTime, revised); @@ -508,10 +488,18 @@ public TableMetadata withHandle(Handle handle) throws CatalogException if (result == null) { return 0; } - sendUpdate(EventType.COLUMNS_UPDATE, result); + sendUpdate(eventType, result); return result.updateTime(); } catch (CallbackFailedException e) { + if (e.getCause() instanceof StaleUpdateException) { + throw new CatalogException( + CatalogException.BAD_STATE, + Response.Status.CONFLICT, + "Table %s: was changed by another update; retry the operation against its current definition", + id.sqlName() + ); + } if (e.getCause() instanceof CatalogException) { throw (CatalogException) e.getCause(); } @@ -519,6 +507,14 @@ public TableMetadata withHandle(Handle handle) throws CatalogException } } + /** + * Signals that the compare-and-set predicate did not match, so the edit was computed against a spec that is no + * longer current. + */ + private static class StaleUpdateException extends RuntimeException + { + } + private static final String MARK_DELETING_STMT = "UPDATE %s\n SET\n" + " state = 'D',\n" + diff --git a/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java b/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java index 771d8afed69b..4db7e38cbf9e 100644 --- a/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java +++ b/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java @@ -51,8 +51,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -219,6 +221,50 @@ public void testUpdateProperties() throws CatalogException ); } + @Test + public void testUpdatePropertiesFailsWhenAnotherWriterCommits() throws CatalogException + { + final TableSpec spec = new TableSpec( + DatasourceDefn.TABLE_TYPE, + ImmutableMap.of(DatasourceDefn.SEGMENT_GRANULARITY_PROPERTY, "P1D"), + null + ); + final TableMetadata table = TableMetadata.newTable(TableId.datasource("racy"), spec); + manager.create(table); + + final AtomicInteger attempts = new AtomicInteger(); + final CatalogException e = assertThrows( + CatalogException.class, + () -> manager.updateProperties(table.id(), t -> { + if (attempts.getAndIncrement() == 0) { + // Another writer lands between this read and its write. + try { + manager.updateProperties(table.id(), other -> { + final Map updated = new HashMap<>(other.spec().properties()); + updated.put("other", "written"); + return other.spec().withProperties(updated); + }); + } + catch (CatalogException ce) { + throw new RuntimeException(ce); + } + } + final Map updated = new HashMap<>(t.spec().properties()); + updated.put("mine", "written"); + return t.spec().withProperties(updated); + }) + ); + assertTrue(e.getMessage().contains("was changed by another update"), e.getMessage()); + + // The transform ran once: the loser is reported rather than re-applied. + assertEquals(1, attempts.get()); + + // The writer that got there first is intact, and the loser wrote nothing. + final TableMetadata read = manager.read(table.id()); + assertEquals("written", read.spec().properties().get("other")); + assertNull(read.spec().properties().get("mine")); + } + @Test public void testUpdateColumns() throws CatalogException { From 7d6155ee9312c10376a399c27e08093bcdbb1daa Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 21 Aug 2026 20:50:28 -0700 Subject: [PATCH 10/18] fix test soft conflict --- .../catalog/model/table/DatasourceTableTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java b/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java index c58db610471e..7e0e35a5be45 100644 --- a/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java +++ b/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java @@ -513,7 +513,7 @@ public void testProjectionValidation() List.of(new DatasourceProjectionMetadata(good), new DatasourceProjectionMetadata(good)) ) .buildSpec(); - assertThrows(DruidException.class, () -> projectionRegistry.resolve(duplicateNames).validate()); + Assertions.assertThrows(DruidException.class, () -> projectionRegistry.resolve(duplicateNames).validate()); // A sealed table declares its whole schema, so a projection over an undeclared column can never be built. final AggregateProjectionSpec undeclared = AggregateProjectionSpec @@ -530,11 +530,11 @@ public void testProjectionValidation() List.of(new DatasourceProjectionMetadata(undeclared)) ) .buildSpec(); - final DruidException e = assertThrows( + final DruidException e = Assertions.assertThrows( DruidException.class, () -> projectionRegistry.resolve(sealedWithUndeclared).validate() ); - assertTrue(e.getMessage().contains("references column [nope]")); + Assertions.assertTrue(e.getMessage().contains("references column [nope]")); // Ingestion may add columns to a table that is not sealed, so the same projection is allowed there. projectionRegistry.resolve( @@ -556,13 +556,13 @@ public void testProjectionValidation() .column("met", Columns.SQL_BIGINT) .property(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, List.of(new DatasourceProjectionMetadata(good))) .buildSpec(); - final DruidException retyped = assertThrows( + final DruidException retyped = Assertions.assertThrows( DruidException.class, () -> projectionRegistry.resolve(retypedGroupingColumn).validate() ); - assertTrue( - retyped.getMessage(), - retyped.getMessage().contains("groups on column [dim] as type [STRING], but the table declares it as type [LONG]") + Assertions.assertTrue( + retyped.getMessage().contains("groups on column [dim] as type [STRING], but the table declares it as type [LONG]"), + retyped.getMessage() ); } From 3fd5d7bd5b59f901f30b1e1e911339646ae7874f Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 24 Aug 2026 02:17:38 -0700 Subject: [PATCH 11/18] edge case --- .../apache/druid/catalog/storage/sql/SQLCatalogManager.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/storage/sql/SQLCatalogManager.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/storage/sql/SQLCatalogManager.java index 5864c10db774..3084d643024a 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/storage/sql/SQLCatalogManager.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/storage/sql/SQLCatalogManager.java @@ -460,7 +460,10 @@ public TableMetadata withHandle(Handle handle) throws CatalogException handle.rollback(); return null; } - final long updateTime = System.currentTimeMillis(); + // The version is also the compare-and-set token, so it must actually change on every write: a commit + // landing in the same millisecond as the write that produced the version it read would otherwise + // leave the token as it was, letting a second writer's predicate match after this one commits. + final long updateTime = Math.max(System.currentTimeMillis(), existing.updateTime() + 1); final int updateCount = handle .createStatement(statement(updateStmt)) .bind(SCHEMA_NAME_COL, id.schema()) From 3e8505cd97607290998a38c668b786e564708cb8 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 25 Aug 2026 00:59:23 -0700 Subject: [PATCH 12/18] more test, adjustments --- .../catalog/CatalogDdlAndIngestTest.java | 66 ++++++++++++++++--- .../druid/server/http/catalog/EditorTest.java | 20 +++--- .../apache/druid/catalog/model/Columns.java | 2 +- .../sql/calcite/CalciteCatalogDdlTest.java | 25 +++---- 4 files changed, 81 insertions(+), 32 deletions(-) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java index 1e51490c8d06..d6b02af167ed 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java @@ -333,15 +333,6 @@ public void testAlterTableAddAndDropProjection() cluster.callApi().runSql("ALTER TABLE \"%s\" DROP PROJECTION IF EXISTS p", tableName); } - private List projectionsOf(String tableName) - { - return TestHelper.JSON_MAPPER.convertValue( - client.readTable(TableId.datasource(tableName)) - .spec().properties().get(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY), - new TypeReference>() {} - ); - } - /** * A clustered table defined entirely in SQL. The declared column order is the physical segment order, so rows come * back grouped by the clustering column, and a computed column is materialized at ingest time from the expression @@ -398,6 +389,54 @@ public void testCreateClusteredBaseTableThenIngestAndQuery() ); } + /** + * The same layout without SEALED: a column the query produces but the table does not declare is stored after the + * declared layout rather than rejected or dropped, and the clustering the table declared is unchanged. + */ + @Test + public void testCreateNonSealedClusteredBaseTableThenIngestExtraColumn() + { + final String tableName = dataSource; + + cluster.callApi().runSql( + "CREATE TABLE \"%s\" (\n" + + " tenant VARCHAR,\n" + + " __time TIMESTAMP,\n" + + " v BIGINT,\n" + + " PROJECTION __base AS (SELECT tenant, __time, v CLUSTERED BY tenant)\n" + + ")\n" + + "PARTITIONED BY DAY", + tableName + ); + + final TableMetadata table = client.readTable(TableId.datasource(tableName)); + assertNull(table.spec().properties().get(DatasourceDefn.SEALED_PROPERTY)); + assertNotNull(table.spec().properties().get(DatasourceDefn.BASE_TABLE_PROPERTY)); + + // 'extra_col' is not declared by the table. + ingest( + "INSERT INTO \"%s\"\n" + + "SELECT TIME_PARSE(a) AS __time, b AS tenant, c AS v, d AS extra_col\n" + + "FROM TABLE(\n" + + " EXTERN(\n" + + " '{\"type\":\"inline\",\"data\":\"2022-12-26T12:34:56,bbb,1,x" + + "\\n2022-12-26T12:34:56,aaa,2,y\\n2022-12-26T12:34:56,aaa,3,z\"}',\n" + + " '{\"type\":\"csv\",\"findColumnsFromHeader\":false,\"columns\":[\"a\",\"b\",\"c\",\"d\"]}'\n" + + " )\n" + + ") EXTEND (a VARCHAR, b VARCHAR, c BIGINT, d VARCHAR)\n", + tableName + ); + + // Declared columns first, the appended extra last; rows still grouped by the clustering column. + cluster.callApi().verifySqlQuery( + "SELECT * FROM %s", + tableName, + "aaa,2022-12-26T12:34:56.000Z,2,y\n" + + "aaa,2022-12-26T12:34:56.000Z,3,z\n" + + "bbb,2022-12-26T12:34:56.000Z,1,x" + ); + } + /** * A column the base table computes cannot be written directly: the catalog rejects the INSERT and says to supply * the expression's inputs instead. @@ -451,6 +490,15 @@ public void testAlterTableAddAndDropBaseProjection() ); } + private List projectionsOf(String tableName) + { + return TestHelper.JSON_MAPPER.convertValue( + client.readTable(TableId.datasource(tableName)) + .spec().properties().get(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY), + new TypeReference>() {} + ); + } + private void ingest(String sqlPattern, String tableName) { final SqlTaskStatus status = msqApis.submitTaskSql(StringUtils.format(sqlPattern, tableName)); diff --git a/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java b/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java index a6c60383d50b..5b95751c049f 100644 --- a/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java +++ b/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java @@ -610,15 +610,6 @@ public void testAlterColumns() throws CatalogException ); } - private String columnType(String tableName, String columnName) throws CatalogException - { - return catalog.tables().read(TableId.datasource(tableName)).spec().columns().stream() - .filter(c -> columnName.equals(c.name())) - .findFirst() - .orElseThrow(() -> new AssertionError("No column [" + columnName + "]")) - .dataType(); - } - /** * A property edit is validated against the parts of the spec it does not touch. Segment granularity and the declared * projections are only meaningful together, so coarsening the segments under a projection has to be caught here @@ -780,11 +771,20 @@ public void testDropColumnsValidatedAgainstBaseTable() throws CatalogException assertTrue(new TableEditor(catalog, table.id(), new DropColumns(ImmutableList.of("met"))).go() > 0); } + + private String columnType(String tableName, String columnName) throws CatalogException + { + return catalog.tables().read(TableId.datasource(tableName)).spec().columns().stream() + .filter(c -> columnName.equals(c.name())) + .findFirst() + .orElseThrow(() -> new AssertionError("No column [" + columnName + "]")) + .dataType(); + } + private List projectionsOf(String tableName) throws CatalogException { return catalog.tableRegistry() .resolve(catalog.tables().read(TableId.datasource(tableName)).spec()) .decodeProperty(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY); } - } diff --git a/server/src/main/java/org/apache/druid/catalog/model/Columns.java b/server/src/main/java/org/apache/druid/catalog/model/Columns.java index 9be0d4ab3122..c9081bc91e38 100644 --- a/server/src/main/java/org/apache/druid/catalog/model/Columns.java +++ b/server/src/main/java/org/apache/druid/catalog/model/Columns.java @@ -78,7 +78,7 @@ private Columns() } /** - * The Druid type of a column, which for {@code __time} is always {@link ColumnType#LONG} whatever was declared. + * The Druid type of a column ({@code __time} is always {@link ColumnType#LONG} regardless of what was declared). * Use {@link #druidTypeFromString} to resolve a declared type on its own terms. */ public static ColumnType druidType(ColumnSpec spec) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java index 26b001847c7f..b347aa019984 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java @@ -562,18 +562,6 @@ public void testProjectionRejectsDuplicateName() assertTrue(e.getMessage().contains("declared more than once"), e.getMessage()); } - @SuppressWarnings("unchecked") - private DatasourceProjectionMetadata projection(int index) - { - return ((List) WRITER.calls.get(0).spec.properties().get("projections")).get(index); - } - - private String projectionsJson() throws Exception - { - return queryFramework().queryJsonMapper() - .writeValueAsString(WRITER.calls.get(0).spec.properties().get("projections")); - } - /** * The reserved {@code __base} projection describes the table's own layout, so it becomes the baseTable property * rather than one of the projections. A computed column becomes a virtual column materializing the declared column @@ -804,6 +792,19 @@ public void testAlterTableDropBaseProjection() assertTrue(WRITER.lastCall("dropBaseTable").ifExists); } + + @SuppressWarnings("unchecked") + private DatasourceProjectionMetadata projection(int index) + { + return ((List) WRITER.calls.get(0).spec.properties().get("projections")).get(index); + } + + private String projectionsJson() throws Exception + { + return queryFramework().queryJsonMapper() + .writeValueAsString(WRITER.calls.get(0).spec.properties().get("projections")); + } + private ClusteredValueGroupsBaseTableMetadata baseTable() { return (ClusteredValueGroupsBaseTableMetadata) From 1390ecb84323eca4d436127d80b032b64cd6add2 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 4 Sep 2026 10:23:00 -0700 Subject: [PATCH 13/18] fixes --- docs/configuration/index.md | 2 +- docs/development/extensions-core/catalog.md | 50 +++++-- .../druid/catalog/http/TableEditor.java | 53 ++----- .../storage/sql/SQLCatalogManager.java | 24 ++- .../catalog/sync/CachedMetadataCatalog.java | 41 ++++++ .../catalog/sync/CatalogSqlTableWriter.java | 10 +- .../druid/catalog/sync/CatalogCacheTest.java | 42 ++++++ .../dart/controller/sql/DartSqlEngine.java | 1 + .../druid/msq/sql/MSQTaskSqlEngine.java | 1 + .../calcite/planner/CatalogDdlHandler.java | 28 +++- .../calcite/planner/ProjectionSqlEngine.java | 1 + .../druid/sql/calcite/run/EngineFeature.java | 7 + .../sql/calcite/run/NativeSqlEngine.java | 1 + .../druid/sql/calcite/view/ViewSqlEngine.java | 1 + .../sql/calcite/CalciteCatalogDdlTest.java | 138 +++++++++++++++++- .../sql/calcite/CalciteInsertDmlTest.java | 22 +++ .../sql/calcite/IngestionTestSqlEngine.java | 1 + .../calcite/parser/DruidSqlDdlParserTest.java | 24 +-- 18 files changed, 363 insertions(+), 84 deletions(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 409938e0f96e..36afbe4e7e4e 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -1900,7 +1900,7 @@ The Druid SQL server is configured through the following properties on the Broke |`druid.sql.planner.metadataSegmentPollPeriod`|How often to poll coordinator for published segments list if `druid.sql.planner.metadataSegmentCacheEnable` is set to true. Poll period is in milliseconds. |60000| |`druid.sql.planner.authorizeSystemTablesDirectly`|If true, Druid authorizes queries against any of the system schema tables (`sys` in SQL) as `SYSTEM_TABLE` resources which require `READ` access, in addition to permissions based content filtering.|false| |`druid.sql.planner.authorizeTableVisibility`|Whether [READ DATASOURCE](../multi-stage-query/security.md) permissions are required for table visibility in the validator. When this is set, users that query unauthorized tables see a "not found" error rather than "forbidden". Additionally, when this is set, INSERT and REPLACE require both READ and WRITE access to the target table. (If this property is not set, they require only WRITE.) Regardless of the value of this property, READ access is required for tables to show up in the INFORMATION_SCHEMA.|true| -|`druid.sql.planner.enableCatalogDdl`|If true, `CREATE TABLE` and `ALTER TABLE` statements may be used to define [catalog](../development/extensions-core/catalog.md) tables. These statements require `WRITE` permission on the datasource, the same permission the catalog API requires, so enabling this lets anyone who can ingest into a datasource also change its catalog definition. Requires the `druid-catalog` extension. Cannot be overridden per query.|false| +|`druid.sql.planner.enableCatalogDdl`|If true, `CREATE TABLE` and `ALTER TABLE` statements may be used to define [catalog](../development/extensions-core/catalog.md) tables. These statements require both `READ` and `WRITE` permission on the datasource, the same permissions the catalog API requires for its write operations, so enabling this lets anyone who can ingest into a datasource also change its catalog definition. Requires the `druid-catalog` extension. Cannot be overridden per query.|false| |`druid.sql.planner.useNativeQueryExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan as a JSON representation of equivalent native query(s), else it will return the original version of explain plan generated by Calcite. It can be overridden per query with `useNativeQueryExplain` context key.|true| |`druid.sql.planner.maxNumericInFilters`|Max limit for the amount of numeric values that can be compared for a string type dimension when the entire SQL WHERE clause of a query translates to an [OR](../querying/filters.md#or) of [Bound filter](../querying/filters.md#bound-filter). By default, Druid does not restrict the amount of numeric Bound Filters on String columns, although this situation may block other queries from running. Set this property to a smaller value to prevent Druid from running queries that have prohibitively long segment processing times. The optimal limit requires some trial and error; we recommend starting with 100. Users who submit a query that exceeds the limit of `maxNumericInFilters` should instead rewrite their queries to use strings in the `WHERE` clause instead of numbers. For example, `WHERE someString IN (‘123’, ‘456’)`. If this value is disabled, `maxNumericInFilters` set through query context is ignored.|`-1` (disabled)| |`druid.sql.approxCountDistinct.function`|Implementation to use for the [`APPROX_COUNT_DISTINCT` function](../querying/sql-aggregations.md). Without extensions loaded, the only valid value is `APPROX_COUNT_DISTINCT_BUILTIN` (a HyperLogLog, or HLL, based implementation). If the [DataSketches extension](../development/extensions-core/datasketches-extension.md) is loaded, this can also be `APPROX_COUNT_DISTINCT_DS_HLL` (alternative HLL implementation) or `APPROX_COUNT_DISTINCT_DS_THETA`.

Theta sketches use significantly more memory than HLL sketches, so you should prefer one of the two HLL implementations.|`APPROX_COUNT_DISTINCT_BUILTIN`| diff --git a/docs/development/extensions-core/catalog.md b/docs/development/extensions-core/catalog.md index af7a01eb1631..123aefe6122c 100644 --- a/docs/development/extensions-core/catalog.md +++ b/docs/development/extensions-core/catalog.md @@ -43,6 +43,24 @@ allowing queries to be more concise, and simpler to write. This also allows the written into a defined column of the table is consistent with that columns definition, minimizing errors where unexpected data is written into a particular column of the table. +### Effects of a table definition + +A table definition takes effect when it is read, not when it is written: changing one, whether through SQL DDL or the +REST API, rewrites nothing. What it affects: + +- **SELECT queries** report each declared column with its declared type, in declared order. Segments that store a + column as a different physical type are converted to the declared type at query time. Columns present in segments + but not declared in the catalog remain queryable, and follow the declared columns with their physical types. +- **INSERT and REPLACE** (the catalog applies only to SQL-based ingestion) validate against the definition rather + than merely defaulting from it. A query column targeting a declared column must be convertible to the declared type + without changing it: inserting a `BIGINT` value into a column declared `VARCHAR` stores the value as a string, while + inserting a `VARCHAR` value into a column declared `BIGINT` is an error. Columns the query produces that the table + does not declare are rejected when the table is [`sealed`](#table-properties) and ingested normally otherwise. + Table properties such as `segmentGranularity` and `clusterKeys` act as defaults that an individual statement may + override, such as with its own `PARTITIONED BY`. +- **Streaming and native batch ingestion** do not consult the catalog; their specs are unaffected by any table + definition. + ### SQL DDL Tables can be defined with SQL instead of by posting a table specification. `CREATE TABLE` and `ALTER TABLE` are @@ -50,23 +68,28 @@ submitted to the Broker like any other SQL statement, and write the same catalog return no rows. These statements change catalog metadata only. They never create, modify, or delete segments: defining a table does -not ingest anything, and altering a column does not rewrite existing data. Column changes take effect for subsequent -ingestion. +not ingest anything, and altering a column does not rewrite existing data. Changes take effect for subsequent queries +and ingestion, as described in [Effects of a table definition](#effects-of-a-table-definition). These statements are disabled by default. Set `druid.sql.planner.enableCatalogDdl` to `true` on the Broker to enable -them. They require `WRITE` permission on the datasource, the same permission the catalog API requires, so enabling -them lets anyone who can ingest into a datasource also change its catalog definition; leave them disabled if you -manage catalog entries with your own tooling. The setting cannot be overridden per query. +them. They require both `READ` and `WRITE` permission on the datasource, the same permissions the catalog API +requires for its write operations, so enabling them lets anyone who can ingest into a datasource also change its +catalog definition; leave them disabled if you manage catalog entries with your own tooling. The setting cannot be +overridden per query. The `druid-catalog` extension must be loaded on both the Broker and the Coordinator; without it, these statements report that the extension is not available. ```sql CREATE [OR REPLACE] TABLE [IF NOT EXISTS]

- [ ( { | PROJECTION AS (
[, ...] ) ] [ PARTITIONED BY ] [ CLUSTERED BY [, ...] ] [ SEALED ] + +
::= + + | PROJECTION AS (
- [ (
[, ...] ) ] + [ [ SEALED ] (
[, ...] ) ] [ PARTITIONED BY ] [ CLUSTERED BY [, ...] ] - [ SEALED ]
::= @@ -95,7 +94,10 @@ CREATE [OR REPLACE] TABLE [IF NOT EXISTS]
`OR REPLACE` replaces the specification of an existing table; `IF NOT EXISTS` leaves an existing table unchanged. The two cannot be combined. `PARTITIONED BY` sets [`segmentGranularity`](#table-properties) and `CLUSTERED BY` sets `clusterKeys`, both of which a later `INSERT` or `REPLACE` inherits unless it states its own. `SEALED` sets -[`sealed`](#table-properties), which requires every ingested column to be declared. +[`sealed`](#table-properties), which requires every ingested column to be declared. It is written just before the +column list because that is what it describes: the list is the table's whole schema, so the statement does not accept +`SEALED` without one. (The `sealed` property itself can still be set on any table through `SET PROPERTIES` or the +REST API.) Note that the table-level `CLUSTERED BY` is a sort order applied to each ingestion, which is a different thing from the `CLUSTERED BY` inside a [`__base` projection](#the-base-table), which defines how segments physically group rows. @@ -197,7 +199,7 @@ Its body lists the columns in the order segments store them, so it must name eve order. An item written as ` AS ` makes that column computed at ingest time, from the columns it reads: ```sql -CREATE TABLE "druid"."events" ( +CREATE TABLE "druid"."events" SEALED ( tenant VARCHAR, bucket BIGINT, __time TIMESTAMP, @@ -209,7 +211,6 @@ CREATE TABLE "druid"."events" ( ) ) PARTITIONED BY DAY -SEALED ``` The clustering columns must be the leading columns of the table, because the declared order is the physical order. diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java index d6b02af167ed..ac83f09fc0b0 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java @@ -344,7 +344,7 @@ public void testCreateClusteredBaseTableThenIngestAndQuery() final String tableName = dataSource; cluster.callApi().runSql( - "CREATE TABLE \"%s\" (\n" + "CREATE TABLE \"%s\" SEALED (\n" + " varchar_col2 VARCHAR,\n" + " __time TIMESTAMP,\n" + " varchar_col1 VARCHAR,\n" @@ -355,7 +355,7 @@ public void testCreateClusteredBaseTableThenIngestAndQuery() + " CLUSTERED BY varchar_col2\n" + " )\n" + ")\n" - + "PARTITIONED BY DAY SEALED", + + "PARTITIONED BY DAY", tableName ); @@ -447,9 +447,9 @@ public void testInsertIntoComputedColumnIsRejected() final String tableName = dataSource; cluster.callApi().runSql( - "CREATE TABLE \"%s\" (t VARCHAR, __time TIMESTAMP, v BIGINT, doubled BIGINT," + "CREATE TABLE \"%s\" SEALED (t VARCHAR, __time TIMESTAMP, v BIGINT, doubled BIGINT," + " PROJECTION __base AS (SELECT t, __time, v, v * 2 AS doubled CLUSTERED BY t))" - + " PARTITIONED BY DAY SEALED", + + " PARTITIONED BY DAY", tableName ); @@ -471,7 +471,7 @@ public void testAlterTableAddAndDropBaseProjection() final String tableName = dataSource; cluster.callApi().runSql( - "CREATE TABLE \"%s\" (t VARCHAR, __time TIMESTAMP, v BIGINT) PARTITIONED BY DAY SEALED", + "CREATE TABLE \"%s\" SEALED (t VARCHAR, __time TIMESTAMP, v BIGINT) PARTITIONED BY DAY", tableName ); cluster.callApi().runSql( diff --git a/sql/src/main/codegen/includes/ddl.ftl b/sql/src/main/codegen/includes/ddl.ftl index b941f53060af..8dde5ae9e17e 100644 --- a/sql/src/main/codegen/includes/ddl.ftl +++ b/sql/src/main/codegen/includes/ddl.ftl @@ -40,6 +40,9 @@ SqlCreate DruidSqlCreateTable(Span s, boolean replace) : [ { ifNotExists = true; } ] id = CompoundTableIdentifier() [ + // SEALED binds to the column list rather than trailing the statement: it declares that the list is the table's + // whole schema, so it is only accepted when there is a list for it to describe. + [ { sealed = true; } ] { elementSpan = span(); } AddDruidTableElement(columns, projections) ( @@ -54,9 +57,6 @@ SqlCreate DruidSqlCreateTable(Span s, boolean replace) : [ clusteredBy = ClusteredBy() ] - [ - { sealed = true; } - ] { final SqlParserPos elementPos = elementSpan == null ? s.pos() : elementSpan.end(this); return new DruidSqlCreateTable( diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java index 38d79ea8e2da..cadab6cf6da9 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java @@ -155,8 +155,12 @@ public void unparse(SqlWriter writer, int leftPrec, int rightPrec) name.unparse(writer, leftPrec, rightPrec); // The element list is optional and, when present, must hold at least one element, so a table declaring nothing - // prints no parentheses at all: "()" is not something the grammar can read back. + // prints no parentheses at all: "()" is not something the grammar can read back. SEALED binds to the list (it + // declares the list to be the whole schema), so it prints just before it and never without it. if (!columnList.isEmpty() || !projectionList.isEmpty()) { + if (sealed) { + writer.keyword("SEALED"); + } final SqlWriter.Frame frame = writer.startList("(", ")"); for (SqlNode column : columnList) { writer.sep(","); @@ -183,10 +187,6 @@ public void unparse(SqlWriter writer, int leftPrec, int rightPrec) } writer.endList(clusterFrame); } - - if (sealed) { - writer.keyword("SEALED"); - } } private static class DruidSqlCreateTableOperator extends SqlSpecialOperator diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java index 2b50f79372b1..c5faa5707cb6 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java @@ -696,7 +696,7 @@ public void testProjectionRejectsDuplicateName() public void testCreateTableWithBaseProjection() throws Exception { execute( - "CREATE TABLE tbl (" + "CREATE TABLE tbl SEALED (" + " tenant VARCHAR," + " bucket BIGINT," + " __time TIMESTAMP," @@ -705,7 +705,7 @@ public void testCreateTableWithBaseProjection() throws Exception + " SELECT tenant, ABS(user_id) AS bucket, __time, user_id" + " CLUSTERED BY tenant, bucket" + " )" - + ") PARTITIONED BY DAY SEALED" + + ") PARTITIONED BY DAY" ); final TableSpec spec = WRITER.calls.get(0).spec; @@ -735,8 +735,8 @@ public void testBaseProjectionRenamedColumnFails() final DruidException e = assertThrows( DruidException.class, () -> execute( - "CREATE TABLE tbl (id BIGINT, copy BIGINT, __time TIMESTAMP," - + " PROJECTION __base AS (SELECT id, id AS copy, __time CLUSTERED BY id)) SEALED" + "CREATE TABLE tbl SEALED (id BIGINT, copy BIGINT, __time TIMESTAMP," + + " PROJECTION __base AS (SELECT id, id AS copy, __time CLUSTERED BY id))" ) ); assertTrue( @@ -749,8 +749,8 @@ public void testBaseProjectionRenamedColumnFails() public void testBaseProjectionWithoutComputedColumns() { execute( - "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT," - + " PROJECTION __base AS (SELECT tenant, __time, v CLUSTERED BY tenant)) SEALED" + "CREATE TABLE tbl SEALED (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + + " PROJECTION __base AS (SELECT tenant, __time, v CLUSTERED BY tenant))" ); final ClusteredValueGroupsBaseTableMetadata baseTable = baseTable(); assertEquals(List.of("tenant"), baseTable.getClusteringColumns()); @@ -764,9 +764,9 @@ public void testBaseProjectionWithoutComputedColumns() public void testBaseProjectionAlongsideAggregateProjection() { execute( - "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + "CREATE TABLE tbl SEALED (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + " PROJECTION __base AS (SELECT tenant, __time, v CLUSTERED BY tenant)," - + " PROJECTION by_tenant AS (SELECT tenant, SUM(v) AS sum_v GROUP BY tenant)) SEALED" + + " PROJECTION by_tenant AS (SELECT tenant, SUM(v) AS sum_v GROUP BY tenant))" ); final TableSpec spec = WRITER.calls.get(0).spec; assertNotNull(spec.properties().get(DatasourceDefn.BASE_TABLE_PROPERTY)); @@ -799,8 +799,8 @@ public void testBaseProjectionColumnOrderMustMatch() final DruidException wrongOrder = assertThrows( DruidException.class, () -> execute( - "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT," - + " PROJECTION __base AS (SELECT __time, tenant, v CLUSTERED BY tenant)) SEALED" + "CREATE TABLE tbl SEALED (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + + " PROJECTION __base AS (SELECT __time, tenant, v CLUSTERED BY tenant))" ) ); assertTrue(wrongOrder.getMessage().contains("the table declares"), wrongOrder.getMessage()); @@ -808,8 +808,8 @@ public void testBaseProjectionColumnOrderMustMatch() final DruidException missing = assertThrows( DruidException.class, () -> execute( - "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT," - + " PROJECTION __base AS (SELECT tenant, __time CLUSTERED BY tenant)) SEALED" + "CREATE TABLE tbl SEALED (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + + " PROJECTION __base AS (SELECT tenant, __time CLUSTERED BY tenant))" ) ); assertTrue(missing.getMessage().contains("must name every declared column"), missing.getMessage()); @@ -825,8 +825,8 @@ public void testBaseProjectionClusteringMustBeLeadingPrefix() final DruidException e = assertThrows( DruidException.class, () -> execute( - "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT," - + " PROJECTION __base AS (SELECT tenant, __time, v CLUSTERED BY v)) SEALED" + "CREATE TABLE tbl SEALED (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + + " PROJECTION __base AS (SELECT tenant, __time, v CLUSTERED BY v))" ) ); assertTrue(e.getMessage().contains("__base"), e.getMessage()); @@ -842,7 +842,7 @@ public void testBaseProjectionRejectsFilterOrGrouping() final DruidException e = assertThrows( DruidException.class, () -> execute( - "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, PROJECTION __base AS (" + body + ")) SEALED" + "CREATE TABLE tbl SEALED (tenant VARCHAR, __time TIMESTAMP, PROJECTION __base AS (" + body + "))" ), body ); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java index da3a49029d9c..9ae1d0186b0b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java @@ -320,7 +320,7 @@ public void testProjectionBodyRejectsUnsupportedClauses() public void testCreateTableWithBaseProjectionAndSealed() { final DruidSqlCreateTable create = parseCreate( - "CREATE TABLE t (\n" + "CREATE TABLE t SEALED (\n" + " tenant VARCHAR,\n" + " bucket BIGINT,\n" + " __time TIMESTAMP,\n" @@ -328,7 +328,7 @@ public void testCreateTableWithBaseProjectionAndSealed() + " SELECT tenant, ABS(user_id) AS bucket, __time\n" + " CLUSTERED BY tenant, bucket\n" + " )\n" - + ") PARTITIONED BY DAY SEALED" + + ") PARTITIONED BY DAY" ); assertTrue(create.isSealed()); @@ -343,10 +343,24 @@ public void testCreateTableWithBaseProjectionAndSealed() @Test public void testSealedWithoutProjection() { - assertTrue(parseCreate("CREATE TABLE t (a VARCHAR) SEALED").isSealed()); + assertTrue(parseCreate("CREATE TABLE t SEALED (a VARCHAR)").isSealed()); assertFalse(parseCreate("CREATE TABLE t (a VARCHAR)").isSealed()); } + /** + * SEALED binds to the column list, so a table declaring no columns cannot be sealed: the odd statement is + * unparseable rather than accepted. + */ + @Test + public void testSealedWithoutColumnListIsRejected() + { + final DruidException e = assertThrows( + DruidException.class, + () -> parse("CREATE TABLE tbl SEALED PARTITIONED BY DAY") + ); + assertTrue(e.getMessage().contains("PARTITIONED"), e.getMessage()); + } + /** * SEALED is a non-reserved keyword, so it remains usable as an identifier. */ @@ -354,7 +368,8 @@ public void testSealedWithoutProjection() public void testSealedUsableAsIdentifier() { assertEquals("sealed VARCHAR", columnsOf(parseCreate("CREATE TABLE t (sealed VARCHAR)"))); - assertTrue(parseCreate("CREATE TABLE sealed (a VARCHAR) SEALED").isSealed()); + // A table named 'sealed' followed by the SEALED keyword: the identifier and the keyword coexist. + assertTrue(parseCreate("CREATE TABLE sealed SEALED (a VARCHAR)").isSealed()); } @Test @@ -387,7 +402,7 @@ public void testUnparseRoundTrip() assertUnparseRoundTrips("ALTER TABLE \"tbl\" DROP COLUMN \"a\""); assertUnparseRoundTrips("ALTER TABLE \"tbl\" ALTER COLUMN \"a\" SET DATA TYPE BIGINT"); assertUnparseRoundTrips("ALTER TABLE \"tbl\" SET PROPERTIES (\"sealed\" = TRUE)"); - assertUnparseRoundTrips("CREATE TABLE \"tbl\" (\"a\" VARCHAR) SEALED"); + assertUnparseRoundTrips("CREATE TABLE \"tbl\" SEALED (\"a\" VARCHAR)"); assertUnparseRoundTrips( "CREATE TABLE \"tbl\" (\"a\" VARCHAR, \"c\" BIGINT," + " PROJECTION \"p\" AS (SELECT \"a\", SUM(\"c\") AS \"total\" GROUP BY \"a\"))" @@ -411,9 +426,9 @@ public void testCloneRoundTrip() "CREATE OR REPLACE TABLE \"tbl\" (\"a\" VARCHAR)", "CREATE TABLE IF NOT EXISTS \"tbl\" (\"a\" VARCHAR)", "CREATE TABLE \"tbl\" (\"a\" VARCHAR) PARTITIONED BY DAY CLUSTERED BY \"a\"", - "CREATE TABLE \"tbl\" (\"a\" VARCHAR) SEALED", + "CREATE TABLE \"tbl\" SEALED (\"a\" VARCHAR)", "CREATE TABLE \"tbl\" (\"a\" VARCHAR, PROJECTION \"p\" AS (SELECT \"a\" GROUP BY \"a\"))", - "CREATE TABLE \"tbl\" (\"a\" VARCHAR, PROJECTION \"__base\" AS (SELECT \"a\" CLUSTERED BY \"a\")) SEALED", + "CREATE TABLE \"tbl\" SEALED (\"a\" VARCHAR, PROJECTION \"__base\" AS (SELECT \"a\" CLUSTERED BY \"a\"))", "ALTER TABLE \"tbl\" ADD COLUMN \"a\" DOUBLE", "ALTER TABLE \"tbl\" DROP COLUMN \"a\"", "ALTER TABLE \"tbl\" ALTER COLUMN \"a\" SET DATA TYPE BIGINT", From c9eb48cdaaef83135f288826f42361d198c8ad5f Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 8 Sep 2026 15:44:02 -0700 Subject: [PATCH 16/18] fix lookahead to resolve ambiguity --- sql/src/main/codegen/includes/ddl.ftl | 8 ++++--- .../calcite/parser/DruidSqlDdlParserTest.java | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/sql/src/main/codegen/includes/ddl.ftl b/sql/src/main/codegen/includes/ddl.ftl index 8dde5ae9e17e..cda3686622bb 100644 --- a/sql/src/main/codegen/includes/ddl.ftl +++ b/sql/src/main/codegen/includes/ddl.ftl @@ -74,15 +74,17 @@ SqlCreate DruidSqlCreateTable(Span s, boolean replace) : } // A table element is either a column declaration or a projection definition. A column may legitimately be named -// "projection" (the keyword is non-reserved) and may have a bare-identifier type, so two tokens are not enough to -// tell the two apart: a projection definition is distinguished by its third token, which is always '(' or AS. +// "projection" (the keyword is non-reserved), and its type may itself start with a non-reserved keyword and a +// parenthesis: "projection TYPE('COMPLEX')" is a column. So neither two nor three tokens are enough to tell +// the two apart. A projection is recognized by what follows its name: AS, or a parenthesized body starting with +// SELECT; a column's TYPE(...) parenthesis holds a string literal, never SELECT. void AddDruidTableElement(List columns, List projections) : { final DruidSqlColumnDeclaration column; final SqlProjectionSpec projection; } { - LOOKAHEAD(3) + LOOKAHEAD( SimpleIdentifier() ( |
ADD COLUMN ALTER TABLE
DROP COLUMN ALTER TABLE
ALTER COLUMN SET DATA TYPE -ALTER TABLE
ADD [IF NOT EXISTS] PROJECTION AS (
ADD PROJECTION [IF NOT EXISTS] AS (
DROP PROJECTION [IF EXISTS] ALTER TABLE
SET PROPERTIES ( = [, ...] ) ``` @@ -180,7 +184,7 @@ aggregates as separate columns and divide at query time. A projection can store Projections may also be added to and removed from an existing table: ```sql -ALTER TABLE "druid"."visits" ADD [IF NOT EXISTS] PROJECTION by_agent AS ( +ALTER TABLE "druid"."visits" ADD PROJECTION [IF NOT EXISTS] by_agent AS ( SELECT user_agent, SUM(pages_visited) AS total_pages GROUP BY user_agent ) ALTER TABLE "druid"."visits" DROP PROJECTION [IF EXISTS] by_agent @@ -206,7 +210,7 @@ CREATE TABLE "druid"."events" SEALED ( user_id BIGINT, payload TYPE('COMPLEX'), PROJECTION __base AS ( - SELECT tenant, ABS(user_id) % 128 AS bucket, __time, user_id, payload + SELECT tenant, MOD(ABS(user_id), 128) AS bucket, __time, user_id, payload CLUSTERED BY tenant, bucket ) ) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java index ac83f09fc0b0..9d021e8e7d5a 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java @@ -231,6 +231,24 @@ public void testCoordinatorValidationErrorSurfacesToSqlUser() assertTrue(e.getMessage().contains("granularity"), e.getMessage()); } + @Test + public void testCreateTableUnregisteredComplexTypeFails() + { + final Exception e = assertThrows( + Exception.class, + () -> cluster.callApi().runSql( + "CREATE TABLE \"%s\" (__time TIMESTAMP, sketch TYPE('COMPLEX'))", + dataSource + ) + ); + assertTrue( + e.getMessage().contains( + "Column [sketch] declares complex type [COMPLEX], which is not registered on this server" + ), + e.getMessage() + ); + } + @Test public void testCreateTableAlreadyExistsFails() { @@ -320,7 +338,7 @@ public void testAlterTableAddAndDropProjection() ) ); cluster.callApi().runSql( - "ALTER TABLE \"%s\" ADD IF NOT EXISTS PROJECTION p AS (SELECT a, SUM(b) AS sum_b GROUP BY a)", + "ALTER TABLE \"%s\" ADD PROJECTION IF NOT EXISTS p AS (SELECT a, SUM(b) AS sum_b GROUP BY a)", tableName ); assertEquals(1, projectionsOf(tableName).size()); diff --git a/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java b/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java index 7be7b400db29..11bf5739f45d 100644 --- a/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java +++ b/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java @@ -40,7 +40,10 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.TypeSignature; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.indexing.DataSchema; +import org.apache.druid.segment.serde.ComplexMetrics; import java.util.ArrayList; import java.util.Arrays; @@ -127,6 +130,34 @@ public void validate(ResolvedTable table) validateProjections(table); } + /** + * A datasource column's declared complex type must be registered by an extension or it will be rejected at catalog + * write time rather than at the first ingest or query that touches the column. + *

+ * Reads never validate, so a spec stored before its type's extension left the load list keeps resolving, and + * surfaces on its next edit like any other invalid stored spec. + */ + @Override + protected void validateColumn(ColumnSpec colSpec) + { + super.validateColumn(colSpec); + TypeSignature type = Columns.druidType(colSpec); + while (type != null && type.isArray()) { + type = type.getElementType(); + } + if (type != null + && type.is(ValueType.COMPLEX) + && type.getComplexTypeName() != null + && ComplexMetrics.getSerdeForType(type.getComplexTypeName()) == null) { + throw InvalidInput.exception( + "Column [%s] declares complex type [%s], which is not registered on this server. Load the extension that" + + " provides the type, or correct the type name", + colSpec.name(), + colSpec.dataType() + ); + } + } + /** * Cross-validate the declared projections. Names must be unique, a projection must not be coarser than the segments * it lives in, and the types it groups by must agree with the types the table declares. For a sealed table the @@ -170,10 +201,9 @@ private void validateProjections(ResolvedTable table) for (String required : requiredColumns(spec)) { if (!available.contains(required)) { throw InvalidInput.exception( - "Projection [%s] references column [%s], which table [%s] does not declare", + "Projection [%s] references column [%s], which the table does not declare", spec.getName(), - required, - table.spec().type() + required ); } } diff --git a/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java b/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java index c722519542b6..a9450de1541a 100644 --- a/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java +++ b/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java @@ -38,6 +38,7 @@ import org.apache.druid.data.input.impl.AggregateProjectionSpec; import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.error.DruidException; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.logger.Logger; @@ -66,6 +67,10 @@ @Tag("CatalogTest") public class DatasourceTableTest extends InitializedNullHandlingTest { + static { + BuiltInTypesModule.registerHandlersAndSerde(); + } + private static final Logger LOG = new Logger(DatasourceTableTest.class); private final ObjectMapper mapper = DefaultObjectMapper.INSTANCE; @@ -469,13 +474,38 @@ public void testColumns() Assertions.assertTrue(e.getMessage().contains("Column [foo] has an unrecognized type [ARRAY]")); } { - // Parse-level validation only: any well-formed complex type is accepted at the logical layer, whether or not - // its serde is registered on this server. + // A registered complex type is accepted: json is registered by the static BuiltInTypesModule call above, as it + // is on any server by the module itself. TableSpec spec = builder.copy() - .column("foo", "COMPLEX") + .column("foo", "COMPLEX") .buildSpec(); expectValidationSucceeds(spec); } + { + // A well-formed complex type that no loaded extension registers is a typo or a missing extension either way, + // and is rejected at write time rather than at the first ingest or query that touches the column. + TableSpec spec = builder.copy() + .column("foo", "COMPLEX") + .buildSpec(); + DruidException e = Assertions.assertThrows(DruidException.class, () -> registry.resolve(spec).validate()); + Assertions.assertEquals( + "Column [foo] declares complex type [COMPLEX], which is not registered on this server." + + " Load the extension that provides the type, or correct the type name", + e.getMessage() + ); + } + { + // The registry check reaches through array wrappers to the element type. + TableSpec spec = builder.copy() + .column("foo", "ARRAY>") + .buildSpec(); + DruidException e = Assertions.assertThrows(DruidException.class, () -> registry.resolve(spec).validate()); + Assertions.assertEquals( + "Column [foo] declares complex type [ARRAY>], which is not registered on this server." + + " Load the extension that provides the type, or correct the type name", + e.getMessage() + ); + } } /** @@ -532,7 +562,10 @@ public void testProjectionValidation() DruidException.class, () -> projectionRegistry.resolve(sealedWithUndeclared).validate() ); - Assertions.assertTrue(e.getMessage().contains("references column [nope]")); + Assertions.assertEquals( + "Projection [bad] references column [nope], which the table does not declare", + e.getMessage() + ); // Ingestion may add columns to a table that is not sealed, so the same projection is allowed there. projectionRegistry.resolve( diff --git a/sql/src/main/codegen/includes/ddl.ftl b/sql/src/main/codegen/includes/ddl.ftl index cda3686622bb..0686985cb206 100644 --- a/sql/src/main/codegen/includes/ddl.ftl +++ b/sql/src/main/codegen/includes/ddl.ftl @@ -85,7 +85,7 @@ void AddDruidTableElement(List columns, List projections) : } { LOOKAHEAD( SimpleIdentifier() ( |