Merge master and resolve conflicts

Author: Caideyipi

RELEASE_NOTES.md

@@ -19,6 +19,44 @@
 
 -->
 
+# Apache IoTDB 2.0.10
+
+## Features & Improvements
+
+- Data Query: Support schema-level and table-level storage space statistics
+- Data Query: Add DEBUG SQL capability and optimize the result set of Explain Analyze
+- Data Query: Support set operations (UNION/INTERSECT/EXCEPT) and Common Table Expressions (CTE) in the table model
+- Data Query: Add IF scalar function, binary functions and APPROX_PERCENTILE aggregate function for the table model
+- Data Query: Optimize display fields of the show queries command, add three new columns: client IP that initiates the query, timeout threshold, and total request waiting latency
+- Storage Management: Provide C-language driver SDK interfaces, including parameter binding, automatic failover for multi-node switching and other fault-tolerant capabilities
+- System Management: Add the capability to manually trigger node load balancing
+- System Management: Add progress query capability for the remove datanode operation
+- System Management: Introduce the show configuration statement to view cluster configuration information
+- System Management: Add statistics for the number of slow write requests
+- AI Management: Add two built-in models: Moirai2 and Toto
+- AI Management: Enable AINode to manage multiple models under the same model_type
+- AI Management: Support loading built-in models for CPU inference with isolated inference processes
+
+## Bugs
+
+- Fix the issue that partition tables get lost when leader switch happens on single-node ConfigNode
+- Fix the bug where configurations delivered via set configuration to a single DataNode take effect on all other DataNodes unexpectedly
+- Fix the occasional continuous accumulation of reservedSize for timeout-only queries across multiple devices in tree model, which triggers query errors after hitting the max capacity limit
+- Fix the parsing error in tree-model-to-table conversion logic when tree paths consist entirely of numeric characters
+- Correct abnormal output of show devices root.db.** when root.db itself is a device, making it display root.db properly
+- Fix the failure of kill query and query timeout mechanism when the client holds the connection open without fully consuming all result sets
+- Fix array index out-of-bounds error caused by null values in attribute fields on the receiver side, which blocks the data synchronization pipeline
+- Fix connection pool leakage exceptions in the C# client
+- Fix exceptions occurring in the C++ client when write redirection is enabled while dn_rpc_address is configured to 0.0.0.0 on DataNode server
+- Fix abnormal repeated broadcasting of deletion markers (Deletions) during the load operation
+- Fix the issue that rows with null inserted fields cannot be synchronized to the receiver in real-time data synchronization scenarios
+- Fix incorrect idempotency logic for out-of-order TTL-expired data on the synchronization receiver, which leads to repeated synchronization of expired data
+- Fix the problem that enable-send-tsfile-limit parameter cannot be manually configured for historical Pipes split from full-data Pipe tasks
+- Fix Pipe temporary stoppage triggered by insufficient memory of a single subtask within synchronization jobs
+- Fix the issue that modifying username/password of source or sink in write-back-sink model generates error logs and prevents data from being received on the sink side
+- Fix permission issue preventing non-root users from stopping the AINode process
+- Mitigate memory leaks that tend to occur during long-running continuous inference of models running on CPU
+
 # Apache IoTDB 2.0.8
 
 ## Features & Improvements

external-service-impl/mqtt/src/main/java/org/apache/iotdb/mqtt/MPPPublishHandler.java

@@ -24,6 +24,7 @@
 import org.apache.iotdb.commons.queryengine.common.SqlDialect;
 import org.apache.iotdb.commons.queryengine.utils.TimestampPrecisionUtils;
 import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.commons.utils.PathUtils;
 import org.apache.iotdb.db.auth.AuthorityChecker;
 import org.apache.iotdb.db.conf.IoTDBConfig;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -274,7 +275,9 @@ private void insertTree(TreeMessage message, MqttClientSession session) {
           DataNodeDevicePathCache.getInstance().getPartialPath(message.getDevice()));
       TimestampPrecisionUtils.checkTimestampPrecision(message.getTimestamp());
       statement.setTime(message.getTimestamp());
-      statement.setMeasurements(message.getMeasurements().toArray(new String[0]));
+      statement.setMeasurements(
+          PathUtils.checkIsLegalSingleMeasurementsAndUpdate(message.getMeasurements())
+              .toArray(new String[0]));
       if (message.getDataTypes() == null) {
         statement.setDataTypes(new TSDataType[message.getMeasurements().size()]);
         statement.setValues(message.getValues().toArray(new Object[0]));

external-service-impl/rest-openapi/src/main/openapi3/iotdb_rest_table_v1.yaml

@@ -98,6 +98,7 @@ components:
         row_limit:
           type: integer
           format: int32
+          description: Maximum rows to return. The effective limit is capped by rest_query_default_row_size_limit.
 
     InsertTabletRequest:
       title: InsertTabletRequest

external-service-impl/rest-openapi/src/main/openapi3/iotdb_rest_v1.yaml

@@ -166,6 +166,7 @@ components:
         rowLimit:
           type: integer
           format: int32
+          description: Maximum rows to return. The effective limit is capped by rest_query_default_row_size_limit.
 
     InsertTabletRequest:
       title: InsertTabletRequest

external-service-impl/rest-openapi/src/main/openapi3/iotdb_rest_v2.yaml

@@ -201,6 +201,7 @@ components:
         row_limit:
           type: integer
           format: int32
+          description: Maximum rows to return. The effective limit is capped by rest_query_default_row_size_limit.
 
     PrefixPathList:
       title: PrefixPathList

external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java

@@ -34,6 +34,9 @@ public final class RestMessages {
   public static final String SQL_SHOULD_NOT_BE_NULL = "sql should not be null";
   public static final String ROW_LIMIT_SHOULD_BE_POSITIVE = "row_limit should be positive";
   public static final String ROW_LIMIT_CAMEL_SHOULD_BE_POSITIVE = "rowLimit should be positive";
+  public static final String
+      MESSAGE_DATASET_ROW_SIZE_EXCEEDED_THE_GIVEN_MAX_ROW_SIZE_ARG_7A3F6452 =
+          "Dataset row size exceeded the given max row size (%d)";
   public static final String PREFIX_PATHS_NOT_NULL = "prefix_paths should not be null";
   public static final String TIMESTAMPS_NOT_NULL = "timestamps should not be null";
   public static final String IS_ALIGNED_NOT_NULL = "is_aligned should not be null";

external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java

@@ -34,6 +34,9 @@ public final class RestMessages {
   public static final String SQL_SHOULD_NOT_BE_NULL = "sql 不能为空";
   public static final String ROW_LIMIT_SHOULD_BE_POSITIVE = "row_limit 应为正数";
   public static final String ROW_LIMIT_CAMEL_SHOULD_BE_POSITIVE = "rowLimit 应为正数";
+  public static final String
+      MESSAGE_DATASET_ROW_SIZE_EXCEEDED_THE_GIVEN_MAX_ROW_SIZE_ARG_7A3F6452 =
+          "数据集行数超过给定的最大行数（%d）";
   public static final String PREFIX_PATHS_NOT_NULL = "prefix_paths 不能为空";
   public static final String TIMESTAMPS_NOT_NULL = "timestamps 不能为空";
   public static final String IS_ALIGNED_NOT_NULL = "is_aligned 不能为空";

external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/handler/QueryRowLimitUtils.java

@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.rest.protocol.handler;
+
+import org.apache.iotdb.rest.i18n.RestMessages;
+import org.apache.iotdb.rest.protocol.model.ExecutionStatus;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import jakarta.ws.rs.core.Response;
+
+public final class QueryRowLimitUtils {
+
+  /**
+   * Fallback used when {@code rest_query_default_row_size_limit} is missing or non-positive.
+   * Matches the built-in default in {@code IoTDBRestServiceConfig}; a non-positive configured value
+   * used to mean "unlimited" and is now treated as this cap instead of being clamped down to 1.
+   */
+  private static final int DEFAULT_ROW_SIZE_LIMIT = 10000;
+
+  private QueryRowLimitUtils() {}
+
+  public static int resolveActualRowSizeLimit(
+      Integer requestedRowSizeLimit, int configuredRowSizeLimit) {
+    int hardLimit = normalizeRowSizeLimit(configuredRowSizeLimit);
+    if (requestedRowSizeLimit == null) {
+      return hardLimit;
+    }
+    return normalizeRowSizeLimit(Math.min(requestedRowSizeLimit, hardLimit));
+  }
+
+  public static int normalizeRowSizeLimit(int rowSizeLimit) {
+    return rowSizeLimit > 0 ? rowSizeLimit : DEFAULT_ROW_SIZE_LIMIT;
+  }
+
+  public static boolean exceedsLimit(
+      int fetchedRowCount, int incomingRowCount, int actualRowSizeLimit) {
+    return incomingRowCount > 0
+        && (long) fetchedRowCount + incomingRowCount > normalizeRowSizeLimit(actualRowSizeLimit);
+  }
+
+  public static Response buildRowSizeLimitExceededResponse(int actualRowSizeLimit) {
+    int rowSizeLimit = normalizeRowSizeLimit(actualRowSizeLimit);
+    return Response.ok()
+        .entity(
+            new ExecutionStatus()
+                .code(TSStatusCode.QUERY_PROCESS_ERROR.getStatusCode())
+                .message(
+                    String.format(
+                        RestMessages
+                            .MESSAGE_DATASET_ROW_SIZE_EXCEEDED_THE_GIVEN_MAX_ROW_SIZE_ARG_7A3F6452,
+                        rowSizeLimit)))
+        .build();
+  }
+}

external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/QueryDataSetHandler.java

@@ -22,9 +22,8 @@
 import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Statement;
 import org.apache.iotdb.db.queryengine.common.header.DatasetHeader;
 import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution;
-import org.apache.iotdb.rest.protocol.model.ExecutionStatus;
+import org.apache.iotdb.rest.protocol.handler.QueryRowLimitUtils;
 import org.apache.iotdb.rest.protocol.table.v1.model.QueryDataSet;
-import org.apache.iotdb.rpc.TSStatusCode;
 
 import org.apache.tsfile.block.column.Column;
 import org.apache.tsfile.common.conf.TSFileConfig;
@@ -45,7 +44,7 @@ public class QueryDataSetHandler {
   private QueryDataSetHandler() {}
 
   /**
-   * @param actualRowSizeLimit max number of rows to return. no limit when actualRowSizeLimit <= 0.
+   * @param actualRowSizeLimit max number of rows to return.
    */
   public static Response fillQueryDataSet(
       IQueryExecution queryExecution, Statement statement, int actualRowSizeLimit)
@@ -61,7 +60,6 @@ public static Response fillQueryDataSet(
       IQueryExecution queryExecution, final int actualRowSizeLimit) throws IoTDBException {
     QueryDataSet targetDataSet = new QueryDataSet();
     int fetched = 0;
-    int columnNum = queryExecution.getOutputValueColumnCount();
 
     DatasetHeader header = queryExecution.getDatasetHeader();
     List<String> resultColumns = header.getRespColumns();
@@ -73,17 +71,6 @@ public static Response fillQueryDataSet(
       targetDataSet.addValuesItem(new ArrayList<>());
     }
     while (true) {
-      if (0 < actualRowSizeLimit && actualRowSizeLimit <= fetched) {
-        return Response.ok()
-            .entity(
-                new ExecutionStatus()
-                    .code(TSStatusCode.QUERY_PROCESS_ERROR.getStatusCode())
-                    .message(
-                        String.format(
-                            "Dataset row size exceeded the given max row size (%d)",
-                            actualRowSizeLimit)))
-            .build();
-      }
       Optional<TsBlock> optionalTsBlock = queryExecution.getBatchResult();
       if (!optionalTsBlock.isPresent() || optionalTsBlock.get().isEmpty()) {
         if (fetched == 0) {
@@ -94,22 +81,22 @@ public static Response fillQueryDataSet(
       }
       TsBlock tsBlock = optionalTsBlock.get();
       int currentCount = tsBlock.getPositionCount();
+      if (QueryRowLimitUtils.exceedsLimit(fetched, currentCount, actualRowSizeLimit)) {
+        return QueryRowLimitUtils.buildRowSizeLimitExceededResponse(actualRowSizeLimit);
+      }
 
       for (int k = 0; k < resultColumns.size(); k++) {
         Column column = tsBlock.getColumn(headerMap.get(resultColumns.get(k)));
         List<Object> targetDataSetColumn = targetDataSet.getValues().get(k);
         for (int i = 0; i < currentCount; i++) {
-          fetched++;
           if (column.isNull(i)) {
             targetDataSetColumn.add(null);
           } else {
             addTypedValueToTarget(targetDataSet.getDataTypes(), k, i, targetDataSetColumn, column);
           }
         }
-        if (k != columnNum - 1) {
-          fetched -= currentCount;
-        }
       }
+      fetched += currentCount;
     }
     targetDataSet.setValues(convertColumnToRow(targetDataSet.getValues()));
     return Response.ok().entity(targetDataSet).build();
@@ -174,17 +161,6 @@ private static Response fillOtherQueryDataSet(
     int fetched = 0;
     int columnNum = queryExecution.getOutputValueColumnCount();
     while (true) {
-      if (0 < actualRowSizeLimit && actualRowSizeLimit <= fetched) {
-        return Response.ok()
-            .entity(
-                new ExecutionStatus()
-                    .code(TSStatusCode.QUERY_PROCESS_ERROR.getStatusCode())
-                    .message(
-                        String.format(
-                            "Dataset row size exceeded the given max row size (%d)",
-                            actualRowSizeLimit)))
-            .build();
-      }
       Optional<TsBlock> optionalTsBlock = queryExecution.getBatchResult();
       if (!optionalTsBlock.isPresent()) {
         if (fetched == 0) {
@@ -199,21 +175,21 @@ private static Response fillOtherQueryDataSet(
         targetDataSet.setValues(new ArrayList<>());
         return Response.ok().entity(targetDataSet).build();
       }
+      if (QueryRowLimitUtils.exceedsLimit(fetched, currentCount, actualRowSizeLimit)) {
+        return QueryRowLimitUtils.buildRowSizeLimitExceededResponse(actualRowSizeLimit);
+      }
       for (int k = 0; k < columnNum; k++) {
         Column column = tsBlock.getColumn(targetDataSetIndexToSourceDataSetIndex[k]);
         List<Object> targetDataSetColumn = targetDataSet.getValues().get(k);
         for (int i = 0; i < currentCount; i++) {
-          fetched++;
           if (column.isNull(i)) {
             targetDataSetColumn.add(null);
           } else {
             addTypedValueToTarget(targetDataSet.getDataTypes(), k, i, targetDataSetColumn, column);
           }
         }
-        if (k != columnNum - 1) {
-          fetched -= currentCount;
-        }
       }
+      fetched += currentCount;
     }
     targetDataSet.setValues(convertColumnToRow(targetDataSet.getValues()));
     return Response.ok().entity(targetDataSet).build();

external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/impl/RestApiServiceImpl.java

@@ -38,6 +38,7 @@
 import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
 import org.apache.iotdb.db.utils.CommonUtils;
 import org.apache.iotdb.db.utils.SetThreadName;
+import org.apache.iotdb.rest.protocol.handler.QueryRowLimitUtils;
 import org.apache.iotdb.rest.protocol.table.v1.NotFoundException;
 import org.apache.iotdb.rest.protocol.table.v1.RestApiService;
 import org.apache.iotdb.rest.protocol.table.v1.handler.ExceptionHandler;
@@ -63,11 +64,12 @@ public class RestApiServiceImpl extends RestApiService {
 
   private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
 
-  private final Integer defaultQueryRowLimit;
+  private final int defaultQueryRowLimit;
 
   public RestApiServiceImpl() {
     defaultQueryRowLimit =
-        IoTDBRestServiceDescriptor.getInstance().getConfig().getRestQueryDefaultRowSizeLimit();
+        QueryRowLimitUtils.normalizeRowSizeLimit(
+            IoTDBRestServiceDescriptor.getInstance().getConfig().getRestQueryDefaultRowSizeLimit());
   }
 
   public Response executeQueryInternal(
@@ -103,7 +105,8 @@ public Response executeQueryInternal(
             QueryDataSetHandler.fillQueryDataSet(
                 queryExecution,
                 statement,
-                sql.getRowLimit() == null ? defaultQueryRowLimit : sql.getRowLimit());
+                QueryRowLimitUtils.resolveActualRowSizeLimit(
+                    sql.getRowLimit(), defaultQueryRowLimit));
         if (queryExecution.getQueryType() == QueryType.READ_WRITE) {
           return responseGenerateHelper(result);
         }