Limit REST write request size on dev/1.3

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java

@@ -33,6 +33,7 @@
 import org.apache.iotdb.confignode.rpc.thrift.TCQConfig;
 import org.apache.iotdb.confignode.rpc.thrift.TGlobalConfig;
 import org.apache.iotdb.confignode.rpc.thrift.TRatisConfig;
+import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor;
 import org.apache.iotdb.db.consensus.DataRegionConsensusImpl;
 import org.apache.iotdb.db.exception.query.QueryProcessException;
 import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
@@ -2062,6 +2063,9 @@ public synchronized void loadHotModifiedProps(TrimProperties properties)
       // update trusted_uri_pattern
       loadTrustedUriPattern(properties);
 
+      // update REST request limits
+      IoTDBRestServiceDescriptor.getInstance().loadHotModifiedProps(properties);
+
       // tvlist_sort_threshold
       conf.setTVListSortThreshold(
           Integer.parseInt(

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java

@@ -59,6 +59,22 @@ public class IoTDBRestServiceConfig {
 
   private int restQueryDefaultRowSizeLimit = 10000;
 
+  /** Maximum accepted REST request body size in bytes. */
+  private long restMaxRequestBodySizeInBytes = 16 * 1024 * 1024L;
+
+  /** Maximum total in-flight REST request body size in bytes across concurrent requests. */
+  private long restMaxTotalConcurrentRequestBodySizeInBytes =
+      getDefaultRequestBodyMemoryLimitInBytes();
+
+  /** Maximum row count accepted by a single REST write request. */
+  private int restMaxInsertRows = 100000;
+
+  /** Maximum column count accepted by a single REST write request. */
+  private int restMaxInsertColumns = 1024;
+
+  /** Maximum value cell count accepted by a single REST write request. */
+  private long restMaxInsertValues = 1000000L;
+
   /** Is client authentication required. */
   private boolean clientAuth = false;
 
@@ -173,4 +189,50 @@ public int getRestQueryDefaultRowSizeLimit() {
   public void setRestQueryDefaultRowSizeLimit(int restQueryDefaultRowSizeLimit) {
     this.restQueryDefaultRowSizeLimit = restQueryDefaultRowSizeLimit;
   }
+
+  public long getRestMaxRequestBodySizeInBytes() {
+    return restMaxRequestBodySizeInBytes;
+  }
+
+  public void setRestMaxRequestBodySizeInBytes(long restMaxRequestBodySizeInBytes) {
+    this.restMaxRequestBodySizeInBytes = restMaxRequestBodySizeInBytes;
+  }
+
+  public long getRestMaxTotalConcurrentRequestBodySizeInBytes() {
+    return restMaxTotalConcurrentRequestBodySizeInBytes;
+  }
+
+  public void setRestMaxTotalConcurrentRequestBodySizeInBytes(
+      long restMaxTotalConcurrentRequestBodySizeInBytes) {
+    this.restMaxTotalConcurrentRequestBodySizeInBytes =
+        restMaxTotalConcurrentRequestBodySizeInBytes;
+  }
+
+  public int getRestMaxInsertRows() {
+    return restMaxInsertRows;
+  }
+
+  public void setRestMaxInsertRows(int restMaxInsertRows) {
+    this.restMaxInsertRows = restMaxInsertRows;
+  }
+
+  public int getRestMaxInsertColumns() {
+    return restMaxInsertColumns;
+  }
+
+  public void setRestMaxInsertColumns(int restMaxInsertColumns) {
+    this.restMaxInsertColumns = restMaxInsertColumns;
+  }
+
+  public long getRestMaxInsertValues() {
+    return restMaxInsertValues;
+  }
+
+  public void setRestMaxInsertValues(long restMaxInsertValues) {
+    this.restMaxInsertValues = restMaxInsertValues;
+  }
+
+  public static long getDefaultRequestBodyMemoryLimitInBytes() {
+    return Runtime.getRuntime().maxMemory() / 20;
+  }
 }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java

@@ -38,6 +38,16 @@
 public class IoTDBRestServiceDescriptor {
   private static final Logger logger = LoggerFactory.getLogger(IoTDBRestServiceDescriptor.class);
 
+  private static final String REST_QUERY_DEFAULT_ROW_SIZE_LIMIT =
+      "rest_query_default_row_size_limit";
+  private static final String REST_MAX_REQUEST_BODY_SIZE_IN_BYTES =
+      "rest_max_request_body_size_in_bytes";
+  private static final String REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES =
+      "rest_max_total_concurrent_request_body_size_in_bytes";
+  private static final String REST_MAX_INSERT_ROWS = "rest_max_insert_rows";
+  private static final String REST_MAX_INSERT_COLUMNS = "rest_max_insert_columns";
+  private static final String REST_MAX_INSERT_VALUES = "rest_max_insert_values";
+
   private final IoTDBRestServiceConfig conf = new IoTDBRestServiceConfig();
 
   protected IoTDBRestServiceDescriptor() {
@@ -86,11 +96,7 @@ private void loadProps(TrimProperties properties) {
         Integer.parseInt(
             properties.getProperty(
                 "rest_service_port", Integer.toString(conf.getRestServicePort()))));
-    conf.setRestQueryDefaultRowSizeLimit(
-        Integer.parseInt(
-            properties.getProperty(
-                "rest_query_default_row_size_limit",
-                Integer.toString(conf.getRestQueryDefaultRowSizeLimit()))));
+    loadRuntimeLimitProps(properties);
     conf.setEnableSwagger(
         Boolean.parseBoolean(
             properties.getProperty("enable_swagger", Boolean.toString(conf.isEnableSwagger()))));
@@ -111,6 +117,72 @@ private void loadProps(TrimProperties properties) {
                 "idle_timeout_in_seconds", Integer.toString(conf.getIdleTimeoutInSeconds()))));
   }
 
+  public synchronized void loadHotModifiedProps(TrimProperties properties) {
+    loadRuntimeLimitProps(properties);
+  }
+
+  private void loadRuntimeLimitProps(TrimProperties properties) {
+    conf.setRestQueryDefaultRowSizeLimit(
+        Integer.parseInt(
+            properties.getProperty(
+                REST_QUERY_DEFAULT_ROW_SIZE_LIMIT,
+                Integer.toString(conf.getRestQueryDefaultRowSizeLimit()))));
+    conf.setRestMaxRequestBodySizeInBytes(
+        Long.parseLong(
+            properties.getProperty(
+                REST_MAX_REQUEST_BODY_SIZE_IN_BYTES,
+                Long.toString(conf.getRestMaxRequestBodySizeInBytes()))));
+    conf.setRestMaxTotalConcurrentRequestBodySizeInBytes(
+        parseMaxTotalConcurrentRequestBodySizeInBytes(properties));
+    conf.setRestMaxInsertRows(
+        Integer.parseInt(
+            properties.getProperty(
+                REST_MAX_INSERT_ROWS, Integer.toString(conf.getRestMaxInsertRows()))));
+    conf.setRestMaxInsertColumns(
+        Integer.parseInt(
+            properties.getProperty(
+                REST_MAX_INSERT_COLUMNS, Integer.toString(conf.getRestMaxInsertColumns()))));
+    conf.setRestMaxInsertValues(
+        Long.parseLong(
+            properties.getProperty(
+                REST_MAX_INSERT_VALUES, Long.toString(conf.getRestMaxInsertValues()))));
+  }
+
+  private long parseMaxTotalConcurrentRequestBodySizeInBytes(TrimProperties properties) {
+    long configuredLimit =
+        Long.parseLong(
+            properties.getProperty(
+                REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES,
+                Long.toString(conf.getRestMaxTotalConcurrentRequestBodySizeInBytes())));
+    return configuredLimit == 0
+        ? calculateRequestBodyMemoryLimitInBytes(properties)
+        : configuredLimit;
+  }
+
+  static long calculateRequestBodyMemoryLimitInBytes(TrimProperties properties) {
+    String memoryAllocateProportion = properties.getProperty("datanode_memory_proportion", null);
+    if (memoryAllocateProportion == null) {
+      memoryAllocateProportion =
+          properties.getProperty("storage_query_schema_consensus_free_memory_proportion", null);
+    }
+    if (memoryAllocateProportion == null) {
+      return IoTDBRestServiceConfig.getDefaultRequestBodyMemoryLimitInBytes();
+    }
+
+    String[] proportions = memoryAllocateProportion.split(":");
+    int proportionSum = 0;
+    for (String proportion : proportions) {
+      proportionSum += Integer.parseInt(proportion.trim());
+    }
+    if (proportionSum == 0 || proportions.length < 6) {
+      return IoTDBRestServiceConfig.getDefaultRequestBodyMemoryLimitInBytes();
+    }
+    return Runtime.getRuntime().maxMemory()
+        * Integer.parseInt(proportions[proportions.length - 1].trim())
+        / proportionSum
+        / 2;
+  }
+
   /**
    * get props url location
    *

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/exception/RequestLimitExceededException.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.db.protocol.rest.exception;
+
+public class RequestLimitExceededException extends IllegalArgumentException {
+
+  public RequestLimitExceededException(String message) {
+    super(message);
+  }
+}

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/filter/RequestBodyMemoryReleaseFilter.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.db.protocol.rest.filter;
+
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerResponseContext;
+import javax.ws.rs.container.ContainerResponseFilter;
+import javax.ws.rs.ext.Provider;
+
+@Provider
+public class RequestBodyMemoryReleaseFilter implements ContainerResponseFilter {
+
+  @Override
+  public void filter(
+      ContainerRequestContext requestContext, ContainerResponseContext responseContext) {
+    RestRequestBodyMemoryManager.releaseReservation(requestContext);
+  }
+}