diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 3feefd72a6f4..5cb613debef0 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -3818,5 +3818,8 @@ private DataNodeQueryMessages() {} public static final String EXCEPTION_VISIBLEALIASES_IS_NULL_630B27F1 = "visibleAliases is null"; public static final String EXCEPTION_HAS_NO_PERMISSION_TO_EXECUTE_ARG_BECAUSE_ONLY_THE_SUPERUSER_CAN_ALTER_HIM_HERSELF_C5902893 = "Has no permission to execute %s, because only the superuser can alter him/herself."; + public static final String + LOG_FAILED_TO_CLEAN_DEVICEENTRY_DATA_SET_ASYNCHRONOUSLY_QUERYID_ARG_PLANNODEID_ARG_9106C4C5 = + "Failed to clean DeviceEntry data set asynchronously: queryId=%s, planNodeId=%s"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 0d4e7cc0071b..89bafbd34ed1 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -4575,5 +4575,8 @@ private DataNodeQueryMessages() {} public static final String EXCEPTION_VISIBLEALIASES_IS_NULL_630B27F1 = "visibleAliases 不能为空"; public static final String EXCEPTION_HAS_NO_PERMISSION_TO_EXECUTE_ARG_BECAUSE_ONLY_THE_SUPERUSER_CAN_ALTER_HIM_HERSELF_C5902893 = "无权执行 %s，因为只有超级用户可以修改其自身。"; + public static final String + LOG_FAILED_TO_CLEAN_DEVICEENTRY_DATA_SET_ASYNCHRONOUSLY_QUERYID_ARG_PLANNODEID_ARG_9106C4C5 = + "异步清理 DeviceEntry 数据集失败：queryId=%s，planNodeId=%s"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 1d726366ddd2..19bfa1bf9e67 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -256,6 +256,9 @@ public class IoTDBConfig { private String queryDir = IoTDBConstant.DN_DEFAULT_DATA_DIR + File.separator + IoTDBConstant.QUERY_FOLDER_NAME; + /** Maximum DeviceEntry bytes kept in memory before a table-query spill. */ + private long tableQueryDeviceEntryBatchSizeInBytes; + /** External lib directory, stores user-uploaded JAR files */ private String extDir = IoTDBConstant.EXT_FOLDER_NAME; @@ -1789,6 +1792,14 @@ public void setQueryDir(String queryDir) { this.queryDir = queryDir; } + public long getTableQueryDeviceEntryBatchSizeInBytes() { + return tableQueryDeviceEntryBatchSizeInBytes; + } + + public void setTableQueryDeviceEntryBatchSizeInBytes(long tableQueryDeviceEntryBatchSizeInBytes) { + this.tableQueryDeviceEntryBatchSizeInBytes = tableQueryDeviceEntryBatchSizeInBytes; + } + public String getRatisDataRegionSnapshotDir() { return ratisDataRegionSnapshotDir; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 15d2b7d003d5..ed46c49e4114 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -352,6 +352,18 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException conf.setQueryDir( FilePathUtils.regularizePath(conf.getSystemDir() + IoTDBConstant.QUERY_FOLDER_NAME)); + long deviceEntryBatchSize = + Long.parseLong( + properties.getProperty( + "table_query_device_entry_batch_size_in_bytes", + Long.toString(conf.getTableQueryDeviceEntryBatchSizeInBytes()))); + if (deviceEntryBatchSize <= 0) { + deviceEntryBatchSize = + memoryConfig.getOperatorsMemoryManager().getTotalMemorySizeInBytes() + / memoryConfig.getQueryThreadCount() + / 4; + } + conf.setTableQueryDeviceEntryBatchSizeInBytes(deviceEntryBatchSize); String[] defaultTierDirs = new String[conf.getTierDataDirs().length]; for (int i = 0; i < defaultTierDirs.length; ++i) { defaultTierDirs[i] = String.join(",", conf.getTierDataDirs()[i]); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 495ebc42b65c..0f274d9c06e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -48,6 +48,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner; import org.apache.iotdb.db.queryengine.plan.planner.memory.NotThreadSafeMemoryReservationManager; import org.apache.iotdb.db.queryengine.plan.relational.function.tvf.read_tsfile.ExternalTsFileQueryResource; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryIOContext; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainOutputFormat; import org.apache.iotdb.db.queryengine.statistics.QueryPlanStatistics; @@ -132,6 +133,8 @@ public enum ExplainType { private QueryPlanStatistics queryPlanStatistics = null; + private DeviceEntryIOContext deviceEntryIOContext; + // To avoid query front-end from consuming too much memory, it needs to reserve memory when // constructing some Expression and PlanNode. private final MemoryReservationManager memoryReservationManager; @@ -403,6 +406,13 @@ public void setStartTime(long startTime) { this.startTime = startTime; } + public DeviceEntryIOContext getOrCreateDeviceEntryIOContext(boolean duringFetchSchema) { + if (deviceEntryIOContext == null) { + deviceEntryIOContext = new DeviceEntryIOContext(this, duringFetchSchema); + } + return deviceEntryIOContext; + } + public void addFailedEndPoint(TEndPoint endPoint) { this.endPointBlackList.add(endPoint); } @@ -528,6 +538,37 @@ public long getDispatchCost() { return queryPlanStatistics.getDispatchCost(); } + public void recordDeviceEntryDiskIODuringFetchSchema(long bytes, long timeCost) { + getOrCreateQueryPlanStatistics().recordDeviceEntryDiskIODuringFetchSchema(bytes, timeCost); + } + + public void recordDeviceEntryCount(long count) { + getOrCreateQueryPlanStatistics().recordDeviceEntryCount(count); + } + + public long getDiskIOSizeForDeviceEntryDuringFetchSchema() { + return queryPlanStatistics == null + ? 0 + : queryPlanStatistics.getDiskIOSizeForDeviceEntryDuringFetchSchema(); + } + + public long getDiskIOTimeCostForDeviceEntryDuringFetchSchema() { + return queryPlanStatistics == null + ? 0 + : queryPlanStatistics.getDiskIOTimeCostForDeviceEntryDuringFetchSchema(); + } + + public long getDeviceEntryCount() { + return queryPlanStatistics == null ? 0 : queryPlanStatistics.getDeviceEntryCount(); + } + + private QueryPlanStatistics getOrCreateQueryPlanStatistics() { + if (queryPlanStatistics == null) { + queryPlanStatistics = new QueryPlanStatistics(); + } + return queryPlanStatistics; + } + public void setAnalyzeCost(long analyzeCost) { if (queryPlanStatistics == null) { queryPlanStatistics = new QueryPlanStatistics(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java index 2d3447dee69f..0e0d6b14838c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java @@ -43,11 +43,14 @@ import org.apache.iotdb.db.queryengine.execution.memory.LocalMemoryManager; import org.apache.iotdb.db.queryengine.metric.DataExchangeCostMetricSet; import org.apache.iotdb.db.queryengine.metric.DataExchangeCountMetricSet; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntrySpillManager; import org.apache.iotdb.db.utils.SetThreadName; import org.apache.iotdb.mpp.rpc.thrift.MPPDataExchangeService; import org.apache.iotdb.mpp.rpc.thrift.TAcknowledgeDataBlockEvent; import org.apache.iotdb.mpp.rpc.thrift.TCloseSinkChannelEvent; import org.apache.iotdb.mpp.rpc.thrift.TEndOfDataBlockEvent; +import org.apache.iotdb.mpp.rpc.thrift.TFetchDeviceEntrySegmentReq; +import org.apache.iotdb.mpp.rpc.thrift.TFetchDeviceEntrySegmentResp; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId; import org.apache.iotdb.mpp.rpc.thrift.TGetDataBlockRequest; import org.apache.iotdb.mpp.rpc.thrift.TGetDataBlockResponse; @@ -96,6 +99,46 @@ class MPPDataExchangeServiceImpl implements MPPDataExchangeService.Iface { private final DataExchangeCountMetricSet DATA_EXCHANGE_COUNT_METRICS = DataExchangeCountMetricSet.getInstance(); + @Override + public TFetchDeviceEntrySegmentResp fetchDeviceEntrySegment( + TFetchDeviceEntrySegmentReq request) { + try { + DeviceEntrySpillManager spillManager = DeviceEntrySpillManager.getInstance(); + byte[] payload = + spillManager.readSegment( + request.getQueryId(), request.getPlanNodeId(), request.getSegmentId()); + if (request.getSegmentId() > 0) { + spillManager.deleteSegment( + request.getQueryId(), request.getPlanNodeId(), request.getSegmentId() - 1); + } + return new TFetchDeviceEntrySegmentResp( + new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())) + .setPayload(payload); + } catch (IOException | RuntimeException e) { + return new TFetchDeviceEntrySegmentResp( + new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode()) + .setMessage(e.getMessage())); + } + } + + @Override + public TSStatus finishDeviceEntrySegment(String queryId, String planNodeId) { + executorService.submit( + () -> { + try { + DeviceEntrySpillManager.getInstance().finishSegmentDataSet(queryId, planNodeId); + } catch (IOException | RuntimeException e) { + LOGGER.warn( + DataNodeQueryMessages + .LOG_FAILED_TO_CLEAN_DEVICEENTRY_DATA_SET_ASYNCHRONOUSLY_QUERYID_ARG_PLANNODEID_ARG_9106C4C5, + queryId, + planNodeId, + e); + } + }); + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } + @Override public TGetDataBlockResponse getDataBlock(TGetDataBlockRequest req) throws TException { long startTime = System.nanoTime(); @@ -623,6 +666,11 @@ public MPPDataExchangeServiceImpl getOrCreateMPPDataExchangeServiceImpl() { return mppDataExchangeService; } + public IClientManager + getMppDataExchangeServiceClientManager() { + return mppDataExchangeServiceClientManager; + } + public void deRegisterFragmentInstanceFromMemoryPool( String queryId, String fragmentInstanceId, boolean forceDeregister) { localMemoryManager diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java index 7bb7478950b9..9b4cc79d9474 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java @@ -48,6 +48,8 @@ import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.queryengine.plan.analyze.cache.partition.PartitionCache; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSet; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryReader; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; import org.apache.iotdb.rpc.TSStatusCode; @@ -57,6 +59,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -207,28 +210,26 @@ public DataPartition getDataPartition( final Map> sgNameToQueryParamsMap) { DataPartition dataPartition = partitionCache.getDataPartition(sgNameToQueryParamsMap); if (null == dataPartition) { - try (ConfigNodeClient client = - configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - final TDataPartitionTableResp dataPartitionTableResp = - client.getDataPartitionTable(constructDataPartitionReqForQuery(sgNameToQueryParamsMap)); - if (dataPartitionTableResp.getStatus().getCode() - == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - dataPartition = parseDataPartitionResp(dataPartitionTableResp); - partitionCache.updateDataPartitionCache(dataPartitionTableResp.getDataPartitionTable()); - } else { - throw new StatementAnalyzeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_AN_ERROR_OCCURRED_WHEN_EXECUTING_GETDATAPARTITION_S_D21A0011, - dataPartitionTableResp.getStatus().getMessage())); - } - } catch (final ClientManagerException | TException e) { - throw new StatementAnalyzeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_AN_ERROR_OCCURRED_WHEN_EXECUTING_GETDATAPARTITION_S_D21A0011, - e.getMessage())); - } + dataPartition = + fetchDataPartition(constructDataPartitionReqForQuery(sgNameToQueryParamsMap), true); + } + return dataPartition; + } + + @Override + public DataPartition getDataPartition( + final String database, + final DeviceEntryDataSet dataSet, + final List timePartitionSlots) { + final Set seriesPartitionSlots = collectSeriesPartitionSlots(dataSet); + DataPartition dataPartition = + partitionCache.getDataPartition(database, seriesPartitionSlots, timePartitionSlots); + if (null == dataPartition) { + dataPartition = + fetchDataPartition( + constructDataPartitionReqForQuery( + database, seriesPartitionSlots, timePartitionSlots, false, false), + true); } return dataPartition; } @@ -239,20 +240,41 @@ public DataPartition getDataPartitionWithUnclosedTimeRange( // In this method, we must fetch from config node because it contains -oo or +oo // and there is no need to update cache because since we will never fetch it from cache, the // update operation will be only time waste + return fetchDataPartition(constructDataPartitionReqForQuery(sgNameToQueryParamsMap), false); + } + + @Override + public DataPartition getDataPartitionWithUnclosedTimeRange( + final String database, + final DeviceEntryDataSet dataSet, + final List timePartitionSlots, + final boolean needLeftAll, + final boolean needRightAll) { + final Set seriesPartitionSlots = collectSeriesPartitionSlots(dataSet); + return fetchDataPartition( + constructDataPartitionReqForQuery( + database, seriesPartitionSlots, timePartitionSlots, needLeftAll, needRightAll), + false); + } + + private DataPartition fetchDataPartition( + final TDataPartitionReq request, final boolean updateCache) { try (final ConfigNodeClient client = configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - final TDataPartitionTableResp dataPartitionTableResp = - client.getDataPartitionTable(constructDataPartitionReqForQuery(sgNameToQueryParamsMap)); + final TDataPartitionTableResp dataPartitionTableResp = client.getDataPartitionTable(request); if (dataPartitionTableResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return parseDataPartitionResp(dataPartitionTableResp); - } else { - throw new StatementAnalyzeException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_AN_ERROR_OCCURRED_WHEN_EXECUTING_GETDATAPARTITION_S_D21A0011, - dataPartitionTableResp.getStatus().getMessage())); + final DataPartition dataPartition = parseDataPartitionResp(dataPartitionTableResp); + if (updateCache) { + partitionCache.updateDataPartitionCache(dataPartitionTableResp.getDataPartitionTable()); + } + return dataPartition; } + throw new StatementAnalyzeException( + String.format( + DataNodeQueryMessages + .QUERY_EXCEPTION_AN_ERROR_OCCURRED_WHEN_EXECUTING_GETDATAPARTITION_S_D21A0011, + dataPartitionTableResp.getStatus().getMessage())); } catch (final ClientManagerException | TException e) { throw new StatementAnalyzeException( String.format( @@ -543,6 +565,34 @@ private TDataPartitionReq constructDataPartitionReqForQuery( return new TDataPartitionReq(partitionSlotsMap); } + private TDataPartitionReq constructDataPartitionReqForQuery( + final String database, + final Set seriesPartitionSlots, + final List timePartitionSlots, + final boolean needLeftAll, + final boolean needRightAll) { + final TTimeSlotList sharedTimeSlotList = + new TTimeSlotList(timePartitionSlots, needLeftAll, needRightAll); + final Map seriesSlotToTimeSlots = new HashMap<>(); + for (final TSeriesPartitionSlot seriesPartitionSlot : seriesPartitionSlots) { + seriesSlotToTimeSlots.put(seriesPartitionSlot, sharedTimeSlotList); + } + return new TDataPartitionReq(Collections.singletonMap(database, seriesSlotToTimeSlots)); + } + + private Set collectSeriesPartitionSlots(final DeviceEntryDataSet dataSet) { + final Set seriesPartitionSlots = new HashSet<>(); + try (final DeviceEntryReader reader = dataSet.openReader()) { + while (reader.hasNext()) { + seriesPartitionSlots.add( + partitionExecutor.getSeriesPartitionSlot(reader.next().getDeviceID())); + } + } catch (final IOException e) { + throw new UncheckedIOException(e); + } + return seriesPartitionSlots; + } + private SchemaPartition parseSchemaPartitionTableResp( final TSchemaPartitionTableResp schemaPartitionTableResp) { final Map> regionReplicaMap = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java index 0549ec396474..56315b45638e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java @@ -19,11 +19,13 @@ package org.apache.iotdb.db.queryengine.plan.analyze; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.partition.SchemaNodeManagementPartition; import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSet; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; import org.apache.tsfile.file.metadata.IDeviceID; @@ -59,6 +61,9 @@ default SchemaPartition getSchemaPartition( */ DataPartition getDataPartition(Map> sgNameToQueryParamsMap); + DataPartition getDataPartition( + String database, DeviceEntryDataSet dataSet, List timePartitionSlots); + /** * Get data partition, used in query scenarios which contains time filter like: time < XX or time * > XX @@ -68,6 +73,13 @@ default SchemaPartition getSchemaPartition( DataPartition getDataPartitionWithUnclosedTimeRange( Map> sgNameToQueryParamsMap); + DataPartition getDataPartitionWithUnclosedTimeRange( + String database, + DeviceEntryDataSet dataSet, + List timePartitionSlots, + boolean needLeftAll, + boolean needRightAll); + /** * Get or create data partition, used in standalone write scenarios. if enableAutoCreateSchema is * true and database/series/time slots not exists, then automatically create. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java index 4b8ee72ffd48..e7bdf88ca6a0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java @@ -873,93 +873,80 @@ public void invalidAllSchemaPartitionCache() { */ public DataPartition getDataPartition( Map> databaseToQueryParamsMap) { + final Map>> querySlots = + new HashMap<>(); + for (final Map.Entry> entry : + databaseToQueryParamsMap.entrySet()) { + if (entry.getValue() == null) { + return null; + } + final Map> seriesSlots = new HashMap<>(); + for (final DataPartitionQueryParam param : entry.getValue()) { + if (param.getDeviceID() == null) { + return null; + } + seriesSlots.put( + partitionExecutor.getSeriesPartitionSlot(param.getDeviceID()), + param.getTimePartitionSlotList()); + } + querySlots.put(entry.getKey(), seriesSlots); + } + return getDataPartitionBySlots(querySlots); + } + + public DataPartition getDataPartition( + final String database, + final Set seriesPartitionSlots, + final List timePartitionSlots) { + final Map> querySlots = new HashMap<>(); + for (final TSeriesPartitionSlot seriesPartitionSlot : seriesPartitionSlots) { + querySlots.put(seriesPartitionSlot, timePartitionSlots); + } + return getDataPartitionBySlots(Collections.singletonMap(database, querySlots)); + } + + private DataPartition getDataPartitionBySlots( + final Map>> querySlots) { dataPartitionCacheLock.readLock().lock(); try { failIfMetadataLeaseFenced(); - if (databaseToQueryParamsMap.isEmpty()) { + if (querySlots.isEmpty()) { cacheMetrics.record(false, CacheMetrics.DATA_PARTITION_CACHE_NAME); return null; } - final Set allConsensusGroupIds = new HashSet<>(); final Map> consensusGroupToTimeSlotMap = new HashMap<>(); - - for (Map.Entry> entry : - databaseToQueryParamsMap.entrySet()) { - String databaseName = entry.getKey(); - List params = entry.getValue(); - - if (null == params || params.isEmpty()) { + for (final Map.Entry>> + databaseEntry : querySlots.entrySet()) { + final DataPartitionTable dataPartitionTable = + dataPartitionCache.getIfPresent(databaseEntry.getKey()); + if (dataPartitionTable == null || databaseEntry.getValue().isEmpty()) { cacheMetrics.record(false, CacheMetrics.DATA_PARTITION_CACHE_NAME); return null; } - - DataPartitionTable dataPartitionTable = dataPartitionCache.getIfPresent(databaseName); - if (null == dataPartitionTable) { - if (logger.isDebugEnabled()) { - logger.debug( - DataNodeQueryMessages.ARG_CACHE_MISS_WHEN_SEARCH_DATABASE_ARG, - CacheMetrics.DATA_PARTITION_CACHE_NAME, - databaseName); - } - cacheMetrics.record(false, CacheMetrics.DATA_PARTITION_CACHE_NAME); - return null; - } - - Map cachedDatabasePartitionMap = - dataPartitionTable.getDataPartitionMap(); - - for (DataPartitionQueryParam param : params) { - TSeriesPartitionSlot seriesPartitionSlot; - if (null != param.getDeviceID()) { - seriesPartitionSlot = partitionExecutor.getSeriesPartitionSlot(param.getDeviceID()); - } else { - return null; - } - - SeriesPartitionTable cachedSeriesPartitionTable = - cachedDatabasePartitionMap.get(seriesPartitionSlot); - if (null == cachedSeriesPartitionTable) { - if (logger.isDebugEnabled()) { - logger.debug( - DataNodeQueryMessages.ARG_CACHE_MISS_WHEN_SEARCH_DEVICE_ARG, - CacheMetrics.DATA_PARTITION_CACHE_NAME, - param.getDeviceID()); - } + for (final Map.Entry> seriesEntry : + databaseEntry.getValue().entrySet()) { + final SeriesPartitionTable cachedSeriesPartitionTable = + dataPartitionTable.getDataPartitionMap().get(seriesEntry.getKey()); + if (cachedSeriesPartitionTable == null || seriesEntry.getValue().isEmpty()) { cacheMetrics.record(false, CacheMetrics.DATA_PARTITION_CACHE_NAME); return null; } - - Map> cachedTimePartitionSlot = - cachedSeriesPartitionTable.getSeriesPartitionMap(); - - if (param.getTimePartitionSlotList().isEmpty()) { - return null; - } - - for (TTimePartitionSlot timePartitionSlot : param.getTimePartitionSlotList()) { - List cacheConsensusGroupIds = - cachedTimePartitionSlot.get(timePartitionSlot); - if (null == cacheConsensusGroupIds - || cacheConsensusGroupIds.isEmpty() - || null == timePartitionSlot) { - if (logger.isDebugEnabled()) { - logger.debug( - DataNodeQueryMessages.ARG_CACHE_MISS_WHEN_SEARCH_TIME_PARTITION_ARG, - CacheMetrics.DATA_PARTITION_CACHE_NAME, - timePartitionSlot); - } + for (final TTimePartitionSlot timePartitionSlot : seriesEntry.getValue()) { + final List cachedConsensusGroupIds = + cachedSeriesPartitionTable.getSeriesPartitionMap().get(timePartitionSlot); + if (cachedConsensusGroupIds == null || cachedConsensusGroupIds.isEmpty()) { cacheMetrics.record(false, CacheMetrics.DATA_PARTITION_CACHE_NAME); return null; } - - for (TConsensusGroupId groupId : cacheConsensusGroupIds) { + for (final TConsensusGroupId groupId : cachedConsensusGroupIds) { allConsensusGroupIds.add(groupId); consensusGroupToTimeSlotMap - .computeIfAbsent(groupId, k -> new HashSet<>()) + .computeIfAbsent(groupId, key -> new HashSet<>()) .add( - new TimeSlotRegionInfo(databaseName, seriesPartitionSlot, timePartitionSlot)); + new TimeSlotRegionInfo( + databaseEntry.getKey(), seriesEntry.getKey(), timePartitionSlot)); } } } @@ -967,23 +954,19 @@ public DataPartition getDataPartition( final List consensusGroupIds = new ArrayList<>(allConsensusGroupIds); final List allRegionReplicaSets = getRegionReplicaSet(consensusGroupIds); - - Map>>> + final Map>>> dataPartitionMap = new HashMap<>(); - for (int i = 0; i < allRegionReplicaSets.size(); i++) { - TConsensusGroupId groupId = consensusGroupIds.get(i); - TRegionReplicaSet replicaSet = allRegionReplicaSets.get(i); - - for (TimeSlotRegionInfo info : consensusGroupToTimeSlotMap.get(groupId)) { + final TConsensusGroupId groupId = consensusGroupIds.get(i); + final TRegionReplicaSet replicaSet = allRegionReplicaSets.get(i); + for (final TimeSlotRegionInfo info : consensusGroupToTimeSlotMap.get(groupId)) { dataPartitionMap - .computeIfAbsent(info.databaseName, k -> new HashMap<>()) - .computeIfAbsent(info.seriesPartitionSlot, k -> new HashMap<>()) - .computeIfAbsent(info.timePartitionSlot, k -> new ArrayList<>()) + .computeIfAbsent(info.databaseName, key -> new HashMap<>()) + .computeIfAbsent(info.seriesPartitionSlot, key -> new HashMap<>()) + .computeIfAbsent(info.timePartitionSlot, key -> new ArrayList<>()) .add(replicaSet); } } - if (logger.isDebugEnabled()) { logger.debug(DataNodeQueryMessages.CACHE_HIT, CacheMetrics.DATA_PARTITION_CACHE_NAME); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java index 4701e0e9c62b..1b5e6ab693b1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/DeviceEntry.java @@ -29,6 +29,7 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; +import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; @@ -98,6 +99,14 @@ public void serialize(final DataOutputStream stream) throws IOException { stream); } + public byte[] serializeToBytes() throws IOException { + final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + try (DataOutputStream output = new DataOutputStream(byteStream)) { + serialize(output); + } + return byteStream.toByteArray(); + } + public static DeviceEntry deserialize(final ByteBuffer byteBuffer) { final IDeviceID iDeviceID = StringArrayDeviceID.deserialize(byteBuffer); int size = readInt(byteBuffer); @@ -109,6 +118,10 @@ public static DeviceEntry deserialize(final ByteBuffer byteBuffer) { return constructDeviceEntry(iDeviceID, attributeColumnValues, readInt(byteBuffer)); } + public static DeviceEntry deserialize(final byte[] bytes) { + return deserialize(ByteBuffer.wrap(bytes)); + } + public static void serializeBinary(final ByteBuffer byteBuffer, final Binary binary) { if (binary == null) { write(-1, byteBuffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java index 8b7f0f32656b..04db571083f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java @@ -20,11 +20,13 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata; import org.apache.iotdb.calc.plan.relational.metadata.ITypeMetadata; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.queryengine.common.SessionInfo; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.queryengine.plan.relational.function.ITableFunctionFactory; import org.apache.iotdb.commons.queryengine.plan.relational.function.OperatorType; import org.apache.iotdb.commons.queryengine.plan.relational.metadata.QualifiedObjectName; @@ -37,13 +39,14 @@ import org.apache.iotdb.db.exception.load.LoadAnalyzeTableColumnDisorderException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSet; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSetResult; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.common.type.Type; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; @@ -86,11 +89,12 @@ default boolean isWindowFunction( * index scanning * @param attributeColumns attribute column names */ - Map> indexScan( + DeviceEntryDataSetResult indexScan( final QualifiedObjectName tableName, final List expressionList, final List attributeColumns, - final MPPQueryContext context); + final MPPQueryContext context, + final PlanNodeId planNodeId); /** * This method is used for table column validation and should be invoked before device validation. @@ -203,6 +207,11 @@ SchemaPartition getOrCreateSchemaPartition( DataPartition getDataPartition( final String database, final List sgNameToQueryParamsMap); + DataPartition getDataPartition( + final String database, + final DeviceEntryDataSet dataSet, + final List timePartitionSlots); + /** * Get data partition, used in query scenarios which contains time filter like: time < XX or time * > XX @@ -212,4 +221,11 @@ DataPartition getDataPartition( */ DataPartition getDataPartitionWithUnclosedTimeRange( final String database, final List sgNameToQueryParamsMap); + + DataPartition getDataPartitionWithUnclosedTimeRange( + final String database, + final DeviceEntryDataSet dataSet, + final List timePartitionSlots, + final boolean needLeftAll, + final boolean needRightAll); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 9e736a35c581..89a53d18bd43 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -21,11 +21,13 @@ import org.apache.iotdb.calc.plan.relational.metadata.CommonMetadataUtils; import org.apache.iotdb.calc.utils.constant.SqlConstant; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.queryengine.common.SessionInfo; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.queryengine.plan.relational.function.OperatorType; import org.apache.iotdb.commons.queryengine.plan.relational.function.TableFunctionFactory; import org.apache.iotdb.commons.queryengine.plan.relational.function.arithmetic.AdditionResolver; @@ -57,6 +59,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaValidator; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSet; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSetResult; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.table.ITableCache; @@ -75,7 +79,6 @@ import java.util.Collections; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -1439,24 +1442,6 @@ && isNumericType(argumentTypes.get(0)) functionName)); } break; - case SqlConstant.IRATE: - validateRateFunctionArguments( - functionName, - argumentTypes, - 2, - DataNodeQueryMessages - .EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_2_ARGUMENTS_VALUE_TIME_E2F55C08); - break; - case SqlConstant.RATE: - case SqlConstant.INCREASE: - case SqlConstant.DELTA: - validateRateFunctionArguments( - functionName, - argumentTypes, - 4, - DataNodeQueryMessages - .EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_4_ARGUMENTS_VALUE_TIME_WINDOW_START_WINDOW_END_FBEC794B); - break; case SqlConstant.COUNT: break; default: @@ -1498,10 +1483,6 @@ && isNumericType(argumentTypes.get(0)) case SqlConstant.REGR_INTERCEPT: case SqlConstant.SKEWNESS: case SqlConstant.KURTOSIS: - case SqlConstant.RATE: - case SqlConstant.INCREASE: - case SqlConstant.IRATE: - case SqlConstant.DELTA: return DOUBLE; case SqlConstant.APPROX_MOST_FREQUENT: return STRING; @@ -1631,33 +1612,6 @@ && isNumericType(argumentTypes.get(0)) throw new SemanticException(DataNodeQueryMessages.UNKNOWN_FUNCTION + functionName); } - private static void validateRateFunctionArguments( - String functionName, - List argumentTypes, - int expectedArgumentCount, - String argumentCountError) { - if (argumentTypes.size() != expectedArgumentCount) { - throw new SemanticException(String.format(argumentCountError, functionName)); - } - if (!CommonMetadataUtils.isSupportedMathNumericType(argumentTypes.get(0))) { - throw new SemanticException( - String.format( - DataNodeQueryMessages - .EXCEPTION_AGGREGATE_FUNCTION_ARG_ONLY_SUPPORTS_INT32_INT64_FLOAT_AND_DOUBLE_AS_THE_FIRST_ARGUMENT_8D201434, - functionName)); - } - for (int i = 1; i < argumentTypes.size(); i++) { - Type argumentType = argumentTypes.get(i); - if (!INT64.equals(argumentType) && !TIMESTAMP.equals(argumentType)) { - throw new SemanticException( - String.format( - DataNodeQueryMessages - .EXCEPTION_THE_TIME_ARGUMENTS_OF_AGGREGATE_FUNCTION_ARG_SHOULD_BE_TIMESTAMP_OR_INT64_TYPE_9C736DE3, - functionName)); - } - } - } - @Override public boolean isAggregationFunction( final SessionInfo session, final String functionName, final AccessControl accessControl) { @@ -1677,18 +1631,20 @@ public boolean canCoerce(final Type from, final Type to) { } @Override - public Map> indexScan( + public DeviceEntryDataSetResult indexScan( final QualifiedObjectName tableName, final List expressionList, final List attributeColumns, - final MPPQueryContext context) { + final MPPQueryContext context, + final PlanNodeId planNodeId) { return TableDeviceSchemaFetcher.getInstance() - .fetchDeviceSchemaForDataQuery( + .fetchDeviceSchemaForDataQueryAsDataSet( tableName.getDatabaseName(), tableName.getObjectName(), expressionList, attributeColumns, - context); + context, + planNodeId); } @Override @@ -1764,10 +1720,29 @@ public DataPartition getDataPartition( Collections.singletonMap(database, sgNameToQueryParamsMap)); } + @Override + public DataPartition getDataPartition( + final String database, + final DeviceEntryDataSet dataSet, + final List timePartitionSlots) { + return partitionFetcher.getDataPartition(database, dataSet, timePartitionSlots); + } + @Override public DataPartition getDataPartitionWithUnclosedTimeRange( String database, List sgNameToQueryParamsMap) { return partitionFetcher.getDataPartitionWithUnclosedTimeRange( Collections.singletonMap(database, sgNameToQueryParamsMap)); } + + @Override + public DataPartition getDataPartitionWithUnclosedTimeRange( + final String database, + final DeviceEntryDataSet dataSet, + final List timePartitionSlots, + final boolean needLeftAll, + final boolean needRightAll) { + return partitionFetcher.getDataPartitionWithUnclosedTimeRange( + database, dataSet, timePartitionSlots, needLeftAll, needRightAll); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java index 024b9ef44ac8..1ac6e1789a4f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java @@ -22,6 +22,8 @@ import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.iotdb.commons.exception.QueryTimeoutException; +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.commons.schema.filter.SchemaFilter; @@ -48,6 +50,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.IDeviceSchema; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceNormalSchema; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSet; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSetResult; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryMaterializer; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AbstractTraverseDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice; @@ -64,6 +69,8 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -238,7 +245,7 @@ public Map> fetchDeviceSchemaForDataQuery( mayContainDuplicateDevice, false)) { fetchMissingDeviceSchemaForQuery( - database, tableInstance, attributeColumns, statement, deviceEntryMap, queryContext); + database, tableInstance, attributeColumns, statement, deviceEntryMap, null, queryContext); } // TODO table metadata: implement deduplicate during schemaRegion execution @@ -253,6 +260,129 @@ public Map> fetchDeviceSchemaForDataQuery( : deviceEntryMap; } + public DeviceEntryDataSetResult fetchDeviceSchemaForDataQueryAsDataSet( + final String database, + final String table, + final List expressionList, + final List attributeColumns, + final MPPQueryContext queryContext, + final PlanNodeId planNodeId) { + final TsTable tableInstance = DataNodeTableCache.getInstance().getTable(database, table); + if (TreeViewSchema.isTreeViewTable(tableInstance)) { + final Map> deviceEntryMap = new HashMap<>(); + final AtomicBoolean mayContainDuplicateDevice = new AtomicBoolean(false); + boolean containsNonAlignedDevice = false; + final ShowDevice statement = new ShowDevice(database, table); + try (DeviceEntryMaterializer materializer = + new DeviceEntryMaterializer( + queryContext.getQueryId().getId(), + planNodeId, + CONFIG.getTableQueryDeviceEntryBatchSizeInBytes(), + true, + queryContext)) { + final boolean needRemoteFetch = + parseFilter4TraverseDevice( + tableInstance, + expressionList, + statement, + deviceEntryMap, + attributeColumns, + queryContext, + mayContainDuplicateDevice, + false); + for (List entries : deviceEntryMap.values()) { + for (DeviceEntry entry : entries) { + appendToMaterializer(materializer, entry, queryContext, true); + if (entry instanceof NonAlignedDeviceEntry) { + containsNonAlignedDevice = true; + } + } + entries.clear(); + } + if (needRemoteFetch) { + containsNonAlignedDevice |= + fetchMissingDeviceSchemaForQuery( + database, + tableInstance, + attributeColumns, + statement, + deviceEntryMap, + materializer, + queryContext); + } + if (deviceEntryMap.size() > 1) { + throw new SemanticException( + DataNodeQueryMessages.TREE_DEVICE_VIEW_WITH_MULTIPLE_DATABASES + + deviceEntryMap.keySet() + + DataNodeQueryMessages.IS_UNSUPPORTED_YET); + } + final String resultDatabase = + deviceEntryMap.isEmpty() ? null : deviceEntryMap.keySet().iterator().next(); + return new DeviceEntryDataSetResult( + resultDatabase, materializer.finish(), containsNonAlignedDevice); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + final Map> cachedEntries = new HashMap<>(); + cachedEntries.put(database, new ArrayList<>()); + final AtomicBoolean mayContainDuplicateDevice = new AtomicBoolean(false); + final ShowDevice statement = new ShowDevice(database, table); + final boolean needRemoteFetch = + parseFilter4TraverseDevice( + tableInstance, + expressionList, + statement, + cachedEntries, + attributeColumns, + queryContext, + mayContainDuplicateDevice, + false); + + if (mayContainDuplicateDevice.get()) { + if (needRemoteFetch) { + fetchMissingDeviceSchemaForQuery( + database, + tableInstance, + attributeColumns, + statement, + cachedEntries, + null, + queryContext); + } + cachedEntries.put( + database, new ArrayList<>(new LinkedHashSet<>(cachedEntries.get(database)))); + } + + try (DeviceEntryMaterializer materializer = + new DeviceEntryMaterializer( + queryContext.getQueryId().getId(), + planNodeId, + CONFIG.getTableQueryDeviceEntryBatchSizeInBytes(), + true, + queryContext)) { + for (DeviceEntry entry : cachedEntries.get(database)) { + appendToMaterializer(materializer, entry, queryContext, true); + } + cachedEntries.get(database).clear(); + if (needRemoteFetch && !mayContainDuplicateDevice.get()) { + fetchMissingDeviceSchemaForQuery( + database, + tableInstance, + attributeColumns, + statement, + cachedEntries, + materializer, + queryContext); + } + final DeviceEntryDataSet dataSet = materializer.finish(); + return new DeviceEntryDataSetResult(database, dataSet, false); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + // Used by show/count device and update device. // Update / Delete device will not access cache public boolean parseFilter4TraverseDevice( @@ -489,14 +619,16 @@ public static IDeviceID convertTagValuesToDeviceID( return IDeviceID.Factory.DEFAULT_FACTORY.create(deviceIdNodes); } - private void fetchMissingDeviceSchemaForQuery( + private boolean fetchMissingDeviceSchemaForQuery( final String database, final TsTable tableInstance, final List attributeColumns, final ShowDevice statement, final Map> deviceEntryMap, + final DeviceEntryMaterializer materializer, final MPPQueryContext mppQueryContext) { Throwable t = null; + boolean containsNonAlignedDevice = false; final long queryId = SessionManager.getInstance().requestQueryId(); // For the correctness of attribute remote update @@ -567,10 +699,17 @@ private void fetchMissingDeviceSchemaForQuery( statement, mppQueryContext, attributeColumns, - deviceEntryMap.get(database)); + deviceEntryMap.get(database), + materializer); } else { - constructTreeResults( - tsBlock.get(), columnHeaderList, tableInstance, mppQueryContext, deviceEntryMap); + containsNonAlignedDevice |= + constructTreeResults( + tsBlock.get(), + columnHeaderList, + tableInstance, + mppQueryContext, + deviceEntryMap, + materializer); } } } else { @@ -583,6 +722,7 @@ private void fetchMissingDeviceSchemaForQuery( TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode()); } } + return containsNonAlignedDevice; } catch (final Throwable throwable) { t = throwable; throw throwable; @@ -602,7 +742,8 @@ private void constructTableResults( final ShowDevice statement, final MPPQueryContext mppQueryContext, final List attributeColumns, - final List deviceEntryList) { + final List deviceEntryList, + final DeviceEntryMaterializer materializer) { final Column[] columns = tsBlock.getValueColumns(); for (int i = 0; i < tsBlock.getPositionCount(); i++) { final String[] nodes = new String[tableInstance.getTagNum() + 1]; @@ -619,22 +760,48 @@ private void constructTableResults( final AlignedDeviceEntry deviceEntry = new AlignedDeviceEntry( deviceID, attributeColumns.stream().map(attributeMap::get).toArray(Binary[]::new)); - mppQueryContext.reserveMemoryForFrontEnd(deviceEntry.ramBytesUsed()); - deviceEntryList.add(deviceEntry); + if (materializer == null) { + mppQueryContext.reserveMemoryForFrontEnd(deviceEntry.ramBytesUsed()); + deviceEntryList.add(deviceEntry); + } else { + appendToMaterializer(materializer, deviceEntry, mppQueryContext, false); + } // Only cache those exact device query - // Fetch paths is null iff there are fuzzy queries related to tag columns + // Fetch paths is null iff there are fuzzy queries related to id columns if (Objects.nonNull(statement.getPartitionKeyList())) { cache.putAttributes(statement.getDatabase(), deviceID, attributeMap); } } } - private void constructTreeResults( + private static void appendToMaterializer( + DeviceEntryMaterializer materializer, + DeviceEntry deviceEntry, + MPPQueryContext queryContext, + boolean memoryAlreadyReserved) { + try { + long releasedRamBytes = materializer.appendWithMemoryControl(deviceEntry); + if (releasedRamBytes > 0) { + queryContext.releaseMemoryReservedForFrontEnd(releasedRamBytes); + } + if (memoryAlreadyReserved && materializer.isSpilled()) { + queryContext.releaseMemoryReservedForFrontEnd(deviceEntry.ramBytesUsed()); + } else if (!memoryAlreadyReserved && !materializer.isSpilled()) { + queryContext.reserveMemoryForFrontEnd(deviceEntry.ramBytesUsed()); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private boolean constructTreeResults( final TsBlock tsBlock, final List columnHeaderList, final TsTable tableInstance, final MPPQueryContext mppQueryContext, - final Map> deviceEntryMap) { + final Map> deviceEntryMap, + final DeviceEntryMaterializer materializer) { + boolean containsNonAlignedDevice = false; final Column[] columns = tsBlock.getValueColumns(); for (int i = 0; i < tsBlock.getPositionCount(); i++) { final String[] nodes = new String[tableInstance.getTagNum()]; @@ -646,13 +813,19 @@ private void constructTreeResults( columns[columns.length - 2].getBoolean(i) ? new AlignedDeviceEntry(deviceID, new Binary[0]) : new NonAlignedDeviceEntry(deviceID, new Binary[0]); - mppQueryContext.reserveMemoryForFrontEnd(deviceEntry.ramBytesUsed()); - deviceEntryMap - .computeIfAbsent( + containsNonAlignedDevice |= deviceEntry instanceof NonAlignedDeviceEntry; + final List deviceEntries = + deviceEntryMap.computeIfAbsent( columns[columns.length - 1].getBinary(i).getStringValue(TSFileConfig.STRING_CHARSET), - k -> new ArrayList<>()) - .add(deviceEntry); + k -> new ArrayList<>()); + if (materializer == null) { + mppQueryContext.reserveMemoryForFrontEnd(deviceEntry.ramBytesUsed()); + deviceEntries.add(deviceEntry); + } else { + appendToMaterializer(materializer, deviceEntry, mppQueryContext, false); + } } + return containsNonAlignedDevice; } private void constructNodesArrayAndAttributeMap( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/AbstractDeviceEntryMaterializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/AbstractDeviceEntryMaterializer.java new file mode 100644 index 000000000000..92886b0036a6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/AbstractDeviceEntryMaterializer.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import org.apache.tsfile.external.commons.io.FileUtils; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +public abstract class AbstractDeviceEntryMaterializer implements AutoCloseable { + + private final String queryId; + private final PlanNodeId planNodeId; + private final long thresholdInBytes; + private final List bufferedEntries = new ArrayList<>(); + + private int entryCount; + private Path ownerDirectory; + private boolean ownerRegistered; + private boolean finished; + private DeviceEntryIOContext ioContext; + private MPPQueryContext queryContext; + + protected AbstractDeviceEntryMaterializer( + String queryId, PlanNodeId planNodeId, long thresholdInBytes) { + if (thresholdInBytes <= 0) { + throw new IllegalArgumentException(); + } + this.queryId = queryId; + this.planNodeId = planNodeId; + this.thresholdInBytes = thresholdInBytes; + } + + /** + * Appends a DeviceEntry to this materializer's in-memory buffer. Spill decisions are external. + */ + public abstract void append(DeviceEntry entry) throws IOException; + + public abstract DeviceEntryDataSet finish() throws IOException; + + protected final String queryId() { + return queryId; + } + + protected final long thresholdInBytes() { + return thresholdInBytes; + } + + protected final void appendToBuffer(DeviceEntry entry) { + bufferedEntries.add(entry); + entryCount++; + } + + protected final void incrementEntryCount() { + entryCount++; + } + + protected final Iterable bufferedEntries() { + return bufferedEntries; + } + + protected final boolean isBufferEmpty() { + return bufferedEntries.isEmpty(); + } + + protected final List copyBufferedEntries() { + return new ArrayList<>(bufferedEntries); + } + + protected final void sortBufferedEntries(Comparator comparator) { + bufferedEntries.sort(comparator); + } + + public abstract void forceSpill() throws IOException; + + protected final void setQueryContext(MPPQueryContext queryContext) { + this.queryContext = queryContext; + } + + protected final DeviceEntryIOContext ioContext() { + return ioContext; + } + + protected final DeviceEntryIOContext createIOContextOnSpill(boolean duringFetchSchema) { + if (ioContext == null && queryContext != null) { + ioContext = queryContext.getOrCreateDeviceEntryIOContext(duringFetchSchema); + } + return ioContext; + } + + protected final int entryCount() { + return entryCount; + } + + protected final void clearBuffer() { + bufferedEntries.clear(); + } + + protected final Path ownerDirectory() { + return ownerDirectory; + } + + protected final Path ensureOwnerDirectory() throws IOException { + if (ownerDirectory == null) { + ownerDirectory = DeviceEntrySpillManager.getInstance().register(queryId, planNodeId); + ownerRegistered = true; + } + return ownerDirectory; + } + + protected final void checkNotFinished() { + if (finished) { + throw new IllegalStateException(); + } + } + + public MPPQueryContext getQueryContext() { + return queryContext; + } + + protected final void markFinished() { + finished = true; + } + + protected final void cleanupOwnerDirectory() throws IOException { + if (ownerDirectory != null) { + if (ownerRegistered) { + DeviceEntrySpillManager.getInstance().deregisterOwner(queryId, ownerDirectory); + } else { + FileUtils.deleteDirectory(ownerDirectory.toFile()); + } + ownerDirectory = null; + ownerRegistered = false; + } + } + + @Override + public void close() throws IOException { + if (!finished) { + cleanupOwnerDirectory(); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/BatchDeviceEntrySource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/BatchDeviceEntrySource.java new file mode 100644 index 000000000000..72f00e7b4865 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/BatchDeviceEntrySource.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import java.io.IOException; +import java.util.List; + +public interface BatchDeviceEntrySource extends AutoCloseable { + + boolean hasNextBatch(); + + List nextBatch() throws IOException; + + @Override + void close() throws IOException; +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryDataSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryDataSet.java new file mode 100644 index 000000000000..9642acdb0041 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryDataSet.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import java.io.IOException; +import java.util.List; + +public interface DeviceEntryDataSet extends AutoCloseable { + + int getEntryCount(); + + boolean isSpilled(); + + DeviceEntryReader openReader() throws IOException; + + default DeviceEntryReader openConsumingReader() throws IOException { + throw new UnsupportedOperationException("Open consuming reader is not supported"); + } + + default List getInlineEntries() { + throw new UnsupportedOperationException( + "Only InMemoryDeviceEntryDataSet supports get inline device entries"); + } + + @Override + void close() throws IOException; +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryDataSetHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryDataSetHandle.java new file mode 100644 index 000000000000..f295cd00a3b0 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryDataSetHandle.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public final class DeviceEntryDataSetHandle { + + private final String queryId; + private final PlanNodeId planNodeId; + private final TEndPoint coordinatorEndPoint; + private final int segmentCount; + private final int entryCount; + private final boolean ordered; + + public DeviceEntryDataSetHandle( + String queryId, + PlanNodeId planNodeId, + TEndPoint coordinatorEndPoint, + int segmentCount, + int entryCount, + boolean ordered) { + this.queryId = queryId; + this.planNodeId = planNodeId; + this.coordinatorEndPoint = coordinatorEndPoint; + this.segmentCount = segmentCount; + this.entryCount = entryCount; + this.ordered = ordered; + } + + public String getQueryId() { + return queryId; + } + + public PlanNodeId getPlanNodeId() { + return planNodeId; + } + + public TEndPoint getCoordinatorEndPoint() { + return coordinatorEndPoint; + } + + public int getSegmentCount() { + return segmentCount; + } + + public int getEntryCount() { + return entryCount; + } + + public boolean isOrdered() { + return ordered; + } + + public void serialize(ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(queryId, byteBuffer); + ReadWriteIOUtils.write(planNodeId.getId(), byteBuffer); + ThriftCommonsSerDeUtils.serializeTEndPoint(coordinatorEndPoint, byteBuffer); + ReadWriteIOUtils.write(segmentCount, byteBuffer); + ReadWriteIOUtils.write(entryCount, byteBuffer); + ReadWriteIOUtils.write(ordered, byteBuffer); + } + + public void serialize(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(queryId, stream); + ReadWriteIOUtils.write(planNodeId.getId(), stream); + ThriftCommonsSerDeUtils.serializeTEndPoint(coordinatorEndPoint, stream); + ReadWriteIOUtils.write(segmentCount, stream); + ReadWriteIOUtils.write(entryCount, stream); + ReadWriteIOUtils.write(ordered, stream); + } + + public static DeviceEntryDataSetHandle deserialize(ByteBuffer byteBuffer) { + return new DeviceEntryDataSetHandle( + ReadWriteIOUtils.readString(byteBuffer), + new PlanNodeId(ReadWriteIOUtils.readString(byteBuffer)), + ThriftCommonsSerDeUtils.deserializeTEndPoint(byteBuffer), + ReadWriteIOUtils.readInt(byteBuffer), + ReadWriteIOUtils.readInt(byteBuffer), + ReadWriteIOUtils.readBool(byteBuffer)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryDataSetResult.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryDataSetResult.java new file mode 100644 index 000000000000..7faed3c5a93f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryDataSetResult.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +public final class DeviceEntryDataSetResult { + + private final String database; + private final DeviceEntryDataSet dataSet; + private final boolean containsNonAlignedDevice; + + public DeviceEntryDataSetResult( + String database, DeviceEntryDataSet dataSet, boolean containsNonAlignedDevice) { + this.database = database; + this.dataSet = dataSet; + this.containsNonAlignedDevice = containsNonAlignedDevice; + } + + public String getDatabase() { + return database; + } + + public DeviceEntryDataSet getDataSet() { + return dataSet; + } + + public boolean containsNonAlignedDevice() { + return containsNonAlignedDevice; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryDiskSpiller.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryDiskSpiller.java new file mode 100644 index 000000000000..a7bcdfbcc387 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryDiskSpiller.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import java.io.BufferedOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; + +public final class DeviceEntryDiskSpiller implements AutoCloseable { + + private final Path directory; + private final long targetSegmentBytes; + private final DeviceEntryIOContext ioContext; + private final List sealedSegments = new ArrayList<>(); + + private DataOutputStream output; + private Path temporaryFile; + private long currentBytes; + private int nextSegmentId; + + public DeviceEntryDiskSpiller(Path directory, long targetSegmentBytes) throws IOException { + this(directory, targetSegmentBytes, null); + } + + public DeviceEntryDiskSpiller( + Path directory, long targetSegmentBytes, DeviceEntryIOContext ioContext) throws IOException { + this.directory = directory; + this.targetSegmentBytes = targetSegmentBytes; + this.ioContext = ioContext; + Files.createDirectories(directory); + } + + public void append(byte[] serializedEntry) throws IOException { + checkTimeout(); + long startNanos = System.nanoTime(); + int recordBytes = Integer.BYTES + serializedEntry.length; + if (currentBytes > 0 && currentBytes + recordBytes > targetSegmentBytes) { + sealCurrentSegment(); + } + ensureOutput(); + output.writeInt(serializedEntry.length); + output.write(serializedEntry); + recordDiskIO(recordBytes, startNanos); + currentBytes += recordBytes; + } + + public List finish() throws IOException { + sealCurrentSegment(); + return List.copyOf(sealedSegments); + } + + private void ensureOutput() throws IOException { + if (output != null) { + return; + } + temporaryFile = directory.resolve(String.format("segment-%06d.tmp", nextSegmentId)); + output = new DataOutputStream(new BufferedOutputStream(Files.newOutputStream(temporaryFile))); + currentBytes = 0; + } + + private void sealCurrentSegment() throws IOException { + if (output == null) { + return; + } + output.close(); + output = null; + Path sealedFile = directory.resolve(String.format("segment-%06d.bin", nextSegmentId++)); + try { + Files.move( + temporaryFile, + sealedFile, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + Files.move(temporaryFile, sealedFile, StandardCopyOption.REPLACE_EXISTING); + } + sealedSegments.add(sealedFile); + temporaryFile = null; + currentBytes = 0; + } + + private void checkTimeout() { + if (ioContext != null) { + ioContext.checkTimeout(); + } + } + + private void recordDiskIO(long bytes, long startNanos) { + if (ioContext != null) { + ioContext.recordDiskIO(bytes, startNanos); + } + } + + @Override + public void close() throws IOException { + if (output != null) { + output.close(); + output = null; + } + if (temporaryFile != null) { + Files.deleteIfExists(temporaryFile); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryFileSpillerReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryFileSpillerReader.java new file mode 100644 index 000000000000..935c9b51b592 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryFileSpillerReader.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.NoSuchElementException; + +public final class DeviceEntryFileSpillerReader implements DeviceEntryReader { + + private final List segments; + private final boolean deleteSegmentAfterRead; + private final DeviceEntryIOContext ioContext; + private int segmentIndex; + private DataInputStream input; + private Path currentSegment; + private DeviceEntry next; + + public DeviceEntryFileSpillerReader(List segments) { + this(segments, false, null); + } + + public DeviceEntryFileSpillerReader(List segments, boolean deleteSegmentAfterRead) { + this(segments, deleteSegmentAfterRead, null); + } + + public DeviceEntryFileSpillerReader( + List segments, boolean deleteSegmentAfterRead, DeviceEntryIOContext ioContext) { + this.segments = segments; + this.deleteSegmentAfterRead = deleteSegmentAfterRead; + this.ioContext = ioContext; + } + + @Override + public boolean hasNext() throws IOException { + if (next != null) { + return true; + } + while (true) { + if (input == null && !openNextSegment()) { + return false; + } + checkTimeout(); + long startNanos = System.nanoTime(); + Integer length = readRecordLength(); + if (length == null) { + closeCurrentSegment(true); + continue; + } + byte[] bytes = new byte[length]; + input.readFully(bytes); + if (ioContext != null) { + ioContext.recordDiskIO(Integer.BYTES + length, startNanos); + } + next = DeviceEntry.deserialize(bytes); + return true; + } + } + + private void checkTimeout() { + if (ioContext != null) { + ioContext.checkTimeout(); + } + } + + @Override + public DeviceEntry next() throws IOException { + if (!hasNext()) { + throw new NoSuchElementException(); + } + DeviceEntry result = next; + next = null; + return result; + } + + private boolean openNextSegment() throws IOException { + if (segmentIndex >= segments.size()) { + return false; + } + currentSegment = segments.get(segmentIndex++); + input = new DataInputStream(new BufferedInputStream(Files.newInputStream(currentSegment))); + return true; + } + + private void closeCurrentSegment(boolean fullyConsumed) throws IOException { + input.close(); + input = null; + if (fullyConsumed && deleteSegmentAfterRead) { + try { + Files.deleteIfExists(currentSegment); + } catch (IOException ignored) { + // Query cleanup will retry deleting a segment that could not be deleted eagerly. + } + } + currentSegment = null; + } + + private Integer readRecordLength() throws IOException { + int firstByte = input.read(); + if (firstByte < 0) { + return null; + } + int length = + (firstByte << 24) + | (input.readUnsignedByte() << 16) + | (input.readUnsignedByte() << 8) + | input.readUnsignedByte(); + if (length < 0) { + throw new IOException(); + } + return length; + } + + @Override + public void close() throws IOException { + if (input != null) { + closeCurrentSegment(false); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryIOContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryIOContext.java new file mode 100644 index 000000000000..5376478a8f88 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryIOContext.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; + +public final class DeviceEntryIOContext { + + private final MPPQueryContext queryContext; + private final boolean duringFetchSchema; + + public DeviceEntryIOContext(MPPQueryContext queryContext, boolean duringFetchSchema) { + this.queryContext = queryContext; + this.duringFetchSchema = duringFetchSchema; + } + + public void checkTimeout() { + queryContext.checkTimeOut(); + } + + public void recordDiskIO(long bytes, long startNanos) { + long timeCost = System.nanoTime() - startNanos; + if (duringFetchSchema) { + queryContext.recordDeviceEntryDiskIODuringFetchSchema(bytes, timeCost); + } + checkTimeout(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryMaterializationMemoryController.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryMaterializationMemoryController.java new file mode 100644 index 000000000000..ef991aadfa12 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryMaterializationMemoryController.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import java.io.IOException; +import java.util.IdentityHashMap; +import java.util.Map; + +/** Controls the total in-memory DeviceEntry buffers owned by all Region materializers. */ +public final class DeviceEntryMaterializationMemoryController { + + private final long memoryLimitInBytes; + private final Map retainedBytesByMaterializer = + new IdentityHashMap<>(); + private long retainedBytes; + + public DeviceEntryMaterializationMemoryController(long memoryLimitInBytes) { + if (memoryLimitInBytes <= 0) { + throw new IllegalArgumentException(); + } + this.memoryLimitInBytes = memoryLimitInBytes; + } + + public void append(AbstractDeviceEntryMaterializer materializer, DeviceEntry deviceEntry) + throws IOException { + materializer.append(deviceEntry); + long entryRamBytes = deviceEntry.ramBytesUsed(); + retainedBytesByMaterializer.merge(materializer, entryRamBytes, Long::sum); + retainedBytes += entryRamBytes; + enforceMemoryLimit(); + } + + public long getRetainedBytes() { + return retainedBytes; + } + + public long getMemoryLimitInBytes() { + return memoryLimitInBytes; + } + + private void enforceMemoryLimit() throws IOException { + while (retainedBytes > memoryLimitInBytes) { + AbstractDeviceEntryMaterializer largest = null; + long largestRetainedBytes = 0; + for (Map.Entry entry : + retainedBytesByMaterializer.entrySet()) { + if (entry.getValue() > largestRetainedBytes) { + largest = entry.getKey(); + largestRetainedBytes = entry.getValue(); + } + } + if (largest == null) { + throw new IllegalStateException(); + } + largest.forceSpill(); + retainedBytesByMaterializer.put(largest, 0L); + retainedBytes -= largestRetainedBytes; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryMaterializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryMaterializer.java new file mode 100644 index 000000000000..82f53bb2caf2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryMaterializer.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import java.io.IOException; +import java.nio.file.Path; + +public final class DeviceEntryMaterializer extends AbstractDeviceEntryMaterializer { + + private final boolean rawSegment; + private DeviceEntryDiskSpiller spiller; + // Only be used in fetchDeviceSchema, manages memory itself + private long rawBufferedRamBytes; + + public DeviceEntryMaterializer( + String queryId, PlanNodeId planNodeId, long thresholdInBytes, boolean rawSegment) { + super(queryId, planNodeId, thresholdInBytes); + this.rawSegment = rawSegment; + } + + public DeviceEntryMaterializer( + String queryId, + PlanNodeId planNodeId, + long thresholdInBytes, + boolean rawSegment, + MPPQueryContext queryContext) { + this(queryId, planNodeId, thresholdInBytes, rawSegment); + setQueryContext(queryContext); + } + + @Override + public void append(DeviceEntry entry) throws IOException { + checkNotFinished(); + appendToBuffer(entry); + } + + /** Returns the RAM bytes released when Coordinator Raw Fetch switches to spill mode. */ + public long appendWithMemoryControl(DeviceEntry entry) throws IOException { + checkNotFinished(); + long ramBytesUsed = entry.ramBytesUsed(); + if (spiller == null && rawBufferedRamBytes + ramBytesUsed <= thresholdInBytes()) { + appendToBuffer(entry); + rawBufferedRamBytes += ramBytesUsed; + return 0; + } + long releasedRamBytes = rawBufferedRamBytes; + ensureSpiller(); + rawBufferedRamBytes = 0; + spiller.append(entry.serializeToBytes()); + incrementEntryCount(); + return releasedRamBytes; + } + + @Override + public void forceSpill() throws IOException { + checkNotFinished(); + if (spiller == null && !isBufferEmpty()) { + ensureSpiller(); + } + rawBufferedRamBytes = 0; + } + + public boolean isSpilled() { + return spiller != null; + } + + @Override + public DeviceEntryDataSet finish() throws IOException { + checkNotFinished(); + DeviceEntryDataSet dataSet; + if (spiller == null) { + dataSet = new InMemoryDeviceEntryDataSet(copyBufferedEntries()); + } else { + dataSet = + new SpilledDeviceEntryDataSet( + queryId(), ownerDirectory(), spiller.finish(), entryCount()); + } + if (rawSegment && getQueryContext() != null) { + getQueryContext().recordDeviceEntryCount(entryCount()); + } + markFinished(); + return dataSet; + } + + private void ensureSpiller() throws IOException { + if (spiller != null) { + return; + } + Path ownerDirectory = ensureOwnerDirectory(); + if (rawSegment) { + createIOContextOnSpill(true); + } + spiller = + new DeviceEntryDiskSpiller( + ownerDirectory.resolve(rawSegment ? "raw" : "fi"), thresholdInBytes(), ioContext()); + for (DeviceEntry entry : bufferedEntries()) { + spiller.append(entry.serializeToBytes()); + } + clearBuffer(); + } + + @Override + public void close() throws IOException { + try { + if (spiller != null) { + spiller.close(); + } + } finally { + super.close(); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryReader.java new file mode 100644 index 000000000000..cf5410df8f8b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryReader.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import java.io.IOException; + +public interface DeviceEntryReader extends AutoCloseable { + + boolean hasNext() throws IOException; + + DeviceEntry next() throws IOException; + + @Override + void close() throws IOException; +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryRpcSegmentFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryRpcSegmentFetcher.java new file mode 100644 index 000000000000..90b33b53bc86 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryRpcSegmentFetcher.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient; +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeService; +import org.apache.iotdb.mpp.rpc.thrift.TFetchDeviceEntrySegmentReq; +import org.apache.iotdb.mpp.rpc.thrift.TFetchDeviceEntrySegmentResp; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.thrift.TException; + +import java.io.IOException; + +public final class DeviceEntryRpcSegmentFetcher implements DeviceEntrySegmentFetcher { + + private static final int MAX_ATTEMPTS = 3; + + private final IClientManager clientManager; + + private DeviceEntryRpcSegmentFetcher() { + this( + MPPDataExchangeService.getInstance() + .getMPPDataExchangeManager() + .getMppDataExchangeServiceClientManager()); + } + + @TestOnly + public DeviceEntryRpcSegmentFetcher( + IClientManager clientManager) { + this.clientManager = clientManager; + } + + public static DeviceEntryRpcSegmentFetcher getInstance() { + return DeviceEntryRpcSegmentFetcherHolder.INSTANCE; + } + + @Override + public byte[] fetch(DeviceEntryDataSetHandle handle, int segmentId) throws IOException { + IOException failure = null; + for (int attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { + TFetchDeviceEntrySegmentResp response; + try { + try (SyncDataNodeMPPDataExchangeServiceClient client = + clientManager.borrowClient(handle.getCoordinatorEndPoint())) { + response = client.fetchDeviceEntrySegment(createFetchRequest(handle, segmentId)); + } + } catch (ClientManagerException | TException e) { + failure = new IOException(e); + continue; + } + if (response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new IOException(response.getStatus().getMessage()); + } + return response.getPayload(); + } + throw failure; + } + + @Override + public void finish(DeviceEntryDataSetHandle handle) { + for (int attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { + try { + try (SyncDataNodeMPPDataExchangeServiceClient client = + clientManager.borrowClient(handle.getCoordinatorEndPoint())) { + org.apache.iotdb.common.rpc.thrift.TSStatus status = + client.finishDeviceEntrySegment(handle.getQueryId(), handle.getPlanNodeId().getId()); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return; + } + } + return; + } catch (ClientManagerException | TException e) { + // Cleanup notification is best effort and must not fail the query. + } + } + } + + private TFetchDeviceEntrySegmentReq createFetchRequest( + DeviceEntryDataSetHandle handle, int segmentId) { + return new TFetchDeviceEntrySegmentReq( + handle.getQueryId(), handle.getPlanNodeId().getId(), segmentId); + } + + private static class DeviceEntryRpcSegmentFetcherHolder { + private static final DeviceEntryRpcSegmentFetcher INSTANCE = new DeviceEntryRpcSegmentFetcher(); + + private DeviceEntryRpcSegmentFetcherHolder() {} + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntrySegmentFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntrySegmentFetcher.java new file mode 100644 index 000000000000..c66e46f2c124 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntrySegmentFetcher.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import java.io.IOException; + +public interface DeviceEntrySegmentFetcher { + + byte[] fetch(DeviceEntryDataSetHandle handle, int segmentId) throws IOException; + + void finish(DeviceEntryDataSetHandle handle); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntrySortedMaterializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntrySortedMaterializer.java new file mode 100644 index 000000000000..d135139f7009 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntrySortedMaterializer.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import org.apache.tsfile.external.commons.io.FileUtils; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.PriorityQueue; + +/** Materializes a sorted data set in memory or through sorted runs and a K-way merge. */ +public final class DeviceEntrySortedMaterializer extends AbstractDeviceEntryMaterializer { + + private static final int MAX_MERGE_FAN_IN = 32; + + private final Comparator comparator; + private final List> sortedRuns = new ArrayList<>(); + + private Path runDirectory; + + public DeviceEntrySortedMaterializer( + String queryId, + PlanNodeId planNodeId, + long bufferSizeInBytes, + Comparator comparator) { + super(queryId, planNodeId, bufferSizeInBytes); + this.comparator = comparator; + } + + public DeviceEntrySortedMaterializer( + String queryId, + PlanNodeId planNodeId, + long bufferSizeInBytes, + Comparator comparator, + MPPQueryContext queryContext) { + this(queryId, planNodeId, bufferSizeInBytes, comparator); + setQueryContext(queryContext); + } + + @Override + public void append(DeviceEntry entry) throws IOException { + checkNotFinished(); + appendToBuffer(entry); + } + + @Override + public void forceSpill() throws IOException { + checkNotFinished(); + flushRun(); + } + + @Override + public DeviceEntryDataSet finish() throws IOException { + checkNotFinished(); + if (entryCount() == 0) { + DeviceEntryDataSet dataSet = new InMemoryDeviceEntryDataSet(copyBufferedEntries()); + markFinished(); + return dataSet; + } + if (sortedRuns.isEmpty()) { + sortBufferedEntries(comparator); + DeviceEntryDataSet dataSet = new InMemoryDeviceEntryDataSet(copyBufferedEntries()); + markFinished(); + return dataSet; + } + + try { + flushRun(); + List> finalRuns = compactRuns(new ArrayList<>(sortedRuns)); + Path finalDirectory = ownerDirectory().resolve("fi"); + List finalSegments; + try (DeviceEntryDiskSpiller outputSpiller = + new DeviceEntryDiskSpiller(finalDirectory, thresholdInBytes(), ioContext())) { + if (finalRuns.size() == 1) { + copyRun(finalRuns.get(0), outputSpiller); + } else { + mergeRuns(finalRuns, outputSpiller); + } + finalSegments = outputSpiller.finish(); + } + DeviceEntryDataSet dataSet = + new SpilledDeviceEntryDataSet(queryId(), ownerDirectory(), finalSegments, entryCount()); + markFinished(); + deleteRunDirectoryBestEffort(); + return dataSet; + } catch (IOException | RuntimeException e) { + try { + cleanupOwnerDirectory(); + } catch (IOException cleanupException) { + e.addSuppressed(cleanupException); + } + throw e; + } + } + + private void flushRun() throws IOException { + if (isBufferEmpty()) { + return; + } + ensureSpillDirectory(); + sortBufferedEntries(comparator); + Path currentRunDirectory = runDirectory.resolve(String.format("run-%06d", sortedRuns.size())); + try (DeviceEntryDiskSpiller runSpiller = + new DeviceEntryDiskSpiller(currentRunDirectory, thresholdInBytes(), ioContext())) { + for (DeviceEntry entry : bufferedEntries()) { + runSpiller.append(entry.serializeToBytes()); + } + sortedRuns.add(runSpiller.finish()); + } + clearBuffer(); + } + + private void ensureSpillDirectory() throws IOException { + if (ownerDirectory() != null) { + return; + } + createIOContextOnSpill(false); + runDirectory = ensureOwnerDirectory().resolve("sort-run"); + } + + private void copyRun(List run, DeviceEntryDiskSpiller outputSpiller) throws IOException { + try (DeviceEntryFileSpillerReader reader = + new DeviceEntryFileSpillerReader(run, true, ioContext())) { + while (reader.hasNext()) { + outputSpiller.append(reader.next().serializeToBytes()); + } + } + } + + private void deleteRunDirectoryBestEffort() { + try { + FileUtils.deleteDirectory(runDirectory.toFile()); + } catch (IOException ignored) { + // Query cleanup removes the published data set and any remaining runs. + } + } + + private List> compactRuns(List> runs) throws IOException { + int level = 1; + while (runs.size() > MAX_MERGE_FAN_IN) { + List> nextRuns = new ArrayList<>(); + for (int from = 0, group = 0; from < runs.size(); from += MAX_MERGE_FAN_IN, group++) { + int to = Math.min(from + MAX_MERGE_FAN_IN, runs.size()); + List> runGroup = new ArrayList<>(runs.subList(from, to)); + if (runGroup.size() == 1) { + nextRuns.add(runGroup.get(0)); + continue; + } + Path outputDirectory = + runDirectory + .resolve(String.format("level-%06d", level)) + .resolve(String.format("run-%06d", group)); + try (DeviceEntryDiskSpiller outputSpiller = + new DeviceEntryDiskSpiller(outputDirectory, thresholdInBytes(), ioContext())) { + mergeRuns(runGroup, outputSpiller); + nextRuns.add(outputSpiller.finish()); + } + } + runs = nextRuns; + level++; + } + return runs; + } + + private void mergeRuns(List> runs, DeviceEntryDiskSpiller outputSpiller) + throws IOException { + List readers = new ArrayList<>(runs.size()); + PriorityQueue queue = + new PriorityQueue<>( + (left, right) -> { + int result = comparator.compare(left.entry, right.entry); + return result != 0 ? result : Integer.compare(left.readerIndex, right.readerIndex); + }); + Throwable failure = null; + try { + for (int i = 0; i < runs.size(); i++) { + DeviceEntryFileSpillerReader reader = + new DeviceEntryFileSpillerReader(runs.get(i), true, ioContext()); + readers.add(reader); + if (reader.hasNext()) { + queue.add(new MergeElement(reader.next(), i)); + } + } + while (!queue.isEmpty()) { + MergeElement element = queue.poll(); + outputSpiller.append(element.entry.serializeToBytes()); + DeviceEntryFileSpillerReader reader = readers.get(element.readerIndex); + if (reader.hasNext()) { + queue.add(new MergeElement(reader.next(), element.readerIndex)); + } + } + } catch (IOException | RuntimeException | Error e) { + failure = e; + throw e; + } finally { + IOException closeException = null; + for (DeviceEntryFileSpillerReader reader : readers) { + try { + reader.close(); + } catch (IOException e) { + if (closeException == null) { + closeException = e; + } else { + closeException.addSuppressed(e); + } + } + } + if (closeException != null) { + if (failure != null) { + failure.addSuppressed(closeException); + } else { + throw closeException; + } + } + } + } + + private static final class MergeElement { + private final DeviceEntry entry; + private final int readerIndex; + + private MergeElement(DeviceEntry entry, int readerIndex) { + this.entry = entry; + this.readerIndex = readerIndex; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntrySpillManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntrySpillManager.java new file mode 100644 index 000000000000..2b28c95ddafd --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntrySpillManager.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.conf.IoTDBDescriptor; + +import org.apache.tsfile.external.commons.io.FileUtils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +public final class DeviceEntrySpillManager { + + private final ConcurrentHashMap> queryDirectories = new ConcurrentHashMap<>(); + + private DeviceEntrySpillManager() {} + + public static DeviceEntrySpillManager getInstance() { + return DeviceEntrySpillManagerHolder.INSTANCE; + } + + public Path register(String queryId, PlanNodeId planNodeId) throws IOException { + Path ownerDirectory = resolveOwnerDirectory(queryId, planNodeId.getId()); + Files.createDirectories(ownerDirectory); + queryDirectories + .computeIfAbsent(queryId, ignored -> ConcurrentHashMap.newKeySet()) + .add(ownerDirectory); + return ownerDirectory; + } + + public void deregisterOwner(String queryId, Path ownerDirectory) throws IOException { + Set owners = queryDirectories.get(queryId); + if (owners != null) { + owners.remove(ownerDirectory); + if (owners.isEmpty()) { + queryDirectories.remove(queryId, owners); + } + } + FileUtils.deleteDirectory(ownerDirectory.toFile()); + } + + public void deregisterQuery(String queryId) throws IOException { + queryDirectories.remove(queryId); + FileUtils.deleteDirectory(resolveQueryDirectory(queryId).toFile()); + } + + @TestOnly + public List listSegments(String queryId, String planNodeId) throws IOException { + Path dataSetDirectory = resolveRegisteredDataSetDirectory(queryId, planNodeId); + try (java.util.stream.Stream stream = Files.list(dataSetDirectory)) { + return stream + .filter(path -> path.getFileName().toString().matches("segment-[0-9]{6,}\\.bin")) + .sorted( + Comparator.comparingInt((Path path) -> path.getFileName().toString().length()) + .thenComparing(path -> path.getFileName().toString())) + .collect(Collectors.toList()); + } + } + + public byte[] readSegment(String queryId, String dataSetId, int segmentId) throws IOException { + return Files.readAllBytes(resolveSegment(queryId, dataSetId, segmentId)); + } + + public Path resolveSegment(String queryId, String dataSetId, int segmentId) throws IOException { + Path segment = getRegisteredSegmentPath(queryId, dataSetId, segmentId); + if (!Files.isRegularFile(segment)) { + throw new java.nio.file.NoSuchFileException(segment.toString()); + } + return segment; + } + + public Path resolveSegment(String queryId, PlanNodeId planNodeId, int segmentId) + throws IOException { + return resolveSegment(queryId, planNodeId.getId(), segmentId); + } + + public void deleteSegment(String queryId, String dataSetId, int segmentId) throws IOException { + Files.deleteIfExists(getRegisteredSegmentPath(queryId, dataSetId, segmentId)); + } + + public void deleteSegment(String queryId, PlanNodeId planNodeId, int segmentId) + throws IOException { + deleteSegment(queryId, planNodeId.getId(), segmentId); + } + + public void finishSegmentDataSet(String queryId, String planNodeId) throws IOException { + Path ownerDirectory = resolveOwnerDirectory(queryId, planNodeId); + deregisterOwner(queryId, ownerDirectory); + } + + private Path resolveRegisteredDataSetDirectory(String queryId, String dataSetId) + throws IOException { + Path relativeDataSetPath = Path.of(dataSetId); + if (relativeDataSetPath.isAbsolute() + || java.util.stream.StreamSupport.stream(relativeDataSetPath.spliterator(), false) + .anyMatch(path -> path.toString().equals("..") || path.toString().equals("."))) { + throw new IllegalArgumentException(); + } + Path queryDirectory = resolveQueryDirectory(queryId); + Path dataSetDirectory = queryDirectory.resolve(relativeDataSetPath).resolve("fi").normalize(); + if (!dataSetDirectory.startsWith(queryDirectory)) { + throw new IllegalArgumentException(); + } + Set owners = queryDirectories.get(queryId); + boolean registered = + owners != null + && owners.stream() + .map(Path::normalize) + .anyMatch(owner -> dataSetDirectory.startsWith(owner) && Files.isDirectory(owner)); + if (!registered || !Files.isDirectory(dataSetDirectory)) { + throw new java.nio.file.NoSuchFileException(dataSetDirectory.toString()); + } + return dataSetDirectory; + } + + private Path getRegisteredSegmentPath(String queryId, String dataSetId, int segmentId) + throws IOException { + if (segmentId < 0) { + throw new IllegalArgumentException(); + } + return resolveRegisteredDataSetDirectory(queryId, dataSetId) + .resolve(String.format("segment-%06d.bin", segmentId)); + } + + public void clearStaleData() throws IOException { + FileUtils.deleteDirectory(rootDirectory().toFile()); + Files.createDirectories(rootDirectory()); + queryDirectories.clear(); + } + + private Path rootDirectory() { + return Path.of(IoTDBDescriptor.getInstance().getConfig().getSortTmpDir(), "device-entry"); + } + + private Path resolveOwnerDirectory(String queryId, String planNodeId) { + Path queryDirectory = resolveQueryDirectory(queryId); + Path ownerDirectory = queryDirectory.resolve(planNodeId).normalize(); + return ownerDirectory; + } + + private Path resolveQueryDirectory(String queryId) { + Path root = rootDirectory().normalize(); + Path queryDirectory = root.resolve(queryId).normalize(); + return queryDirectory; + } + + private static class DeviceEntrySpillManagerHolder { + private static final DeviceEntrySpillManager INSTANCE = new DeviceEntrySpillManager(); + + private DeviceEntrySpillManagerHolder() {} + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/InMemoryDeviceEntryDataSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/InMemoryDeviceEntryDataSet.java new file mode 100644 index 000000000000..7c5564fcbf12 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/InMemoryDeviceEntryDataSet.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +public final class InMemoryDeviceEntryDataSet implements DeviceEntryDataSet { + + private final List entries; + + public InMemoryDeviceEntryDataSet(List entries) { + this.entries = Collections.unmodifiableList(entries); + } + + @Override + public int getEntryCount() { + return entries.size(); + } + + @Override + public boolean isSpilled() { + return false; + } + + @Override + public DeviceEntryReader openReader() { + Iterator iterator = entries.iterator(); + return new DeviceEntryReader() { + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public DeviceEntry next() { + return iterator.next(); + } + + @Override + public void close() { + // No resource to release. + } + }; + } + + @Override + public List getInlineEntries() { + return entries; + } + + @Override + public void close() { + // The query context owns memory accounting for inline entries. + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/InMemoryDeviceEntrySource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/InMemoryDeviceEntrySource.java new file mode 100644 index 000000000000..8d67e760e9b3 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/InMemoryDeviceEntrySource.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import java.util.Collections; +import java.util.List; + +public final class InMemoryDeviceEntrySource implements BatchDeviceEntrySource { + + private List entries; + + public InMemoryDeviceEntrySource(List entries) { + this.entries = entries; + } + + @Override + public boolean hasNextBatch() { + return entries != null; + } + + @Override + public List nextBatch() { + if (entries == null) { + return Collections.emptyList(); + } + List result = entries; + entries = null; + return result; + } + + @Override + public void close() { + entries = null; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/LocalSegmentDeviceEntrySource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/LocalSegmentDeviceEntrySource.java new file mode 100644 index 000000000000..684b8806c699 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/LocalSegmentDeviceEntrySource.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +public final class LocalSegmentDeviceEntrySource extends SegmentDeviceEntrySource { + + private final DeviceEntrySpillManager spillManager; + + public LocalSegmentDeviceEntrySource(DeviceEntryDataSetHandle handle) { + this(handle, DeviceEntrySpillManager.getInstance()); + } + + public LocalSegmentDeviceEntrySource( + DeviceEntryDataSetHandle handle, DeviceEntrySpillManager spillManager) { + super(handle); + this.spillManager = spillManager; + } + + @Override + public List nextBatch() throws IOException { + int segmentId = nextSegmentId; + Path segment = acquireSegment(segmentId); + List result = deserialize(Files.readAllBytes(segment)); + releaseSegment(segmentId); + nextSegmentId++; + return result; + } + + private Path acquireSegment(int segmentId) throws IOException { + return spillManager.resolveSegment(handle.getQueryId(), handle.getPlanNodeId(), segmentId); + } + + private void releaseSegment(int segmentId) throws IOException { + if (segmentId + 1 == handle.getSegmentCount()) { + spillManager.finishSegmentDataSet(handle.getQueryId(), handle.getPlanNodeId().getId()); + } else { + spillManager.deleteSegment(handle.getQueryId(), handle.getPlanNodeId(), segmentId); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/RemoteSegmentDeviceEntrySource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/RemoteSegmentDeviceEntrySource.java new file mode 100644 index 000000000000..d43d7afa9cec --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/RemoteSegmentDeviceEntrySource.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import java.io.IOException; +import java.util.List; + +public final class RemoteSegmentDeviceEntrySource extends SegmentDeviceEntrySource { + + private final DeviceEntrySegmentFetcher fetcher; + private boolean finished; + + public RemoteSegmentDeviceEntrySource(DeviceEntryDataSetHandle handle) { + this(handle, DeviceEntryRpcSegmentFetcher.getInstance()); + } + + public RemoteSegmentDeviceEntrySource( + DeviceEntryDataSetHandle handle, DeviceEntrySegmentFetcher fetcher) { + super(handle); + this.fetcher = fetcher; + } + + @Override + public List nextBatch() throws IOException { + int segmentId = nextSegmentId; + byte[] payload = fetcher.fetch(handle, segmentId); + List result = deserialize(payload); + nextSegmentId++; + if (!hasNextBatch()) { + finish(); + } + return result; + } + + private void finish() { + fetcher.finish(handle); + finished = true; + } + + @Override + public void close() throws IOException { + if (!finished) { + finish(); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/SegmentDeviceEntrySource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/SegmentDeviceEntrySource.java new file mode 100644 index 000000000000..0a00e959e349 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/SegmentDeviceEntrySource.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.db.queryengine.common.DataNodeEndPoints; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public abstract class SegmentDeviceEntrySource implements BatchDeviceEntrySource { + + protected final DeviceEntryDataSetHandle handle; + protected int nextSegmentId; + + protected SegmentDeviceEntrySource(DeviceEntryDataSetHandle handle) { + this.handle = handle; + } + + public static SegmentDeviceEntrySource create(DeviceEntryDataSetHandle handle) { + return handle.getCoordinatorEndPoint().equals(DataNodeEndPoints.LOCAL_HOST_INTERNAL_ENDPOINT) + ? new LocalSegmentDeviceEntrySource(handle) + : new RemoteSegmentDeviceEntrySource(handle); + } + + @Override + public final boolean hasNextBatch() { + return nextSegmentId < handle.getSegmentCount(); + } + + protected final List deserialize(byte[] segmentBytes) throws IOException { + List result = new ArrayList<>(); + int segmentLength = segmentBytes.length; + try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(segmentBytes))) { + int consumedBytes = 0; + while (consumedBytes < segmentLength) { + if (segmentLength - consumedBytes < Integer.BYTES) { + throw new IOException(); + } + int length = input.readInt(); + consumedBytes += Integer.BYTES; + if (length < 0 || length > segmentLength - consumedBytes) { + throw new IOException(); + } + byte[] bytes = new byte[length]; + input.readFully(bytes); + consumedBytes += length; + result.add(DeviceEntry.deserialize(bytes)); + } + } + return result; + } + + @Override + public void close() throws IOException { + // No local cache is created by a segment source. + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/SpilledDeviceEntryDataSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/SpilledDeviceEntryDataSet.java new file mode 100644 index 000000000000..99803c5c0fba --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/SpilledDeviceEntryDataSet.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +public final class SpilledDeviceEntryDataSet implements DeviceEntryDataSet { + + private final String queryId; + private final Path ownerDirectory; + private final List segments; + private final int entryCount; + + public SpilledDeviceEntryDataSet( + String queryId, Path ownerDirectory, List segments, int entryCount) { + this.queryId = queryId; + this.ownerDirectory = ownerDirectory; + this.segments = segments; + this.entryCount = entryCount; + } + + @Override + public int getEntryCount() { + return entryCount; + } + + @Override + public boolean isSpilled() { + return true; + } + + public Path getOwnerDirectory() { + return ownerDirectory; + } + + public List getSegments() { + return segments; + } + + @Override + public DeviceEntryReader openReader() { + return new DeviceEntryFileSpillerReader(segments); + } + + @Override + public DeviceEntryReader openConsumingReader() { + return new DeviceEntryFileSpillerReader(segments, true); + } + + @Override + public void close() throws IOException { + DeviceEntrySpillManager.getInstance().deregisterOwner(queryId, ownerDirectory); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 25845758a081..7e4024efe87c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -90,6 +90,13 @@ import org.apache.iotdb.db.queryengine.plan.relational.function.tvf.read_tsfile.ExternalTsFileQueryResource.DeviceTaskPartition; import org.apache.iotdb.db.queryengine.plan.relational.metadata.AlignedDeviceEntry; import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSet; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSetHandle; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryMaterializationMemoryController; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryMaterializer; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryReader; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntrySortedMaterializer; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.SpilledDeviceEntryDataSet; import org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolAllocator; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTreeDeviceViewScanNode; @@ -129,6 +136,8 @@ import javax.annotation.Nonnull; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -147,10 +156,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static org.apache.iotdb.calc.utils.constant.SqlConstant.COUNT; -import static org.apache.iotdb.calc.utils.constant.SqlConstant.DELTA; -import static org.apache.iotdb.calc.utils.constant.SqlConstant.INCREASE; -import static org.apache.iotdb.calc.utils.constant.SqlConstant.IRATE; -import static org.apache.iotdb.calc.utils.constant.SqlConstant.RATE; import static org.apache.iotdb.commons.partition.DataPartition.NOT_ASSIGNED; import static org.apache.iotdb.commons.queryengine.plan.relational.function.FunctionKind.AGGREGATE; import static org.apache.iotdb.commons.queryengine.plan.relational.metadata.FunctionNullability.getAggregationFunctionNullability; @@ -975,6 +980,11 @@ private List constructDeviceTableScanByRegionReplicaSet( String.format(DataNodeQueryMessages.GIVEN_QUERIED_DATABASE_S_IS_NOT_EXIST, dbName)); } + if (node.getCoordinatorDeviceEntryDataSet().isSpilled()) { + return constructSpilledDeviceTableScanByRegionReplicaSet( + node, context, dataPartition, seriesSlotMap); + } + final Map tableScanNodeMap = new HashMap<>(); Map> cachedSeriesSlotWithRegions = new HashMap<>(); @@ -1048,6 +1058,197 @@ private List constructDeviceTableScanByRegionReplicaSet( return resultTableScanNodeList; } + private List constructSpilledDeviceTableScanByRegionReplicaSet( + DeviceTableScanNode node, + PlanContext context, + DataPartition dataPartition, + Map>> seriesSlotMap) { + Optional sortPropertyContext = + context.hasSortProperty ? analyzeSortProperty(node, context) : Optional.empty(); + Comparator comparator = + sortPropertyContext.map(property -> property.comparator).orElse(null); + long batchSize = + IoTDBDescriptor.getInstance().getConfig().getTableQueryDeviceEntryBatchSizeInBytes(); + Map scanNodes = new HashMap<>(); + Map materializers = new HashMap<>(); + Map sortedMaterializers = new HashMap<>(); + Map regionEntryCounts = new HashMap<>(); + Map> cachedSeriesSlotWithRegions = new HashMap<>(); + DeviceEntryMaterializationMemoryController memoryController = + new DeviceEntryMaterializationMemoryController(batchSize); + + try (DeviceEntryReader reader = node.getCoordinatorDeviceEntryDataSet().openConsumingReader()) { + while (reader.hasNext()) { + DeviceEntry deviceEntry = reader.next(); + List regionReplicaSets = + getDeviceReplicaSets( + dataPartition, + seriesSlotMap, + deviceEntry.getDeviceID(), + node.getTimeFilter(), + cachedSeriesSlotWithRegions); + if (regionReplicaSets.size() > 1) { + context.deviceCrossRegion = true; + } + for (TRegionReplicaSet regionReplicaSet : regionReplicaSets) { + DeviceTableScanNode scanNode = + scanNodes.computeIfAbsent( + regionReplicaSet, + ignored -> createRegionDeviceTableScanNode(node, regionReplicaSet)); + PlanNodeId ownerId = scanNode.getPlanNodeId(); + if (comparator == null) { + DeviceEntryMaterializer materializer = + materializers.computeIfAbsent( + regionReplicaSet, + ignored -> + new DeviceEntryMaterializer( + queryId.getId(), ownerId, batchSize, false, queryContext)); + memoryController.append(materializer, deviceEntry); + } else { + DeviceEntrySortedMaterializer sortedMaterializer = + sortedMaterializers.get(regionReplicaSet); + if (sortedMaterializer == null) { + sortedMaterializer = + new DeviceEntrySortedMaterializer( + queryId.getId(), ownerId, batchSize, comparator, queryContext); + sortedMaterializers.put(regionReplicaSet, sortedMaterializer); + } + memoryController.append(sortedMaterializer, deviceEntry); + } + regionEntryCounts.merge(regionReplicaSet, 1, Integer::sum); + } + } + + for (Map.Entry entry : scanNodes.entrySet()) { + DeviceEntryDataSet dataSet = + comparator == null + ? materializers.get(entry.getKey()).finish() + : sortedMaterializers.get(entry.getKey()).finish(); + installDataSet(entry.getValue(), dataSet, comparator != null); + } + } catch (IOException e) { + closeSpillWriters(materializers.values(), sortedMaterializers.values()); + throw new UncheckedIOException(e); + } + + if (scanNodes.isEmpty()) { + node.setRegionReplicaSet(NOT_ASSIGNED); + return Collections.singletonList(node); + } + + List result = new ArrayList<>(); + TRegionReplicaSet mostUsedRegion = null; + int maxEntryCount = -1; + for (Map.Entry entry : + topology.filterReachableCandidates(scanNodes.entrySet())) { + result.add(entry.getValue()); + int entryCount = regionEntryCounts.getOrDefault(entry.getKey(), 0); + if (entryCount > maxEntryCount) { + mostUsedRegion = entry.getKey(); + maxEntryCount = entryCount; + } + } + if (mostUsedRegion == null) { + throw new RootFIPlacementException(scanNodes.keySet()); + } + context.mostUsedRegion = mostUsedRegion; + sortPropertyContext.ifPresent(property -> applySortProperty(node, result, property, false)); + return result; + } + + private DeviceTableScanNode createRegionDeviceTableScanNode( + DeviceTableScanNode node, TRegionReplicaSet regionReplicaSet) { + DeviceTableScanNode scanNode = + new DeviceTableScanNode( + queryId.genPlanNodeId(), + node.getQualifiedObjectName(), + node.getOutputSymbols(), + node.getAssignments(), + new ArrayList<>(), + node.getTagAndAttributeIndexMap(), + node.getScanOrder(), + node.getTimePredicate().orElse(null), + node.getPushDownPredicate(), + node.getPushDownLimit(), + node.getPushDownOffset(), + node.isPushLimitToEachDevice(), + node.containsNonAlignedDevice()); + scanNode.setRegionReplicaSet(regionReplicaSet); + return scanNode; + } + + private void installDataSet( + DeviceTableScanNode scanNode, DeviceEntryDataSet dataSet, boolean ordered) { + scanNode.setCoordinatorDeviceEntryDataSet(dataSet); + if (!dataSet.isSpilled()) { + return; + } + SpilledDeviceEntryDataSet spilled = (SpilledDeviceEntryDataSet) dataSet; + scanNode.setDeviceEntryDataSetHandle( + new DeviceEntryDataSetHandle( + queryId.getId(), + scanNode.getPlanNodeId(), + DataNodeEndPoints.getLocalDataNodeLocation().getInternalEndPoint(), + spilled.getSegments().size(), + spilled.getEntryCount(), + ordered)); + } + + private void closeSpillWriters( + Collection materializers, + Collection sortedMaterializers) { + for (DeviceEntryMaterializer writer : materializers) { + try { + writer.close(); + } catch (Exception ignored) { + // The original planning exception is more useful than a cleanup failure. + } + } + for (DeviceEntrySortedMaterializer writer : sortedMaterializers) { + try { + writer.close(); + } catch (Exception ignored) { + // The original planning exception is more useful than a cleanup failure. + } + } + } + + private DeviceEntryDataSet finishRegionStagingDataSet( + DeviceEntryDataSet stagingDataSet, + PlanNodeId ownerId, + long batchSize, + Comparator comparator) + throws IOException { + if (comparator == null) { + return stagingDataSet; + } + + try (DeviceEntrySortedMaterializer sortedMaterializer = + new DeviceEntrySortedMaterializer( + queryId.getId(), ownerId, batchSize, comparator, queryContext); + DeviceEntryReader reader = + stagingDataSet.isSpilled() + ? stagingDataSet.openConsumingReader() + : stagingDataSet.openReader()) { + DeviceEntryMaterializationMemoryController memoryController = + new DeviceEntryMaterializationMemoryController(batchSize); + while (reader.hasNext()) { + memoryController.append(sortedMaterializer, reader.next()); + } + return sortedMaterializer.finish(); + } + } + + private void closeDeviceEntryDataSets(Collection dataSets) { + for (DeviceEntryDataSet dataSet : dataSets) { + try { + dataSet.close(); + } catch (Exception ignored) { + // The original planning exception is more useful than a cleanup failure. + } + } + } + @Override public List visitTreeDeviceViewScan(TreeDeviceViewScanNode node, PlanContext context) { DataPartition dataPartition = analysis.getDataPartitionInfo(); @@ -1066,6 +1267,11 @@ public List visitTreeDeviceViewScan(TreeDeviceViewScanNode node, PlanC String.format(DataNodeQueryMessages.GIVEN_QUERIED_DATABASE_S_IS_NOT_EXIST, dbName)); } + if (node.getCoordinatorDeviceEntryDataSet().isSpilled()) { + return constructSpilledTreeDeviceViewScanByRegionReplicaSet( + node, context, dataPartition, seriesSlotMap); + } + Map> tableScanNodeMap = new HashMap<>(); Map> cachedSeriesSlotWithRegions = new HashMap<>(); @@ -1105,7 +1311,6 @@ public List visitTreeDeviceViewScan(TreeDeviceViewScanNode node, PlanC node.getTreeDBName(), node.getMeasurementColumnNameMap()); scanNode.setRegionReplicaSet(regionReplicaSet); - scanNode.setTopKRuntimeFilterSourceId(node.getTopKRuntimeFilterSourceId()); pair.left = scanNode; } @@ -1128,7 +1333,6 @@ public List visitTreeDeviceViewScan(TreeDeviceViewScanNode node, PlanC node.getTreeDBName(), node.getMeasurementColumnNameMap()); scanNode.setRegionReplicaSet(regionReplicaSet); - scanNode.setTopKRuntimeFilterSourceId(node.getTopKRuntimeFilterSourceId()); pair.right = scanNode; } @@ -1186,6 +1390,201 @@ public List visitTreeDeviceViewScan(TreeDeviceViewScanNode node, PlanC return resultTableScanNodeList; } + private List constructSpilledTreeDeviceViewScanByRegionReplicaSet( + TreeDeviceViewScanNode node, + PlanContext context, + DataPartition dataPartition, + Map>> seriesSlotMap) { + Optional sortPropertyContext = + context.hasSortProperty ? analyzeSortProperty(node, context) : Optional.empty(); + Comparator comparator = + sortPropertyContext.map(property -> property.comparator).orElse(null); + long batchSize = + IoTDBDescriptor.getInstance().getConfig().getTableQueryDeviceEntryBatchSizeInBytes(); + Map> + scanNodes = new HashMap<>(); + Map materializers = new HashMap<>(); + Map sortedMaterializers = new HashMap<>(); + Map regionEntryCounts = new HashMap<>(); + Map> cachedSeriesSlotWithRegions = new HashMap<>(); + DeviceEntryMaterializationMemoryController memoryController = + new DeviceEntryMaterializationMemoryController(batchSize); + + try (DeviceEntryReader reader = node.getCoordinatorDeviceEntryDataSet().openConsumingReader()) { + while (reader.hasNext()) { + DeviceEntry deviceEntry = reader.next(); + List regionReplicaSets = + getDeviceReplicaSets( + dataPartition, + seriesSlotMap, + deviceEntry.getDeviceID(), + node.getTimeFilter(), + cachedSeriesSlotWithRegions); + if (regionReplicaSets.size() > 1) { + context.deviceCrossRegion = true; + } + boolean aligned = deviceEntry instanceof AlignedDeviceEntry; + for (TRegionReplicaSet regionReplicaSet : regionReplicaSets) { + Pair pair = + scanNodes.computeIfAbsent(regionReplicaSet, ignored -> new Pair<>(null, null)); + DeviceTableScanNode scanNode; + if (aligned) { + if (pair.left == null) { + pair.left = createTreeAlignedScanNode(node, regionReplicaSet); + } + scanNode = pair.left; + } else { + if (pair.right == null) { + pair.right = createTreeNonAlignedScanNode(node, regionReplicaSet); + } + scanNode = pair.right; + } + appendToRegionDataSet( + scanNode, + deviceEntry, + comparator, + batchSize, + materializers, + sortedMaterializers, + memoryController); + regionEntryCounts.merge(regionReplicaSet, 1, Integer::sum); + } + } + + for (Pair pair : + scanNodes.values()) { + if (pair.left != null) { + finishAndInstallDataSet(pair.left, comparator, materializers, sortedMaterializers); + } + if (pair.right != null) { + finishAndInstallDataSet(pair.right, comparator, materializers, sortedMaterializers); + } + } + } catch (IOException e) { + closeSpillWriters(materializers.values(), sortedMaterializers.values()); + throw new UncheckedIOException(e); + } + + if (scanNodes.isEmpty()) { + node.setRegionReplicaSet(NOT_ASSIGNED); + node.setTreeDBName(null); + return Collections.singletonList(node); + } + + List result = new ArrayList<>(); + TRegionReplicaSet mostUsedRegion = null; + int maxEntryCount = -1; + for (Map.Entry< + TRegionReplicaSet, + Pair> + entry : topology.filterReachableCandidates(scanNodes.entrySet())) { + if (entry.getValue().left != null) { + result.add(entry.getValue().left); + } + if (entry.getValue().right != null) { + result.add(entry.getValue().right); + } + int entryCount = regionEntryCounts.getOrDefault(entry.getKey(), 0); + if (entryCount > maxEntryCount) { + mostUsedRegion = entry.getKey(); + maxEntryCount = entryCount; + } + } + if (mostUsedRegion == null) { + throw new RootFIPlacementException(scanNodes.keySet()); + } + context.mostUsedRegion = mostUsedRegion; + sortPropertyContext.ifPresent(property -> applySortProperty(node, result, property, false)); + return result; + } + + private TreeAlignedDeviceViewScanNode createTreeAlignedScanNode( + TreeDeviceViewScanNode node, TRegionReplicaSet regionReplicaSet) { + TreeAlignedDeviceViewScanNode scanNode = + new TreeAlignedDeviceViewScanNode( + queryId.genPlanNodeId(), + node.getQualifiedObjectName(), + node.getOutputSymbols(), + node.getAssignments(), + new ArrayList<>(), + node.getTagAndAttributeIndexMap(), + node.getScanOrder(), + node.getTimePredicate().orElse(null), + node.getPushDownPredicate(), + node.getPushDownLimit(), + node.getPushDownOffset(), + node.isPushLimitToEachDevice(), + node.containsNonAlignedDevice(), + node.getTreeDBName(), + node.getMeasurementColumnNameMap()); + scanNode.setRegionReplicaSet(regionReplicaSet); + return scanNode; + } + + private TreeNonAlignedDeviceViewScanNode createTreeNonAlignedScanNode( + TreeDeviceViewScanNode node, TRegionReplicaSet regionReplicaSet) { + TreeNonAlignedDeviceViewScanNode scanNode = + new TreeNonAlignedDeviceViewScanNode( + queryId.genPlanNodeId(), + node.getQualifiedObjectName(), + node.getOutputSymbols(), + node.getAssignments(), + new ArrayList<>(), + node.getTagAndAttributeIndexMap(), + node.getScanOrder(), + node.getTimePredicate().orElse(null), + node.getPushDownPredicate(), + node.getPushDownLimit(), + node.getPushDownOffset(), + node.isPushLimitToEachDevice(), + node.containsNonAlignedDevice(), + node.getTreeDBName(), + node.getMeasurementColumnNameMap()); + scanNode.setRegionReplicaSet(regionReplicaSet); + return scanNode; + } + + private void appendToRegionDataSet( + DeviceTableScanNode scanNode, + DeviceEntry deviceEntry, + Comparator comparator, + long batchSize, + Map materializers, + Map sortedMaterializers, + DeviceEntryMaterializationMemoryController memoryController) + throws IOException { + if (comparator == null) { + materializers.computeIfAbsent( + scanNode, + ignored -> + new DeviceEntryMaterializer( + queryId.getId(), scanNode.getPlanNodeId(), batchSize, false, queryContext)); + memoryController.append(materializers.get(scanNode), deviceEntry); + return; + } + DeviceEntrySortedMaterializer sortedMaterializer = sortedMaterializers.get(scanNode); + if (sortedMaterializer == null) { + sortedMaterializer = + new DeviceEntrySortedMaterializer( + queryId.getId(), scanNode.getPlanNodeId(), batchSize, comparator, queryContext); + sortedMaterializers.put(scanNode, sortedMaterializer); + } + memoryController.append(sortedMaterializer, deviceEntry); + } + + private void finishAndInstallDataSet( + DeviceTableScanNode scanNode, + Comparator comparator, + Map materializers, + Map sortedMaterializers) + throws IOException { + DeviceEntryDataSet dataSet = + comparator == null + ? materializers.get(scanNode).finish() + : sortedMaterializers.get(scanNode).finish(); + installDataSet(scanNode, dataSet, comparator != null); + } + @Override public List visitInformationSchemaTableScan( InformationSchemaTableScanNode node, PlanContext context) { @@ -1324,10 +1723,9 @@ public List visitAggregation(AggregationNode node, PlanContext context // push down aggregation if the child of aggregation node only has the union Node if (childrenNodes.size() == 1) { node.setChild(childrenNodes.get(0)); - AggregationNode physicalAggregation = withRateFunctionInputOrdering(node, childOrdering); if (childrenNodes.get(0) instanceof UnionNode - && physicalAggregation.getAggregations().values().stream() + && node.getAggregations().values().stream() .noneMatch(aggregation -> aggregation.isDistinct() || aggregation.hasMask())) { UnionNode unionNode = (UnionNode) childrenNodes.get(0); List children = unionNode.getChildren(); @@ -1348,8 +1746,7 @@ public List visitAggregation(AggregationNode node, PlanContext context } // 2. split the aggregation into partial and final - Pair splitResult = - split(physicalAggregation, symbolAllocator, queryId); + Pair splitResult = split(node, symbolAllocator, queryId); AggregationNode intermediate = splitResult.right; // 3. add the aggregation node above the project node @@ -1368,7 +1765,7 @@ public List visitAggregation(AggregationNode node, PlanContext context intermediate.getStep(), intermediate.getHashSymbol(), intermediate.getGroupIdSymbol()); - if (physicalAggregation.isStreamable() && childOrdering != null) { + if (node.isStreamable() && childOrdering != null) { nodeOrderingMap.put(planNodeId, expectedOrderingSchema); } return aggregationNode; @@ -1384,7 +1781,7 @@ public List visitAggregation(AggregationNode node, PlanContext context return Collections.singletonList(splitResult.left); } - return Collections.singletonList(physicalAggregation); + return Collections.singletonList(node); } // We cannot do multi-stage Aggregate if any aggregation-function is distinct. @@ -1392,12 +1789,10 @@ public List visitAggregation(AggregationNode node, PlanContext context // MarkDistinctNode will merge all data from different child. if (node.getAggregations().values().stream() .anyMatch(aggregation -> aggregation.isDistinct() || aggregation.hasMask())) { - PlanNode physicalChild = + node.setChild( mergeChildrenViaCollectOrMergeSort( - nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()), childrenNodes); - node.setChild(physicalChild); - return Collections.singletonList( - withRateFunctionInputOrdering(node, nodeOrderingMap.get(physicalChild.getPlanNodeId()))); + nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()), childrenNodes)); + return Collections.singletonList(node); } Pair splitResult = split(node, symbolAllocator, queryId); AggregationNode intermediate = splitResult.right; @@ -1429,58 +1824,6 @@ public List visitAggregation(AggregationNode node, PlanContext context return Collections.singletonList(splitResult.left); } - private static AggregationNode withRateFunctionInputOrdering( - AggregationNode node, OrderingScheme childOrdering) { - Map aggregations = new LinkedHashMap<>(); - node.getAggregations() - .forEach( - (symbol, aggregation) -> - aggregations.put( - symbol, - new AggregationNode.Aggregation( - aggregation.getResolvedFunction(), - aggregation.getArguments(), - aggregation.isDistinct(), - aggregation.getFilter(), - aggregation.getOrderingScheme(), - aggregation.getMask(), - isInputOrderedByTimeAscending( - aggregation, node.getStep(), node.getGroupingKeys(), childOrdering)))); - return AggregationNode.builderFrom(node).setAggregations(aggregations).build(); - } - - static boolean isInputOrderedByTimeAscending( - AggregationNode.Aggregation aggregation, - AggregationNode.Step step, - List groupingKeys, - OrderingScheme childOrdering) { - String functionName = aggregation.getResolvedFunction().getSignature().getName(); - if (step != SINGLE - || childOrdering == null - || aggregation.getArguments().size() < 2 - || !(RATE.equalsIgnoreCase(functionName) - || INCREASE.equalsIgnoreCase(functionName) - || IRATE.equalsIgnoreCase(functionName) - || DELTA.equalsIgnoreCase(functionName))) { - return false; - } - - Symbol timeSymbol = Symbol.from(aggregation.getArguments().get(1)); - List orderBy = childOrdering.getOrderBy(); - int timeIndex = orderBy.indexOf(timeSymbol); - if (timeIndex < 0 || !childOrdering.getOrdering(timeSymbol).isAscending()) { - return false; - } - - Set groupingKeySet = new HashSet<>(groupingKeys); - for (int i = 0; i < timeIndex; i++) { - if (!groupingKeySet.contains(orderBy.get(i))) { - return false; - } - } - return true; - } - private boolean prefixMatched(OrderingScheme childOrdering, List preGroupedSymbols) { List orderKeys = childOrdering.getOrderBy(); if (orderKeys.size() < preGroupedSymbols.size()) { @@ -1505,6 +1848,11 @@ public List visitAggregationTableScan( return Collections.singletonList(node); } + if (node.getCoordinatorDeviceEntryDataSet().isSpilled()) { + return constructSpilledAggregationTableScanByRegionReplicaSet( + node, context, dataPartition, dbName); + } + AggregationDistributionInfo distributionInfo = prepareAggregationDistribution(node, dbName, dataPartition, context); @@ -1554,6 +1902,185 @@ public List visitAggregationTableScan( return resultTableScanNodeList; } + private List constructSpilledAggregationTableScanByRegionReplicaSet( + AggregationTableScanNode node, + PlanContext context, + DataPartition dataPartition, + String dbName) { + Map>> seriesSlotMap = + dataPartition.getDataPartitionMap().get(dbName); + if (seriesSlotMap == null) { + throw new SemanticException( + String.format(DataNodeQueryMessages.GIVEN_QUERIED_DATABASE_S_IS_NOT_EXIST, dbName)); + } + + long batchSize = + IoTDBDescriptor.getInstance().getConfig().getTableQueryDeviceEntryBatchSizeInBytes(); + Map> cachedSeriesSlotWithRegions = new HashMap<>(); + Map crossRegionDeviceCounts = + node.mayUseLastCache() ? new HashMap<>() : Collections.emptyMap(); + Map regionPlanNodeIds = new HashMap<>(); + Map stagingMaterializers = new HashMap<>(); + Map stagingDataSets = new HashMap<>(); + Map regionEntryCounts = new HashMap<>(); + DeviceEntryMaterializationMemoryController memoryController = + new DeviceEntryMaterializationMemoryController(batchSize); + boolean hasCrossRegionDevice = false; + try (DeviceEntryReader reader = node.getCoordinatorDeviceEntryDataSet().openConsumingReader()) { + while (reader.hasNext()) { + DeviceEntry deviceEntry = reader.next(); + List regions = + getDeviceReplicaSets( + dataPartition, + seriesSlotMap, + deviceEntry.getDeviceID(), + node.getTimeFilter(), + cachedSeriesSlotWithRegions); + if (regions.size() > 1) { + hasCrossRegionDevice = true; + context.deviceCrossRegion = true; + if (node.mayUseLastCache()) { + crossRegionDeviceCounts.put(deviceEntry, regions.size()); + } + } + for (TRegionReplicaSet region : regions) { + PlanNodeId regionPlanNodeId = + regionPlanNodeIds.computeIfAbsent(region, ignored -> queryId.genPlanNodeId()); + stagingMaterializers.computeIfAbsent( + region, + ignored -> + new DeviceEntryMaterializer( + queryId.getId(), regionPlanNodeId, batchSize, false, queryContext)); + memoryController.append(stagingMaterializers.get(region), deviceEntry); + regionEntryCounts.merge(region, 1, Integer::sum); + } + } + for (Map.Entry entry : + stagingMaterializers.entrySet()) { + stagingDataSets.put(entry.getKey(), entry.getValue().finish()); + } + } catch (IOException e) { + closeSpillWriters(stagingMaterializers.values(), Collections.emptyList()); + throw new UncheckedIOException(e); + } catch (RuntimeException e) { + closeSpillWriters(stagingMaterializers.values(), Collections.emptyList()); + throw e; + } + + boolean needSplit = hasCrossRegionDevice && node.getStep() == SINGLE; + AggregationTableScanNode templateNode = node; + AggregationNode finalAggregation = null; + try { + if (needSplit) { + Pair splitResult = + split(node, symbolAllocator, queryId); + finalAggregation = splitResult.left; + templateNode = splitResult.right; + if (!context.hasSortProperty && finalAggregation.isStreamable()) { + context.setExpectedOrderingScheme(constructOrderingSchema(node.getPreGroupedSymbols())); + } + } + } catch (RuntimeException e) { + closeDeviceEntryDataSets(stagingDataSets.values()); + throw e; + } + if (hasCrossRegionDevice && node.mayUseLastCache()) { + queryContext.setNeedUpdateScanNumForLastQuery(true); + } + + Optional sortPropertyContext; + try { + sortPropertyContext = + context.hasSortProperty ? analyzeSortProperty(node, context) : Optional.empty(); + } catch (RuntimeException e) { + closeDeviceEntryDataSets(stagingDataSets.values()); + throw e; + } + Comparator comparator = + sortPropertyContext.map(property -> property.comparator).orElse(null); + Map scanNodes = new HashMap<>(); + try { + for (Map.Entry entry : stagingDataSets.entrySet()) { + TRegionReplicaSet region = entry.getKey(); + PlanNodeId regionPlanNodeId = regionPlanNodeIds.get(region); + AggregationTableScanNode scanNode = + createAggregationScanNode(templateNode, regionPlanNodeId, region); + DeviceEntryDataSet dataSet = + finishRegionStagingDataSet(entry.getValue(), regionPlanNodeId, batchSize, comparator); + installDataSet(scanNode, dataSet, comparator != null); + if (!crossRegionDeviceCounts.isEmpty()) { + scanNode.setDeviceCountMap(crossRegionDeviceCounts); + } + scanNodes.put(region, scanNode); + } + } catch (IOException e) { + closeDeviceEntryDataSets(stagingDataSets.values()); + throw new UncheckedIOException(e); + } catch (RuntimeException e) { + closeDeviceEntryDataSets(stagingDataSets.values()); + throw e; + } + + List result = new ArrayList<>(); + TRegionReplicaSet mostUsedRegion = null; + int maxEntryCount = -1; + for (Map.Entry entry : + topology.filterReachableCandidates(scanNodes.entrySet())) { + result.add(entry.getValue()); + int entryCount = regionEntryCounts.getOrDefault(entry.getKey(), 0); + if (entryCount > maxEntryCount) { + mostUsedRegion = entry.getKey(); + maxEntryCount = entryCount; + } + } + if (mostUsedRegion == null) { + throw new RootFIPlacementException(scanNodes.keySet()); + } + context.mostUsedRegion = mostUsedRegion; + sortPropertyContext.ifPresent(property -> applySortProperty(node, result, property, false)); + + if (needSplit) { + if (result.size() == 1) { + finalAggregation.setChild(result.get(0)); + } else { + finalAggregation.setChild( + mergeChildrenViaCollectOrMergeSort( + nodeOrderingMap.get(result.get(0).getPlanNodeId()), result)); + } + return Collections.singletonList(finalAggregation); + } + return result; + } + + private AggregationTableScanNode createAggregationScanNode( + AggregationTableScanNode template, + PlanNodeId planNodeId, + TRegionReplicaSet regionReplicaSet) { + AggregationTableScanNode scanNode = + new AggregationTableScanNode( + planNodeId, + template.getQualifiedObjectName(), + template.getOutputSymbols(), + template.getAssignments(), + new ArrayList<>(), + template.getTagAndAttributeIndexMap(), + template.getScanOrder(), + template.getTimePredicate().orElse(null), + template.getPushDownPredicate(), + template.getPushDownLimit(), + template.getPushDownOffset(), + template.isPushLimitToEachDevice(), + template.containsNonAlignedDevice(), + template.getProjection(), + template.getAggregations(), + template.getGroupingSets(), + template.getPreGroupedSymbols(), + template.getStep(), + template.getGroupIdSymbol()); + scanNode.setRegionReplicaSet(regionReplicaSet); + return scanNode; + } + @Override public List visitAggregationTreeDeviceViewScan( AggregationTreeDeviceViewScanNode node, PlanContext context) { @@ -1586,6 +2113,11 @@ public List visitAggregationTreeDeviceViewScan( node.getMeasurementColumnNameMap())); } + if (node.getCoordinatorDeviceEntryDataSet().isSpilled()) { + return constructSpilledAggregationTreeDeviceViewScanByRegionReplicaSet( + node, context, dataPartition, dbName); + } + AggregationDistributionInfo distributionInfo = prepareAggregationDistribution(node, dbName, dataPartition, context); @@ -1737,6 +2269,249 @@ public List visitAggregationTreeDeviceViewScan( return resultTableScanNodeList; } + private List constructSpilledAggregationTreeDeviceViewScanByRegionReplicaSet( + AggregationTreeDeviceViewScanNode node, + PlanContext context, + DataPartition dataPartition, + String dbName) { + Map>> seriesSlotMap = + dataPartition.getDataPartitionMap().get(dbName); + if (seriesSlotMap == null) { + throw new SemanticException( + String.format(DataNodeQueryMessages.GIVEN_QUERIED_DATABASE_S_IS_NOT_EXIST, dbName)); + } + + long batchSize = + IoTDBDescriptor.getInstance().getConfig().getTableQueryDeviceEntryBatchSizeInBytes(); + boolean hasCrossRegionDevice = false; + Map> cachedSeriesSlotWithRegions = new HashMap<>(); + Map> regionPlanNodeIds = new HashMap<>(); + Map stagingMaterializers = new HashMap<>(); + Map stagingDataSets = new HashMap<>(); + Map regionEntryCounts = new HashMap<>(); + DeviceEntryMaterializationMemoryController memoryController = + new DeviceEntryMaterializationMemoryController(batchSize); + try (DeviceEntryReader reader = node.getCoordinatorDeviceEntryDataSet().openConsumingReader()) { + while (reader.hasNext()) { + DeviceEntry deviceEntry = reader.next(); + List regions = + getDeviceReplicaSets( + dataPartition, + seriesSlotMap, + deviceEntry.getDeviceID(), + node.getTimeFilter(), + cachedSeriesSlotWithRegions); + if (regions.size() > 1) { + hasCrossRegionDevice = true; + context.deviceCrossRegion = true; + } + boolean aligned = deviceEntry instanceof AlignedDeviceEntry; + for (TRegionReplicaSet region : regions) { + Pair planNodeIds = + regionPlanNodeIds.computeIfAbsent(region, ignored -> new Pair<>(null, null)); + PlanNodeId planNodeId; + if (aligned) { + if (planNodeIds.left == null) { + planNodeIds.left = queryId.genPlanNodeId(); + } + planNodeId = planNodeIds.left; + } else { + if (planNodeIds.right == null) { + planNodeIds.right = queryId.genPlanNodeId(); + } + planNodeId = planNodeIds.right; + } + stagingMaterializers.computeIfAbsent( + planNodeId, + ignored -> + new DeviceEntryMaterializer( + queryId.getId(), planNodeId, batchSize, false, queryContext)); + memoryController.append(stagingMaterializers.get(planNodeId), deviceEntry); + regionEntryCounts.merge(region, 1, Integer::sum); + } + } + for (Map.Entry entry : stagingMaterializers.entrySet()) { + stagingDataSets.put(entry.getKey(), entry.getValue().finish()); + } + } catch (IOException e) { + closeSpillWriters(stagingMaterializers.values(), Collections.emptyList()); + throw new UncheckedIOException(e); + } catch (RuntimeException e) { + closeSpillWriters(stagingMaterializers.values(), Collections.emptyList()); + throw e; + } + + boolean needSplit = hasCrossRegionDevice && node.getStep() == SINGLE; + AggregationTableScanNode templateNode = node; + AggregationNode finalAggregation = null; + try { + if (needSplit) { + Pair splitResult = + split(node, symbolAllocator, queryId); + finalAggregation = splitResult.left; + templateNode = splitResult.right; + if (!context.hasSortProperty && finalAggregation.isStreamable()) { + context.setExpectedOrderingScheme(constructOrderingSchema(node.getPreGroupedSymbols())); + } + } + } catch (RuntimeException e) { + closeDeviceEntryDataSets(stagingDataSets.values()); + throw e; + } + + Optional sortPropertyContext; + try { + sortPropertyContext = + context.hasSortProperty ? analyzeSortProperty(node, context) : Optional.empty(); + } catch (RuntimeException e) { + closeDeviceEntryDataSets(stagingDataSets.values()); + throw e; + } + Comparator comparator = + sortPropertyContext.map(property -> property.comparator).orElse(null); + Map< + TRegionReplicaSet, + Pair< + AlignedAggregationTreeDeviceViewScanNode, + NonAlignedAggregationTreeDeviceViewScanNode>> + scanNodes = new HashMap<>(); + try { + for (Map.Entry> entry : + regionPlanNodeIds.entrySet()) { + TRegionReplicaSet region = entry.getKey(); + Pair + scanNodePair = new Pair<>(null, null); + if (entry.getValue().left != null) { + PlanNodeId planNodeId = entry.getValue().left; + scanNodePair.left = + createAlignedAggregationTreeScanNode(node, templateNode, planNodeId, region); + DeviceEntryDataSet dataSet = + finishRegionStagingDataSet( + stagingDataSets.get(planNodeId), planNodeId, batchSize, comparator); + installDataSet(scanNodePair.left, dataSet, comparator != null); + } + if (entry.getValue().right != null) { + PlanNodeId planNodeId = entry.getValue().right; + scanNodePair.right = + createNonAlignedAggregationTreeScanNode(node, templateNode, planNodeId, region); + DeviceEntryDataSet dataSet = + finishRegionStagingDataSet( + stagingDataSets.get(planNodeId), planNodeId, batchSize, comparator); + installDataSet(scanNodePair.right, dataSet, comparator != null); + } + scanNodes.put(region, scanNodePair); + } + } catch (IOException e) { + closeDeviceEntryDataSets(stagingDataSets.values()); + throw new UncheckedIOException(e); + } catch (RuntimeException e) { + closeDeviceEntryDataSets(stagingDataSets.values()); + throw e; + } + + List result = new ArrayList<>(); + TRegionReplicaSet mostUsedRegion = null; + int maxEntryCount = -1; + for (Map.Entry< + TRegionReplicaSet, + Pair< + AlignedAggregationTreeDeviceViewScanNode, + NonAlignedAggregationTreeDeviceViewScanNode>> + entry : topology.filterReachableCandidates(scanNodes.entrySet())) { + if (entry.getValue().left != null) { + result.add(entry.getValue().left); + } + if (entry.getValue().right != null) { + result.add(entry.getValue().right); + } + int entryCount = regionEntryCounts.getOrDefault(entry.getKey(), 0); + if (entryCount > maxEntryCount) { + mostUsedRegion = entry.getKey(); + maxEntryCount = entryCount; + } + } + if (mostUsedRegion == null) { + throw new RootFIPlacementException(scanNodes.keySet()); + } + context.mostUsedRegion = mostUsedRegion; + sortPropertyContext.ifPresent(property -> applySortProperty(node, result, property, false)); + if (needSplit) { + if (result.size() == 1) { + finalAggregation.setChild(result.get(0)); + } else { + finalAggregation.setChild( + mergeChildrenViaCollectOrMergeSort( + nodeOrderingMap.get(result.get(0).getPlanNodeId()), result)); + } + return Collections.singletonList(finalAggregation); + } + return result; + } + + private AlignedAggregationTreeDeviceViewScanNode createAlignedAggregationTreeScanNode( + AggregationTreeDeviceViewScanNode source, + AggregationTableScanNode template, + PlanNodeId planNodeId, + TRegionReplicaSet region) { + AlignedAggregationTreeDeviceViewScanNode scanNode = + new AlignedAggregationTreeDeviceViewScanNode( + planNodeId, + template.getQualifiedObjectName(), + template.getOutputSymbols(), + template.getAssignments(), + new ArrayList<>(), + template.getTagAndAttributeIndexMap(), + template.getScanOrder(), + template.getTimePredicate().orElse(null), + template.getPushDownPredicate(), + template.getPushDownLimit(), + template.getPushDownOffset(), + template.isPushLimitToEachDevice(), + template.containsNonAlignedDevice(), + template.getProjection(), + template.getAggregations(), + template.getGroupingSets(), + template.getPreGroupedSymbols(), + template.getStep(), + template.getGroupIdSymbol(), + source.getTreeDBName(), + source.getMeasurementColumnNameMap()); + scanNode.setRegionReplicaSet(region); + return scanNode; + } + + private NonAlignedAggregationTreeDeviceViewScanNode createNonAlignedAggregationTreeScanNode( + AggregationTreeDeviceViewScanNode source, + AggregationTableScanNode template, + PlanNodeId planNodeId, + TRegionReplicaSet region) { + NonAlignedAggregationTreeDeviceViewScanNode scanNode = + new NonAlignedAggregationTreeDeviceViewScanNode( + planNodeId, + template.getQualifiedObjectName(), + template.getOutputSymbols(), + template.getAssignments(), + new ArrayList<>(), + template.getTagAndAttributeIndexMap(), + template.getScanOrder(), + template.getTimePredicate().orElse(null), + template.getPushDownPredicate(), + template.getPushDownLimit(), + template.getPushDownOffset(), + template.isPushLimitToEachDevice(), + template.containsNonAlignedDevice(), + template.getProjection(), + template.getAggregations(), + template.getGroupingSets(), + template.getPreGroupedSymbols(), + template.getStep(), + template.getGroupIdSymbol(), + source.getTreeDBName(), + source.getMeasurementColumnNameMap()); + scanNode.setRegionReplicaSet(region); + return scanNode; + } + private static class AggregationDistributionInfo { private final List> regionReplicaSetsList; private final AggregationTableScanNode templateNode; @@ -2162,7 +2937,7 @@ private void applySortProperty( sortPropertyContext.sortOrders, sortPropertyContext.lastIsTimeRelated, resultTableScanNodeList.size() == 1 - && ((DeviceTableScanNode) resultTableScanNodeList.get(0)).getDeviceEntries().size() + && ((DeviceTableScanNode) resultTableScanNodeList.get(0)).getDeviceEntryCount() == 1); for (final PlanNode planNode : resultTableScanNodeList) { final DeviceTableScanNode scanNode = (DeviceTableScanNode) planNode; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java index 819efbbc4aff..526c20abdd17 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java @@ -117,7 +117,7 @@ public static Optional pruneColumns(TableScanNode node, Set re treeDeviceViewScanNode.getTreeDBName(), treeDeviceViewScanNode.getMeasurementColumnNameMap()); prunedNode.setRegionReplicaSet(deviceTableScanNode.getRegionReplicaSet()); - return Optional.of(prunedNode); + return Optional.of(deviceTableScanNode.copyDeviceEntryDataSetTo(prunedNode)); } else if (node instanceof TreeNonAlignedDeviceViewScanNode) { TreeNonAlignedDeviceViewScanNode treeDeviceViewScanNode = (TreeNonAlignedDeviceViewScanNode) deviceTableScanNode; @@ -139,11 +139,11 @@ public static Optional pruneColumns(TableScanNode node, Set re treeDeviceViewScanNode.getTreeDBName(), treeDeviceViewScanNode.getMeasurementColumnNameMap()); prunedNode.setRegionReplicaSet(deviceTableScanNode.getRegionReplicaSet()); - return Optional.of(prunedNode); + return Optional.of(deviceTableScanNode.copyDeviceEntryDataSetTo(prunedNode)); } else if (node instanceof TreeDeviceViewScanNode) { TreeDeviceViewScanNode treeDeviceViewScanNode = (TreeDeviceViewScanNode) deviceTableScanNode; - return Optional.of( + TreeDeviceViewScanNode prunedNode = new TreeDeviceViewScanNode( deviceTableScanNode.getPlanNodeId(), deviceTableScanNode.getQualifiedObjectName(), @@ -159,7 +159,9 @@ public static Optional pruneColumns(TableScanNode node, Set re deviceTableScanNode.isPushLimitToEachDevice(), deviceTableScanNode.containsNonAlignedDevice(), treeDeviceViewScanNode.getTreeDBName(), - treeDeviceViewScanNode.getMeasurementColumnNameMap())); + treeDeviceViewScanNode.getMeasurementColumnNameMap()); + prunedNode.setRegionReplicaSet(deviceTableScanNode.getRegionReplicaSet()); + return Optional.of(deviceTableScanNode.copyDeviceEntryDataSetTo(prunedNode)); } else if (node instanceof ExternalTsFileScanNode externalTsFileScanNode) { ExternalTsFileScanNode prunedNode = new ExternalTsFileScanNode( @@ -179,7 +181,7 @@ public static Optional pruneColumns(TableScanNode node, Set re externalTsFileScanNode.getDeviceTaskPartitionIndex(), externalTsFileScanNode.getSchemaFilter()); prunedNode.setRegionReplicaSet(deviceTableScanNode.getRegionReplicaSet()); - return Optional.of(prunedNode); + return Optional.of(deviceTableScanNode.copyDeviceEntryDataSetTo(prunedNode)); } else { DeviceTableScanNode prunedNode = new DeviceTableScanNode( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTableScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTableScanNode.java index 70a4198fff93..58d85026b7d5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTableScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTableScanNode.java @@ -268,26 +268,27 @@ public Optional getGroupIdSymbol() { @Override public AggregationTableScanNode clone() { - return new AggregationTableScanNode( - id, - qualifiedObjectName, - outputSymbols, - assignments, - deviceEntries, - tagAndAttributeIndexMap, - scanOrder, - timePredicate, - pushDownPredicate, - pushDownLimit, - pushDownOffset, - pushLimitToEachDevice, - containsNonAlignedDevice, - projection, - aggregations, - groupingSets, - preGroupedSymbols, - step, - groupIdSymbol); + return copyDeviceEntryDataSetTo( + new AggregationTableScanNode( + id, + qualifiedObjectName, + outputSymbols, + assignments, + deviceEntries, + tagAndAttributeIndexMap, + scanOrder, + timePredicate, + pushDownPredicate, + pushDownLimit, + pushDownOffset, + pushLimitToEachDevice, + containsNonAlignedDevice, + projection, + aggregations, + groupingSets, + preGroupedSymbols, + step, + groupIdSymbol)); } @Override @@ -330,50 +331,52 @@ public static AggregationTableScanNode combineAggregationAndTableScan( } if (tableScanNode instanceof TreeDeviceViewScanNode) { TreeDeviceViewScanNode treeDeviceViewScanNode = (TreeDeviceViewScanNode) tableScanNode; - return new AggregationTreeDeviceViewScanNode( - id, - tableScanNode.getQualifiedObjectName(), - tableScanNode.getOutputSymbols(), - tableScanNode.getAssignments(), - tableScanNode.getDeviceEntries(), - tableScanNode.getTagAndAttributeIndexMap(), - tableScanNode.getScanOrder(), - tableScanNode.getTimePredicate().orElse(null), - tableScanNode.getPushDownPredicate(), - tableScanNode.getPushDownLimit(), - tableScanNode.getPushDownOffset(), - tableScanNode.isPushLimitToEachDevice(), - tableScanNode.containsNonAlignedDevice(), - projectNode == null ? null : projectNode.getAssignments(), - aggregationNode.getAggregations(), - aggregationNode.getGroupingSets(), - aggregationNode.getPreGroupedSymbols(), - aggregationNode.getStep(), - aggregationNode.getGroupIdSymbol(), - treeDeviceViewScanNode.getTreeDBName(), - treeDeviceViewScanNode.getMeasurementColumnNameMap()); + return tableScanNode.copyDeviceEntryDataSetTo( + new AggregationTreeDeviceViewScanNode( + id, + tableScanNode.getQualifiedObjectName(), + tableScanNode.getOutputSymbols(), + tableScanNode.getAssignments(), + tableScanNode.getDeviceEntries(), + tableScanNode.getTagAndAttributeIndexMap(), + tableScanNode.getScanOrder(), + tableScanNode.getTimePredicate().orElse(null), + tableScanNode.getPushDownPredicate(), + tableScanNode.getPushDownLimit(), + tableScanNode.getPushDownOffset(), + tableScanNode.isPushLimitToEachDevice(), + tableScanNode.containsNonAlignedDevice(), + projectNode == null ? null : projectNode.getAssignments(), + aggregationNode.getAggregations(), + aggregationNode.getGroupingSets(), + aggregationNode.getPreGroupedSymbols(), + aggregationNode.getStep(), + aggregationNode.getGroupIdSymbol(), + treeDeviceViewScanNode.getTreeDBName(), + treeDeviceViewScanNode.getMeasurementColumnNameMap())); } - return new AggregationTableScanNode( - id, - tableScanNode.getQualifiedObjectName(), - tableScanNode.getOutputSymbols(), - tableScanNode.getAssignments(), - tableScanNode.getDeviceEntries(), - tableScanNode.getTagAndAttributeIndexMap(), - tableScanNode.getScanOrder(), - tableScanNode.getTimePredicate().orElse(null), - tableScanNode.getPushDownPredicate(), - tableScanNode.getPushDownLimit(), - tableScanNode.getPushDownOffset(), - tableScanNode.isPushLimitToEachDevice(), - tableScanNode.containsNonAlignedDevice(), - projectNode == null ? null : projectNode.getAssignments(), - aggregationNode.getAggregations(), - aggregationNode.getGroupingSets(), - aggregationNode.getPreGroupedSymbols(), - aggregationNode.getStep(), - aggregationNode.getGroupIdSymbol()); + return tableScanNode.copyDeviceEntryDataSetTo( + new AggregationTableScanNode( + id, + tableScanNode.getQualifiedObjectName(), + tableScanNode.getOutputSymbols(), + tableScanNode.getAssignments(), + tableScanNode.getDeviceEntries(), + tableScanNode.getTagAndAttributeIndexMap(), + tableScanNode.getScanOrder(), + tableScanNode.getTimePredicate().orElse(null), + tableScanNode.getPushDownPredicate(), + tableScanNode.getPushDownLimit(), + tableScanNode.getPushDownOffset(), + tableScanNode.isPushLimitToEachDevice(), + tableScanNode.containsNonAlignedDevice(), + projectNode == null ? null : projectNode.getAssignments(), + aggregationNode.getAggregations(), + aggregationNode.getGroupingSets(), + aggregationNode.getPreGroupedSymbols(), + aggregationNode.getStep(), + aggregationNode.getGroupIdSymbol())); } public static AggregationTableScanNode combineAggregationAndTableScan( @@ -410,50 +413,52 @@ public static AggregationTableScanNode combineAggregationAndTableScan( } if (tableScanNode instanceof TreeDeviceViewScanNode) { TreeDeviceViewScanNode treeDeviceViewScanNode = (TreeDeviceViewScanNode) tableScanNode; - return new AggregationTreeDeviceViewScanNode( - id, - tableScanNode.getQualifiedObjectName(), - tableScanNode.getOutputSymbols(), - tableScanNode.getAssignments(), - tableScanNode.getDeviceEntries(), - tableScanNode.getTagAndAttributeIndexMap(), - tableScanNode.getScanOrder(), - tableScanNode.getTimePredicate().orElse(null), - tableScanNode.getPushDownPredicate(), - tableScanNode.getPushDownLimit(), - tableScanNode.getPushDownOffset(), - tableScanNode.isPushLimitToEachDevice(), - tableScanNode.containsNonAlignedDevice(), - projectNode == null ? null : projectNode.getAssignments(), - aggregationNode.getAggregations(), - aggregationNode.getGroupingSets(), - aggregationNode.getPreGroupedSymbols(), - step, - aggregationNode.getGroupIdSymbol(), - treeDeviceViewScanNode.getTreeDBName(), - treeDeviceViewScanNode.getMeasurementColumnNameMap()); + return tableScanNode.copyDeviceEntryDataSetTo( + new AggregationTreeDeviceViewScanNode( + id, + tableScanNode.getQualifiedObjectName(), + tableScanNode.getOutputSymbols(), + tableScanNode.getAssignments(), + tableScanNode.getDeviceEntries(), + tableScanNode.getTagAndAttributeIndexMap(), + tableScanNode.getScanOrder(), + tableScanNode.getTimePredicate().orElse(null), + tableScanNode.getPushDownPredicate(), + tableScanNode.getPushDownLimit(), + tableScanNode.getPushDownOffset(), + tableScanNode.isPushLimitToEachDevice(), + tableScanNode.containsNonAlignedDevice(), + projectNode == null ? null : projectNode.getAssignments(), + aggregationNode.getAggregations(), + aggregationNode.getGroupingSets(), + aggregationNode.getPreGroupedSymbols(), + step, + aggregationNode.getGroupIdSymbol(), + treeDeviceViewScanNode.getTreeDBName(), + treeDeviceViewScanNode.getMeasurementColumnNameMap())); } - return new AggregationTableScanNode( - id, - tableScanNode.getQualifiedObjectName(), - tableScanNode.getOutputSymbols(), - tableScanNode.getAssignments(), - tableScanNode.getDeviceEntries(), - tableScanNode.getTagAndAttributeIndexMap(), - tableScanNode.getScanOrder(), - tableScanNode.getTimePredicate().orElse(null), - tableScanNode.getPushDownPredicate(), - tableScanNode.getPushDownLimit(), - tableScanNode.getPushDownOffset(), - tableScanNode.isPushLimitToEachDevice(), - tableScanNode.containsNonAlignedDevice(), - projectNode == null ? null : projectNode.getAssignments(), - aggregationNode.getAggregations(), - aggregationNode.getGroupingSets(), - aggregationNode.getPreGroupedSymbols(), - step, - aggregationNode.getGroupIdSymbol()); + return tableScanNode.copyDeviceEntryDataSetTo( + new AggregationTableScanNode( + id, + tableScanNode.getQualifiedObjectName(), + tableScanNode.getOutputSymbols(), + tableScanNode.getAssignments(), + tableScanNode.getDeviceEntries(), + tableScanNode.getTagAndAttributeIndexMap(), + tableScanNode.getScanOrder(), + tableScanNode.getTimePredicate().orElse(null), + tableScanNode.getPushDownPredicate(), + tableScanNode.getPushDownLimit(), + tableScanNode.getPushDownOffset(), + tableScanNode.isPushLimitToEachDevice(), + tableScanNode.containsNonAlignedDevice(), + projectNode == null ? null : projectNode.getAssignments(), + aggregationNode.getAggregations(), + aggregationNode.getGroupingSets(), + aggregationNode.getPreGroupedSymbols(), + step, + aggregationNode.getGroupIdSymbol())); } public boolean mayUseLastCache() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTreeDeviceViewScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTreeDeviceViewScanNode.java index 3b35876a2c3e..f6e65beddc53 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTreeDeviceViewScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTreeDeviceViewScanNode.java @@ -137,28 +137,29 @@ public String toString() { @Override public AggregationTreeDeviceViewScanNode clone() { - return new AggregationTreeDeviceViewScanNode( - id, - qualifiedObjectName, - outputSymbols, - assignments, - deviceEntries, - tagAndAttributeIndexMap, - scanOrder, - timePredicate, - pushDownPredicate, - pushDownLimit, - pushDownOffset, - pushLimitToEachDevice, - containsNonAlignedDevice, - projection, - aggregations, - groupingSets, - preGroupedSymbols, - step, - groupIdSymbol, - treeDBName, - measurementColumnNameMap); + return copyDeviceEntryDataSetTo( + new AggregationTreeDeviceViewScanNode( + id, + qualifiedObjectName, + outputSymbols, + assignments, + deviceEntries, + tagAndAttributeIndexMap, + scanOrder, + timePredicate, + pushDownPredicate, + pushDownLimit, + pushDownOffset, + pushLimitToEachDevice, + containsNonAlignedDevice, + projection, + aggregations, + groupingSets, + preGroupedSymbols, + step, + groupIdSymbol, + treeDBName, + measurementColumnNameMap)); } protected PlanNodeType getPlanNodeType() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AlignedAggregationTreeDeviceViewScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AlignedAggregationTreeDeviceViewScanNode.java index 1e577e812800..19d1013420f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AlignedAggregationTreeDeviceViewScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AlignedAggregationTreeDeviceViewScanNode.java @@ -94,28 +94,29 @@ public R accept(IPlanVisitor visitor, C context) { @Override public AlignedAggregationTreeDeviceViewScanNode clone() { - return new AlignedAggregationTreeDeviceViewScanNode( - getPlanNodeId(), - qualifiedObjectName, - outputSymbols, - assignments, - deviceEntries, - tagAndAttributeIndexMap, - scanOrder, - timePredicate, - pushDownPredicate, - pushDownLimit, - pushDownOffset, - pushLimitToEachDevice, - containsNonAlignedDevice, - projection, - aggregations, - groupingSets, - preGroupedSymbols, - step, - groupIdSymbol, - getTreeDBName(), - getMeasurementColumnNameMap()); + return copyDeviceEntryDataSetTo( + new AlignedAggregationTreeDeviceViewScanNode( + getPlanNodeId(), + qualifiedObjectName, + outputSymbols, + assignments, + deviceEntries, + tagAndAttributeIndexMap, + scanOrder, + timePredicate, + pushDownPredicate, + pushDownLimit, + pushDownOffset, + pushLimitToEachDevice, + containsNonAlignedDevice, + projection, + aggregations, + groupingSets, + preGroupedSymbols, + step, + groupIdSymbol, + getTreeDBName(), + getMeasurementColumnNameMap())); } protected PlanNodeType getPlanNodeType() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java index 3b7b365c3292..50a8e59bd172 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java @@ -30,6 +30,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.relational.metadata.AlignedDeviceEntry; import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSet; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSetHandle; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; import org.apache.tsfile.read.filter.basic.Filter; @@ -41,6 +43,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -48,7 +51,12 @@ public class DeviceTableScanNode extends TableScanNode { - protected List deviceEntries; + protected List deviceEntries = Collections.emptyList(); + + @Nullable protected DeviceEntryDataSetHandle deviceEntryDataSetHandle; + + // Only used on the FE before distributed planning and is not serialized to the BE. + protected transient DeviceEntryDataSet coordinatorDeviceEntryDataSet; // Indicates the respective index order of tag and attribute columns in DeviceEntry. // For example, for DeviceEntry `table1.tag1.tag2.attribute1.attribute2.s1.s2`, the content of @@ -148,6 +156,8 @@ public DeviceTableScanNode clone() { pushLimitToEachDevice, containsNonAlignedDevice); cloned.topKRuntimeFilterSourceId = topKRuntimeFilterSourceId; + cloned.deviceEntryDataSetHandle = deviceEntryDataSetHandle; + cloned.coordinatorDeviceEntryDataSet = coordinatorDeviceEntryDataSet; return cloned; } @@ -155,9 +165,14 @@ protected static void serializeMemberVariables( DeviceTableScanNode node, ByteBuffer byteBuffer, boolean serializeOutputSymbols) { TableScanNode.serializeMemberVariables(node, byteBuffer, serializeOutputSymbols); - ReadWriteIOUtils.write(node.deviceEntries.size(), byteBuffer); - for (DeviceEntry entry : node.deviceEntries) { - entry.serialize(byteBuffer); + ReadWriteIOUtils.write(node.deviceEntryDataSetHandle != null, byteBuffer); + if (node.deviceEntryDataSetHandle != null) { + node.deviceEntryDataSetHandle.serialize(byteBuffer); + } else { + ReadWriteIOUtils.write(node.deviceEntries.size(), byteBuffer); + for (DeviceEntry entry : node.deviceEntries) { + entry.serialize(byteBuffer); + } } ReadWriteIOUtils.write(node.tagAndAttributeIndexMap.size(), byteBuffer); @@ -185,9 +200,14 @@ protected static void serializeMemberVariables( throws IOException { TableScanNode.serializeMemberVariables(node, stream, serializeOutputSymbols); - ReadWriteIOUtils.write(node.deviceEntries.size(), stream); - for (DeviceEntry entry : node.deviceEntries) { - entry.serialize(stream); + ReadWriteIOUtils.write(node.deviceEntryDataSetHandle != null, stream); + if (node.deviceEntryDataSetHandle != null) { + node.deviceEntryDataSetHandle.serialize(stream); + } else { + ReadWriteIOUtils.write(node.deviceEntries.size(), stream); + for (DeviceEntry entry : node.deviceEntries) { + entry.serialize(stream); + } } ReadWriteIOUtils.write(node.tagAndAttributeIndexMap.size(), stream); @@ -214,12 +234,18 @@ protected static void deserializeMemberVariables( ByteBuffer byteBuffer, DeviceTableScanNode node, boolean deserializeOutputSymbols) { TableScanNode.deserializeMemberVariables(byteBuffer, node, deserializeOutputSymbols); - int size = ReadWriteIOUtils.readInt(byteBuffer); - List deviceEntries = new ArrayList<>(size); - while (size-- > 0) { - deviceEntries.add(AlignedDeviceEntry.deserialize(byteBuffer)); + int size; + if (ReadWriteIOUtils.readBool(byteBuffer)) { + node.deviceEntryDataSetHandle = DeviceEntryDataSetHandle.deserialize(byteBuffer); + node.deviceEntries = new ArrayList<>(); + } else { + size = ReadWriteIOUtils.readInt(byteBuffer); + List deviceEntries = new ArrayList<>(size); + while (size-- > 0) { + deviceEntries.add(AlignedDeviceEntry.deserialize(byteBuffer)); + } + node.deviceEntries = deviceEntries; } - node.deviceEntries = deviceEntries; size = ReadWriteIOUtils.readInt(byteBuffer); Map tagAndAttributeIndexMap = new HashMap<>(size); @@ -265,6 +291,48 @@ public static DeviceTableScanNode deserialize(ByteBuffer byteBuffer) { public void setDeviceEntries(List deviceEntries) { this.deviceEntries = deviceEntries; + this.deviceEntryDataSetHandle = null; + } + + public void setDeviceEntryDataSet(final DeviceEntryDataSet deviceEntryDataSet) { + this.coordinatorDeviceEntryDataSet = deviceEntryDataSet; + this.deviceEntries = + deviceEntryDataSet.isSpilled() + ? Collections.emptyList() + : deviceEntryDataSet.getInlineEntries(); + } + + public void setDeviceEntryDataSetHandle(DeviceEntryDataSetHandle deviceEntryDataSetHandle) { + this.deviceEntryDataSetHandle = deviceEntryDataSetHandle; + this.deviceEntries = Collections.emptyList(); + } + + public Optional getDeviceEntryDataSetHandle() { + return Optional.ofNullable(deviceEntryDataSetHandle); + } + + public boolean hasSpilledDeviceEntries() { + return deviceEntryDataSetHandle != null; + } + + public T copyDeviceEntryDataSetTo(final T target) { + target.deviceEntryDataSetHandle = deviceEntryDataSetHandle; + target.coordinatorDeviceEntryDataSet = coordinatorDeviceEntryDataSet; + return target; + } + + public int getDeviceEntryCount() { + return deviceEntryDataSetHandle == null + ? deviceEntries.size() + : deviceEntryDataSetHandle.getEntryCount(); + } + + public void setCoordinatorDeviceEntryDataSet(DeviceEntryDataSet dataSet) { + setDeviceEntryDataSet(dataSet); + } + + public DeviceEntryDataSet getCoordinatorDeviceEntryDataSet() { + return coordinatorDeviceEntryDataSet; } public Map getTagAndAttributeIndexMap() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/NonAlignedAggregationTreeDeviceViewScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/NonAlignedAggregationTreeDeviceViewScanNode.java index af851b82c3be..592b4a5c381c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/NonAlignedAggregationTreeDeviceViewScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/NonAlignedAggregationTreeDeviceViewScanNode.java @@ -95,28 +95,29 @@ public R accept(IPlanVisitor visitor, C context) { @Override public NonAlignedAggregationTreeDeviceViewScanNode clone() { - return new NonAlignedAggregationTreeDeviceViewScanNode( - getPlanNodeId(), - qualifiedObjectName, - outputSymbols, - assignments, - deviceEntries, - tagAndAttributeIndexMap, - scanOrder, - timePredicate, - pushDownPredicate, - pushDownLimit, - pushDownOffset, - pushLimitToEachDevice, - containsNonAlignedDevice, - projection, - aggregations, - groupingSets, - preGroupedSymbols, - step, - groupIdSymbol, - getTreeDBName(), - getMeasurementColumnNameMap()); + return copyDeviceEntryDataSetTo( + new NonAlignedAggregationTreeDeviceViewScanNode( + getPlanNodeId(), + qualifiedObjectName, + outputSymbols, + assignments, + deviceEntries, + tagAndAttributeIndexMap, + scanOrder, + timePredicate, + pushDownPredicate, + pushDownLimit, + pushDownOffset, + pushLimitToEachDevice, + containsNonAlignedDevice, + projection, + aggregations, + groupingSets, + preGroupedSymbols, + step, + groupIdSymbol, + getTreeDBName(), + getMeasurementColumnNameMap())); } protected PlanNodeType getPlanNodeType() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TreeAlignedDeviceViewScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TreeAlignedDeviceViewScanNode.java index 7246f04dacc6..162de5084211 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TreeAlignedDeviceViewScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TreeAlignedDeviceViewScanNode.java @@ -81,22 +81,23 @@ public R accept(IPlanVisitor visitor, C context) { @Override public TreeAlignedDeviceViewScanNode clone() { - return new TreeAlignedDeviceViewScanNode( - getPlanNodeId(), - qualifiedObjectName, - outputSymbols, - assignments, - deviceEntries, - tagAndAttributeIndexMap, - scanOrder, - timePredicate, - pushDownPredicate, - pushDownLimit, - pushDownOffset, - pushLimitToEachDevice, - containsNonAlignedDevice, - treeDBName, - measurementColumnNameMap); + return copyDeviceEntryDataSetTo( + new TreeAlignedDeviceViewScanNode( + getPlanNodeId(), + qualifiedObjectName, + outputSymbols, + assignments, + deviceEntries, + tagAndAttributeIndexMap, + scanOrder, + timePredicate, + pushDownPredicate, + pushDownLimit, + pushDownOffset, + pushLimitToEachDevice, + containsNonAlignedDevice, + treeDBName, + measurementColumnNameMap)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TreeDeviceViewScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TreeDeviceViewScanNode.java index da308b749914..59109d5df369 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TreeDeviceViewScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TreeDeviceViewScanNode.java @@ -112,22 +112,23 @@ public R accept(IPlanVisitor visitor, C context) { @Override public TreeDeviceViewScanNode clone() { - return new TreeDeviceViewScanNode( - getPlanNodeId(), - qualifiedObjectName, - outputSymbols, - assignments, - deviceEntries, - tagAndAttributeIndexMap, - scanOrder, - timePredicate, - pushDownPredicate, - pushDownLimit, - pushDownOffset, - pushLimitToEachDevice, - containsNonAlignedDevice, - treeDBName, - measurementColumnNameMap); + return copyDeviceEntryDataSetTo( + new TreeDeviceViewScanNode( + getPlanNodeId(), + qualifiedObjectName, + outputSymbols, + assignments, + deviceEntries, + tagAndAttributeIndexMap, + scanOrder, + timePredicate, + pushDownPredicate, + pushDownLimit, + pushDownOffset, + pushLimitToEachDevice, + containsNonAlignedDevice, + treeDBName, + measurementColumnNameMap)); } protected static void serializeMemberVariables( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TreeNonAlignedDeviceViewScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TreeNonAlignedDeviceViewScanNode.java index 60db4f93f499..bd56fdc0e77a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TreeNonAlignedDeviceViewScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TreeNonAlignedDeviceViewScanNode.java @@ -81,22 +81,23 @@ public R accept(IPlanVisitor visitor, C context) { @Override public TreeNonAlignedDeviceViewScanNode clone() { - return new TreeNonAlignedDeviceViewScanNode( - getPlanNodeId(), - qualifiedObjectName, - outputSymbols, - assignments, - deviceEntries, - tagAndAttributeIndexMap, - scanOrder, - timePredicate, - pushDownPredicate, - pushDownLimit, - pushDownOffset, - pushLimitToEachDevice, - containsNonAlignedDevice, - treeDBName, - measurementColumnNameMap); + return copyDeviceEntryDataSetTo( + new TreeNonAlignedDeviceViewScanNode( + getPlanNodeId(), + qualifiedObjectName, + outputSymbols, + assignments, + deviceEntries, + tagAndAttributeIndexMap, + scanOrder, + timePredicate, + pushDownPredicate, + pushDownLimit, + pushDownOffset, + pushLimitToEachDevice, + containsNonAlignedDevice, + treeDBName, + measurementColumnNameMap)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java index 9f3c5c32e25a..984a711cc1e7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java @@ -221,7 +221,7 @@ private boolean isSingleDeviceEntry(DeviceTableScanNode tableScanNode) { // optimizer cannot safely use the single-device shortcut here. return false; } - return tableScanNode.getDeviceEntries().size() < 2; + return tableScanNode.getDeviceEntryCount() < 2; } private List getTagColumnsInTableStore( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 1ceb4c86f6c1..f24d48db78fd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -22,7 +22,6 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.partition.DataPartition; -import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.TableScanNode; import org.apache.iotdb.commons.queryengine.plan.relational.analyzer.NodeRef; @@ -67,9 +66,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicateCombineIntoTableScanChecker; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoMetadataChecker; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.schema.ConvertSchemaPredicateToFilterVisitor; -import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; -import org.apache.iotdb.db.queryengine.plan.relational.metadata.NonAlignedDeviceEntry; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSet; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSetResult; import org.apache.iotdb.db.queryengine.plan.relational.planner.EqualityInference; import org.apache.iotdb.db.queryengine.plan.relational.planner.IrExpressionInterpreter; import org.apache.iotdb.db.queryengine.plan.relational.planner.IrTypeAnalyzer; @@ -91,6 +90,8 @@ import javax.annotation.Nullable; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -746,7 +747,7 @@ private void getDeviceEntriesWithDataPartitions( } long startTime = System.nanoTime(); - final Map> deviceEntriesMap = + final DeviceEntryDataSetResult deviceEntryDataSetResult = metadata.indexScan( tableScanNode.getQualifiedObjectName(), metadataExpressions.stream() @@ -756,84 +757,85 @@ private void getDeviceEntriesWithDataPartitions( expression, tableScanNode.getAssignments())) .collect(Collectors.toList()), attributeColumns, - queryContext); - if (deviceEntriesMap.size() > 1) { - throw new SemanticException( - DataNodeQueryMessages.TREE_DEVICE_VIEW_WITH_MULTIPLE_DATABASES - + deviceEntriesMap.keySet() - + DataNodeQueryMessages.IS_UNSUPPORTED_YET); - } - final String deviceDatabase = - !deviceEntriesMap.isEmpty() ? deviceEntriesMap.keySet().iterator().next() : null; - final List deviceEntries = - Objects.nonNull(deviceDatabase) - ? deviceEntriesMap.get(deviceDatabase) - : Collections.emptyList(); - - tableScanNode.setDeviceEntries(deviceEntries); - if (deviceEntries.stream() - .anyMatch(deviceEntry -> deviceEntry instanceof NonAlignedDeviceEntry)) { - tableScanNode.setContainsNonAlignedDevice(); - } - - if (tableScanNode instanceof TreeDeviceViewScanNode) { - ((TreeDeviceViewScanNode) tableScanNode).setTreeDBName(deviceDatabase); - } - - final long schemaFetchCost = System.nanoTime() - startTime; - QueryPlanCostMetricSet.getInstance().recordTablePlanCost(SCHEMA_FETCHER, schemaFetchCost); - queryContext.setFetchSchemaCost(schemaFetchCost); - - if (deviceEntries.isEmpty()) { - if (analysis.noAggregates() && !analysis.hasJoinNode()) { - // no device entries, queries(except aggregation and join) can be finished - analysis.setEmptyDataSource(true); - analysis.setFinishQueryAfterAnalyze(); + queryContext, + tableScanNode.getPlanNodeId()); + final DeviceEntryDataSet deviceEntryDataSet = deviceEntryDataSetResult.getDataSet(); + boolean dataSetTransferred = false; + try { + final String deviceDatabase = deviceEntryDataSetResult.getDatabase(); + if (deviceEntryDataSetResult.containsNonAlignedDevice()) { + tableScanNode.setContainsNonAlignedDevice(); } - } else { - final Filter timeFilter = - tableScanNode - .getTimePredicate() - .map( - value -> - value.accept( - new ConvertPredicateToTimeFilterVisitor( - queryContext.getZoneId(), TimestampPrecisionUtils.currPrecision), - null)) - .orElse(null); - - tableScanNode.setTimeFilter(timeFilter); - - startTime = System.nanoTime(); - final DataPartition dataPartition = - fetchDataPartitionByDevices( - // for tree view, we need to pass actual tree db name to this method - tableScanNode instanceof TreeDeviceViewScanNode - ? deviceDatabase - : tableScanNode.getQualifiedObjectName().getDatabaseName(), - deviceEntries, - timeFilter); - - if (dataPartition.getDataPartitionMap().size() > 1) { - throw new IllegalStateException( - DataNodeQueryMessages - .QUERY_EXCEPTION_TABLE_MODEL_CAN_ONLY_PROCESS_DATA_ONLY_IN_ONE_DATABASE_YET_AB8C1EF5); + if (tableScanNode instanceof TreeDeviceViewScanNode) { + ((TreeDeviceViewScanNode) tableScanNode).setTreeDBName(deviceDatabase); } - if (dataPartition.getDataPartitionMap().isEmpty()) { + final long schemaFetchCost = System.nanoTime() - startTime; + QueryPlanCostMetricSet.getInstance().recordTablePlanCost(SCHEMA_FETCHER, schemaFetchCost); + queryContext.setFetchSchemaCost(schemaFetchCost); + + if (deviceEntryDataSet.getEntryCount() == 0) { if (analysis.noAggregates() && !analysis.hasJoinNode()) { - // no data partitions, queries(except aggregation and join) can be finished + // no device entries, queries(except aggregation and join) can be finished analysis.setEmptyDataSource(true); analysis.setFinishQueryAfterAnalyze(); } } else { - analysis.upsertDataPartition(dataPartition); + final Filter timeFilter = + tableScanNode + .getTimePredicate() + .map( + value -> + value.accept( + new ConvertPredicateToTimeFilterVisitor( + queryContext.getZoneId(), TimestampPrecisionUtils.currPrecision), + null)) + .orElse(null); + + tableScanNode.setTimeFilter(timeFilter); + + startTime = System.nanoTime(); + final DataPartition dataPartition = + fetchDataPartitionByDeviceDataSet( + // for tree view, we need to pass actual tree db name to this method + tableScanNode instanceof TreeDeviceViewScanNode + ? deviceDatabase + : tableScanNode.getQualifiedObjectName().getDatabaseName(), + deviceEntryDataSet, + timeFilter); + + if (dataPartition.getDataPartitionMap().size() > 1) { + throw new IllegalStateException( + DataNodeQueryMessages + .QUERY_EXCEPTION_TABLE_MODEL_CAN_ONLY_PROCESS_DATA_ONLY_IN_ONE_DATABASE_YET_AB8C1EF5); + } + + if (dataPartition.getDataPartitionMap().isEmpty()) { + if (analysis.noAggregates() && !analysis.hasJoinNode()) { + // no data partitions, queries(except aggregation and join) can be finished + analysis.setEmptyDataSource(true); + analysis.setFinishQueryAfterAnalyze(); + } + } else { + analysis.upsertDataPartition(dataPartition); + } + + final long fetchPartitionCost = System.nanoTime() - startTime; + QueryPlanCostMetricSet.getInstance() + .recordTablePlanCost(PARTITION_FETCHER, fetchPartitionCost); + queryContext.setFetchPartitionCost(fetchPartitionCost); } - final long fetchPartitionCost = System.nanoTime() - startTime; - QueryPlanCostMetricSet.getInstance() - .recordTablePlanCost(PARTITION_FETCHER, fetchPartitionCost); - queryContext.setFetchPartitionCost(fetchPartitionCost); + tableScanNode.setDeviceEntryDataSet(deviceEntryDataSet); + dataSetTransferred = true; + } finally { + if (!dataSetTransferred) { + try { + deviceEntryDataSet.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } } } @@ -1363,10 +1365,10 @@ public PlanNode visitUnion(UnionNode node, RewriteContext context) { return node; } - private DataPartition fetchDataPartitionByDevices( + private DataPartition fetchDataPartitionByDeviceDataSet( final String database, // for tree view, database should be the real tree db name with `root.` prefix - final List deviceEntries, + final DeviceEntryDataSet deviceEntryDataSet, final Filter globalTimeFilter) { final Pair, Pair> res = getTimePartitionSlotList(globalTimeFilter, queryContext); @@ -1379,18 +1381,11 @@ private DataPartition fetchDataPartitionByDevices( CONFIG.getSeriesPartitionSlotNum()); } - final List dataPartitionQueryParams = - deviceEntries.stream() - .map( - deviceEntry -> - new DataPartitionQueryParam( - deviceEntry.getDeviceID(), res.left, res.right.left, res.right.right)) - .collect(Collectors.toList()); - if (res.right.left || res.right.right) { - return metadata.getDataPartitionWithUnclosedTimeRange(database, dataPartitionQueryParams); + return metadata.getDataPartitionWithUnclosedTimeRange( + database, deviceEntryDataSet, res.left, res.right.left, res.right.right); } else { - return metadata.getDataPartition(database, dataPartitionQueryParams); + return metadata.getDataPartition(database, deviceEntryDataSet, res.left); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/QueryCardinalityUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/QueryCardinalityUtil.java index 0d8427c985f9..6b49aeb88aa0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/QueryCardinalityUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/QueryCardinalityUtil.java @@ -218,10 +218,10 @@ public Range visitAggregationTableScan(AggregationTableScanNode node, Void && !node.getProjection().getMap().isEmpty()) { // also exist date_bin return Range.atLeast(0L); } else { - return Range.atMost((long) node.getDeviceEntries().size()); + return Range.atMost((long) node.getDeviceEntryCount()); } } else { - return Range.singleton((long) node.getDeviceEntries().size()); + return Range.singleton((long) node.getDeviceEntryCount()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java index b5ed662cdae4..f4af36eebea8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java @@ -141,7 +141,7 @@ public PlanNode visitStreamSort(StreamSortNode node, Context context) { @Override public PlanNode visitDeviceTableScan(DeviceTableScanNode node, Context context) { - context.addDeviceEntrySize(node.getDeviceEntries().size()); + context.addDeviceEntrySize(node.getDeviceEntryCount()); context.setTimeColumnName(node.getTimeColumn().map(Symbol::getName).orElse(null)); return node; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index a938793cd2a5..131ba8af383c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -200,7 +200,7 @@ public PlanAndMappings visitTreeDeviceViewScan( newAssignments.put(newSymbol, handle); }); - return new PlanAndMappings( + TreeDeviceViewScanNode rewrittenNode = new TreeDeviceViewScanNode( node.getPlanNodeId(), node.getQualifiedObjectName(), @@ -216,8 +216,8 @@ public PlanAndMappings visitTreeDeviceViewScan( node.isPushLimitToEachDevice(), node.containsNonAlignedDevice(), node.getTreeDBName(), - node.getMeasurementColumnNameMap()), - mapping); + node.getMeasurementColumnNameMap()); + return new PlanAndMappings(node.copyDeviceEntryDataSetTo(rewrittenNode), mapping); } @Override @@ -235,7 +235,7 @@ public PlanAndMappings visitDeviceTableScan(DeviceTableScanNode node, UnaliasCon newAssignments.put(newSymbol, handle); }); - return new PlanAndMappings( + DeviceTableScanNode rewrittenNode = new DeviceTableScanNode( node.getPlanNodeId(), node.getQualifiedObjectName(), @@ -249,8 +249,8 @@ public PlanAndMappings visitDeviceTableScan(DeviceTableScanNode node, UnaliasCon node.getPushDownLimit(), node.getPushDownOffset(), node.isPushLimitToEachDevice(), - node.containsNonAlignedDevice()), - mapping); + node.containsNonAlignedDevice()); + return new PlanAndMappings(node.copyDeviceEntryDataSetTo(rewrittenNode), mapping); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java index b18af7913851..9922c8d78969 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java @@ -214,6 +214,7 @@ public static Pair split( node.getGroupIdSymbol(), aggregationTreeDeviceViewScanNode.getTreeDBName(), aggregationTreeDeviceViewScanNode.getMeasurementColumnNameMap()); + node.copyDeviceEntryDataSetTo(rightResult); return new Pair<>( new AggregationNode( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsDrawer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsDrawer.java index 02999d57b010..7c4a45cfcacd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsDrawer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsDrawer.java @@ -53,6 +53,19 @@ public void renderPlanStatistics(MPPQueryContext context) { 0, String.format( "Fetch Schema Cost: %.3f ms", context.getFetchSchemaCost() * NS_TO_MS_FACTOR)); + addLine( + planHeader, + 1, + String.format( + "Disk IO Size for DeviceEntry During FetchSchema: %d bytes", + context.getDiskIOSizeForDeviceEntryDuringFetchSchema())); + addLine( + planHeader, + 1, + String.format( + "Disk IO Time Cost for DeviceEntry During FetchSchema: %.3f ms", + context.getDiskIOTimeCostForDeviceEntryDuringFetchSchema() * NS_TO_MS_FACTOR)); + addLine(planHeader, 1, String.format("DeviceEntry Count: %d", context.getDeviceEntryCount())); addLine( planHeader, 0, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java index 74571ad152f9..ef887034dac1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawer.java @@ -55,6 +55,13 @@ public void renderPlanStatistics(MPPQueryContext context) { "fetchPartitionCostMs", formatMs(context.getFetchPartitionCost() * NS_TO_MS_FACTOR)); planStatistics.addProperty( "fetchSchemaCostMs", formatMs(context.getFetchSchemaCost() * NS_TO_MS_FACTOR)); + planStatistics.addProperty( + "diskIOSizeForDeviceEntryDuringFetchSchema", + context.getDiskIOSizeForDeviceEntryDuringFetchSchema()); + planStatistics.addProperty( + "diskIOTimeCostForDeviceEntryDuringFetchSchemaMs", + formatMs(context.getDiskIOTimeCostForDeviceEntryDuringFetchSchema() * NS_TO_MS_FACTOR)); + planStatistics.addProperty("deviceEntryCount", context.getDeviceEntryCount()); planStatistics.addProperty( "logicalPlanCostMs", formatMs(context.getLogicalPlanCost() * NS_TO_MS_FACTOR)); planStatistics.addProperty( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/QueryPlanStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/QueryPlanStatistics.java index edb13217db22..aa46addce37c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/QueryPlanStatistics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/QueryPlanStatistics.java @@ -27,6 +27,9 @@ public class QueryPlanStatistics { private long logicalOptimizationCost; private long distributionPlanCost; private long dispatchCost = 0; + private long diskIOSizeForDeviceEntryDuringFetchSchema; + private long diskIOTimeCostForDeviceEntryDuringFetchSchema; + private long deviceEntryCount; public void setAnalyzeCost(long analyzeCost) { this.analyzeCost = analyzeCost; @@ -83,4 +86,25 @@ public void recordDispatchCost(long dispatchCost) { public long getDispatchCost() { return dispatchCost; } + + public void recordDeviceEntryDiskIODuringFetchSchema(long bytes, long timeCost) { + diskIOSizeForDeviceEntryDuringFetchSchema += bytes; + diskIOTimeCostForDeviceEntryDuringFetchSchema += timeCost; + } + + public void recordDeviceEntryCount(long count) { + deviceEntryCount += count; + } + + public long getDiskIOSizeForDeviceEntryDuringFetchSchema() { + return diskIOSizeForDeviceEntryDuringFetchSchema; + } + + public long getDiskIOTimeCostForDeviceEntryDuringFetchSchema() { + return diskIOTimeCostForDeviceEntryDuringFetchSchema; + } + + public long getDeviceEntryCount() { + return deviceEntryCount; + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakePartitionFetcherImpl.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakePartitionFetcherImpl.java index 4c0c39107fef..85c2673cc91b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakePartitionFetcherImpl.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakePartitionFetcherImpl.java @@ -33,6 +33,7 @@ import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSet; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; import org.apache.tsfile.file.metadata.IDeviceID; @@ -216,12 +217,28 @@ public DataPartition getDataPartition( return dataPartition; } + @Override + public DataPartition getDataPartition( + String database, DeviceEntryDataSet dataSet, List timePartitionSlots) { + throw new UnsupportedOperationException(); + } + @Override public DataPartition getDataPartitionWithUnclosedTimeRange( Map> sgNameToQueryParamsMap) { return getDataPartition(sgNameToQueryParamsMap); } + @Override + public DataPartition getDataPartitionWithUnclosedTimeRange( + String database, + DeviceEntryDataSet dataSet, + List timePartitionSlots, + boolean needLeftAll, + boolean needRightAll) { + throw new UnsupportedOperationException(); + } + @Override public DataPartition getOrCreateDataPartition( Map> sgNameToQueryParamsMap) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java index e4d3a0c4ba9c..03615e558065 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java @@ -53,6 +53,7 @@ import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand; import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator; import org.apache.iotdb.db.queryengine.plan.planner.LogicalPlanner; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSet; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; @@ -406,12 +407,30 @@ public DataPartition getDataPartition( return ANALYSIS.getDataPartitionInfo(); } + @Override + public DataPartition getDataPartition( + String database, + DeviceEntryDataSet dataSet, + List timePartitionSlots) { + return ANALYSIS.getDataPartitionInfo(); + } + @Override public DataPartition getDataPartitionWithUnclosedTimeRange( Map> sgNameToQueryParamsMap) { return ANALYSIS.getDataPartitionInfo(); } + @Override + public DataPartition getDataPartitionWithUnclosedTimeRange( + String database, + DeviceEntryDataSet dataSet, + List timePartitionSlots, + boolean needLeftAll, + boolean needRightAll) { + return ANALYSIS.getDataPartitionInfo(); + } + @Override public DataPartition getOrCreateDataPartition( Map> sgNameToQueryParamsMap) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java index bb37c554eab7..303c2268911d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java @@ -50,6 +50,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher; import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator; import org.apache.iotdb.db.queryengine.plan.planner.LogicalPlanner; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSet; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; @@ -299,12 +300,30 @@ public DataPartition getDataPartition( return ANALYSIS.getDataPartitionInfo(); } + @Override + public DataPartition getDataPartition( + String database, + DeviceEntryDataSet dataSet, + List timePartitionSlots) { + return ANALYSIS.getDataPartitionInfo(); + } + @Override public DataPartition getDataPartitionWithUnclosedTimeRange( Map> sgNameToQueryParamsMap) { return ANALYSIS.getDataPartitionInfo(); } + @Override + public DataPartition getDataPartitionWithUnclosedTimeRange( + String database, + DeviceEntryDataSet dataSet, + List timePartitionSlots, + boolean needLeftAll, + boolean needRightAll) { + return ANALYSIS.getDataPartitionInfo(); + } + @Override public DataPartition getOrCreateDataPartition( Map> sgNameToQueryParamsMap) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java index 4794b19994e8..4750ae6867b0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java @@ -19,12 +19,14 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.partition.SchemaNodeManagementPartition; import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.queryengine.common.SessionInfo; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.queryengine.plan.relational.function.OperatorType; import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnMetadata; import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema; @@ -46,6 +48,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.metadata.OperatorNotFoundException; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSet; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSetResult; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.InMemoryDeviceEntryDataSet; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; import org.apache.iotdb.udf.api.relational.TableFunction; @@ -279,7 +284,20 @@ public boolean canCoerce(Type from, Type to) { } @Override - public Map> indexScan( + public DeviceEntryDataSetResult indexScan( + QualifiedObjectName tableName, + List expressionList, + List attributeColumns, + MPPQueryContext context, + PlanNodeId planNodeId) { + final Map> deviceEntries = + indexScanEntries(tableName, expressionList, attributeColumns, context); + final String database = deviceEntries.keySet().iterator().next(); + return new DeviceEntryDataSetResult( + database, new InMemoryDeviceEntryDataSet(deviceEntries.get(database)), false); + } + + private Map> indexScanEntries( QualifiedObjectName tableName, List expressionList, List attributeColumns, @@ -387,12 +405,28 @@ public DataPartition getDataPartition( return DATA_PARTITION; } + @Override + public DataPartition getDataPartition( + String database, DeviceEntryDataSet dataSet, List timePartitionSlots) { + return DATA_PARTITION; + } + @Override public DataPartition getDataPartitionWithUnclosedTimeRange( String database, List sgNameToQueryParamsMap) { return DATA_PARTITION; } + @Override + public DataPartition getDataPartitionWithUnclosedTimeRange( + String database, + DeviceEntryDataSet dataSet, + List timePartitionSlots, + boolean needLeftAll, + boolean needRightAll) { + return DATA_PARTITION; + } + @Override public TableFunction getTableFunction(String functionName) { return null; @@ -430,12 +464,30 @@ public DataPartition getDataPartition( return DATA_PARTITION; } + @Override + public DataPartition getDataPartition( + String database, + DeviceEntryDataSet dataSet, + List timePartitionSlots) { + return DATA_PARTITION; + } + @Override public DataPartition getDataPartitionWithUnclosedTimeRange( Map> sgNameToQueryParamsMap) { return DATA_PARTITION; } + @Override + public DataPartition getDataPartitionWithUnclosedTimeRange( + String database, + DeviceEntryDataSet dataSet, + List timePartitionSlots, + boolean needLeftAll, + boolean needRightAll) { + return DATA_PARTITION; + } + @Override public DataPartition getOrCreateDataPartition( Map> sgNameToQueryParamsMap) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java index ac266f19398c..b371bbaa45a0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; @@ -26,6 +27,7 @@ import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.queryengine.common.SessionInfo; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.queryengine.plan.relational.function.OperatorType; import org.apache.iotdb.commons.queryengine.plan.relational.function.TableBuiltinTableFunction; import org.apache.iotdb.commons.queryengine.plan.relational.function.arithmetic.SubtractionResolver; @@ -59,6 +61,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.OperatorNotFoundException; import org.apache.iotdb.db.queryengine.plan.relational.metadata.TreeDeviceViewSchema; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSet; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.DeviceEntryDataSetResult; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.spill.InMemoryDeviceEntryDataSet; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.schemaengine.table.InformationSchemaUtils; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; @@ -332,7 +337,23 @@ public boolean canCoerce(final Type from, final Type to) { } @Override - public Map> indexScan( + public DeviceEntryDataSetResult indexScan( + final QualifiedObjectName tableName, + final List expressionList, + final List attributeColumns, + final MPPQueryContext context, + final PlanNodeId planNodeId) { + final Map> deviceEntries = + indexScanEntries(tableName, expressionList, attributeColumns, context); + final String database = deviceEntries.keySet().iterator().next(); + final List entries = deviceEntries.get(database); + return new DeviceEntryDataSetResult( + database, + new InMemoryDeviceEntryDataSet(entries), + entries.stream().anyMatch(NonAlignedDeviceEntry.class::isInstance)); + } + + private Map> indexScanEntries( final QualifiedObjectName tableName, final List expressionList, final List attributeColumns, @@ -541,12 +562,30 @@ public DataPartition getDataPartition( return TREE_DB1.equals(database) ? TREE_VIEW_DATA_PARTITION : TABLE_DATA_PARTITION; } + @Override + public DataPartition getDataPartition( + final String database, + final DeviceEntryDataSet dataSet, + final List timePartitionSlots) { + return TREE_DB1.equals(database) ? TREE_VIEW_DATA_PARTITION : TABLE_DATA_PARTITION; + } + @Override public DataPartition getDataPartitionWithUnclosedTimeRange( final String database, final List sgNameToQueryParamsMap) { return TREE_DB1.equals(database) ? TREE_VIEW_DATA_PARTITION : TABLE_DATA_PARTITION; } + @Override + public DataPartition getDataPartitionWithUnclosedTimeRange( + final String database, + final DeviceEntryDataSet dataSet, + final List timePartitionSlots, + final boolean needLeftAll, + final boolean needRightAll) { + return TREE_DB1.equals(database) ? TREE_VIEW_DATA_PARTITION : TABLE_DATA_PARTITION; + } + @Override public TableFunction getTableFunction(String functionName) { if ("EXCLUDE".equalsIgnoreCase(functionName)) { @@ -606,6 +645,14 @@ public DataPartition getDataPartition( : TABLE_DATA_PARTITION; } + @Override + public DataPartition getDataPartition( + String database, + DeviceEntryDataSet dataSet, + List timePartitionSlots) { + return TREE_VIEW_DB.equals(database) ? TREE_VIEW_DATA_PARTITION : TABLE_DATA_PARTITION; + } + @Override public DataPartition getDataPartitionWithUnclosedTimeRange( Map> sgNameToQueryParamsMap) { @@ -614,6 +661,16 @@ public DataPartition getDataPartitionWithUnclosedTimeRange( : TABLE_DATA_PARTITION; } + @Override + public DataPartition getDataPartitionWithUnclosedTimeRange( + String database, + DeviceEntryDataSet dataSet, + List timePartitionSlots, + boolean needLeftAll, + boolean needRightAll) { + return TREE_VIEW_DB.equals(database) ? TREE_VIEW_DATA_PARTITION : TABLE_DATA_PARTITION; + } + @Override public DataPartition getOrCreateDataPartition( Map> sgNameToQueryParamsMap) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryMaterializerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryMaterializerTest.java new file mode 100644 index 000000000000..5350341698d3 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryMaterializerTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.AlignedDeviceEntry; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.Binary; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class DeviceEntryMaterializerTest { + + private Path queryDirectory; + private String originalSortTmpDir; + + @Before + public void setUp() throws Exception { + queryDirectory = Files.createTempDirectory("device-entry-spill-test"); + originalSortTmpDir = IoTDBDescriptor.getInstance().getConfig().getSortTmpDir(); + IoTDBDescriptor.getInstance().getConfig().setSortTmpDir(queryDirectory.toString()); + } + + @After + public void tearDown() throws Exception { + DeviceEntrySpillManager.getInstance().clearStaleData(); + Files.deleteIfExists(queryDirectory.resolve("device-entry")); + Files.deleteIfExists(queryDirectory); + IoTDBDescriptor.getInstance().getConfig().setSortTmpDir(originalSortTmpDir); + } + + @Test + public void testKeepSmallDataSetInline() throws Exception { + List expected = createEntries(3); + try (DeviceEntryMaterializer materializer = + new DeviceEntryMaterializer("q-inline", new PlanNodeId("scan-0"), Long.MAX_VALUE, true)) { + for (DeviceEntry entry : expected) { + materializer.append(entry); + } + try (DeviceEntryDataSet dataSet = materializer.finish()) { + assertFalse(dataSet.isSpilled()); + assertEquals(expected, dataSet.getInlineEntries()); + } + } + } + + @Test + public void testSpillAndReadMultipleSegments() throws Exception { + List expected = createEntries(20); + DeviceEntryDataSet dataSet; + try (DeviceEntryMaterializer materializer = + new DeviceEntryMaterializer("q-spill", new PlanNodeId("scan-0"), 128, true)) { + for (DeviceEntry entry : expected) { + materializer.append(entry); + } + dataSet = materializer.finish(); + } + + assertTrue(dataSet.isSpilled()); + assertEquals(expected.size(), dataSet.getEntryCount()); + List actual = new ArrayList<>(); + try (DeviceEntryReader reader = dataSet.openReader()) { + while (reader.hasNext()) { + actual.add(reader.next()); + } + } + assertEquals(expected, actual); + + dataSet.close(); + assertFalse(Files.exists(queryDirectory.resolve("device-entry/q-spill/scan-0"))); + } + + @Test + public void testControlledSegmentAccess() throws Exception { + DeviceEntryDataSet dataSet; + try (DeviceEntryMaterializer materializer = + new DeviceEntryMaterializer("q-segment", new PlanNodeId("scan-0"), 128, true)) { + for (DeviceEntry entry : createEntries(20)) { + materializer.append(entry); + } + dataSet = materializer.finish(); + } + + DeviceEntrySpillManager manager = DeviceEntrySpillManager.getInstance(); + Path rawDirectory = queryDirectory.resolve("device-entry/q-segment/scan-0/raw"); + List segments; + try (java.util.stream.Stream stream = Files.list(rawDirectory)) { + segments = + stream + .filter(path -> path.getFileName().toString().endsWith(".bin")) + .sorted() + .collect(Collectors.toList()); + } + assertTrue(segments.size() > 1); + assertTrue(Files.size(segments.get(0)) > 0); + dataSet.close(); + } + + @Test + public void testSpillFileUsesLengthPrefixedRecordsWithoutHeaderOrCrc() throws Exception { + DeviceEntry entry = createEntries(1).get(0); + DeviceEntryDataSet dataSet; + try (DeviceEntryMaterializer materializer = + new DeviceEntryMaterializer("q-format", new PlanNodeId("scan-0"), 1, true)) { + materializer.append(entry); + dataSet = materializer.finish(); + } + + Path segment = + DeviceEntrySpillManager.getInstance().listSegments("q-format", "scan-0/raw").get(0); + byte[] fileBytes = Files.readAllBytes(segment); + byte[] payload = entry.serializeToBytes(); + assertEquals(Integer.BYTES + payload.length, fileBytes.length); + assertEquals(payload.length, ByteBuffer.wrap(fileBytes).getInt()); + dataSet.close(); + } + + @Test + public void testMemoryControllerSpillsLargestMaterializer() throws Exception { + DeviceEntry first = createEntries(1).get(0); + DeviceEntry second = createEntries(2).get(1); + try (DeviceEntryMaterializer firstMaterializer = + new DeviceEntryMaterializer("q-controller", new PlanNodeId("scan-0"), 128, false); + DeviceEntryMaterializer secondMaterializer = + new DeviceEntryMaterializer("q-controller", new PlanNodeId("scan-1"), 128, false)) { + DeviceEntryMaterializationMemoryController controller = + new DeviceEntryMaterializationMemoryController( + first.ramBytesUsed() + second.ramBytesUsed() - 1); + controller.append(firstMaterializer, first); + controller.append(secondMaterializer, second); + + assertTrue(firstMaterializer.isSpilled() || secondMaterializer.isSpilled()); + } + } + + @Test + public void testSortedMaterializerMergesRunsInOrder() throws Exception { + List input = createEntries(40); + input.sort(Comparator.comparing(entry -> entry.getDeviceID().toString()).reversed()); + List actual = new ArrayList<>(); + try (DeviceEntrySortedMaterializer materializer = + new DeviceEntrySortedMaterializer( + "q-sorted", + new PlanNodeId("scan-0"), + 128, + Comparator.comparing(entry -> entry.getDeviceID().toString()))) { + DeviceEntryMaterializationMemoryController controller = + new DeviceEntryMaterializationMemoryController(128); + for (DeviceEntry entry : input) { + controller.append(materializer, entry); + } + try (DeviceEntryDataSet dataSet = materializer.finish(); + DeviceEntryReader reader = dataSet.openReader()) { + while (reader.hasNext()) { + actual.add(reader.next()); + } + } + } + List expected = new ArrayList<>(input); + expected.sort(Comparator.comparing(entry -> entry.getDeviceID().toString())); + assertEquals(expected, actual); + } + + private static List createEntries(int count) { + List entries = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + entries.add( + new AlignedDeviceEntry( + IDeviceID.Factory.DEFAULT_FACTORY.create(new String[] {"table", "device" + i}), + new Binary[] {new Binary(("attribute" + i).getBytes(TSFileConfig.STRING_CHARSET))})); + } + return entries; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryRpcSegmentFetcherTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryRpcSegmentFetcherTest.java new file mode 100644 index 000000000000..8572f007c08b --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/DeviceEntryRpcSegmentFetcherTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.mpp.rpc.thrift.TFetchDeviceEntrySegmentResp; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.thrift.TException; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import static org.junit.Assert.assertArrayEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DeviceEntryRpcSegmentFetcherTest { + + private IClientManager clientManager; + private SyncDataNodeMPPDataExchangeServiceClient client; + private DeviceEntryRpcSegmentFetcher fetcher; + private DeviceEntryDataSetHandle handle; + + @Before + @SuppressWarnings("unchecked") + public void setUp() throws Exception { + clientManager = Mockito.mock(IClientManager.class); + client = Mockito.mock(SyncDataNodeMPPDataExchangeServiceClient.class); + when(clientManager.borrowClient(any())).thenReturn(client); + fetcher = new DeviceEntryRpcSegmentFetcher(clientManager); + handle = + new DeviceEntryDataSetHandle( + "query", new PlanNodeId("scan"), new TEndPoint("127.0.0.1", 10740), 1, 1, false); + } + + @Test + public void testFetchRetriesNetworkFailure() throws Exception { + byte[] payload = new byte[] {1, 2, 3}; + when(client.fetchDeviceEntrySegment(any())) + .thenThrow(new TException()) + .thenThrow(new TException()) + .thenReturn( + new TFetchDeviceEntrySegmentResp( + new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())) + .setPayload(payload)); + + assertArrayEquals(payload, fetcher.fetch(handle, 0)); + verify(client, times(3)).fetchDeviceEntrySegment(any()); + } + + @Test + public void testFinishRetriesNetworkFailureAtMostThreeTimes() throws Exception { + when(client.finishDeviceEntrySegment(any(), any())).thenThrow(new TException()); + + fetcher.finish(handle); + + verify(client, times(3)).finishDeviceEntrySegment(any(), any()); + } + + @Test + public void testFinishDoesNotRetryServerFailure() throws Exception { + when(client.finishDeviceEntrySegment(any(), any())) + .thenReturn(new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode())); + + fetcher.finish(handle); + + verify(client, times(1)).finishDeviceEntrySegment(any(), any()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/SegmentDeviceEntrySourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/SegmentDeviceEntrySourceTest.java new file mode 100644 index 000000000000..2eba72dfef12 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/spill/SegmentDeviceEntrySourceTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.spill; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.AlignedDeviceEntry; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; + +import org.apache.tsfile.file.metadata.IDeviceID; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class SegmentDeviceEntrySourceTest { + + private Path queryDirectory; + private String originalSortTmpDir; + + @Before + public void setUp() throws Exception { + queryDirectory = Files.createTempDirectory("device-entry-source-test"); + originalSortTmpDir = IoTDBDescriptor.getInstance().getConfig().getSortTmpDir(); + IoTDBDescriptor.getInstance().getConfig().setSortTmpDir(queryDirectory.toString()); + } + + @After + public void tearDown() throws Exception { + DeviceEntrySpillManager.getInstance().clearStaleData(); + Files.deleteIfExists(queryDirectory.resolve("device-entry")); + Files.deleteIfExists(queryDirectory); + IoTDBDescriptor.getInstance().getConfig().setSortTmpDir(originalSortTmpDir); + } + + @Test + public void testLocalSourceConsumesSegmentsAndCleansDataSet() throws Exception { + List expected = createEntries(20); + PlanNodeId planNodeId = new PlanNodeId("scan-local"); + SpilledDeviceEntryDataSet dataSet; + try (DeviceEntryMaterializer materializer = + new DeviceEntryMaterializer("q-local", planNodeId, 128, false)) { + for (DeviceEntry entry : expected) { + materializer.append(entry); + } + materializer.forceSpill(); + dataSet = (SpilledDeviceEntryDataSet) materializer.finish(); + } + + DeviceEntryDataSetHandle handle = + new DeviceEntryDataSetHandle( + "q-local", + planNodeId, + new TEndPoint("127.0.0.1", 1), + dataSet.getSegments().size(), + expected.size(), + false); + List actual = new ArrayList<>(); + try (LocalSegmentDeviceEntrySource source = new LocalSegmentDeviceEntrySource(handle)) { + while (source.hasNextBatch()) { + actual.addAll(source.nextBatch()); + } + } + + assertEquals(expected, actual); + assertFalse(Files.exists(queryDirectory.resolve("device-entry/q-local/scan-local"))); + } + + @Test + public void testRemoteSourceFetchesSegmentsAndFinishes() throws Exception { + List expected = createEntries(3); + RecordingFetcher fetcher = new RecordingFetcher(expected); + DeviceEntryDataSetHandle handle = + new DeviceEntryDataSetHandle( + "q-remote", + new PlanNodeId("scan-remote"), + new TEndPoint("127.0.0.2", 2), + expected.size(), + expected.size(), + false); + List actual = new ArrayList<>(); + try (RemoteSegmentDeviceEntrySource source = + new RemoteSegmentDeviceEntrySource(handle, fetcher)) { + while (source.hasNextBatch()) { + actual.addAll(source.nextBatch()); + } + } + + assertEquals(expected, actual); + assertEquals(List.of(0, 1, 2), fetcher.segmentIds); + assertTrue(fetcher.finished); + } + + @Test + public void testFinishUnregisteredDataSetIsIdempotent() throws Exception { + DeviceEntrySpillManager.getInstance() + .finishSegmentDataSet("unregistered-query", "unregistered-scan"); + DeviceEntrySpillManager.getInstance() + .finishSegmentDataSet("unregistered-query", "unregistered-scan"); + } + + private static List createEntries(int count) { + List entries = new ArrayList<>(); + for (int i = 0; i < count; i++) { + entries.add( + new AlignedDeviceEntry( + IDeviceID.Factory.DEFAULT_FACTORY.create(new String[] {"table", "device" + i}), + new org.apache.tsfile.utils.Binary[0])); + } + return entries; + } + + private static byte[] serializeSegment(DeviceEntry entry) throws Exception { + byte[] payload = entry.serializeToBytes(); + try (ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + DataOutputStream output = new DataOutputStream(bytes)) { + output.writeInt(payload.length); + output.write(payload); + return bytes.toByteArray(); + } + } + + private static final class RecordingFetcher implements DeviceEntrySegmentFetcher { + + private final List entries; + private final List segmentIds = new ArrayList<>(); + private boolean finished; + + private RecordingFetcher(List entries) { + this.entries = entries; + } + + @Override + public byte[] fetch(DeviceEntryDataSetHandle handle, int segmentId) throws java.io.IOException { + segmentIds.add(segmentId); + try { + return serializeSegment(entries.get(segmentId)); + } catch (Exception e) { + throw new java.io.IOException(e); + } + } + + @Override + public void finish(DeviceEntryDataSetHandle handle) { + finished = true; + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawerTest.java index b83501a277c4..4dd52b45d710 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsJsonDrawerTest.java @@ -60,6 +60,8 @@ public void testRenderPlanStatistics() { context.setLogicalPlanCost(4000000L); // 4ms context.setLogicalOptimizationCost(5000000L); // 5ms context.setDistributionPlanCost(6000000L); // 6ms + context.recordDeviceEntryDiskIODuringFetchSchema(8192L, 7000000L); // 8 KiB, 7ms + context.recordDeviceEntryCount(3); drawer.renderPlanStatistics(context); @@ -78,6 +80,10 @@ public void testRenderPlanStatistics() { assertEquals(4.0, planStats.get("logicalPlanCostMs").getAsDouble(), 0.01); assertEquals(5.0, planStats.get("logicalOptimizationCostMs").getAsDouble(), 0.01); assertEquals(6.0, planStats.get("distributionPlanCostMs").getAsDouble(), 0.01); + assertEquals(8192L, planStats.get("diskIOSizeForDeviceEntryDuringFetchSchema").getAsLong()); + assertEquals( + 7.0, planStats.get("diskIOTimeCostForDeviceEntryDuringFetchSchemaMs").getAsDouble(), 0.01); + assertEquals(3L, planStats.get("deviceEntryCount").getAsLong()); } @Test diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index c4a06b68b149..81573e47f541 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -2445,3 +2445,7 @@ enable_retry_for_unknown_error=false # effectiveMode: hot_reload # Datatype: Boolean include_null_value_in_write_throughput_metric=false +# Maximum DeviceEntry bytes retained in memory before spilling for a table query. +# <= 0 uses query execution memory / query_thread_count / 4. +# Datatype: long; effectiveMode: hot_reload; unit: byte. +# table_query_device_entry_batch_size_in_bytes=0 diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index d66ed10ccf9c..f55f87a0fd90 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -879,6 +879,17 @@ struct TKillQueryInstanceReq { 2: optional string allowedUsername } +struct TFetchDeviceEntrySegmentReq { + 1: required string queryId + 2: required string planNodeId + 3: required i32 segmentId +} + +struct TFetchDeviceEntrySegmentResp { + 1: required common.TSStatus status + 2: optional binary payload +} + /** * END: Used for EXPLAIN ANALYZE **/ @@ -1440,6 +1451,10 @@ service MPPDataExchangeService { void onEndOfDataBlockEvent(TEndOfDataBlockEvent e); + TFetchDeviceEntrySegmentResp fetchDeviceEntrySegment(TFetchDeviceEntrySegmentReq req); + + common.TSStatus finishDeviceEntrySegment(1: string queryId, 2: string planNodeId); + /** Empty rpc, only for connection test */ common.TSStatus testConnectionEmptyRPC() }