diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/DataPartitionTableIntegrityCheckProcedureIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/DataPartitionTableIntegrityCheckProcedureIT.java new file mode 100644 index 000000000000..da54c39479e0 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/DataPartitionTableIntegrityCheckProcedureIT.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.partition; + +import org.apache.iotdb.commons.enums.RepairDataPartitionTableProgressState; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.consensus.ConsensusFactory.RATIS_CONSENSUS; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class DataPartitionTableIntegrityCheckProcedureIT { + private static final Logger LOGGER = + LoggerFactory.getLogger(DataPartitionTableIntegrityCheckProcedureIT.class); + + @Before + public void setUp() { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setConfigNodeConsensusProtocolClass(RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(RATIS_CONSENSUS) + .setDataReplicationFactor(1); + EnvFactory.getEnv().initClusterEnvironment(1, 1); + } + + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testConcurrentSubmitDataPartitionTableIntegrityCheckProcedure() + throws InterruptedException { + final int threadCount = 10; + final CountDownLatch startLatch = new CountDownLatch(1); + final CountDownLatch finishLatch = new CountDownLatch(threadCount); + final ExecutorService executor = Executors.newFixedThreadPool(threadCount); + + final AtomicInteger successCount = new AtomicInteger(0); + final AtomicInteger failCount = new AtomicInteger(0); + final List failureMessages = Collections.synchronizedList(new ArrayList<>()); + + // Concurrently submit the DataPartitionTableIntegrityCheckProcedure + for (int i = 0; i < threadCount; i++) { + final int threadId = i; + executor.submit( + () -> { + try { + startLatch.await(); + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement stmt = connection.createStatement()) { + stmt.execute("REPAIR DATA PARTITION TABLE"); + successCount.incrementAndGet(); + LOGGER.info("Thread {} submitted integrity check successfully", threadId); + } + } catch (final SQLException e) { + failCount.incrementAndGet(); + failureMessages.add("Thread " + threadId + " failed: " + e.getMessage()); + LOGGER.info( + "Thread {} failed to submit integrity check: {}", threadId, e.getMessage()); + } catch (final Exception e) { + failCount.incrementAndGet(); + failureMessages.add("Thread " + threadId + " failed unexpectedly: " + e.getMessage()); + LOGGER.error("Thread {} unexpected error: {}", threadId, e.getMessage(), e); + } finally { + finishLatch.countDown(); + } + }); + } + + startLatch.countDown(); + + final boolean completed = finishLatch.await(60, TimeUnit.SECONDS); + Assert.assertTrue("Not all threads completed within timeout", completed); + + executor.shutdown(); + Assert.assertTrue( + "Executor did not terminate", executor.awaitTermination(10, TimeUnit.SECONDS)); + + LOGGER.info("Success count: {}, Fail count: {}", successCount.get(), failCount.get()); + LOGGER.info("Failure messages: {}", failureMessages); + + Assert.assertEquals( + "Only one procedure should be submitted successfully", 1, successCount.get()); + Assert.assertEquals( + "The other concurrent submissions should be rejected", threadCount - 1, failCount.get()); + } + + @Test + public void testShowRepairDataPartitionTableProgress() throws Exception { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + assertRepairProgress(statement, RepairDataPartitionTableProgressState.IDLE.name(), 0.0, 0.0); + + statement.execute("REPAIR DATA PARTITION TABLE"); + assertRepairProgress(statement, null, 0.0, 100.0); + } + } + + private static void assertRepairProgress( + final Statement statement, + final String expectedStatus, + final double minProgress, + final double maxProgress) + throws SQLException { + try (final ResultSet resultSet = + statement.executeQuery("SHOW REPAIR DATA PARTITION TABLE PROGRESS")) { + Assert.assertTrue(resultSet.next()); + if (expectedStatus != null) { + Assert.assertEquals(expectedStatus, resultSet.getString("Status")); + } else { + Assert.assertNotEquals( + RepairDataPartitionTableProgressState.UNKNOWN.name(), resultSet.getString("Status")); + } + final double progress = resultSet.getDouble("Progress(%)"); + Assert.assertTrue(progress >= minProgress); + Assert.assertTrue(progress <= maxProgress); + Assert.assertNotNull(resultSet.getString("Message")); + Assert.assertFalse(resultSet.next()); + } + } +} diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 index 0087a5335db7..e5e765372217 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 @@ -176,6 +176,7 @@ keyWords | PRIVILEGES | PRIVILEGE_VALUE | PROCESSLIST + | PROGRESS | PROCESSOR | PROPERTY | PRUNE @@ -225,6 +226,7 @@ keyWords | SUBSCRIPTIONS | SUBSTRING | SYSTEM + | TABLE | TAGS | TAIL | TASK diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index a8897f992247..308bdcdfd67d 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -89,6 +89,7 @@ utilityStatement | showQueries | showCurrentTimestamp | killQuery | grantWatermarkEmbedding | revokeWatermarkEmbedding | loadConfiguration | loadTimeseries | loadFile | removeFile | unloadFile + | repairDataPartitionTable | showRepairDataPartitionTableProgress ; /** @@ -1088,6 +1089,16 @@ stopRepairData : STOP REPAIR DATA (ON (LOCAL | CLUSTER))? ; +// Repair Data Partition Table +repairDataPartitionTable + : REPAIR DATA PARTITION TABLE + ; + +// Show Repair Data Partition Table Progress +showRepairDataPartitionTableProgress + : SHOW REPAIR DATA PARTITION TABLE PROGRESS + ; + // Explain explain : EXPLAIN (ANALYZE VERBOSE?)? selectStatement? @@ -1432,4 +1443,4 @@ subStringExpression signedIntegerLiteral : (PLUS|MINUS)?INTEGER_LITERAL - ; \ No newline at end of file + ; diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 index d5847b2d47da..12fa1de93282 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 @@ -806,6 +806,10 @@ SYSTEM : S Y S T E M ; +TABLE + : T A B L E + ; + TAGS : T A G S ; @@ -1082,6 +1086,10 @@ REPAIR : R E P A I R ; +PROGRESS + : P R O G R E S S + ; + SCHEMA_REPLICATION_FACTOR : S C H E M A '_' R E P L I C A T I O N '_' F A C T O R ; @@ -1277,4 +1285,4 @@ fragment V: [vV]; fragment W: [wW]; fragment X: [xX]; fragment Y: [yY]; -fragment Z: [zZ]; \ No newline at end of file +fragment Z: [zZ]; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/CnToDnSyncRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/CnToDnSyncRequestType.java index 14d0d60fc8dc..7feb5d419d29 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/CnToDnSyncRequestType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/CnToDnSyncRequestType.java @@ -36,6 +36,12 @@ public enum CnToDnSyncRequestType { DELETE_OLD_REGION_PEER, RESET_PEER_LIST, + // Data Partition Table Maintenance + COLLECT_EARLIEST_TIMESLOTS, + GENERATE_DATA_PARTITION_TABLE, + GENERATE_DATA_PARTITION_TABLE_HEART_BEAT, + GET_DATA_PARTITION_TABLE_GENERATOR_PROGRESS, + // PartitionCache INVALIDATE_PARTITION_CACHE, INVALIDATE_PERMISSION_CACHE, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java index c1dc83c1dfd9..25d61f0c54c8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java @@ -32,6 +32,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TCreateDataRegionReq; import org.apache.iotdb.mpp.rpc.thrift.TCreatePeerReq; import org.apache.iotdb.mpp.rpc.thrift.TCreateSchemaRegionReq; +import org.apache.iotdb.mpp.rpc.thrift.TGenerateDataPartitionTableReq; import org.apache.iotdb.mpp.rpc.thrift.TInvalidateCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TInvalidatePermissionCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TMaintainPeerReq; @@ -131,6 +132,19 @@ private void buildActionMap() { (req, client) -> client.resetPeerList((TResetPeerListReq) req)); actionMapBuilder.put( CnToDnSyncRequestType.SHOW_CONFIGURATION, (req, client) -> client.showConfiguration()); + actionMapBuilder.put( + CnToDnSyncRequestType.COLLECT_EARLIEST_TIMESLOTS, + (req, client) -> client.getEarliestTimeslots()); + actionMapBuilder.put( + CnToDnSyncRequestType.GENERATE_DATA_PARTITION_TABLE, + (req, client) -> client.generateDataPartitionTable((TGenerateDataPartitionTableReq) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.GENERATE_DATA_PARTITION_TABLE_HEART_BEAT, + (req, client) -> + client.generateDataPartitionTableHeartbeat((TGenerateDataPartitionTableReq) req)); + actionMapBuilder.put( + CnToDnSyncRequestType.GET_DATA_PARTITION_TABLE_GENERATOR_PROGRESS, + (req, client) -> client.getDataPartitionTableGeneratorProgress()); actionMap = actionMapBuilder.build(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 61819bd65097..53900bdc7329 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -44,6 +44,7 @@ import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.conf.TrimProperties; +import org.apache.iotdb.commons.enums.RepairDataPartitionTableProgressState; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.PartialPath; @@ -211,6 +212,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowModelResp; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowRepairDataPartitionTableProgressResp; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp; import org.apache.iotdb.confignode.rpc.thrift.TShowThrottleReq; @@ -400,15 +402,15 @@ protected void setLoadManager() { } public void close() throws IOException { - if (consensusManager.get() != null) { - consensusManager.get().close(); - } if (partitionManager != null) { partitionManager.getRegionMaintainer().shutdown(); } if (procedureManager != null) { procedureManager.stopExecutor(); } + if (consensusManager.get() != null) { + consensusManager.get().close(); + } } @Override @@ -1044,6 +1046,28 @@ public TDataPartitionTableResp getOrCreateDataPartition( return resp; } + @Override + public TSStatus dataPartitionTableIntegrityCheck() { + TSStatus status = confirmLeader(); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return status; + } + + return partitionManager.dataPartitionTableIntegrityCheck(); + } + + @Override + public TShowRepairDataPartitionTableProgressResp showRepairDataPartitionTableProgress() { + TSStatus status = confirmLeader(); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return new TShowRepairDataPartitionTableProgressResp( + status, RepairDataPartitionTableProgressState.UNKNOWN.name(), 0.0) + .setMessage(status.getMessage()); + } + + return partitionManager.showRepairDataPartitionTableProgress(); + } + private void printNewCreatedDataPartition( GetOrCreateDataPartitionPlan getOrCreateDataPartitionPlan, TDataPartitionTableResp resp) { final String lineSeparator = System.lineSeparator(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 326182fbef36..ef2588e5e7e0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -136,6 +136,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowModelResp; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowRepairDataPartitionTableProgressResp; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp; import org.apache.iotdb.confignode.rpc.thrift.TShowTopicReq; @@ -448,6 +449,10 @@ TSchemaNodeManagementResp getNodePathsPartition( TDataPartitionTableResp getOrCreateDataPartition( GetOrCreateDataPartitionPlan getOrCreateDataPartitionPlan); + TSStatus dataPartitionTableIntegrityCheck(); + + TShowRepairDataPartitionTableProgressResp showRepairDataPartitionTableProgress(); + /** * Operate Permission. * diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index f656444ca19e..fd7a16ebf999 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -57,6 +57,7 @@ import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.env.RegionMaintainHandler; import org.apache.iotdb.confignode.procedure.env.RemoveDataNodeHandler; +import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; import org.apache.iotdb.confignode.procedure.impl.cq.CreateCQProcedure; import org.apache.iotdb.confignode.procedure.impl.model.CreateModelProcedure; import org.apache.iotdb.confignode.procedure.impl.model.DropModelProcedure; @@ -64,6 +65,7 @@ import org.apache.iotdb.confignode.procedure.impl.node.RemoveAINodeProcedure; import org.apache.iotdb.confignode.procedure.impl.node.RemoveConfigNodeProcedure; import org.apache.iotdb.confignode.procedure.impl.node.RemoveDataNodesProcedure; +import org.apache.iotdb.confignode.procedure.impl.partition.DataPartitionTableIntegrityCheckProcedure; import org.apache.iotdb.confignode.procedure.impl.pipe.AbstractOperatePipeProcedureV2; import org.apache.iotdb.confignode.procedure.impl.pipe.plugin.CreatePipePluginProcedure; import org.apache.iotdb.confignode.procedure.impl.pipe.plugin.DropPipePluginProcedure; @@ -1743,6 +1745,37 @@ public static void sleepWithoutInterrupt(final long timeToSleep) { } } + public boolean isExistUnfinishedProcedure( + Class> procedureClass) { + if (procedureClass == null) { + return false; + } + + for (Procedure procedure : getExecutor().getProcedures().values()) { + if (!procedure.isFinished() && procedureClass.isInstance(procedure)) { + LOGGER.info( + "[{}] procedure details are {}", + procedureClass.getSimpleName(), + procedure.toStringDetails()); + return true; + } + } + + return false; + } + + public Optional + getUnfinishedDataPartitionTableIntegrityCheckProcedure() { + for (Procedure procedure : getExecutor().getProcedures().values()) { + if (!procedure.isFinished() + && procedure instanceof DataPartitionTableIntegrityCheckProcedure) { + return Optional.of((DataPartitionTableIntegrityCheckProcedure) procedure); + } + } + + return Optional.empty(); + } + // ====================================================== /* GET-SET Region diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java index e5e3473eedf2..67b5654070fd 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java @@ -378,6 +378,25 @@ public TConfigNodeLocation getLeaderLocation() { return null; } + public TConfigNodeLocation getNotNullLeaderLocation() { + Peer leaderPeer = getLeaderPeer(); + + while (leaderPeer == null) { + try { + Thread.sleep(1000); + } catch (InterruptedException ignored) { + + } + leaderPeer = getLeaderPeer(); + } + + Peer finalLeaderPeer = leaderPeer; + return getNodeManager().getRegisteredConfigNodes().stream() + .filter(leader -> leader.getConfigNodeId() == finalLeaderPeer.getNodeId()) + .findFirst() + .orElse(null); + } + /** * @return true if ConfigNode-leader is elected, false otherwise. */ diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java index 7db7ac50d625..9855e4f58da4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java @@ -32,6 +32,7 @@ import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.enums.RepairDataPartitionTableProgressState; import org.apache.iotdb.commons.log.LoggerPeriodicalLogReducer; import org.apache.iotdb.commons.partition.DataPartitionTable; import org.apache.iotdb.commons.partition.SchemaPartitionTable; @@ -83,10 +84,12 @@ import org.apache.iotdb.confignode.persistence.partition.maintainer.RegionDeleteTask; import org.apache.iotdb.confignode.persistence.partition.maintainer.RegionMaintainTask; import org.apache.iotdb.confignode.persistence.partition.maintainer.RegionMaintainType; +import org.apache.iotdb.confignode.procedure.impl.partition.DataPartitionTableIntegrityCheckProcedure; import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListReq; import org.apache.iotdb.confignode.rpc.thrift.TGetRegionIdReq; import org.apache.iotdb.confignode.rpc.thrift.TGetSeriesSlotListReq; import org.apache.iotdb.confignode.rpc.thrift.TGetTimeSlotListReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRepairDataPartitionTableProgressResp; import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.mpp.rpc.thrift.TCreateDataRegionReq; @@ -113,6 +116,7 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -149,6 +153,9 @@ public class PartitionManager { private final ScheduledExecutorService regionMaintainer; private Future currentRegionMaintainerFuture; + private final AtomicBoolean dataPartitionTableIntegrityCheckProcedureRunning = + new AtomicBoolean(false); + public PartitionManager(IManager configManager, PartitionInfo partitionInfo) { this.configManager = configManager; this.partitionInfo = partitionInfo; @@ -468,6 +475,43 @@ public DataPartitionResp getOrCreateDataPartition(final GetOrCreateDataPartition return resp; } + /** Used to repair the lost data partition table */ + public TSStatus dataPartitionTableIntegrityCheck() { + if (configManager + .getProcedureManager() + .isExistUnfinishedProcedure(DataPartitionTableIntegrityCheckProcedure.class) + || !dataPartitionTableIntegrityCheckProcedureRunning.compareAndSet(false, true)) { + return RpcUtils.getStatus( + TSStatusCode.OVERLAP_WITH_EXISTING_TASK, + "DataPartitionTableIntegrityCheckProcedure is already submitted."); + } + + synchronized (this) { + DataPartitionTableIntegrityCheckProcedure procedure = + new DataPartitionTableIntegrityCheckProcedure(); + getProcedureManager().getExecutor().submitProcedure(procedure); + } + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } + + public void markDataPartitionTableIntegrityCheckProcedureFinished() { + dataPartitionTableIntegrityCheckProcedureRunning.set(false); + } + + public TShowRepairDataPartitionTableProgressResp showRepairDataPartitionTableProgress() { + return configManager + .getProcedureManager() + .getUnfinishedDataPartitionTableIntegrityCheckProcedure() + .map(DataPartitionTableIntegrityCheckProcedure::getProgress) + .orElseGet( + () -> + new TShowRepairDataPartitionTableProgressResp( + RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS), + RepairDataPartitionTableProgressState.IDLE.name(), + 0.0) + .setMessage("No running DataPartitionTable integrity check procedure")); + } + private TSStatus consensusWritePartitionResult(ConfigPhysicalPlan plan) { TSStatus status = getConsensusManager().confirmLeader(); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/partition/ConfigNodeProcedureEnv.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/partition/ConfigNodeProcedureEnv.java new file mode 100644 index 000000000000..c1ebd7ffccde --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/partition/ConfigNodeProcedureEnv.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.partition; + +import org.apache.iotdb.confignode.manager.ConfigManager; + +/** + * Environment object for ConfigNode procedures. Provides access to ConfigManager and other + * necessary components. + */ +public class ConfigNodeProcedureEnv { + + private final ConfigManager configManager; + + public ConfigNodeProcedureEnv(ConfigManager configManager) { + this.configManager = configManager; + } + + public ConfigManager getConfigManager() { + return configManager; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/partition/DataPartitionTableIntegrityCheckProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/partition/DataPartitionTableIntegrityCheckProcedure.java new file mode 100644 index 000000000000..dfc5a28ce38a --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/partition/DataPartitionTableIntegrityCheckProcedure.java @@ -0,0 +1,1219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.partition; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.enums.DataPartitionTableGeneratorState; +import org.apache.iotdb.commons.enums.RepairDataPartitionTableProgressState; +import org.apache.iotdb.commons.partition.DataPartitionTable; +import org.apache.iotdb.commons.partition.DatabaseScopedDataPartitionTable; +import org.apache.iotdb.commons.partition.SeriesPartitionTable; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.confignode.client.sync.CnToDnSyncRequestType; +import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool; +import org.apache.iotdb.confignode.consensus.request.read.partition.GetDataPartitionPlan; +import org.apache.iotdb.confignode.consensus.request.write.partition.CreateDataPartitionPlan; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.node.NodeManager; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; +import org.apache.iotdb.confignode.procedure.state.DataPartitionTableIntegrityCheckProcedureState; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.confignode.rpc.thrift.TShowRepairDataPartitionTableProgressResp; +import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; +import org.apache.iotdb.mpp.rpc.thrift.TGenerateDataPartitionTableHeartbeatResp; +import org.apache.iotdb.mpp.rpc.thrift.TGenerateDataPartitionTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TGenerateDataPartitionTableResp; +import org.apache.iotdb.mpp.rpc.thrift.TGetEarliestTimeslotsResp; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.transport.TIOStreamTransport; +import org.apache.thrift.transport.TTransport; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Procedure for checking and restoring data partition table integrity. This procedure scans all + * DataNodes to detect missing data partitions and restores the DataPartitionTable on the ConfigNode + * Leader. + */ +public class DataPartitionTableIntegrityCheckProcedure + extends StateMachineProcedure< + ConfigNodeProcedureEnv, DataPartitionTableIntegrityCheckProcedureState> { + + private static final Logger LOG = + LoggerFactory.getLogger(DataPartitionTableIntegrityCheckProcedure.class); + + // how many times will retry after rpc request failed + private static final int MAX_RETRY_COUNT = 3; + + // how long to start a heartbeat request, the unit is ms + private static final long HEART_BEAT_REQUEST_INTERVAL = 10000; + + // how long to check all datanode are alive, the unit is ms + private static final long CHECK_ALL_DATANODE_IS_ALIVE_INTERVAL = 10000; + + // how long to roll back the next state, the unit is ms + private static final long ROLL_BACK_NEXT_STATE_INTERVAL = 60000; + + NodeManager dataNodeManager; + LoadManager loadManager; + private List allDataNodes = new ArrayList<>(); + + // ============Need serialize BEGIN=============/ + /** Collected earliest timeslots from DataNodes: database -> earliest timeslot */ + private Map earliestTimeslots = new ConcurrentHashMap<>(); + + /** DataPartitionTables collected from DataNodes: dataNodeId -> */ + private Map> dataPartitionTables = + new ConcurrentHashMap<>(); + + /** + * Collect all database names that those database lost data partition, the string in the Set + * collection is database name + */ + private Set databasesWithLostDataPartition = new HashSet<>(); + + /** + * Final merged DataPartitionTable for every database Map key(String): + * database name + */ + private Map finalDataPartitionTables; + + private Set skipDataNodes = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + private Set failedDataNodes = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + + // ============Need serialize END=============/ + + private final Map dataNodeGeneratorProgress = new ConcurrentHashMap<>(); + private volatile Set dataNodeGeneratorTargetDataNodeIds = Collections.emptySet(); + + public DataPartitionTableIntegrityCheckProcedure() { + super(); + } + + @Override + protected void updateMetricsOnFinish( + final ConfigNodeProcedureEnv env, final long runtime, final boolean success) { + super.updateMetricsOnFinish(env, runtime, success); + env.getConfigManager() + .getPartitionManager() + .markDataPartitionTableIntegrityCheckProcedureFinished(); + } + + @Override + protected Flow executeFromState( + final ConfigNodeProcedureEnv env, final DataPartitionTableIntegrityCheckProcedureState state) + throws InterruptedException { + try { + // Ensure to get the real-time DataNodes in the current cluster at every step + dataNodeManager = env.getConfigManager().getNodeManager(); + loadManager = env.getConfigManager().getLoadManager(); + allDataNodes = new ArrayList<>(dataNodeManager.getRegisteredDataNodes()); + + switch (state) { + case COLLECT_EARLIEST_TIMESLOTS: + failedDataNodes = new HashSet<>(); + return collectEarliestTimeslots(); + case ANALYZE_MISSING_PARTITIONS: + databasesWithLostDataPartition = new HashSet<>(); + return analyzeMissingPartitions(env); + case REQUEST_PARTITION_TABLES: + return requestPartitionTables(); + case REQUEST_PARTITION_TABLES_HEART_BEAT: + return requestPartitionTablesHeartBeat(); + case MERGE_PARTITION_TABLES: + finalDataPartitionTables = new HashMap<>(); + return mergePartitionTables(env); + case WRITE_PARTITION_TABLE_TO_CONSENSUS: + return writePartitionTableToConsensus(env); + default: + throw new ProcedureException("Unknown state: " + state); + } + } catch (Exception e) { + LOG.error("[DataPartitionIntegrity] Error executing state {}: {}", state, e.getMessage(), e); + setFailure("DataPartitionTableIntegrityCheckProcedure", e); + return Flow.NO_MORE_STATE; + } + } + + @Override + protected void rollbackState( + final ConfigNodeProcedureEnv env, final DataPartitionTableIntegrityCheckProcedureState state) + throws IOException, InterruptedException, ProcedureException { + // Cleanup resources + switch (state) { + case COLLECT_EARLIEST_TIMESLOTS: + earliestTimeslots.clear(); + break; + case ANALYZE_MISSING_PARTITIONS: + databasesWithLostDataPartition.clear(); + break; + case REQUEST_PARTITION_TABLES: + case REQUEST_PARTITION_TABLES_HEART_BEAT: + dataPartitionTables.clear(); + break; + case MERGE_PARTITION_TABLES: + finalDataPartitionTables.clear(); + break; + case WRITE_PARTITION_TABLE_TO_CONSENSUS: + allDataNodes.clear(); + earliestTimeslots.clear(); + dataPartitionTables.clear(); + finalDataPartitionTables.clear(); + break; + default: + allDataNodes.clear(); + earliestTimeslots.clear(); + dataPartitionTables.clear(); + finalDataPartitionTables.clear(); + throw new ProcedureException("Unknown state for rollback: " + state); + } + } + + @Override + protected DataPartitionTableIntegrityCheckProcedureState getState(final int stateId) { + return DataPartitionTableIntegrityCheckProcedureState.values()[stateId]; + } + + @Override + protected int getStateId(final DataPartitionTableIntegrityCheckProcedureState state) { + return state.ordinal(); + } + + @Override + protected DataPartitionTableIntegrityCheckProcedureState getInitialState() { + skipDataNodes = new HashSet<>(); + failedDataNodes = new HashSet<>(); + return DataPartitionTableIntegrityCheckProcedureState.COLLECT_EARLIEST_TIMESLOTS; + } + + /** + * Collect earliest timeslot information from all DataNodes. Each DataNode returns a Map where key is database name and value is the earliest timeslot id. + */ + /** + * Collect earliest timeslot information from all DataNodes. Each DataNode returns a Map where key is database name and value is the earliest timeslot id. + */ + private Flow collectEarliestTimeslots() { + if (LOG.isDebugEnabled()) { + LOG.debug("Collecting earliest timeslots from all DataNodes..."); + } + + if (allDataNodes.isEmpty()) { + LOG.error( + "[DataPartitionIntegrity] No DataNodes registered, no way to collect earliest timeslots, waiting for them to go up"); + sleep( + CHECK_ALL_DATANODE_IS_ALIVE_INTERVAL, + "[DataPartitionIntegrity] Error waiting for DataNode startup due to thread interruption."); + setNextState(DataPartitionTableIntegrityCheckProcedureState.COLLECT_EARLIEST_TIMESLOTS); + return Flow.HAS_MORE_STATE; + } + + // Collect earliest timeslots from all DataNodes + final List targetDataNodes = new ArrayList<>(allDataNodes); + targetDataNodes.removeAll(skipDataNodes); + for (TDataNodeConfiguration dataNode : targetDataNodes) { + // Check if DataNode is alive before sending request + NodeStatus nodeStatus = loadManager.getNodeStatus(dataNode.getLocation().getDataNodeId()); + if (!NodeStatus.Running.equals(nodeStatus)) { + failedDataNodes.add(dataNode); + continue; + } + + try { + Object response = + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithGivenRetry( + dataNode.getLocation().getInternalEndPoint(), + null, + CnToDnSyncRequestType.COLLECT_EARLIEST_TIMESLOTS, + MAX_RETRY_COUNT); + + if (response instanceof TSStatus) { + failedDataNodes.add(dataNode); + LOG.error( + "[DataPartitionIntegrity] Failed to collected earliest timeslots from the DataNode[id={}], already out of max retry time", + dataNode.getLocation().getDataNodeId()); + continue; + } + + TGetEarliestTimeslotsResp resp = (TGetEarliestTimeslotsResp) response; + if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + failedDataNodes.add(dataNode); + LOG.error( + "[DataPartitionIntegrity] Failed to collected earliest timeslots from the DataNode[id={}], response status is {}", + dataNode.getLocation().getDataNodeId(), + resp.getStatus()); + continue; + } + + Map nodeTimeslots = resp.getDatabaseToEarliestTimeslot(); + + // Merge with existing timeslots (take minimum) + for (Map.Entry entry : nodeTimeslots.entrySet()) { + earliestTimeslots.merge(entry.getKey(), entry.getValue(), Math::min); + } + + if (LOG.isDebugEnabled()) { + LOG.debug( + "Collected earliest timeslots from the DataNode[id={}]: {}", + dataNode.getLocation().getDataNodeId(), + nodeTimeslots); + } + } catch (Exception e) { + LOG.error( + "[DataPartitionIntegrity] Failed to collect earliest timeslots from the DataNode[id={}]: {}", + dataNode.getLocation().getDataNodeId(), + e.getMessage(), + e); + failedDataNodes.add(dataNode); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug( + "Collected earliest timeslots from {} DataNodes: {}, the number of successful DataNodes is {}", + targetDataNodes.size(), + earliestTimeslots, + targetDataNodes.size() - failedDataNodes.size()); + } + + if (countFailedTargetDataNodes(targetDataNodes) == targetDataNodes.size()) { + delayRollbackNextState( + DataPartitionTableIntegrityCheckProcedureState.COLLECT_EARLIEST_TIMESLOTS); + } else { + setNextState(DataPartitionTableIntegrityCheckProcedureState.ANALYZE_MISSING_PARTITIONS); + } + return Flow.HAS_MORE_STATE; + } + + /** + * Analyze which data partitions are missing based on earliest timeslots. Identify data partitions + * of databases need to be repaired. + */ + private Flow analyzeMissingPartitions(final ConfigNodeProcedureEnv env) { + if (LOG.isDebugEnabled()) { + LOG.debug("Analyzing missing data partitions..."); + } + + if (earliestTimeslots.isEmpty()) { + LOG.warn( + "[DataPartitionIntegrity] No missing data partitions detected, nothing needs to be repaired, terminating procedure"); + return Flow.NO_MORE_STATE; + } + + // Find all databases that have lost data partition tables + for (Map.Entry entry : earliestTimeslots.entrySet()) { + String database = entry.getKey(); + long earliestTimeslot = entry.getValue(); + + // Get current DataPartitionTable from ConfigManager + Map>>> + localDataPartitionTable = getLocalDataPartitionTable(env, database); + + // Check if ConfigNode has a data partition that is associated with the earliestTimeslot + if ((localDataPartitionTable == null + || localDataPartitionTable.isEmpty() + || localDataPartitionTable.get(database) == null + || localDataPartitionTable.get(database).isEmpty()) + && database.startsWith("root.")) { + databasesWithLostDataPartition.add(database); + LOG.warn( + "[DataPartitionIntegrity] No data partition table related to database {} was found from the ConfigNode, and this issue needs to be repaired", + database); + continue; + } + + Map>> + seriesPartitionMap = localDataPartitionTable.get(database); + long localEarliestSlotStartTime = Long.MAX_VALUE; + for (Map.Entry>> + seriesPartitionEntry : seriesPartitionMap.entrySet()) { + Map> tTimePartitionSlotListMap = + seriesPartitionEntry.getValue(); + + if (tTimePartitionSlotListMap.isEmpty()) { + continue; + } + + TTimePartitionSlot localEarliestSlot = + tTimePartitionSlotListMap.keySet().stream() + .min(Comparator.comparingLong(TTimePartitionSlot::getStartTime)) + .orElse(null); + + localEarliestSlotStartTime = + Math.min(localEarliestSlotStartTime, localEarliestSlot.getStartTime()); + } + + if (localEarliestSlotStartTime + > TimePartitionUtils.getStartTimeByPartitionId(earliestTimeslot)) { + databasesWithLostDataPartition.add(database); + LOG.warn( + "[DataPartitionIntegrity] Database {} has lost timeslot {} in its data table partition, and this issue needs to be repaired", + database, + earliestTimeslot); + } + } + + if (databasesWithLostDataPartition.isEmpty()) { + LOG.info( + "[DataPartitionIntegrity] No databases have lost data partitions, terminating procedure"); + return Flow.NO_MORE_STATE; + } + + LOG.info( + "[DataPartitionIntegrity] Identified {} databases have lost data partitions, will request DataPartitionTable generation from {} DataNodes", + databasesWithLostDataPartition.size(), + allDataNodes.size() - failedDataNodes.size()); + setNextState(DataPartitionTableIntegrityCheckProcedureState.REQUEST_PARTITION_TABLES); + return Flow.HAS_MORE_STATE; + } + + private Map>>> + getLocalDataPartitionTable(final ConfigNodeProcedureEnv env, final String database) { + PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendPathPattern(new PartialPath(database, false).concatNode("**")); + patternTree.constructTree(); + Map> schemaPartitionTable = + env.getConfigManager().getSchemaPartition(patternTree).getSchemaPartitionTable(); + + // Construct request for getting data partition + final Map> partitionSlotsMap = new HashMap<>(); + schemaPartitionTable.forEach( + (key, value) -> { + Map slotListMap = new HashMap<>(); + value + .keySet() + .forEach( + slot -> + slotListMap.put( + slot, new TTimeSlotList(Collections.emptyList(), true, true))); + partitionSlotsMap.put(key, slotListMap); + }); + final GetDataPartitionPlan getDataPartitionPlan = new GetDataPartitionPlan(partitionSlotsMap); + return env.getConfigManager().getDataPartition(getDataPartitionPlan).getDataPartitionTable(); + } + + /** + * Request DataPartitionTable generation from target DataNodes. Each DataNode scans its tsfile + * resources and generates a DataPartitionTable. + */ + private Flow requestPartitionTables() { + if (LOG.isDebugEnabled()) { + LOG.debug( + "Requesting DataPartitionTable generation from {} DataNodes...", allDataNodes.size()); + } + + if (allDataNodes.isEmpty()) { + LOG.error( + "[DataPartitionIntegrity] No DataNodes registered, no way to requested DataPartitionTable generation, terminating procedure"); + sleep( + CHECK_ALL_DATANODE_IS_ALIVE_INTERVAL, + "[DataPartitionIntegrity] Error waiting for DataNode startup due to thread interruption."); + setNextState(DataPartitionTableIntegrityCheckProcedureState.COLLECT_EARLIEST_TIMESLOTS); + return Flow.HAS_MORE_STATE; + } + + final List targetDataNodes = new ArrayList<>(allDataNodes); + targetDataNodes.removeAll(skipDataNodes); + targetDataNodes.removeAll(failedDataNodes); + refreshDataNodeGeneratorTarget(targetDataNodes); + for (TDataNodeConfiguration dataNode : targetDataNodes) { + int dataNodeId = dataNode.getLocation().getDataNodeId(); + // Check if DataNode is alive before sending request + NodeStatus nodeStatus = loadManager.getNodeStatus(dataNodeId); + if (!NodeStatus.Running.equals(nodeStatus)) { + failedDataNodes.add(dataNode); + dataNodeGeneratorProgress.put(dataNodeId, 1.0); + continue; + } + + if (!dataPartitionTables.containsKey(dataNodeId)) { + dataNodeGeneratorProgress.put(dataNodeId, 0.0); + try { + TGenerateDataPartitionTableReq req = new TGenerateDataPartitionTableReq(); + req.setDatabases(databasesWithLostDataPartition); + Object response = + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithGivenRetry( + dataNode.getLocation().getInternalEndPoint(), + req, + CnToDnSyncRequestType.GENERATE_DATA_PARTITION_TABLE, + MAX_RETRY_COUNT); + + if (response instanceof TSStatus) { + failedDataNodes.add(dataNode); + dataNodeGeneratorProgress.put(dataNodeId, 1.0); + LOG.error( + "[DataPartitionIntegrity] Failed to request DataPartitionTable generation from the DataNode[id={}], already out of max retry time", + dataNode.getLocation().getDataNodeId()); + continue; + } + + TGenerateDataPartitionTableResp resp = (TGenerateDataPartitionTableResp) response; + if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + failedDataNodes.add(dataNode); + dataNodeGeneratorProgress.put(dataNodeId, 1.0); + LOG.error( + "[DataPartitionIntegrity] Failed to request DataPartitionTable generation from the DataNode[id={}], response status is {}", + dataNode.getLocation().getDataNodeId(), + resp.getStatus()); + } + } catch (Exception e) { + failedDataNodes.add(dataNode); + dataNodeGeneratorProgress.put(dataNodeId, 1.0); + LOG.error( + "[DataPartitionIntegrity] Failed to request DataPartitionTable generation from DataNode[id={}]: {}", + dataNodeId, + e.getMessage(), + e); + } + } + } + + if (countFailedTargetDataNodes(targetDataNodes) == targetDataNodes.size()) { + delayRollbackNextState( + DataPartitionTableIntegrityCheckProcedureState.COLLECT_EARLIEST_TIMESLOTS); + return Flow.HAS_MORE_STATE; + } + + setNextState( + DataPartitionTableIntegrityCheckProcedureState.REQUEST_PARTITION_TABLES_HEART_BEAT); + return Flow.HAS_MORE_STATE; + } + + private Flow requestPartitionTablesHeartBeat() { + if (LOG.isDebugEnabled()) { + LOG.debug("Checking DataPartitionTable generation completion status..."); + } + + final List targetDataNodes = new ArrayList<>(allDataNodes); + targetDataNodes.removeAll(skipDataNodes); + targetDataNodes.removeAll(failedDataNodes); + refreshDataNodeGeneratorTarget(targetDataNodes); + + int completeCount = 0; + for (TDataNodeConfiguration dataNode : targetDataNodes) { + int dataNodeId = dataNode.getLocation().getDataNodeId(); + // Check if DataNode is alive before sending request + NodeStatus nodeStatus = loadManager.getNodeStatus(dataNodeId); + if (!NodeStatus.Running.equals(nodeStatus)) { + failedDataNodes.add(dataNode); + dataNodeGeneratorProgress.put(dataNodeId, 1.0); + continue; + } + + if (!dataPartitionTables.containsKey(dataNodeId)) { + try { + TGenerateDataPartitionTableReq req = new TGenerateDataPartitionTableReq(); + req.setDatabases(databasesWithLostDataPartition); + Object response = + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithGivenRetry( + dataNode.getLocation().getInternalEndPoint(), + req, + CnToDnSyncRequestType.GENERATE_DATA_PARTITION_TABLE_HEART_BEAT, + MAX_RETRY_COUNT); + + if (response instanceof TSStatus) { + failedDataNodes.add(dataNode); + dataNodeGeneratorProgress.put(dataNodeId, 1.0); + LOG.error( + "[DataPartitionIntegrity] Failed to request DataPartitionTable generation heart beat from the DataNode[id={}], already out of max retry time", + dataNode.getLocation().getDataNodeId()); + continue; + } + + TGenerateDataPartitionTableHeartbeatResp resp = + (TGenerateDataPartitionTableHeartbeatResp) response; + DataPartitionTableGeneratorState state = + DataPartitionTableGeneratorState.getStateByCode(resp.getErrorCode()); + + if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOG.error( + "[DataPartitionIntegrity] Failed to request DataPartitionTable generation heart beat from the DataNode[id={}], state is {}, response status is {}", + dataNode.getLocation().getDataNodeId(), + state, + resp.getStatus()); + continue; + } + + switch (state) { + case SUCCESS: + List byteBufferList = resp.getDatabaseScopedDataPartitionTables(); + List databaseScopedDataPartitionTableList = + deserializeDatabaseScopedTableList(byteBufferList); + dataPartitionTables.put(dataNodeId, databaseScopedDataPartitionTableList); + dataNodeGeneratorProgress.put(dataNodeId, 1.0); + LOG.info( + "[DataPartitionIntegrity] DataNode {} completed DataPartitionTable generation, terminating heart beat", + dataNodeId); + completeCount++; + break; + case IN_PROGRESS: + dataNodeGeneratorProgress.put(dataNodeId, clampProgress(resp.getProgress())); + LOG.info( + "[DataPartitionIntegrity] DataNode {} still generating DataPartitionTable", + dataNodeId); + break; + default: + failedDataNodes.add(dataNode); + dataNodeGeneratorProgress.put(dataNodeId, 1.0); + LOG.error( + "[DataPartitionIntegrity] DataNode {} returned unknown error code: {}", + dataNodeId, + resp.getErrorCode()); + break; + } + } catch (Exception e) { + LOG.error( + "[DataPartitionIntegrity] Error checking DataPartitionTable status from DataNode {}: {}, terminating heart beat", + dataNodeId, + e.getMessage(), + e); + dataNodeGeneratorProgress.put(dataNodeId, 1.0); + completeCount++; + } + } else { + dataNodeGeneratorProgress.put(dataNodeId, 1.0); + completeCount++; + } + } + + if (completeCount >= targetDataNodes.size()) { + setNextState(DataPartitionTableIntegrityCheckProcedureState.MERGE_PARTITION_TABLES); + return Flow.HAS_MORE_STATE; + } + + // Don't find any one data partition table generation task on all registered DataNodes, go back + // to the REQUEST_PARTITION_TABLES step and re-execute + if (countFailedTargetDataNodes(targetDataNodes) == targetDataNodes.size()) { + delayRollbackNextState( + DataPartitionTableIntegrityCheckProcedureState.REQUEST_PARTITION_TABLES); + return Flow.HAS_MORE_STATE; + } + + sleep( + HEART_BEAT_REQUEST_INTERVAL, + "[DataPartitionIntegrity] Error checking DataPartitionTable status due to thread interruption."); + setNextState( + DataPartitionTableIntegrityCheckProcedureState.REQUEST_PARTITION_TABLES_HEART_BEAT); + return Flow.HAS_MORE_STATE; + } + + private static void sleep(long intervalTime, String logMessage) { + try { + Thread.sleep(intervalTime); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.error(logMessage); + } + } + + /** Merge DataPartitionTables from all DataNodes into a final table. */ + private Flow mergePartitionTables(final ConfigNodeProcedureEnv env) { + if (LOG.isDebugEnabled()) { + LOG.debug("Merging DataPartitionTables from {} DataNodes...", dataPartitionTables.size()); + } + + if (dataPartitionTables.isEmpty()) { + LOG.error( + "[DataPartitionIntegrity] No DataPartitionTables to merge, dataPartitionTables is empty"); + delayRollbackNextState( + DataPartitionTableIntegrityCheckProcedureState.COLLECT_EARLIEST_TIMESLOTS); + return Flow.HAS_MORE_STATE; + } + + for (String database : databasesWithLostDataPartition) { + Map finalDataPartitionMap = new HashMap<>(); + + // Get current DataPartitionTable from ConfigManager + Map>>> + localDataPartitionTableMap = getLocalDataPartitionTable(env, database); + + // Check if ConfigNode has a data partition that is associated with the earliestTimeslot + if (localDataPartitionTableMap == null + || localDataPartitionTableMap.isEmpty() + || localDataPartitionTableMap.get(database) == null + || localDataPartitionTableMap.get(database).isEmpty()) { + LOG.warn( + "[DataPartitionIntegrity] No data partition table related to database {} was found from the ConfigNode, use data partition table of DataNode directly", + database); + } else { + localDataPartitionTableMap + .values() + .forEach( + map -> + map.forEach( + (tSeriesPartitionSlot, seriesPartitionTableMap) -> { + if (tSeriesPartitionSlot == null + || seriesPartitionTableMap == null + || seriesPartitionTableMap.isEmpty()) { + return; + } + finalDataPartitionMap.computeIfAbsent( + tSeriesPartitionSlot, + k -> new SeriesPartitionTable(seriesPartitionTableMap)); + })); + } + + dataPartitionTables.forEach( + (k, v) -> + v.forEach( + databaseScopedDataPartitionTable -> { + if (!databaseScopedDataPartitionTable.getDatabase().equals(database)) { + return; + } + finalDataPartitionTables.put( + database, + new DataPartitionTable(finalDataPartitionMap) + .merge(databaseScopedDataPartitionTable.getDataPartitionTable())); + })); + } + + LOG.info("[DataPartitionIntegrity] DataPartitionTables merge completed successfully"); + setNextState(DataPartitionTableIntegrityCheckProcedureState.WRITE_PARTITION_TABLE_TO_CONSENSUS); + return Flow.HAS_MORE_STATE; + } + + /** Write the final DataPartitionTable to consensus log. */ + private Flow writePartitionTableToConsensus(final ConfigNodeProcedureEnv env) { + if (LOG.isDebugEnabled()) { + LOG.debug("Writing DataPartitionTable to consensus log..."); + } + + if (databasesWithLostDataPartition.isEmpty()) { + LOG.error("[DataPartitionIntegrity] No database lost data partition table"); + setFailure( + "DataPartitionTableIntegrityCheckProcedure", + new ProcedureException("No database lost data partition table for consensus write")); + return getFlow(); + } + + if (finalDataPartitionTables.isEmpty()) { + LOG.error("[DataPartitionIntegrity] DataPartitionTable to write to consensus"); + setFailure( + "DataPartitionTableIntegrityCheckProcedure", + new ProcedureException("No DataPartitionTable available for consensus write")); + return getFlow(); + } + + int failedCnt = 0; + final int maxRetryCountForConsensus = 3; + while (failedCnt < maxRetryCountForConsensus) { + try { + CreateDataPartitionPlan createPlan = new CreateDataPartitionPlan(); + Map assignedDataPartition = new HashMap<>(); + for (String database : databasesWithLostDataPartition) { + assignedDataPartition.put(database, finalDataPartitionTables.get(database)); + } + createPlan.setAssignedDataPartition(assignedDataPartition); + TSStatus tsStatus = env.getConfigManager().getConsensusManager().write(createPlan); + + if (tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOG.info( + "[DataPartitionIntegrity] DataPartitionTable successfully written to consensus log"); + break; + } else { + LOG.error("[DataPartitionIntegrity] Failed to write DataPartitionTable to consensus log"); + setFailure( + "DataPartitionTableIntegrityCheckProcedure", + new ProcedureException("Failed to write DataPartitionTable to consensus log")); + } + } catch (Exception e) { + LOG.error("[DataPartitionIntegrity] Error writing DataPartitionTable to consensus log", e); + setFailure("DataPartitionTableIntegrityCheckProcedure", e); + } + failedCnt++; + } + + return getFlow(); + } + + /** + * Determine whether there are still DataNode nodes with failed execution of a certain step in + * this round. If such nodes exist, calculate the skipDataNodes and exclude these nodes when + * requesting the list of DataNode nodes in the cluster for the next round; if no such nodes + * exist, it means the procedure has been completed + */ + private Flow getFlow() { + if (!failedDataNodes.isEmpty()) { + allDataNodes.removeAll(failedDataNodes); + skipDataNodes = new HashSet<>(allDataNodes); + delayRollbackNextState( + DataPartitionTableIntegrityCheckProcedureState.COLLECT_EARLIEST_TIMESLOTS); + return Flow.HAS_MORE_STATE; + } else { + skipDataNodes.clear(); + return Flow.NO_MORE_STATE; + } + } + + /** Delay to jump to next state, avoid write raft logs frequently when exception occur */ + private void delayRollbackNextState(DataPartitionTableIntegrityCheckProcedureState state) { + sleep( + ROLL_BACK_NEXT_STATE_INTERVAL, + String.format( + "[DataPartitionIntegrity] Error waiting for roll back the %s state due to thread interruption.", + state)); + setNextState(state); + } + + @Override + public void serialize(final DataOutputStream stream) throws IOException { + stream.writeShort(ProcedureType.DATA_PARTITION_TABLE_INTEGRITY_CHECK_PROCEDURE.getTypeCode()); + super.serialize(stream); + + // Serialize earliestTimeslots + stream.writeInt(earliestTimeslots.size()); + for (Map.Entry entry : earliestTimeslots.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), stream); + stream.writeLong(entry.getValue()); + } + + // Serialize dataPartitionTables count + stream.writeInt(dataPartitionTables.size()); + for (Map.Entry> entry : + dataPartitionTables.entrySet()) { + stream.writeInt(entry.getKey()); + + List tableList = entry.getValue(); + stream.writeInt(tableList.size()); + + for (DatabaseScopedDataPartitionTable table : tableList) { + try (final PublicBAOS publicBAOS = new PublicBAOS(); + final DataOutputStream tmpStream = new DataOutputStream(publicBAOS)) { + + TTransport transport = new TIOStreamTransport(tmpStream); + TBinaryProtocol protocol = new TBinaryProtocol(transport); + + table.serialize(tmpStream, protocol); + + byte[] buf = publicBAOS.getBuf(); + int size = publicBAOS.size(); + ReadWriteIOUtils.write(size, stream); + stream.write(buf, 0, size); + } catch (IOException | TException e) { + LOG.error( + "[DataPartitionIntegrity] {} serialize failed for dataNodeId: {}", + this.getClass().getSimpleName(), + entry.getKey(), + e); + throw new IOException("Failed to serialize dataPartitionTables", e); + } + } + } + + stream.writeInt(databasesWithLostDataPartition.size()); + for (String database : databasesWithLostDataPartition) { + ReadWriteIOUtils.write(database, stream); + } + + if (finalDataPartitionTables != null && !finalDataPartitionTables.isEmpty()) { + stream.writeInt(finalDataPartitionTables.size()); + + for (Map.Entry entry : finalDataPartitionTables.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), stream); + + try (final PublicBAOS publicBAOS = new PublicBAOS(); + final DataOutputStream tmpStream = new DataOutputStream(publicBAOS)) { + TTransport transport = new TIOStreamTransport(tmpStream); + TBinaryProtocol protocol = new TBinaryProtocol(transport); + + entry.getValue().serialize(tmpStream, protocol); + + byte[] buf = publicBAOS.getBuf(); + int size = publicBAOS.size(); + ReadWriteIOUtils.write(size, stream); + stream.write(buf, 0, size); + } catch (IOException | TException e) { + LOG.error( + "[DataPartitionIntegrity] {} serialize finalDataPartitionTables failed", + this.getClass().getSimpleName(), + e); + throw new IOException("Failed to serialize finalDataPartitionTables", e); + } + } + } else { + stream.writeInt(0); + } + + stream.writeInt(skipDataNodes.size()); + for (TDataNodeConfiguration skipDataNode : skipDataNodes) { + try (final PublicBAOS publicBAOS = new PublicBAOS(); + final DataOutputStream tmpStream = new DataOutputStream(publicBAOS)) { + TTransport transport = new TIOStreamTransport(tmpStream); + TBinaryProtocol protocol = new TBinaryProtocol(transport); + skipDataNode.write(protocol); + + byte[] buf = publicBAOS.getBuf(); + int size = publicBAOS.size(); + ReadWriteIOUtils.write(size, stream); + stream.write(buf, 0, size); + } catch (TException e) { + LOG.error("[DataPartitionIntegrity] Failed to serialize skipDataNode", e); + throw new IOException("Failed to serialize skipDataNode", e); + } + } + + stream.writeInt(failedDataNodes.size()); + for (TDataNodeConfiguration failedDataNode : failedDataNodes) { + try (final PublicBAOS publicBAOS = new PublicBAOS(); + final DataOutputStream tmpStream = new DataOutputStream(publicBAOS)) { + TTransport transport = new TIOStreamTransport(tmpStream); + TBinaryProtocol protocol = new TBinaryProtocol(transport); + failedDataNode.write(protocol); + + byte[] buf = publicBAOS.getBuf(); + int size = publicBAOS.size(); + ReadWriteIOUtils.write(size, stream); + stream.write(buf, 0, size); + } catch (TException e) { + LOG.error("[DataPartitionIntegrity] Failed to serialize failedDataNode", e); + throw new IOException("Failed to serialize failedDataNode", e); + } + } + } + + @Override + public void deserialize(final ByteBuffer byteBuffer) { + super.deserialize(byteBuffer); + + // Deserialize earliestTimeslots + int earliestTimeslotsSize = byteBuffer.getInt(); + earliestTimeslots = new ConcurrentHashMap<>(); + for (int i = 0; i < earliestTimeslotsSize; i++) { + String database = ReadWriteIOUtils.readString(byteBuffer); + long timeslot = byteBuffer.getLong(); + earliestTimeslots.put(database, timeslot); + } + + // Deserialize dataPartitionTables count + int dataPartitionTablesSize = byteBuffer.getInt(); + dataPartitionTables = new ConcurrentHashMap<>(); + for (int i = 0; i < dataPartitionTablesSize; i++) { + int dataNodeId = byteBuffer.getInt(); + int listSize = byteBuffer.getInt(); + + List tableList = new ArrayList<>(listSize); + + for (int j = 0; j < listSize; j++) { + int dataSize = byteBuffer.getInt(); + byte[] bytes = new byte[dataSize]; + byteBuffer.get(bytes); + + try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + DataInputStream dis = new DataInputStream(bais)) { + + TTransport transport = new TIOStreamTransport(dis); + TBinaryProtocol protocol = new TBinaryProtocol(transport); + + DatabaseScopedDataPartitionTable table = + DatabaseScopedDataPartitionTable.deserialize(dis, protocol); + tableList.add(table); + + } catch (IOException | TException e) { + LOG.error( + "[DataPartitionIntegrity] {} deserialize failed for dataNodeId: {}", + this.getClass().getSimpleName(), + dataNodeId, + e); + throw new RuntimeException("Failed to deserialize dataPartitionTables", e); + } + } + + dataPartitionTables.put(dataNodeId, tableList); + } + + int databasesWithLostDataPartitionSize = byteBuffer.getInt(); + for (int i = 0; i < databasesWithLostDataPartitionSize; i++) { + String database = ReadWriteIOUtils.readString(byteBuffer); + databasesWithLostDataPartition.add(database); + } + + // Deserialize finalDataPartitionTable size + int finalDataPartitionTablesSize = byteBuffer.getInt(); + finalDataPartitionTables = new ConcurrentHashMap<>(); + + for (int i = 0; i < finalDataPartitionTablesSize; i++) { + String database = ReadWriteIOUtils.readString(byteBuffer); + + int dataSize = byteBuffer.getInt(); + byte[] bytes = new byte[dataSize]; + byteBuffer.get(bytes); + + try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + DataInputStream dis = new DataInputStream(bais)) { + + TTransport transport = new TIOStreamTransport(dis); + TBinaryProtocol protocol = new TBinaryProtocol(transport); + + DataPartitionTable dataPartitionTable = new DataPartitionTable(); + dataPartitionTable.deserialize(dis, protocol); + + finalDataPartitionTables.put(database, dataPartitionTable); + + } catch (IOException | TException e) { + LOG.error( + "[DataPartitionIntegrity] {} deserialize finalDataPartitionTables failed", + this.getClass().getSimpleName(), + e); + throw new RuntimeException("Failed to deserialize finalDataPartitionTables", e); + } + } + + skipDataNodes = new HashSet<>(); + int skipDataNodesSize = byteBuffer.getInt(); + for (int i = 0; i < skipDataNodesSize; i++) { + int size = byteBuffer.getInt(); + byte[] bytes = new byte[size]; + byteBuffer.get(bytes); + + try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes)) { + TTransport transport = new TIOStreamTransport(bais); + TBinaryProtocol protocol = new TBinaryProtocol(transport); + + TDataNodeConfiguration dataNode = new TDataNodeConfiguration(); + dataNode.read(protocol); + skipDataNodes.add(dataNode); + } catch (TException | IOException e) { + LOG.error("[DataPartitionIntegrity] Failed to deserialize skipDataNode", e); + throw new RuntimeException(e); + } + } + + failedDataNodes = new HashSet<>(); + int failedDataNodesSize = byteBuffer.getInt(); + for (int i = 0; i < failedDataNodesSize; i++) { + int size = byteBuffer.getInt(); + byte[] bytes = new byte[size]; + byteBuffer.get(bytes); + + try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes)) { + TTransport transport = new TIOStreamTransport(bais); + TBinaryProtocol protocol = new TBinaryProtocol(transport); + + TDataNodeConfiguration dataNode = new TDataNodeConfiguration(); + dataNode.read(protocol); + failedDataNodes.add(dataNode); + } catch (TException | IOException e) { + LOG.error("[DataPartitionIntegrity] Failed to deserialize failedDataNode", e); + throw new RuntimeException(e); + } + } + } + + private List deserializeDatabaseScopedTableList( + List dataList) { + if (dataList == null || dataList.isEmpty()) { + return Collections.emptyList(); + } + + List result = new ArrayList<>(dataList.size()); + + for (ByteBuffer data : dataList) { + if (data == null || data.remaining() == 0) { + LOG.warn("[DataPartitionIntegrity] Skipping empty ByteBuffer during deserialization"); + continue; + } + + try { + DatabaseScopedDataPartitionTable table = DatabaseScopedDataPartitionTable.deserialize(data); + result.add(table); + } catch (Exception e) { + LOG.error( + "[DataPartitionIntegrity] Failed to deserialize DatabaseScopedDataPartitionTable", e); + } + } + + return result; + } + + public Map getEarliestTimeslots() { + return earliestTimeslots; + } + + public Map> getDataPartitionTables() { + return dataPartitionTables; + } + + public Set getDatabasesWithLostDataPartition() { + return databasesWithLostDataPartition; + } + + public Map getFinalDataPartitionTables() { + return finalDataPartitionTables; + } + + public Set getSkipDataNodes() { + return skipDataNodes; + } + + public Set getFailedDataNodes() { + return failedDataNodes; + } + + public void setEarliestTimeslots(Map earliestTimeslots) { + this.earliestTimeslots = earliestTimeslots; + } + + public void setDataPartitionTables( + Map> dataPartitionTables) { + this.dataPartitionTables = dataPartitionTables; + } + + public void setDatabasesWithLostDataPartition(Set databasesWithLostDataPartition) { + this.databasesWithLostDataPartition = databasesWithLostDataPartition; + } + + public void setFinalDataPartitionTables( + Map finalDataPartitionTables) { + this.finalDataPartitionTables = finalDataPartitionTables; + } + + public void setSkipDataNodes(Set skipDataNodes) { + this.skipDataNodes = skipDataNodes; + } + + public void setFailedDataNodes(Set failedDataNodes) { + this.failedDataNodes = failedDataNodes; + } + + public TShowRepairDataPartitionTableProgressResp getProgress() { + try { + final DataPartitionTableIntegrityCheckProcedureState currentState = getCurrentState(); + final String state = getProgressStateName(currentState); + final double progress = + currentState == null ? 0.0 : calculateProgressByState(currentState) * 100; + + return new TShowRepairDataPartitionTableProgressResp( + RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS), state, progress) + .setMessage( + String.format("DataPartitionTable integrity check progress: %.1f%%", progress)); + } catch (Exception e) { + LOG.warn("Failed to show DataPartitionTable integrity check progress", e); + return new TShowRepairDataPartitionTableProgressResp( + RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS), + RepairDataPartitionTableProgressState.UNKNOWN.name(), + 0.0) + .setMessage("Failed to show DataPartitionTable integrity check progress"); + } + } + + private double calculateProgressByState( + final DataPartitionTableIntegrityCheckProcedureState currentState) { + switch (currentState) { + case COLLECT_EARLIEST_TIMESLOTS: + return 0.0; + case ANALYZE_MISSING_PARTITIONS: + return 0.05; + case REQUEST_PARTITION_TABLES: + return 0.1; + case REQUEST_PARTITION_TABLES_HEART_BEAT: + return 0.1 + 0.8 * calculateDataNodeGeneratorProgress(); + case MERGE_PARTITION_TABLES: + return 0.95; + case WRITE_PARTITION_TABLE_TO_CONSENSUS: + return 0.99; + default: + LOG.warn( + "Encountered unexpected DataPartitionTableIntegrityCheckProcedureState {} when showing progress", + currentState); + return 0.0; + } + } + + private String getProgressStateName( + final DataPartitionTableIntegrityCheckProcedureState currentState) { + if (currentState == null) { + return RepairDataPartitionTableProgressState.UNKNOWN.name(); + } + try { + return RepairDataPartitionTableProgressState.valueOf(currentState.name()).name(); + } catch (IllegalArgumentException e) { + LOG.warn( + "Unexpected DataPartitionTableIntegrityCheckProcedureState {} when showing progress", + currentState); + return RepairDataPartitionTableProgressState.UNKNOWN.name(); + } + } + + private double calculateDataNodeGeneratorProgress() { + final Set currentTargetDataNodeIds = dataNodeGeneratorTargetDataNodeIds; + if (currentTargetDataNodeIds.isEmpty()) { + return dataPartitionTables.isEmpty() ? 0.0 : 1.0; + } + + double progressSum = 0.0; + for (int dataNodeId : currentTargetDataNodeIds) { + progressSum += clampProgress(dataNodeGeneratorProgress.getOrDefault(dataNodeId, 0.0)); + } + return clampProgress(progressSum / currentTargetDataNodeIds.size()); + } + + private void refreshDataNodeGeneratorTarget(final List targetDataNodes) { + final Set targetDataNodeIds = new HashSet<>(); + for (TDataNodeConfiguration dataNode : targetDataNodes) { + targetDataNodeIds.add(dataNode.getLocation().getDataNodeId()); + } + dataNodeGeneratorProgress.keySet().retainAll(targetDataNodeIds); + dataNodeGeneratorTargetDataNodeIds = Collections.unmodifiableSet(targetDataNodeIds); + } + + private long countFailedTargetDataNodes(final List targetDataNodes) { + return targetDataNodes.stream().filter(failedDataNodes::contains).count(); + } + + private double clampProgress(final double progress) { + return Math.max(0.0, Math.min(1.0, progress)); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/DataPartitionTableIntegrityCheckProcedureState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/DataPartitionTableIntegrityCheckProcedureState.java new file mode 100644 index 000000000000..bf302db755ba --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/DataPartitionTableIntegrityCheckProcedureState.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.state; + +public enum DataPartitionTableIntegrityCheckProcedureState { + /** Collect earliest timeslot information from all DataNodes */ + COLLECT_EARLIEST_TIMESLOTS, + /** Analyze missing data partitions */ + ANALYZE_MISSING_PARTITIONS, + /** Request DataPartitionTable generation from DataNodes */ + REQUEST_PARTITION_TABLES, + /** Round robin get DataPartitionTable generation result from DataNodes */ + REQUEST_PARTITION_TABLES_HEART_BEAT, + /** Merge DataPartitionTables from all DataNodes */ + MERGE_PARTITION_TABLES, + /** Write final DataPartitionTable to raft log */ + WRITE_PARTITION_TABLE_TO_CONSENSUS +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java index fc88b54f3d56..07035a525820 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java @@ -28,6 +28,7 @@ import org.apache.iotdb.confignode.procedure.impl.node.RemoveAINodeProcedure; import org.apache.iotdb.confignode.procedure.impl.node.RemoveConfigNodeProcedure; import org.apache.iotdb.confignode.procedure.impl.node.RemoveDataNodesProcedure; +import org.apache.iotdb.confignode.procedure.impl.partition.DataPartitionTableIntegrityCheckProcedure; import org.apache.iotdb.confignode.procedure.impl.pipe.plugin.CreatePipePluginProcedure; import org.apache.iotdb.confignode.procedure.impl.pipe.plugin.DropPipePluginProcedure; import org.apache.iotdb.confignode.procedure.impl.pipe.runtime.PipeHandleLeaderChangeProcedure; @@ -286,6 +287,9 @@ public Procedure create(ByteBuffer buffer) throws IOException { case ADD_NEVER_FINISH_SUB_PROCEDURE_PROCEDURE: procedure = new AddNeverFinishSubProcedureProcedure(); break; + case DATA_PARTITION_TABLE_INTEGRITY_CHECK_PROCEDURE: + procedure = new DataPartitionTableIntegrityCheckProcedure(); + break; default: LOGGER.error("Unknown Procedure type: {}", typeCode); throw new IOException("Unknown Procedure type: " + typeCode); @@ -403,6 +407,8 @@ public static ProcedureType getProcedureType(Procedure procedure) { return ProcedureType.NEVER_FINISH_PROCEDURE; } else if (procedure instanceof AddNeverFinishSubProcedureProcedure) { return ProcedureType.ADD_NEVER_FINISH_SUB_PROCEDURE_PROCEDURE; + } else if (procedure instanceof DataPartitionTableIntegrityCheckProcedure) { + return ProcedureType.DATA_PARTITION_TABLE_INTEGRITY_CHECK_PROCEDURE; } throw new UnsupportedOperationException( "Procedure type " + procedure.getClass() + " is not supported"); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java index 48ccca42d44a..71af76e0ee28 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java @@ -127,7 +127,10 @@ public enum ProcedureType { @TestOnly NEVER_FINISH_PROCEDURE((short) 30000), @TestOnly - ADD_NEVER_FINISH_SUB_PROCEDURE_PROCEDURE((short) 30001); + ADD_NEVER_FINISH_SUB_PROCEDURE_PROCEDURE((short) 30001), + + /** Data Partition Table Integrity Check */ + DATA_PARTITION_TABLE_INTEGRITY_CHECK_PROCEDURE((short) 1600); private final short typeCode; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index 9fbb6b2e6579..dbce42c04ba6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -190,6 +190,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowRepairDataPartitionTableProgressResp; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp; import org.apache.iotdb.confignode.rpc.thrift.TShowTTLResp; @@ -674,6 +675,16 @@ public TDataPartitionTableResp getOrCreateDataPartitionTable(TDataPartitionReq r return configManager.getOrCreateDataPartition(getOrCreateDataPartitionReq); } + @Override + public TSStatus dataPartitionTableIntegrityCheck() { + return configManager.dataPartitionTableIntegrityCheck(); + } + + @Override + public TShowRepairDataPartitionTableProgressResp showRepairDataPartitionTableProgress() { + return configManager.showRepairDataPartitionTableProgress(); + } + @Override public TSStatus operatePermission(final TAuthorizerReq req) { if (req.getAuthorType() < 0 || req.getAuthorType() >= AuthorType.values().length) { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/partition/DataPartitionTableIntegrityCheckProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/partition/DataPartitionTableIntegrityCheckProcedureTest.java new file mode 100644 index 000000000000..4f2fcfe27519 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/partition/DataPartitionTableIntegrityCheckProcedureTest.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.partition; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TNodeResource; +import org.apache.iotdb.commons.enums.RepairDataPartitionTableProgressState; +import org.apache.iotdb.commons.partition.DataPartitionTable; +import org.apache.iotdb.commons.partition.DatabaseScopedDataPartitionTable; +import org.apache.iotdb.confignode.procedure.Procedure; +import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; +import org.apache.iotdb.confignode.rpc.thrift.TShowRepairDataPartitionTableProgressResp; + +import org.apache.tsfile.utils.PublicBAOS; +import org.junit.Assert; +import org.junit.Test; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class DataPartitionTableIntegrityCheckProcedureTest { + @Test + public void serDeTest() throws IOException { + DataPartitionTableIntegrityCheckProcedure original = createTestProcedureWithData(); + + try (PublicBAOS baos = new PublicBAOS(); + DataOutputStream dos = new DataOutputStream(baos)) { + + original.serialize(dos); + + System.out.println("Serialized bytes length: " + baos.size()); + + ByteBuffer buffer = ByteBuffer.wrap(baos.getBuf(), 0, baos.size()); + + Procedure recreated = ProcedureFactory.getInstance().create(buffer); + + if (recreated instanceof DataPartitionTableIntegrityCheckProcedure) { + DataPartitionTableIntegrityCheckProcedure actual = + (DataPartitionTableIntegrityCheckProcedure) recreated; + assertProcedureEquals(original, actual); + System.out.println("All checked fields match!"); + } else { + Assert.fail("Recreated is not DataPartitionTableIntegrityCheckProcedure"); + } + } + } + + @Test + public void progressStateTest() { + DataPartitionTableIntegrityCheckProcedure procedure = + new DataPartitionTableIntegrityCheckProcedure(); + TShowRepairDataPartitionTableProgressResp progress = procedure.getProgress(); + Assert.assertEquals( + RepairDataPartitionTableProgressState.COLLECT_EARLIEST_TIMESLOTS.name(), + progress.getState()); + Assert.assertTrue(progress.getProgress() >= 0.0 && progress.getProgress() <= 100.0); + } + + private DataPartitionTableIntegrityCheckProcedure createTestProcedureWithData() { + DataPartitionTableIntegrityCheckProcedure proc = + new DataPartitionTableIntegrityCheckProcedure(); + String database = "root.test"; + + Map earliestTimeslots = new HashMap<>(); + earliestTimeslots.put(database, 0L); + proc.setEarliestTimeslots(earliestTimeslots); + + Map> dataPartitionTables = new HashMap<>(); + DataPartitionTable dataPartitionTable = new DataPartitionTable(); + dataPartitionTables.put( + 1, + Collections.singletonList( + new DatabaseScopedDataPartitionTable(database, dataPartitionTable))); + proc.setDataPartitionTables(dataPartitionTables); + + Set databasesWithLostDataPartition = new HashSet<>(); + databasesWithLostDataPartition.add(database); + proc.setDatabasesWithLostDataPartition(databasesWithLostDataPartition); + + Map finalDataPartitionTables = new HashMap<>(); + finalDataPartitionTables.put(database, dataPartitionTable); + proc.setFinalDataPartitionTables(finalDataPartitionTables); + + Set skipNodes = getTDataNodeConfigurations(1); + proc.setSkipDataNodes(skipNodes); + + Set failedNodes = getTDataNodeConfigurations(2); + proc.setFailedDataNodes(failedNodes); + + return proc; + } + + private static Set getTDataNodeConfigurations(int dataNodeId) { + Set nodes = new HashSet<>(); + TDataNodeLocation tDataNodeConfiguration = + new TDataNodeLocation( + dataNodeId, + new TEndPoint("127.0.0.1", 5), + new TEndPoint("127.0.0.1", 6), + new TEndPoint("127.0.0.1", 7), + new TEndPoint("127.0.0.1", 8), + new TEndPoint("127.0.0.1", 9)); + TNodeResource resource = new TNodeResource(16, 34359738368L); + TDataNodeConfiguration skipDataNodeConfiguration = + new TDataNodeConfiguration(tDataNodeConfiguration, resource); + nodes.add(skipDataNodeConfiguration); + return nodes; + } + + private void assertProcedureEquals( + DataPartitionTableIntegrityCheckProcedure expected, + DataPartitionTableIntegrityCheckProcedure actual) { + Assert.assertEquals("procId mismatch", expected.getProcId(), actual.getProcId()); + Assert.assertEquals("state mismatch", expected.getState(), actual.getState()); + Assert.assertEquals( + "earliestTimeslots mismatch", + expected.getEarliestTimeslots(), + actual.getEarliestTimeslots()); + Assert.assertEquals( + "dataPartitionTables mismatch", + expected.getDataPartitionTables(), + actual.getDataPartitionTables()); + Assert.assertEquals( + "databasesWithLostDataPartition mismatch", + expected.getDatabasesWithLostDataPartition(), + actual.getDatabasesWithLostDataPartition()); + Assert.assertEquals( + "finalDataPartitionTables mismatch", + expected.getFinalDataPartitionTables(), + actual.getFinalDataPartitionTables()); + Assert.assertEquals( + "skipDataNodes mismatch", expected.getSkipDataNodes(), actual.getSkipDataNodes()); + Assert.assertEquals( + "failedDataNodes mismatch", expected.getFailedDataNodes(), actual.getFailedDataNodes()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index a3130f49f343..0b3bafb89350 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1256,6 +1256,11 @@ public class IoTDBConfig { private long cacheLastValuesMemoryBudgetInByte = 4 * 1024 * 1024; + /* Need use these parameters when repair data partition table */ + private int partitionTableRecoverWorkerNum = 10; + // Rate limit set to 10 MB/s + private int partitionTableRecoverMaxReadMBsPerSecond = 10; + IoTDBConfig() {} public int getMaxLogEntriesNumPerBatch() { @@ -4542,4 +4547,21 @@ public long getCacheLastValuesMemoryBudgetInByte() { public void setCacheLastValuesMemoryBudgetInByte(long cacheLastValuesMemoryBudgetInByte) { this.cacheLastValuesMemoryBudgetInByte = cacheLastValuesMemoryBudgetInByte; } + + public int getPartitionTableRecoverWorkerNum() { + return partitionTableRecoverWorkerNum; + } + + public void setPartitionTableRecoverWorkerNum(int partitionTableRecoverWorkerNum) { + this.partitionTableRecoverWorkerNum = partitionTableRecoverWorkerNum; + } + + public int getPartitionTableRecoverMaxReadMBsPerSecond() { + return partitionTableRecoverMaxReadMBsPerSecond; + } + + public void setPartitionTableRecoverMaxReadMBsPerSecond( + int partitionTableRecoverMaxReadMBsPerSecond) { + this.partitionTableRecoverMaxReadMBsPerSecond = partitionTableRecoverMaxReadMBsPerSecond; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 5658e10cc607..155fa0e38d12 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1089,6 +1089,16 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException loadQuerySampleThroughput(properties); // update trusted_uri_pattern loadTrustedUriPattern(properties); + conf.setPartitionTableRecoverWorkerNum( + Integer.parseInt( + properties.getProperty( + "partition_table_recover_worker_num", + String.valueOf(conf.getPartitionTableRecoverWorkerNum())))); + conf.setPartitionTableRecoverMaxReadMBsPerSecond( + Integer.parseInt( + properties.getProperty( + "partition_table_recover_max_read_megabytes_per_second", + String.valueOf(conf.getPartitionTableRecoverMaxReadMBsPerSecond())))); } private void loadFixedSizeLimitForQuery( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/partition/DataPartitionTableGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/partition/DataPartitionTableGenerator.java new file mode 100644 index 000000000000..0261cc348892 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/partition/DataPartitionTableGenerator.java @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.partition; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.partition.DataPartitionTable; +import org.apache.iotdb.commons.partition.SeriesPartitionTable; +import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor; +import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.storageengine.StorageEngine; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; + +import com.google.common.util.concurrent.RateLimiter; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.PlainDeviceID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Generator for DataPartitionTable by scanning tsfile resources. This class scans the data + * directory structure and builds a complete DataPartitionTable based on existing tsfiles. + */ +public class DataPartitionTableGenerator { + + private static final Logger LOG = LoggerFactory.getLogger(DataPartitionTableGenerator.class); + + // Task status + private volatile TaskStatus status = TaskStatus.NOT_STARTED; + private volatile String errorMessage; + private Map databasePartitionTableMap = new ConcurrentHashMap<>(); + + // Progress tracking + private final AtomicInteger processedTimePartitions = new AtomicInteger(0); + private final AtomicInteger failedTimePartitions = new AtomicInteger(0); + private long totalTimePartitions = 0; + + // Configuration + private final ExecutorService executor; + private final Set databases; + private final int seriesSlotNum; + private final String seriesPartitionExecutorClass; + + private final RateLimiter limiter = + RateLimiter.create( + (long) + IoTDBDescriptor.getInstance() + .getConfig() + .getPartitionTableRecoverMaxReadMBsPerSecond() + * 1024 + * 1024); + + public static final Set IGNORE_DATABASE = + new HashSet() { + { + add("root.__audit"); + add("root.__system"); + } + }; + + public DataPartitionTableGenerator( + ExecutorService executor, + Set databases, + int seriesSlotNum, + String seriesPartitionExecutorClass) { + this.executor = executor; + this.databases = databases; + this.seriesSlotNum = seriesSlotNum; + this.seriesPartitionExecutorClass = seriesPartitionExecutorClass; + } + + public Map getDatabasePartitionTableMap() { + return databasePartitionTableMap; + } + + public enum TaskStatus { + NOT_STARTED, + IN_PROGRESS, + COMPLETED, + FAILED + } + + /** Start generating DataPartitionTable asynchronously. */ + public CompletableFuture startGeneration() { + if (status != TaskStatus.NOT_STARTED) { + throw new IllegalStateException("Task is already started or completed"); + } + + status = TaskStatus.IN_PROGRESS; + return CompletableFuture.runAsync(this::generateDataPartitionTableByMemory); + } + + private void generateDataPartitionTableByMemory() { + List> futures = new ArrayList<>(); + + SeriesPartitionExecutor seriesPartitionExecutor = + SeriesPartitionExecutor.getSeriesPartitionExecutor( + seriesPartitionExecutorClass, seriesSlotNum); + + try { + totalTimePartitions = + StorageEngine.getInstance().getAllDataRegions().stream() + .mapToLong( + dataRegion -> + (dataRegion == null) + ? 0 + : dataRegion.getTsFileManager().getTimePartitions().size()) + .sum(); + for (DataRegion dataRegion : StorageEngine.getInstance().getAllDataRegions()) { + CompletableFuture regionFuture = + CompletableFuture.runAsync( + () -> { + try { + TsFileManager tsFileManager = dataRegion.getTsFileManager(); + String databaseName = dataRegion.getDatabaseName(); + if (!databases.contains(databaseName) + || IGNORE_DATABASE.contains(databaseName)) { + return; + } + + Map dataPartitionMap = + new ConcurrentHashMap<>(); + + tsFileManager.readLock(); + List seqTsFileList = tsFileManager.getTsFileList(true); + List unseqTsFileList = tsFileManager.getTsFileList(false); + tsFileManager.readUnlock(); + + constructDataPartitionMap( + seqTsFileList, seriesPartitionExecutor, dataPartitionMap); + constructDataPartitionMap( + unseqTsFileList, seriesPartitionExecutor, dataPartitionMap); + + if (dataPartitionMap.isEmpty()) { + LOG.error("Failed to generate DataPartitionTable, dataPartitionMap is empty"); + status = TaskStatus.FAILED; + errorMessage = "DataPartitionMap is empty after processing resource file"; + return; + } + + DataPartitionTable dataPartitionTable = + new DataPartitionTable(dataPartitionMap); + + databasePartitionTableMap.compute( + databaseName, + (k, v) -> { + if (v == null) { + return new DataPartitionTable(dataPartitionMap); + } + v.merge(dataPartitionTable); + return v; + }); + } catch (Exception e) { + LOG.error("Error processing data region: {}", dataRegion.getDatabaseName(), e); + failedTimePartitions.incrementAndGet(); + errorMessage = "Failed to process data region: " + e.getMessage(); + } + }, + executor); + futures.add(regionFuture); + } + + // Wait for all tasks to complete + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + + status = TaskStatus.COMPLETED; + LOG.info( + "DataPartitionTable generation completed successfully. Processed: {}, Failed: {}", + processedTimePartitions.get(), + failedTimePartitions.get()); + } catch (Exception e) { + LOG.error("Failed to generate DataPartitionTable", e); + status = TaskStatus.FAILED; + errorMessage = "Generation failed: " + e.getMessage(); + } + } + + private void constructDataPartitionMap( + List seqTsFileList, + SeriesPartitionExecutor seriesPartitionExecutor, + Map dataPartitionMap) { + Set timeSlotIds = Collections.newSetFromMap(new ConcurrentHashMap<>()); + + for (TsFileResource tsFileResource : seqTsFileList) { + long timeSlotId = tsFileResource.getTsFileID().timePartitionId; + try { + Set devices = tsFileResource.getDevices(limiter); + int regionId = tsFileResource.getTsFileID().regionId; + + TConsensusGroupId consensusGroupId = new TConsensusGroupId(); + consensusGroupId.setId(regionId); + consensusGroupId.setType(TConsensusGroupType.DataRegion); + + for (IDeviceID deviceId : devices) { + TSeriesPartitionSlot seriesSlotId = + seriesPartitionExecutor.getSeriesPartitionSlot( + ((PlainDeviceID) deviceId).toStringID()); + TTimePartitionSlot timePartitionSlot = + new TTimePartitionSlot(TimePartitionUtils.getStartTimeByPartitionId(timeSlotId)); + dataPartitionMap + .computeIfAbsent( + seriesSlotId, empty -> newSeriesPartitionTable(consensusGroupId, timeSlotId)) + .putDataPartition(timePartitionSlot, consensusGroupId); + } + if (!timeSlotIds.contains(timeSlotId)) { + timeSlotIds.add(timeSlotId); + processedTimePartitions.incrementAndGet(); + } + } catch (Exception e) { + if (!timeSlotIds.contains(timeSlotId)) { + timeSlotIds.add(timeSlotId); + failedTimePartitions.incrementAndGet(); + } + LOG.error("Failed to process tsfile {}, {}", tsFileResource.getTsFileID(), e.getMessage()); + } + } + + timeSlotIds.clear(); + } + + private static SeriesPartitionTable newSeriesPartitionTable( + TConsensusGroupId consensusGroupId, long timeSlotId) { + SeriesPartitionTable seriesPartitionTable = new SeriesPartitionTable(); + TTimePartitionSlot timePartitionSlot = + new TTimePartitionSlot(TimePartitionUtils.getStartTimeByPartitionId(timeSlotId)); + seriesPartitionTable.putDataPartition(timePartitionSlot, consensusGroupId); + return seriesPartitionTable; + } + + // Getters + public TaskStatus getStatus() { + return status; + } + + public String getErrorMessage() { + return errorMessage; + } + + public double getProgress() { + if (totalTimePartitions == 0) { + return 0.0; + } + return (double) (processedTimePartitions.get() + failedTimePartitions.get()) + / totalTimePartitions; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 948afb1d563c..cf83a81cd489 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -150,6 +150,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowRepairDataPartitionTableProgressResp; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp; import org.apache.iotdb.confignode.rpc.thrift.TShowTTLResp; @@ -635,6 +636,20 @@ public TDataPartitionTableResp getOrCreateDataPartitionTable(TDataPartitionReq r resp -> !updateConfigNodeLeader(resp.status)); } + @Override + public TSStatus dataPartitionTableIntegrityCheck() throws TException { + return executeRemoteCallWithRetry( + () -> client.dataPartitionTableIntegrityCheck(), status -> !updateConfigNodeLeader(status)); + } + + @Override + public TShowRepairDataPartitionTableProgressResp showRepairDataPartitionTableProgress() + throws TException { + return executeRemoteCallWithRetry( + () -> client.showRepairDataPartitionTableProgress(), + resp -> !updateConfigNodeLeader(resp.status)); + } + @Override public TSStatus operatePermission(TAuthorizerReq req) throws TException { return executeRemoteCallWithRetry( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/OperationType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/OperationType.java index 5838dac25bfb..f9329c661a98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/OperationType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/OperationType.java @@ -51,7 +51,10 @@ public enum OperationType { CHECK_AUTHORITY("checkAuthority"), EXECUTE_NON_QUERY_PLAN("executeNonQueryPlan"), QUERY_LATENCY("queryLatency"), - DISPATCH_FRAGMENT_INSTANCE("dispatchFragmentInstance"); + DISPATCH_FRAGMENT_INSTANCE("dispatchFragmentInstance"), + GET_EARLIEST_TIMESLOTS("getEarliestTimeslots"), + GENERATE_DATA_PARTITION_TABLE("generateDataPartitionTable"), + CHECK_DATA_PARTITION_TABLE_STATUS("checkDataPartitionTableStatus"); private final String name; OperationType(String name) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 99e314affa04..b8701f41fb78 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -53,8 +53,11 @@ import org.apache.iotdb.commons.consensus.SchemaRegionId; import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.consensus.index.ProgressIndexType; +import org.apache.iotdb.commons.enums.DataPartitionTableGeneratorState; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.partition.DataPartitionTable; +import org.apache.iotdb.commons.partition.DatabaseScopedDataPartitionTable; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; import org.apache.iotdb.commons.path.PathPatternTree; @@ -82,6 +85,7 @@ import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl; import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.partition.DataPartitionTableGenerator; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.protocol.client.cn.DnToCnInternalServiceAsyncRequestManager; @@ -153,10 +157,13 @@ import org.apache.iotdb.db.service.RegionMigrateService; import org.apache.iotdb.db.service.metrics.FileMetrics; import org.apache.iotdb.db.storageengine.StorageEngine; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.compaction.repair.RepairTaskStatus; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleTaskManager; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; import org.apache.iotdb.db.storageengine.dataregion.compaction.settle.SettleRequestHandler; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeSpaceQuotaManager; import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeThrottleQuotaManager; import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; @@ -208,6 +215,11 @@ import org.apache.iotdb.mpp.rpc.thrift.TFireTriggerReq; import org.apache.iotdb.mpp.rpc.thrift.TFireTriggerResp; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceInfoResp; +import org.apache.iotdb.mpp.rpc.thrift.TGenerateDataPartitionTableHeartbeatResp; +import org.apache.iotdb.mpp.rpc.thrift.TGenerateDataPartitionTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TGenerateDataPartitionTableResp; +import org.apache.iotdb.mpp.rpc.thrift.TGetDataPartitionTableGeneratorProgressResp; +import org.apache.iotdb.mpp.rpc.thrift.TGetEarliestTimeslotsResp; import org.apache.iotdb.mpp.rpc.thrift.TInactiveTriggerInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TInvalidateCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TInvalidateMatchedSchemaCacheReq; @@ -257,9 +269,13 @@ import com.google.common.collect.ImmutableList; import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.transport.TIOStreamTransport; +import org.apache.thrift.transport.TTransport; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NotImplementedException; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.record.Tablet; @@ -284,11 +300,14 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -301,10 +320,10 @@ import static org.apache.iotdb.commons.client.request.TestConnectionUtils.testConnectionsImpl; import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; import static org.apache.iotdb.db.service.RegionMigrateService.REGION_MIGRATE_PROCESS; +import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onIoTDBException; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onQueryException; public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface { - private static final Logger LOGGER = LoggerFactory.getLogger(DataNodeInternalRPCServiceImpl.class); @@ -2556,4 +2575,365 @@ public TSStatus stopAndClearDataNode() { public void handleClientExit() { // Do nothing } + + // ==================================================== + // Data Partition Table Integrity Check Implementation + // ==================================================== + + private volatile DataPartitionTableGenerator currentGenerator; + private volatile CompletableFuture currentGeneratorFuture; + private volatile long currentTaskId = 0; + + @Override + public TGetEarliestTimeslotsResp getEarliestTimeslots() { + TGetEarliestTimeslotsResp resp = new TGetEarliestTimeslotsResp(); + + try { + Map earliestTimeslots = new ConcurrentHashMap<>(); + processDataRegionForEarliestTimeslots(earliestTimeslots); + + resp.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + resp.setDatabaseToEarliestTimeslot(earliestTimeslots); + + LOGGER.info("Retrieved earliest timeslots for {} databases", earliestTimeslots.size()); + } catch (Exception e) { + LOGGER.error("Failed to get earliest timeslots", e); + resp.setStatus( + onIoTDBException( + e, + OperationType.GET_EARLIEST_TIMESLOTS, + TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode())); + } + + return resp; + } + + @Override + public TGenerateDataPartitionTableResp generateDataPartitionTable( + TGenerateDataPartitionTableReq req) { + TGenerateDataPartitionTableResp resp = new TGenerateDataPartitionTableResp(); + + try { + // Check if there's already a task in the progress + final DataPartitionTableGenerator runningGenerator = currentGenerator; + if (runningGenerator != null + && runningGenerator.getStatus() == DataPartitionTableGenerator.TaskStatus.IN_PROGRESS) { + resp.setErrorCode(DataPartitionTableGeneratorState.IN_PROGRESS.getCode()); + resp.setMessage( + String.format( + "DataPartitionTable generation is already in the progress: %.1f%%", + runningGenerator.getProgress() * 100)); + resp.setStatus(RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR)); + return resp; + } + + // Create generator for all data directories + int seriesSlotNum = IoTDBDescriptor.getInstance().getConfig().getSeriesPartitionSlotNum(); + String seriesPartitionExecutorClass = + IoTDBDescriptor.getInstance().getConfig().getSeriesPartitionExecutorClass(); + + final ExecutorService partitionTableRecoverExecutor = + new WrappedThreadPoolExecutor( + 0, + IoTDBDescriptor.getInstance().getConfig().getPartitionTableRecoverWorkerNum(), + 0L, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>( + IoTDBDescriptor.getInstance().getConfig().getPartitionTableRecoverWorkerNum()), + new IoTThreadFactory(ThreadName.DATA_PARTITION_RECOVER_PARALLEL_POOL.getName()), + ThreadName.DATA_PARTITION_RECOVER_PARALLEL_POOL.getName(), + new ThreadPoolExecutor.CallerRunsPolicy()); + + final DataPartitionTableGenerator generator = + new DataPartitionTableGenerator( + partitionTableRecoverExecutor, + req.getDatabases(), + seriesSlotNum, + seriesPartitionExecutorClass); + currentGenerator = generator; + currentTaskId = System.currentTimeMillis(); + + // Start generation synchronously for now to return the data partition table immediately + currentGeneratorFuture = generator.startGeneration(); + parseGenerationStatus(resp, generator); + } catch (Exception e) { + LOGGER.error("Failed to generate DataPartitionTable", e); + resp.setStatus( + onIoTDBException( + e, + OperationType.GENERATE_DATA_PARTITION_TABLE, + TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode())); + } + + return resp; + } + + @Override + public TGenerateDataPartitionTableHeartbeatResp generateDataPartitionTableHeartbeat( + TGenerateDataPartitionTableReq req) { + TGenerateDataPartitionTableHeartbeatResp resp = new TGenerateDataPartitionTableHeartbeatResp(); + // Must be lower than the RPC request timeout, in milliseconds + final long timeoutMs = 50000; + // Set default value + resp.setDatabaseScopedDataPartitionTables(Collections.emptyList()); + try { + // To resolve this situation that the DataNode is registered and didn't request + // generateDataPartitionTable interface yet. + CompletableFuture generatorFuture = currentGeneratorFuture; + DataPartitionTableGenerator generator = currentGenerator; + if (generatorFuture == null || generator == null) { + generateDataPartitionTable(req); + generatorFuture = currentGeneratorFuture; + generator = currentGenerator; + if (generatorFuture == null || generator == null) { + resp.setErrorCode(DataPartitionTableGeneratorState.UNKNOWN.getCode()); + resp.setMessage("No DataPartitionTable generation task found"); + resp.setStatus(RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR)); + return resp; + } + } + + try { + generatorFuture.get(timeoutMs, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + parseGenerationStatus(resp, generator); + return resp; + } + + parseGenerationStatus(resp, generator); + if (generator.getStatus().equals(DataPartitionTableGenerator.TaskStatus.COMPLETED)) { + boolean success = false; + List databaseScopedDataPartitionTableList = + new ArrayList<>(); + Map dataPartitionTableMap = + generator.getDatabasePartitionTableMap(); + if (!dataPartitionTableMap.isEmpty()) { + for (Map.Entry entry : dataPartitionTableMap.entrySet()) { + String database = entry.getKey(); + DataPartitionTable dataPartitionTable = entry.getValue(); + if (database != null && !database.isEmpty() && dataPartitionTable != null) { + DatabaseScopedDataPartitionTable databaseScopedDataPartitionTable = + new DatabaseScopedDataPartitionTable(database, dataPartitionTable); + databaseScopedDataPartitionTableList.add(databaseScopedDataPartitionTable); + success = true; + } + } + } + + if (success) { + List result = + serializeDatabaseScopedTableList(databaseScopedDataPartitionTableList); + resp.setDatabaseScopedDataPartitionTables(result); + + // Clear current generator + currentGenerator = null; + } + } + } catch (Exception e) { + LOGGER.error("Failed to check DataPartitionTable generation status", e); + resp.setStatus( + onIoTDBException( + e, + OperationType.CHECK_DATA_PARTITION_TABLE_STATUS, + TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode())); + } + return resp; + } + + @Override + public TGetDataPartitionTableGeneratorProgressResp getDataPartitionTableGeneratorProgress() { + TGetDataPartitionTableGeneratorProgressResp resp = + new TGetDataPartitionTableGeneratorProgressResp(); + final DataPartitionTableGenerator generator = currentGenerator; + + if (generator == null) { + resp.setErrorCode(DataPartitionTableGeneratorState.UNKNOWN.getCode()); + resp.setProgress(0.0); + resp.setMessage("No DataPartitionTable generation task found"); + resp.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + return resp; + } + + switch (generator.getStatus()) { + case IN_PROGRESS: + resp.setErrorCode(DataPartitionTableGeneratorState.IN_PROGRESS.getCode()); + resp.setProgress(generator.getProgress()); + resp.setMessage( + String.format( + "DataPartitionTable generation in progress: %.1f%%", + generator.getProgress() * 100)); + resp.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + break; + case COMPLETED: + resp.setErrorCode(DataPartitionTableGeneratorState.SUCCESS.getCode()); + resp.setProgress(1.0); + resp.setMessage("DataPartitionTable generation completed successfully"); + resp.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + break; + case FAILED: + resp.setErrorCode(DataPartitionTableGeneratorState.FAILED.getCode()); + resp.setProgress(generator.getProgress()); + resp.setMessage("DataPartitionTable generation failed: " + generator.getErrorMessage()); + resp.setStatus(RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR)); + break; + default: + resp.setErrorCode(DataPartitionTableGeneratorState.UNKNOWN.getCode()); + resp.setProgress(generator.getProgress()); + resp.setMessage("Unknown task status: " + generator.getStatus()); + resp.setStatus(RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR)); + break; + } + return resp; + } + + private void parseGenerationStatus(Object resp, DataPartitionTableGenerator generator) { + switch (generator.getStatus()) { + case IN_PROGRESS: + setResponseFields( + resp, + DataPartitionTableGeneratorState.IN_PROGRESS.getCode(), + String.format( + "DataPartitionTable generation in progress: %.1f%%", generator.getProgress() * 100), + generator.getProgress(), + RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + LOGGER.info( + String.format( + "DataPartitionTable generation with task ID: %s in progress: %.1f%%", + currentTaskId, generator.getProgress() * 100)); + break; + case COMPLETED: + setResponseFields( + resp, + DataPartitionTableGeneratorState.SUCCESS.getCode(), + "DataPartitionTable generation completed successfully", + 1.0, + RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + LOGGER.info("DataPartitionTable generation completed with task ID: {}", currentTaskId); + break; + case FAILED: + setResponseFields( + resp, + DataPartitionTableGeneratorState.FAILED.getCode(), + "DataPartitionTable generation failed: " + generator.getErrorMessage(), + generator.getProgress(), + RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR)); + LOGGER.info("DataPartitionTable generation failed with task ID: {}", currentTaskId); + break; + default: + setResponseFields( + resp, + DataPartitionTableGeneratorState.UNKNOWN.getCode(), + "Unknown task status: " + generator.getStatus(), + generator.getProgress(), + RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR)); + LOGGER.info("DataPartitionTable generation failed with task ID: {}", currentTaskId); + break; + } + } + + private void setResponseFields( + Object resp, int errorCode, String message, double progress, TSStatus status) { + if (resp instanceof TGenerateDataPartitionTableResp) { + ((TGenerateDataPartitionTableResp) resp).setErrorCode(errorCode); + ((TGenerateDataPartitionTableResp) resp).setMessage(message); + ((TGenerateDataPartitionTableResp) resp).setStatus(status); + } else if (resp instanceof TGenerateDataPartitionTableHeartbeatResp) { + ((TGenerateDataPartitionTableHeartbeatResp) resp).setErrorCode(errorCode); + ((TGenerateDataPartitionTableHeartbeatResp) resp).setMessage(message); + ((TGenerateDataPartitionTableHeartbeatResp) resp).setProgress(progress); + ((TGenerateDataPartitionTableHeartbeatResp) resp).setStatus(status); + } + } + + /** + * Scan the seq and unseq directory on every data region, then compute the earliest time slot id + * of database + */ + private void processDataRegionForEarliestTimeslots(Map earliestTimeslots) { + final Set ignoreDatabase = + new HashSet() { + { + add("root.__audit"); + add("root.__system"); + } + }; + List> futures = new ArrayList<>(); + final ExecutorService findEarliestTimeSlotExecutor = + new WrappedThreadPoolExecutor( + 0, + IoTDBDescriptor.getInstance().getConfig().getPartitionTableRecoverWorkerNum(), + 0L, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>( + IoTDBDescriptor.getInstance().getConfig().getPartitionTableRecoverWorkerNum()), + new IoTThreadFactory(ThreadName.FIND_EARLIEST_TIME_SLOT_PARALLEL_POOL.getName()), + ThreadName.FIND_EARLIEST_TIME_SLOT_PARALLEL_POOL.getName(), + new ThreadPoolExecutor.CallerRunsPolicy()); + + for (DataRegion dataRegion : StorageEngine.getInstance().getAllDataRegions()) { + CompletableFuture regionFuture = + CompletableFuture.runAsync( + () -> { + TsFileManager tsFileManager = dataRegion.getTsFileManager(); + String databaseName = dataRegion.getDatabaseName(); + if (ignoreDatabase.contains(databaseName)) { + return; + } + + Set timePartitionIds = tsFileManager.getTimePartitions(); + if (timePartitionIds.isEmpty()) { + return; + } + final long earliestTimeSlotId = Collections.min(timePartitionIds); + earliestTimeslots.compute( + databaseName, + (k, v) -> v == null ? earliestTimeSlotId : Math.min(earliestTimeSlotId, v)); + }, + findEarliestTimeSlotExecutor); + futures.add(regionFuture); + } + + // Wait for all tasks to complete + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + LOGGER.info("Process data directory for earliestTimeslots completed successfully"); + } + + private long findEarliestTimeslotInFiles( + List seqTsFileList, long earliestTimeSlotId) { + for (TsFileResource tsFileResource : seqTsFileList) { + long timeSlotId = tsFileResource.getTsFileID().timePartitionId; + earliestTimeSlotId = + earliestTimeSlotId == Long.MIN_VALUE + ? timeSlotId + : Math.min(earliestTimeSlotId, timeSlotId); + } + + return earliestTimeSlotId; + } + + private List serializeDatabaseScopedTableList( + List list) { + if (list == null || list.isEmpty()) { + return Collections.emptyList(); + } + + List result = new ArrayList<>(list.size()); + + for (DatabaseScopedDataPartitionTable table : list) { + try (PublicBAOS baos = new PublicBAOS(); + DataOutputStream oos = new DataOutputStream(baos)) { + TTransport transport = new TIOStreamTransport(oos); + TBinaryProtocol protocol = new TBinaryProtocol(transport); + table.serialize(oos, protocol); + result.add(ByteBuffer.wrap(baos.getBuf(), 0, baos.size())); + } catch (IOException | TException e) { + LOGGER.error( + "Failed to serialize DatabaseScopedDataPartitionTable for database: {}", + table.getDatabase(), + e); + } + } + + return result; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java index 92712daee529..1eaa1683fde8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java @@ -183,6 +183,11 @@ private ColumnHeaderConstant() { public static final String ESTIMATED_REMAINING_SECONDS = "EstimatedRemainingSeconds"; public static final String RECENT_FAILURES = "RecentFailures"; + // column names for show repair data partition table progress + public static final String REPAIR_DATA_PARTITION_TABLE_STATUS = "Status"; + public static final String REPAIR_DATA_PARTITION_TABLE_PROGRESS = "Progress(%)"; + public static final String REPAIR_DATA_PARTITION_TABLE_MESSAGE = "Message"; + // column names for select into public static final String SOURCE_DEVICE = "SourceDevice"; public static final String SOURCE_COLUMN = "SourceColumn"; @@ -454,6 +459,12 @@ private ColumnHeaderConstant() { new ColumnHeader(ESTIMATED_REMAINING_SECONDS, TSDataType.TEXT), new ColumnHeader(RECENT_FAILURES, TSDataType.TEXT)); + public static final List showRepairDataPartitionTableProgressColumnHeaders = + ImmutableList.of( + new ColumnHeader(REPAIR_DATA_PARTITION_TABLE_STATUS, TSDataType.TEXT), + new ColumnHeader(REPAIR_DATA_PARTITION_TABLE_PROGRESS, TSDataType.DOUBLE), + new ColumnHeader(REPAIR_DATA_PARTITION_TABLE_MESSAGE, TSDataType.TEXT)); + public static final List showTopicColumnHeaders = ImmutableList.of( new ColumnHeader(TOPIC_NAME, TSDataType.TEXT), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java index 132dafd246d9..0eb7a789ef9b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java @@ -159,6 +159,11 @@ public static DatasetHeader getShowPipeHeader() { return new DatasetHeader(ColumnHeaderConstant.showPipeColumnHeaders, true); } + public static DatasetHeader getShowRepairDataPartitionTableProgressHeader() { + return new DatasetHeader( + ColumnHeaderConstant.showRepairDataPartitionTableProgressColumnHeaders, true); + } + public static DatasetHeader getShowTopicHeader() { return new DatasetHeader(ColumnHeaderConstant.showTopicColumnHeaders, true); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java index 7a0556007ca3..d6adc41f1b77 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java @@ -80,8 +80,10 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.sys.KillQueryTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.LoadConfigurationTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.MergeTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.RepairDataPartitionTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetConfigurationTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetSystemStatusTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.ShowRepairDataPartitionTableProgressTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.StartRepairDataTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.StopRepairDataTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.TestConnectionTask; @@ -172,8 +174,10 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.MergeStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.RepairDataPartitionTable; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowRepairDataPartitionTableProgressStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.TestConnectionStatement; @@ -305,6 +309,19 @@ public IConfigTask visitStartRepairData( return new StartRepairDataTask(startRepairDataStatement); } + @Override + public IConfigTask visitRepairDataPartitionTable( + RepairDataPartitionTable repairDataPartitionTable, MPPQueryContext context) { + return new RepairDataPartitionTableTask(); + } + + @Override + public IConfigTask visitShowRepairDataPartitionTableProgress( + ShowRepairDataPartitionTableProgressStatement showRepairDataPartitionTableProgressStatement, + MPPQueryContext context) { + return new ShowRepairDataPartitionTableProgressTask(); + } + @Override public IConfigTask visitStopRepairData( StopRepairDataStatement stopRepairDataStatement, MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index e2a850231865..a0c85d9a72ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -123,6 +123,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowRepairDataPartitionTableProgressResp; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp; import org.apache.iotdb.confignode.rpc.thrift.TShowTTLResp; @@ -176,6 +177,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.template.ShowNodesInSchemaTemplateTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.template.ShowPathSetTemplateTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.template.ShowSchemaTemplateTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.ShowRepairDataPartitionTableProgressTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.TestConnectionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.pipe.ShowPipeTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.quota.ShowSpaceQuotaTask; @@ -1221,6 +1223,47 @@ public SettableFuture stopRepairData(boolean onCluster) { return future; } + @Override + public SettableFuture repairDataPartitionTable() { + SettableFuture future = SettableFuture.create(); + TSStatus tsStatus = new TSStatus(); + + try (ConfigNodeClient client = + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + // Send request to ConfigNode to trigger DataPartitionTableIntegrityCheckProcedure + tsStatus = client.dataPartitionTableIntegrityCheck(); + } catch (ClientManagerException | TException e) { + future.setException(e); + } + + if (tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } else { + future.setException(new IoTDBException(tsStatus)); + } + return future; + } + + @Override + public SettableFuture showRepairDataPartitionTableProgress() { + SettableFuture future = SettableFuture.create(); + + try (ConfigNodeClient client = + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + TShowRepairDataPartitionTableProgressResp resp = + client.showRepairDataPartitionTableProgress(); + if (resp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + ShowRepairDataPartitionTableProgressTask.buildTsBlock(resp, future); + } else { + future.setException(new IoTDBException(resp.getStatus())); + } + } catch (ClientManagerException | TException e) { + future.setException(e); + } + + return future; + } + @Override public SettableFuture loadConfiguration(boolean onCluster) { SettableFuture future = SettableFuture.create(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index b663017fadd9..3b377db6bd97 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -129,6 +129,10 @@ public interface IConfigTaskExecutor { SettableFuture stopRepairData(boolean onCluster); + SettableFuture repairDataPartitionTable(); + + SettableFuture showRepairDataPartitionTableProgress(); + SettableFuture flush(TFlushReq tFlushReq, boolean onCluster); SettableFuture clearCache(boolean onCluster, Set options); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/RepairDataPartitionTableTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/RepairDataPartitionTableTask.java new file mode 100644 index 000000000000..f3675e9a0d97 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/RepairDataPartitionTableTask.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.sys; + +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; + +import com.google.common.util.concurrent.ListenableFuture; + +public class RepairDataPartitionTableTask implements IConfigTask { + + public RepairDataPartitionTableTask() {} + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.repairDataPartitionTable(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/ShowRepairDataPartitionTableProgressTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/ShowRepairDataPartitionTableProgressTask.java new file mode 100644 index 000000000000..d0ccbaa9f342 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/ShowRepairDataPartitionTableProgressTask.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.sys; + +import org.apache.iotdb.confignode.rpc.thrift.TShowRepairDataPartitionTableProgressResp; +import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.rpc.TSStatusCode; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.utils.Binary; + +import java.util.Arrays; + +public class ShowRepairDataPartitionTableProgressTask implements IConfigTask { + + public ShowRepairDataPartitionTableProgressTask() { + // Empty constructor + } + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.showRepairDataPartitionTableProgress(); + } + + public static void buildTsBlock( + TShowRepairDataPartitionTableProgressResp resp, SettableFuture future) { + TsBlockBuilder builder = + new TsBlockBuilder(Arrays.asList(TSDataType.TEXT, TSDataType.DOUBLE, TSDataType.TEXT)); + + builder.getTimeColumnBuilder().writeLong(0L); + builder + .getColumnBuilder(0) + .writeBinary(new Binary(resp.getState(), TSFileConfig.STRING_CHARSET)); + builder.getColumnBuilder(1).writeDouble(resp.getProgress()); + builder + .getColumnBuilder(2) + .writeBinary( + new Binary(resp.isSetMessage() ? resp.getMessage() : "", TSFileConfig.STRING_CHARSET)); + builder.declarePosition(); + + future.set( + new ConfigTaskResult( + TSStatusCode.SUCCESS_STATUS, + builder.build(), + DatasetHeaderFactory.getShowRepairDataPartitionTableProgressHeader())); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 75c841d4140a..d000f62de327 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -213,9 +213,11 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.RepairDataPartitionTable; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowRepairDataPartitionTableProgressStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; @@ -3398,6 +3400,20 @@ public Statement visitStartRepairData(IoTDBSqlParser.StartRepairDataContext ctx) return startRepairDataStatement; } + // Repair Data Partition Table + + @Override + public Statement visitRepairDataPartitionTable( + IoTDBSqlParser.RepairDataPartitionTableContext ctx) { + return new RepairDataPartitionTable(); + } + + @Override + public Statement visitShowRepairDataPartitionTableProgress( + IoTDBSqlParser.ShowRepairDataPartitionTableProgressContext ctx) { + return new ShowRepairDataPartitionTableProgressStatement(); + } + // Stop Repair Data @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java index 4d5fe5dbf3fb..4be624bc8ee5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java @@ -178,6 +178,8 @@ public enum StatementType { PIPE_ENRICHED, START_REPAIR_DATA, STOP_REPAIR_DATA, + REPAIR_DATA_PARTITION_TABLE, + SHOW_REPAIR_DATA_PARTITION_TABLE_PROGRESS, CREATE_TOPIC, DROP_TOPIC, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index 1792d4eb2687..1630c73dc582 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -123,9 +123,11 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.MergeStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.RepairDataPartitionTable; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowRepairDataPartitionTableProgressStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; @@ -439,6 +441,17 @@ public R visitStopRepairData(StopRepairDataStatement stopRepairDataStatement, C return visitStatement(stopRepairDataStatement, context); } + public R visitRepairDataPartitionTable( + RepairDataPartitionTable repairDataPartitionTable, C context) { + return visitStatement(repairDataPartitionTable, context); + } + + public R visitShowRepairDataPartitionTableProgress( + ShowRepairDataPartitionTableProgressStatement showRepairDataPartitionTableProgressStatement, + C context) { + return visitStatement(showRepairDataPartitionTableProgressStatement, context); + } + public R visitLoadConfiguration( LoadConfigurationStatement loadConfigurationStatement, C context) { return visitStatement(loadConfigurationStatement, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/RepairDataPartitionTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/RepairDataPartitionTable.java new file mode 100644 index 000000000000..4c2890d01f7e --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/RepairDataPartitionTable.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.sys; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.auth.AuthorityChecker; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.Collections; +import java.util.List; + +public class RepairDataPartitionTable extends Statement implements IConfigStatement { + + public RepairDataPartitionTable() { + this.statementType = StatementType.REPAIR_DATA_PARTITION_TABLE; + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } + + @Override + public QueryType getQueryType() { + return QueryType.WRITE; + } + + @Override + public TSStatus checkPermissionBeforeProcess(String userName) { + return AuthorityChecker.checkSuperUserOrMaintain(userName); + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitRepairDataPartitionTable(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowRepairDataPartitionTableProgressStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowRepairDataPartitionTableProgressStatement.java new file mode 100644 index 000000000000..823755156063 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowRepairDataPartitionTableProgressStatement.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.sys; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.auth.AuthorityChecker; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.Collections; +import java.util.List; + +public class ShowRepairDataPartitionTableProgressStatement extends Statement + implements IConfigStatement { + + public ShowRepairDataPartitionTableProgressStatement() { + this.statementType = StatementType.SHOW_REPAIR_DATA_PARTITION_TABLE_PROGRESS; + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } + + @Override + public QueryType getQueryType() { + return QueryType.READ; + } + + @Override + public TSStatus checkPermissionBeforeProcess(String userName) { + return AuthorityChecker.checkSuperUserOrMaintain(userName); + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitShowRepairDataPartitionTableProgress(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index f5df3066809f..9c1320503a31 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -41,6 +41,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.TimeIndexLevel; import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; +import com.google.common.util.concurrent.RateLimiter; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; @@ -500,6 +501,10 @@ public Set getDevices() { return timeIndex.getDevices(file.getPath(), this); } + public Set getDevices(RateLimiter limiter) { + return timeIndex.getDevices(file.getPath(), this, limiter); + } + public DeviceTimeIndex buildDeviceTimeIndex() throws IOException { readLock(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java index cc2e100a71d4..2e492647073b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import com.google.common.util.concurrent.RateLimiter; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.PlainDeviceID; import org.apache.tsfile.utils.FilePathUtils; @@ -170,6 +171,24 @@ public Set getDevices(String tsFilePath, TsFileResource tsFileResourc return deviceToIndex.keySet(); } + @Override + public Set getDevices( + String tsFilePath, TsFileResource tsFileResource, RateLimiter limiter) { + return deviceToIndex.keySet(); + } + + public Map getDeviceToIndex() { + return deviceToIndex; + } + + public long[] getEndTimes() { + return endTimes; + } + + public long[] getStartTimes() { + return startTimes; + } + /** * Deserialize TimeIndex and get devices only. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java index 768f81a05d62..72cd3d1c7977 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java @@ -21,10 +21,12 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; +import org.apache.iotdb.commons.utils.IOUtils; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.exception.load.PartitionViolationException; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import com.google.common.util.concurrent.RateLimiter; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.fileSystem.FSFactoryProducer; import org.apache.tsfile.utils.FilePathUtils; @@ -115,6 +117,40 @@ public Set getDevices(String tsFilePath, TsFileResource tsFileResourc } } + @Override + public Set getDevices( + String tsFilePath, TsFileResource tsFileResource, RateLimiter limiter) { + tsFileResource.readLock(); + try { + try (IOUtils.RatelimitedInputStream inputStream = + new IOUtils.RatelimitedInputStream( + FSFactoryProducer.getFSFactory() + .getBufferedInputStream(tsFilePath + TsFileResource.RESOURCE_SUFFIX), + limiter)) { + // The first byte is VERSION_NUMBER, second byte is timeIndexType. + ReadWriteIOUtils.readBytes(inputStream, 2); + return DeviceTimeIndex.getDevices(inputStream); + } + } catch (NoSuchFileException e) { + // deleted by ttl + if (tsFileResource.isDeleted()) { + return Collections.emptySet(); + } else { + logger.error( + "Can't read file {} from disk ", tsFilePath + TsFileResource.RESOURCE_SUFFIX, e); + throw new RuntimeException( + "Can't read file " + tsFilePath + TsFileResource.RESOURCE_SUFFIX + " from disk"); + } + } catch (Exception e) { + logger.error( + "Failed to get devices from tsfile: {}", tsFilePath + TsFileResource.RESOURCE_SUFFIX, e); + throw new RuntimeException( + "Failed to get devices from tsfile: " + tsFilePath + TsFileResource.RESOURCE_SUFFIX); + } finally { + tsFileResource.readUnlock(); + } + } + @Override public boolean endTimeEmpty() { return endTime == Long.MIN_VALUE; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java index 6cccc2fa3a0a..57f79bbd54a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.exception.load.PartitionViolationException; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import com.google.common.util.concurrent.RateLimiter; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -72,6 +73,13 @@ public interface ITimeIndex { */ Set getDevices(String tsFilePath, TsFileResource tsFileResource); + /** + * get devices in TimeIndex and limit files reading rate + * + * @return device names + */ + Set getDevices(String tsFilePath, TsFileResource tsFileResource, RateLimiter limiter); + /** * @return whether end time is empty (Long.MIN_VALUE) */ diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java index ea36bd2d7888..3ad249ab42ff 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java @@ -774,6 +774,27 @@ public void testCreateView() throws IllegalPathException { assertEquals(null, stmt.getQueryStatement()); } + @Test + public void testShowRepairDataPartitionTableProgress() { + Statement statement = + StatementGenerator.createStatement( + "SHOW REPAIR DATA PARTITION TABLE PROGRESS;", ZonedDateTime.now().getOffset()); + assertEquals(StatementType.SHOW_REPAIR_DATA_PARTITION_TABLE_PROGRESS, statement.getType()); + + QueryStatement queryStatement = + (QueryStatement) + StatementGenerator.createStatement( + "SELECT progress FROM root.sg.d1;", ZonedDateTime.now().getOffset()); + assertEquals( + "progress", + queryStatement + .getSelectComponent() + .getResultColumns() + .get(0) + .getExpression() + .getExpressionString()); + } + // TODO: add more tests private void checkQueryStatement( diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 2bc03179bd14..6ffc52ae24e9 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -722,6 +722,18 @@ failure_detector_phi_acceptable_pause_in_ms=10000 # Datatype: double(percentage) disk_space_warning_threshold=0.05 +# Purpose: for data partition repair +# The number of threads used for parallel scanning in the partition table recovery +# effectiveMode: restart +# Datatype: Integer +partition_table_recover_worker_num=10 + +# Purpose: for data partition repair +# Limit the number of bytes read per second from a file, the unit is MB +# effectiveMode: restart +# Datatype: Integer +partition_table_recover_max_read_megabytes_per_second=10 + #################### ### Memory Control Configuration #################### diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java index a90289309405..49061c2ef222 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java @@ -194,6 +194,8 @@ public enum ThreadName { REPAIR_DATA("RepairData"), FILE_TIME_INDEX_RECORD("FileTimeIndexRecord"), BINARY_ALLOCATOR_SAMPLE_EVICTOR("BinaryAllocator-SampleEvictor"), + FIND_EARLIEST_TIME_SLOT_PARALLEL_POOL("FindEarliestTimeSlot-Parallel-Pool"), + DATA_PARTITION_RECOVER_PARALLEL_POOL("DataPartitionRecover-Parallel-Pool"), // the unknown thread name is used for metrics UNKOWN("UNKNOWN"); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/enums/DataPartitionTableGeneratorState.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/enums/DataPartitionTableGeneratorState.java new file mode 100644 index 000000000000..93cca687799f --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/enums/DataPartitionTableGeneratorState.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.enums; + +public enum DataPartitionTableGeneratorState { + SUCCESS(0), + FAILED(1), + IN_PROGRESS(2), + UNKNOWN(-1); + + private final int code; + + DataPartitionTableGeneratorState(int code) { + this.code = code; + } + + public int getCode() { + return code; + } + + /** + * get DataPartitionTableGeneratorState by code + * + * @param code code + * @return DataPartitionTableGeneratorState + */ + public static DataPartitionTableGeneratorState getStateByCode(int code) { + for (DataPartitionTableGeneratorState state : DataPartitionTableGeneratorState.values()) { + if (state.code == code) { + return state; + } + } + return UNKNOWN; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/enums/RepairDataPartitionTableProgressState.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/enums/RepairDataPartitionTableProgressState.java new file mode 100644 index 000000000000..7afa3cbbdc8d --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/enums/RepairDataPartitionTableProgressState.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.enums; + +public enum RepairDataPartitionTableProgressState { + UNKNOWN, + IDLE, + COLLECT_EARLIEST_TIMESLOTS, + ANALYZE_MISSING_PARTITIONS, + REQUEST_PARTITION_TABLES, + REQUEST_PARTITION_TABLES_HEART_BEAT, + MERGE_PARTITION_TABLES, + WRITE_PARTITION_TABLE_TO_CONSENSUS +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionTable.java index 91346f0c69c8..d154f1813e1b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionTable.java @@ -282,6 +282,48 @@ public Set autoCleanPartitionTable( return removedTimePartitionSlots; } + /** + * Merge a complete DataPartitionTable from the partition tables received from multiple DataNodes + * (supports cross-database merging, which is exactly the logic implemented in the current PR) + * + * @param sourceMap Map + * @return The complete merged partition table + */ + public DataPartitionTable merge(Map sourceMap) { + DataPartitionTable merged = new DataPartitionTable(this.dataPartitionMap); + for (DataPartitionTable table : sourceMap.values()) { + for (Map.Entry entry : + table.dataPartitionMap.entrySet()) { + TSeriesPartitionSlot slot = entry.getKey(); + SeriesPartitionTable seriesTable = entry.getValue(); + merged + .dataPartitionMap + .computeIfAbsent(slot, k -> new SeriesPartitionTable()) + .merge(seriesTable); + } + } + return merged; + } + + /** + * Support single table merging Merge another DataPartitionTable into the current object (used for + * incremental merging) + */ + public DataPartitionTable merge(DataPartitionTable sourcePartitionTable) { + DataPartitionTable merged = new DataPartitionTable(this.dataPartitionMap); + if (sourcePartitionTable == null) { + return merged; + } + for (Map.Entry entry : + sourcePartitionTable.dataPartitionMap.entrySet()) { + merged + .dataPartitionMap + .computeIfAbsent(entry.getKey(), k -> new SeriesPartitionTable()) + .merge(entry.getValue()); + } + return merged; + } + public void serialize(OutputStream outputStream, TProtocol protocol) throws IOException, TException { ReadWriteIOUtils.write(dataPartitionMap.size(), outputStream); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DatabaseScopedDataPartitionTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DatabaseScopedDataPartitionTable.java new file mode 100644 index 000000000000..a47f4024eac8 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DatabaseScopedDataPartitionTable.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.partition; + +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TProtocol; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class DatabaseScopedDataPartitionTable { + private final String database; + private DataPartitionTable dataPartitionTable; + + public DatabaseScopedDataPartitionTable(String database, DataPartitionTable dataPartitionTable) { + this.database = database; + this.dataPartitionTable = dataPartitionTable; + } + + public String getDatabase() { + return database; + } + + public DataPartitionTable getDataPartitionTable() { + return dataPartitionTable; + } + + public void serialize(OutputStream outputStream, TProtocol protocol) + throws IOException, TException { + ReadWriteIOUtils.write(database, outputStream); + + ReadWriteIOUtils.write(dataPartitionTable != null, outputStream); + + if (dataPartitionTable != null) { + dataPartitionTable.serialize(outputStream, protocol); + } + } + + public static DatabaseScopedDataPartitionTable deserialize(ByteBuffer buffer) { + String database = ReadWriteIOUtils.readString(buffer); + + boolean hasDataPartitionTable = ReadWriteIOUtils.readBool(buffer); + + DataPartitionTable dataPartitionTable = null; + if (hasDataPartitionTable) { + dataPartitionTable = new DataPartitionTable(); + dataPartitionTable.deserialize(buffer); + } + + return new DatabaseScopedDataPartitionTable(database, dataPartitionTable); + } + + public static DatabaseScopedDataPartitionTable deserialize( + InputStream inputStream, TProtocol protocol) throws IOException, TException { + String database = ReadWriteIOUtils.readString(inputStream); + + boolean hasDataPartitionTable = ReadWriteIOUtils.readBool(inputStream); + + DataPartitionTable dataPartitionTable = null; + if (hasDataPartitionTable) { + dataPartitionTable = new DataPartitionTable(); + dataPartitionTable.deserialize(inputStream, protocol); + } + + return new DatabaseScopedDataPartitionTable(database, dataPartitionTable); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DatabaseScopedDataPartitionTable that = (DatabaseScopedDataPartitionTable) o; + return Objects.equals(database, that.database) + && Objects.equals(dataPartitionTable, that.dataPartitionTable); + } + + @Override + public int hashCode() { + return Objects.hash(database, dataPartitionTable); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java index f46344566dc3..da8952051e51 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java @@ -37,10 +37,12 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.Vector; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListMap; @@ -73,7 +75,13 @@ public Map> getSeriesPartitionMap() } public void putDataPartition(TTimePartitionSlot timePartitionSlot, TConsensusGroupId groupId) { - seriesPartitionMap.computeIfAbsent(timePartitionSlot, empty -> new Vector<>()).add(groupId); + List groupList = + seriesPartitionMap.computeIfAbsent(timePartitionSlot, empty -> new Vector<>()); + synchronized (groupList) { + if (!groupList.contains(groupId)) { + groupList.add(groupId); + } + } } /** @@ -270,6 +278,23 @@ public List autoCleanPartitionTable( return removedTimePartitions; } + public void merge(SeriesPartitionTable sourceMap) { + if (sourceMap == null) return; + sourceMap.seriesPartitionMap.forEach( + (timeSlot, groups) -> { + List groupList = + this.seriesPartitionMap.computeIfAbsent(timeSlot, k -> new ArrayList<>()); + synchronized (groupList) { + Set groupSet = new HashSet<>(groupList); + for (TConsensusGroupId groupId : groups) { + if (!groupSet.contains(groupId)) { + groupList.add(groupId); + } + } + } + }); + } + public void serialize(OutputStream outputStream, TProtocol protocol) throws IOException, TException { ReadWriteIOUtils.write(seriesPartitionMap.size(), outputStream); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/IOUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/IOUtils.java index 047dd6bfea5b..da6544717b3b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/IOUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/IOUtils.java @@ -25,12 +25,14 @@ import org.apache.iotdb.commons.path.PartialPath; import com.google.common.base.Supplier; +import com.google.common.util.concurrent.RateLimiter; import org.apache.tsfile.utils.Pair; import java.io.DataInputStream; import java.io.EOFException; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; @@ -327,4 +329,37 @@ public static Optional retryNoException( } return Optional.empty(); } + + public static class RatelimitedInputStream extends InputStream { + private RateLimiter rateLimiter; + private InputStream inner; + + public RatelimitedInputStream(InputStream inner, RateLimiter limiter) { + this.inner = inner; + this.rateLimiter = limiter; + } + + @Override + public int read() throws IOException { + rateLimiter.acquire(1); + return inner.read(); + } + + @Override + public int read(byte[] b) throws IOException { + rateLimiter.acquire(b.length); + return inner.read(b); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + rateLimiter.acquire(len); + return inner.read(b, off, len); + } + + @Override + public void close() throws IOException { + inner.close(); + } + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java index 7b331fddaac0..0dc6eed8af40 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java @@ -122,6 +122,10 @@ public static long getTimePartitionIdWithoutOverflow(long time) { return partitionId.longValue(); } + public static long getStartTimeByPartitionId(long partitionId) { + return (partitionId * timePartitionInterval) + timePartitionOrigin; + } + public static boolean satisfyPartitionId(long startTime, long endTime, long partitionId) { long startPartition = originMayCauseOverflow diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index b04274945daa..317b592c2b32 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -254,6 +254,13 @@ struct TDataPartitionTableResp { 2: optional map>>> dataPartitionTable } +struct TShowRepairDataPartitionTableProgressResp { + 1: required common.TSStatus status + 2: required string state + 3: required double progress + 4: optional string message +} + struct TGetRegionIdReq { 1: required common.TConsensusGroupType type 2: optional string database @@ -1246,6 +1253,10 @@ service IConfigNodeRPCService { */ TDataPartitionTableResp getOrCreateDataPartitionTable(TDataPartitionReq req) + common.TSStatus dataPartitionTableIntegrityCheck() + + TShowRepairDataPartitionTableProgressResp showRepairDataPartitionTableProgress() + // ====================================================== // Authorize // ====================================================== diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 0d1f60d61a87..4fde5d0a349c 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -566,6 +566,44 @@ struct TFetchTimeseriesResp { 6: optional list tsDataset 7: optional bool hasMoreData } +/** +* BEGIN: Data Partition Table Integrity Check Structures +**/ + +struct TGetEarliestTimeslotsResp { + 1: required common.TSStatus status + 2: optional map databaseToEarliestTimeslot +} + +struct TGenerateDataPartitionTableReq { + 1: required set databases +} + +struct TGenerateDataPartitionTableResp { + 1: required common.TSStatus status + 2: required i32 errorCode + 3: optional string message +} + +struct TGenerateDataPartitionTableHeartbeatResp { + 1: required common.TSStatus status + 2: required i32 errorCode + 3: optional string message + 4: optional list databaseScopedDataPartitionTables + 5: optional double progress +} + +struct TGetDataPartitionTableGeneratorProgressResp { + 1: required common.TSStatus status + 2: required i32 errorCode + 3: required double progress + 4: optional string message +} + +/** +* END: Data Partition Table Integrity Check Structures +**/ + /** * BEGIN: Used for EXPLAIN ANALYZE **/ @@ -1062,6 +1100,34 @@ service IDataNodeRPCService { /** Empty rpc, only for connection test */ common.TSStatus testConnectionEmptyRPC() + /** + * BEGIN: Data Partition Table Integrity Check + **/ + + /** + * Get earliest timeslot information from DataNode + * Returns map of database name to earliest timeslot id + */ + TGetEarliestTimeslotsResp getEarliestTimeslots() + + /** + * Request DataNode to generate DataPartitionTable by scanning tsfile resources + */ + TGenerateDataPartitionTableResp generateDataPartitionTable(TGenerateDataPartitionTableReq req) + + /** + * Check the status of DataPartitionTable generation task + */ + TGenerateDataPartitionTableHeartbeatResp generateDataPartitionTableHeartbeat(TGenerateDataPartitionTableReq req) + + /** + * Get the progress of DataPartitionTable generation task without consuming the generated table. + */ + TGetDataPartitionTableGeneratorProgressResp getDataPartitionTableGeneratorProgress() + + /** + * END: Data Partition Table Integrity Check + **/ } service MPPDataExchangeService {