diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java index 1f70341892b3..cca439265662 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java @@ -29,12 +29,16 @@ import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.record.Tablet.ColumnCategory; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.v4.ITsFileWriter; +import org.apache.tsfile.write.v4.TsFileWriterBuilder; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -43,6 +47,8 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.io.File; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.util.ArrayList; @@ -1450,6 +1456,124 @@ public void insertRelationalRowWithAutoCastTest() } } + @Test + public void insertMinMaxTimeTest() throws IoTDBConnectionException, StatementExecutionException { + try { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION timestamp_precision_check_enabled='false'"); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + session.executeNonQueryStatement("USE db1"); + session.executeNonQueryStatement("CREATE TABLE test_insert_min_max (id1 TAG, s1 INT32)"); + + session.executeNonQueryStatement( + String.format( + "INSERT INTO test_insert_min_max(time, id1, s1) VALUES (%d, 'd1', 1)", + Long.MIN_VALUE)); + session.executeNonQueryStatement( + String.format( + "INSERT INTO test_insert_min_max(time, id1, s1) VALUES (%d, 'd1', 1)", + Long.MAX_VALUE)); + + SessionDataSet dataSet = session.executeQueryStatement("SELECT * FROM test_insert_min_max"); + RowRecord record = dataSet.next(); + assertEquals(Long.MIN_VALUE, record.getFields().get(0).getLongV()); + record = dataSet.next(); + assertEquals(Long.MAX_VALUE, record.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + + session.executeNonQueryStatement("FLUSH"); + dataSet = session.executeQueryStatement("SELECT * FROM test_insert_min_max"); + record = dataSet.next(); + assertEquals(Long.MIN_VALUE, record.getFields().get(0).getLongV()); + record = dataSet.next(); + assertEquals(Long.MAX_VALUE, record.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + } finally { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION timestamp_precision_check_enabled='true'"); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + } + } + } + + @Test + public void loadMinMaxTimeAlignedTest() + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + File file = new File("target", "test.tsfile"); + TableSchema tableSchema = + new TableSchema( + "load_min_max", + Arrays.asList("id1", "s1"), + Arrays.asList(TSDataType.STRING, TSDataType.INT32), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); + + try (ITsFileWriter writer = + new TsFileWriterBuilder().file(file).tableSchema(tableSchema).build()) { + Tablet tablet = + new Tablet( + Arrays.asList("id1", "s1"), Arrays.asList(TSDataType.STRING, TSDataType.INT32)); + tablet.addTimestamp(0, Long.MIN_VALUE); + tablet.addTimestamp(1, Long.MAX_VALUE); + tablet.addValue(0, 0, "d1"); + tablet.addValue(1, 0, "d1"); + tablet.addValue(0, 1, 1); + tablet.addValue(1, 1, 1); + writer.write(tablet); + } + + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE db1"); + try { + session.executeNonQueryStatement( + "SET CONFIGURATION timestamp_precision_check_enabled='false'"); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + session.executeNonQueryStatement("LOAD \'" + file.getAbsolutePath() + "\'"); + + SessionDataSet dataSet = session.executeQueryStatement("SELECT * FROM load_min_max"); + RowRecord record = dataSet.next(); + assertEquals(Long.MIN_VALUE, record.getFields().get(0).getLongV()); + record = dataSet.next(); + assertEquals(Long.MAX_VALUE, record.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } finally { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION timestamp_precision_check_enabled='false'"); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + } + file.delete(); + } + } + @Test public void autoCreateTagColumnTest2() throws IoTDBConnectionException, StatementExecutionException { diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java index 7cfbed9fabe0..8769a26de55e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java @@ -36,12 +36,17 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -55,6 +60,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.IOException; import java.security.SecureRandom; import java.util.ArrayList; import java.util.Arrays; @@ -72,6 +79,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +@SuppressWarnings({"ThrowFromFinallyBlock", "ResultOfMethodCallIgnored"}) @RunWith(IoTDBTestRunner.class) public class IoTDBSessionSimpleIT { @@ -336,7 +344,7 @@ public void insertByObjAndNotInferTypeTest() { expected.add(TSDataType.TEXT.name()); Set actual = new HashSet<>(); - SessionDataSet dataSet = session.executeQueryStatement("show timeseries root.**"); + SessionDataSet dataSet = session.executeQueryStatement("show timeseries root.sg1.**"); while (dataSet.hasNext()) { actual.add(dataSet.next().getFields().get(3).getStringValue()); } @@ -748,7 +756,7 @@ public void createWrongTimeSeriesTest() { LOGGER.error("", e); } - final SessionDataSet dataSet = session.executeQueryStatement("SHOW TIMESERIES"); + final SessionDataSet dataSet = session.executeQueryStatement("SHOW TIMESERIES root.sg.**"); assertFalse(dataSet.hasNext()); session.deleteStorageGroup(storageGroup); @@ -1859,4 +1867,166 @@ public void convertRecordsToTabletsTest() { e.printStackTrace(); } } + + @Test + @Category({LocalStandaloneIT.class, ClusterIT.class}) + public void insertMinMaxTimeTest() throws IoTDBConnectionException, StatementExecutionException { + try { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION \"timestamp_precision_check_enabled\"=\"false\""); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + + session.executeNonQueryStatement( + String.format( + "INSERT INTO root.testInsertMinMax.d1(timestamp, s1) VALUES (%d, 1)", + Long.MIN_VALUE)); + session.executeNonQueryStatement( + String.format( + "INSERT INTO root.testInsertMinMax.d1(timestamp, s1) VALUES (%d, 1)", + Long.MAX_VALUE)); + + SessionDataSet dataSet = + session.executeQueryStatement("SELECT * FROM root.testInsertMinMax.d1"); + RowRecord record = dataSet.next(); + assertEquals(Long.MIN_VALUE, record.getTimestamp()); + record = dataSet.next(); + assertEquals(Long.MAX_VALUE, record.getTimestamp()); + assertFalse(dataSet.hasNext()); + + session.executeNonQueryStatement("FLUSH"); + dataSet = session.executeQueryStatement("SELECT * FROM root.testInsertMinMax.d1"); + record = dataSet.next(); + assertEquals(Long.MIN_VALUE, record.getTimestamp()); + record = dataSet.next(); + assertEquals(Long.MAX_VALUE, record.getTimestamp()); + assertFalse(dataSet.hasNext()); + } + } finally { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION \"timestamp_precision_check_enabled\"=\"true\""); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + } + } + } + + @Test + @Category({LocalStandaloneIT.class, ClusterIT.class}) + public void loadMinMaxTimeNonAlignedTest() + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + File file = new File("target", "test.tsfile"); + try (TsFileWriter writer = new TsFileWriter(file)) { + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.testLoadMinMax.d1"); + writer.registerTimeseries(deviceID, new MeasurementSchema("s1", TSDataType.INT32)); + TSRecord record = new TSRecord(deviceID, Long.MIN_VALUE); + record.addPoint("s1", 1); + writer.writeRecord(record); + record.setTime(Long.MAX_VALUE); + writer.writeRecord(record); + } + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION \"timestamp_precision_check_enabled\"=\"false\""); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + session.executeNonQueryStatement("LOAD \"" + file.getAbsolutePath() + "\""); + + SessionDataSet dataSet = + session.executeQueryStatement("SELECT * FROM root.testLoadMinMax.d1"); + RowRecord record = dataSet.next(); + assertEquals(Long.MIN_VALUE, record.getTimestamp()); + record = dataSet.next(); + assertEquals(Long.MAX_VALUE, record.getTimestamp()); + assertFalse(dataSet.hasNext()); + } finally { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION \"timestamp_precision_check_enabled\"=\"true\""); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + } + file.delete(); + } + } + + @Test + @Category({LocalStandaloneIT.class, ClusterIT.class}) + public void loadMinMaxTimeAlignedTest() + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + File file = new File("target", "test.tsfile"); + try (TsFileWriter writer = new TsFileWriter(file)) { + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.testLoadMinMaxAligned.d1"); + writer.registerAlignedTimeseries( + deviceID, Collections.singletonList(new MeasurementSchema("s1", TSDataType.INT32))); + TSRecord record = new TSRecord(deviceID, Long.MIN_VALUE); + record.addPoint("s1", 1); + writer.writeRecord(record); + record.setTime(Long.MAX_VALUE); + writer.writeRecord(record); + } + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION \"timestamp_precision_check_enabled\"=\"false\""); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + session.executeNonQueryStatement("LOAD \"" + file.getAbsolutePath() + "\""); + + SessionDataSet dataSet = + session.executeQueryStatement("SELECT * FROM root.testLoadMinMaxAligned.d1"); + RowRecord record = dataSet.next(); + assertEquals(Long.MIN_VALUE, record.getTimestamp()); + record = dataSet.next(); + assertEquals(Long.MAX_VALUE, record.getTimestamp()); + assertFalse(dataSet.hasNext()); + } finally { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + try { + session.executeNonQueryStatement( + "SET CONFIGURATION \"timestamp_precision_check_enabled\"=\"true\""); + } catch (StatementExecutionException e) { + // run in IDE will trigger this, ignore it + if (!e.getMessage().contains("Unable to find the configuration file")) { + throw e; + } + } + } + file.delete(); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index f1e160a16e1b..8d396f65207a 100755 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -2053,6 +2053,15 @@ public synchronized void loadHotModifiedProps(TrimProperties properties) BinaryAllocator.getInstance().close(true); } + commonDescriptor + .getConfig() + .setTimestampPrecisionCheckEnabled( + Boolean.parseBoolean( + properties.getProperty( + "timestamp_precision_check_enabled", + ConfigurationFileUtils.getConfigurationDefaultValue( + "timestamp_precision_check_enabled")))); + conf.setEnablePartialInsert( Boolean.parseBoolean( Optional.ofNullable( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 156ceb5a73b3..e1ebf4a22d46 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -2293,7 +2293,11 @@ public static Pair, Pair> getTimePart result.add(timePartitionSlot); // next init timePartitionSlot = new TTimePartitionSlot(endTime); - endTime = endTime + TimePartitionUtils.getTimePartitionInterval(); + // beware of overflow + endTime = + endTime + TimePartitionUtils.getTimePartitionInterval() > endTime + ? endTime + TimePartitionUtils.getTimePartitionInterval() + : Long.MAX_VALUE; } else { index++; if (index < size) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java index 776bc63d823c..0d7c6f96f03e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java @@ -116,7 +116,7 @@ public void setDatabase(String database) { public void autoCreateAndVerify(IDeviceID device) { try { - if (isDeviceDeletedByMods(device)) { + if (ModificationUtils.isDeviceDeletedByMods(currentModifications, currentTimeIndex, device)) { return; } } catch (IllegalPathException e) { @@ -133,15 +133,6 @@ public void autoCreateAndVerify(IDeviceID device) { } } - private boolean isDeviceDeletedByMods(IDeviceID device) throws IllegalPathException { - return currentTimeIndex != null - && ModificationUtils.isAllDeletedByMods( - currentModifications, - device, - currentTimeIndex.getStartTime(device), - currentTimeIndex.getEndTime(device)); - } - private void addDevice(final IDeviceID device) { final String tableName = device.getTableName(); long memoryUsageSizeInBytes = 0; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java index ff9ea28224c4..f0400a92643c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java @@ -188,12 +188,7 @@ public void setCurrentModificationsAndTimeIndex( } public boolean isDeviceDeletedByMods(IDeviceID device) throws IllegalPathException { - return currentTimeIndex != null - && ModificationUtils.isAllDeletedByMods( - currentModifications, - device, - currentTimeIndex.getStartTime(device), - currentTimeIndex.getEndTime(device)); + return ModificationUtils.isDeviceDeletedByMods(currentModifications, currentTimeIndex, device); } public boolean isTimeseriesDeletedByMods(IDeviceID device, TimeseriesMetadata timeseriesMetadata) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index e21aaa3096b9..b9d3d9c6c9a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -2002,15 +2002,18 @@ private long parseTimeValue(ConstantContext constant) { if (constant.INTEGER_LITERAL() != null) { try { if (constant.MINUS() != null) { - return -Long.parseLong(constant.INTEGER_LITERAL().getText()); + return Long.parseLong("-" + constant.INTEGER_LITERAL().getText()); } return Long.parseLong(constant.INTEGER_LITERAL().getText()); } catch (NumberFormatException e) { throw new SemanticException( String.format( - "Current system timestamp precision is %s, " + "Failed to parse the timestamp: " + + e.getMessage() + + "Current system timestamp precision is %s, " + "please check whether the timestamp %s is correct.", - TIMESTAMP_PRECISION, constant.INTEGER_LITERAL().getText())); + TIMESTAMP_PRECISION, + constant.INTEGER_LITERAL().getText())); } } else if (constant.dateExpression() != null) { return parseDateExpression(constant.dateExpression(), CommonDateTimeUtils.currentTime()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 9a6314fbf293..c8691868c826 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -85,18 +85,22 @@ public boolean isTsFileEmpty() { return resource.getDevices().isEmpty(); } + @SuppressWarnings("OptionalGetWithoutIsPresent") public boolean needDecodeTsFile( - Function>, List> partitionFetcher) - throws IOException { + Function>, List> + partitionFetcher) { List> slotList = new ArrayList<>(); resource .getDevices() .forEach( o -> { + // iterating the index, must present slotList.add( - new Pair<>(o, TimePartitionUtils.getTimePartitionSlot(resource.getStartTime(o)))); + new Pair<>( + o, TimePartitionUtils.getTimePartitionSlot(resource.getStartTime(o).get()))); slotList.add( - new Pair<>(o, TimePartitionUtils.getTimePartitionSlot(resource.getEndTime(o)))); + new Pair<>( + o, TimePartitionUtils.getTimePartitionSlot(resource.getEndTime(o).get()))); }); if (slotList.isEmpty()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 84ff12e3dc7b..292f6026d63a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -666,7 +666,8 @@ protected void updateDeviceLastFlushTime(TsFileResource resource) { long timePartitionId = resource.getTimePartition(); Map endTimeMap = new HashMap<>(); for (IDeviceID deviceId : resource.getDevices()) { - long endTime = resource.getEndTime(deviceId); + @SuppressWarnings("OptionalGetWithoutIsPresent") // checked above + long endTime = resource.getEndTime(deviceId).get(); endTimeMap.put(deviceId, endTime); } if (config.isEnableSeparateData()) { @@ -682,7 +683,9 @@ protected void upgradeAndUpdateDeviceLastFlushTime( Map endTimeMap = new HashMap<>(); for (TsFileResource resource : resources) { for (IDeviceID deviceId : resource.getDevices()) { - long endTime = resource.getEndTime(deviceId); + // checked above + //noinspection OptionalGetWithoutIsPresent + long endTime = resource.getEndTime(deviceId).get(); endTimeMap.put(deviceId, endTime); } } @@ -2455,6 +2458,7 @@ public void insertSeparatorToWAL() { } } + @SuppressWarnings("OptionalGetWithoutIsPresent") private boolean canSkipDelete(TsFileResource tsFileResource, ModEntry deletion) { long fileStartTime = tsFileResource.getTimeIndex().getMinStartTime(); long fileEndTime = @@ -2478,10 +2482,11 @@ private boolean canSkipDelete(TsFileResource tsFileResource, ModEntry deletion) } for (IDeviceID device : tsFileResource.getDevices()) { - long startTime = tsFileResource.getTimeIndex().getStartTime(device); + // we are iterating the time index so the times are definitely present + long startTime = tsFileResource.getTimeIndex().getStartTime(device).get(); long endTime = tsFileResource.isClosed() - ? tsFileResource.getTimeIndex().getEndTime(device) + ? tsFileResource.getTimeIndex().getEndTime(device).get() : Long.MAX_VALUE; if (deletion.affects(device, startTime, endTime)) { return false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java index 303fc049f9d2..a1c441e171c7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java @@ -139,7 +139,9 @@ public void perform() throws Exception { long ttl = deviceIterator.getTTLForCurrentDevice(); sortedSourceFiles.removeIf( x -> x.definitelyNotContains(device) || !x.isDeviceAlive(device, ttl)); - sortedSourceFiles.sort(Comparator.comparingLong(x -> x.getStartTime(device))); + // checked above + //noinspection OptionalGetWithoutIsPresent + sortedSourceFiles.sort(Comparator.comparingLong(x -> x.getStartTime(device).get())); if (ttl != Long.MAX_VALUE) { ModEntry ttlDeletion = CompactionUtils.convertTtlToDeletion( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index 32a1593c4a65..b4deff34b2aa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@ -54,6 +54,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -428,7 +429,8 @@ private void applyModificationForAlignedChunkMetadataList( } IDeviceID device = currentDevice.getLeft(); ModEntry ttlDeletion = null; - if (tsFileResource.getStartTime(device) < timeLowerBoundForCurrentDevice) { + Optional startTime = tsFileResource.getStartTime(device); + if (startTime.isPresent() && startTime.get() < timeLowerBoundForCurrentDevice) { ttlDeletion = CompactionUtils.convertTtlToDeletion(device, timeLowerBoundForCurrentDevice); } @@ -665,7 +667,8 @@ public String nextSeries() throws IllegalPathException { Map> chunkMetadataListMap = chunkMetadataCacheMap.get(reader); ModEntry ttlDeletion = null; - if (resource.getStartTime(device) < timeLowerBoundForCurrentDevice) { + Optional startTime = resource.getStartTime(device); + if (startTime.isPresent() && startTime.get() < timeLowerBoundForCurrentDevice) { ttlDeletion = new TreeDeletionEntry( new MeasurementPath(device, IoTDBConstant.ONE_LEVEL_PATH_WILDCARD), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java index 83dab2e76907..30395b67c555 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java @@ -49,6 +49,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.PriorityQueue; import java.util.stream.Collectors; @@ -350,14 +351,19 @@ private void checkAndCompactOverlapPage(PageElement nextPageElement, TimeValuePa * list is ordered according to the startTime of the current device in the file from small to * large, so that each file can be compacted in order. */ - protected List findOverlapFiles(FileElement file) { + protected List findOverlapFiles(FileElement fileToCheck) { List overlappedFiles = new ArrayList<>(); - long endTime = file.resource.getEndTime(deviceId); - for (FileElement fileElement : fileList) { - if (fileElement.resource.getStartTime(deviceId) <= endTime) { - if (!fileElement.isSelected) { - overlappedFiles.add(fileElement); - fileElement.isSelected = true; + Optional endTimeInCheckingFile = fileToCheck.resource.getEndTime(deviceId); + for (FileElement otherFile : fileList) { + if (!endTimeInCheckingFile.isPresent()) { + continue; + } + Optional startTimeInOtherFile = otherFile.resource.getStartTime(deviceId); + if (startTimeInOtherFile.isPresent() + && startTimeInOtherFile.get() <= endTimeInCheckingFile.get()) { + if (!otherFile.isSelected) { + overlappedFiles.add(otherFile); + otherFile.isSelected = true; } } else { break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/AlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/AlignedSeriesCompactionExecutor.java index 02628027ea02..1abef8e85d65 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/AlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/AlignedSeriesCompactionExecutor.java @@ -61,6 +61,7 @@ public class AlignedSeriesCompactionExecutor { private final List schemaList; private long remainingPointInChunkWriter = 0L; private final CompactionTaskSummary summary; + private boolean lastWriteTimestampSet = false; private long lastWriteTimestamp = Long.MIN_VALUE; private final long chunkSizeThreshold = @@ -210,11 +211,12 @@ private void flushChunkWriterIfLargeEnough() throws IOException { } private void checkAndUpdatePreviousTimestamp(long currentWritingTimestamp) { - if (currentWritingTimestamp <= lastWriteTimestamp) { + if (lastWriteTimestampSet && currentWritingTimestamp <= lastWriteTimestamp) { throw new CompactionLastTimeCheckFailedException( device.toString(), currentWritingTimestamp, lastWriteTimestamp); } else { lastWriteTimestamp = currentWritingTimestamp; + lastWriteTimestampSet = true; } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java index 12c205c472ed..a22f9627ed01 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java @@ -80,7 +80,8 @@ public class ReadChunkAlignedSeriesCompactionExecutor { protected final CompactionTaskSummary summary; protected final boolean ignoreAllNullRows; - private long lastWriteTimestamp = Long.MIN_VALUE; + private long lastWriteTimestamp; + private boolean lastWriteTimestampSet = false; public ReadChunkAlignedSeriesCompactionExecutor( IDeviceID device, @@ -416,11 +417,12 @@ protected IPointReader getPointReader( } protected void checkAndUpdatePreviousTimestamp(long currentWritingTimestamp) { - if (currentWritingTimestamp <= lastWriteTimestamp) { + if (lastWriteTimestampSet && currentWritingTimestamp <= lastWriteTimestamp) { throw new CompactionLastTimeCheckFailedException( device.toString(), currentWritingTimestamp, lastWriteTimestamp); } else { lastWriteTimestamp = currentWritingTimestamp; + lastWriteTimestampSet = true; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java index a8bdcf985b99..69ce89d5af37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java @@ -57,9 +57,12 @@ public class SingleSeriesCompactionExecutor { private Chunk cachedChunk; private ChunkMetadata cachedChunkMetadata; // record the min time and max time to update the target resource - private long minStartTimestamp = Long.MAX_VALUE; - private long maxEndTimestamp = Long.MIN_VALUE; - private long lastWriteTimestamp = Long.MIN_VALUE; + private boolean minStartTimestampSet = false; + private long minStartTimestamp; + private boolean maxEndTimestampSet = false; + private long maxEndTimestamp; + private boolean lastWriteTimestampSet = false; + private long lastWriteTimestamp; private long pointCountInChunkWriter = 0; private final CompactionTaskSummary summary; @@ -158,8 +161,10 @@ public void execute() throws IOException { flushChunkWriter(); } fileWriter.checkMetadataSizeAndMayFlush(); - targetResource.updateStartTime(device, minStartTimestamp); - targetResource.updateEndTime(device, maxEndTimestamp); + if (minStartTimestampSet) { + targetResource.updateStartTime(device, minStartTimestamp); + targetResource.updateEndTime(device, maxEndTimestamp); + } } private void constructChunkWriterFromReadChunk(Chunk chunk) { @@ -269,11 +274,13 @@ private void writeChunkIntoChunkWriter(Chunk chunk) throws IOException { TimeValuePair timeValuePair = batchIterator.nextTimeValuePair(); checkAndUpdatePreviousTimestamp(timeValuePair.getTimestamp()); writeTimeAndValueToChunkWriter(timeValuePair); - if (timeValuePair.getTimestamp() > maxEndTimestamp) { + if (!maxEndTimestampSet || timeValuePair.getTimestamp() > maxEndTimestamp) { maxEndTimestamp = timeValuePair.getTimestamp(); + maxEndTimestampSet = true; } - if (timeValuePair.getTimestamp() < minStartTimestamp) { + if (!minStartTimestampSet || timeValuePair.getTimestamp() < minStartTimestamp) { minStartTimestamp = timeValuePair.getTimestamp(); + minStartTimestampSet = true; } } } @@ -336,11 +343,13 @@ private void writeTimeAndValueToChunkWriter(TimeValuePair timeValuePair) { private void flushChunkToFileWriter(Chunk chunk, ChunkMetadata chunkMetadata) throws IOException { checkAndUpdatePreviousTimestamp(chunkMetadata.getStartTime()); - if (chunkMetadata.getStartTime() < minStartTimestamp) { + if (!minStartTimestampSet || chunkMetadata.getStartTime() < minStartTimestamp) { minStartTimestamp = chunkMetadata.getStartTime(); + minStartTimestampSet = true; } - if (chunkMetadata.getEndTime() > maxEndTimestamp) { + if (!maxEndTimestampSet || chunkMetadata.getEndTime() > maxEndTimestamp) { maxEndTimestamp = chunkMetadata.getEndTime(); + maxEndTimestampSet = true; } fileWriter.writeChunk(chunk, chunkMetadata); } @@ -368,11 +377,12 @@ private void flushChunkWriter() throws IOException { } private void checkAndUpdatePreviousTimestamp(long currentWritingTimestamp) { - if (currentWritingTimestamp <= lastWriteTimestamp) { + if (lastWriteTimestampSet && currentWritingTimestamp <= lastWriteTimestamp) { throw new CompactionLastTimeCheckFailedException( device, measurement, currentWritingTimestamp, lastWriteTimestamp); } else { lastWriteTimestamp = currentWritingTimestamp; + lastWriteTimestampSet = true; } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java index 42e89fd8bbb9..364e21e22737 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java @@ -52,6 +52,7 @@ public abstract class AbstractCompactionWriter implements AutoCloseable { // check if there is unseq error point during writing protected long[] lastTime = new long[subTaskNum]; + protected boolean[] lastTimeSet = new boolean[subTaskNum]; // Each sub task has its own chunk writer. // The index of the array corresponds to subTaskId. @@ -104,7 +105,7 @@ public abstract class AbstractCompactionWriter implements AutoCloseable { public void startMeasurement(String measurement, IChunkWriter chunkWriter, int subTaskId) { lastCheckIndex = 0; - lastTime[subTaskId] = Long.MIN_VALUE; + lastTimeSet[subTaskId] = false; chunkWriters[subTaskId] = chunkWriter; measurementId[subTaskId] = measurement; } @@ -313,7 +314,7 @@ protected long getChunkSize(Chunk chunk) { } protected void checkPreviousTimestamp(long currentWritingTimestamp, int subTaskId) { - if (currentWritingTimestamp <= lastTime[subTaskId]) { + if (lastTimeSet[subTaskId] && currentWritingTimestamp <= lastTime[subTaskId]) { throw new CompactionLastTimeCheckFailedException( deviceId.toString() + IoTDBConstant.PATH_SEPARATOR + measurementId[subTaskId], currentWritingTimestamp, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java index d0e3698bd736..9d074b7fd46f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java @@ -137,6 +137,7 @@ public void write(TimeValuePair timeValuePair, int subTaskId) throws IOException isDeviceExistedInTargetFiles[fileIndex] = true; isEmptyFile[fileIndex] = false; lastTime[subTaskId] = timestamp; + lastTimeSet[subTaskId] = true; } /** Write data in batch, only used for aligned device. */ @@ -207,7 +208,11 @@ private void checkIsDeviceExistAndGetDeviceEndTime() throws IOException { while (fileIndex < seqTsFileResources.size()) { if (seqTsFileResources.get(fileIndex).getTimeIndexType() == 1) { // the timeIndexType of resource is deviceTimeIndex - currentDeviceEndTime[fileIndex] = seqTsFileResources.get(fileIndex).getEndTime(deviceId); + int finalFileIndex = fileIndex; + seqTsFileResources + .get(fileIndex) + .getEndTime(deviceId) + .ifPresent(endTime -> currentDeviceEndTime[finalFileIndex] = endTime); } else { long endTime = Long.MIN_VALUE; // Fast compaction get reader from cache map, while read point compaction get reader from diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java index a06d3f6a63c8..9d5e6f498254 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java @@ -121,6 +121,7 @@ public void write(TimeValuePair timeValuePair, int subTaskId) throws IOException chunkPointNumArray[subTaskId]++; checkChunkSizeAndMayOpenANewChunk(fileWriter, chunkWriters[subTaskId], subTaskId); lastTime[subTaskId] = timeValuePair.getTimestamp(); + lastTimeSet[subTaskId] = true; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java index ae5bc2e75911..611eceed2f29 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java @@ -85,6 +85,7 @@ public boolean flushNonAlignedChunk(Chunk chunk, ChunkMetadata chunkMetadata, in isDeviceExistedInTargetFiles[fileIndex] = true; isEmptyFile[fileIndex] = false; lastTime[subTaskId] = chunkMetadata.getEndTime(); + lastTimeSet[subTaskId] = true; return true; } @@ -123,6 +124,7 @@ public boolean flushAlignedChunk(ChunkMetadataElement chunkMetadataElement, int isDeviceExistedInTargetFiles[fileIndex] = true; isEmptyFile[fileIndex] = false; lastTime[subTaskId] = timeChunkMetadata.getEndTime(); + lastTimeSet[subTaskId] = true; return true; } @@ -159,6 +161,7 @@ public boolean flushBatchedValueChunk( isDeviceExistedInTargetFiles[fileIndex] = true; isEmptyFile[fileIndex] = false; lastTime[subTaskId] = timeChunkMetadata.getEndTime(); + lastTimeSet[subTaskId] = true; return true; } @@ -200,6 +203,7 @@ public boolean flushAlignedPage(AlignedPageElement alignedPageElement, int subTa isDeviceExistedInTargetFiles[fileIndex] = true; isEmptyFile[fileIndex] = false; lastTime[subTaskId] = timePageHeader.getEndTime(); + lastTimeSet[subTaskId] = true; return true; } @@ -239,6 +243,7 @@ public boolean flushBatchedValuePage( isDeviceExistedInTargetFiles[fileIndex] = true; isEmptyFile[fileIndex] = false; lastTime[subTaskId] = timePageHeader.getEndTime(); + lastTimeSet[subTaskId] = true; return true; } @@ -270,6 +275,7 @@ public boolean flushNonAlignedPage( isDeviceExistedInTargetFiles[fileIndex] = true; isEmptyFile[fileIndex] = false; lastTime[subTaskId] = pageHeader.getEndTime(); + lastTimeSet[subTaskId] = true; return true; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java index e9709af3dbf6..6519101781cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java @@ -75,6 +75,7 @@ public boolean flushNonAlignedChunk(Chunk chunk, ChunkMetadata chunkMetadata, in flushNonAlignedChunkToFileWriter(fileWriter, chunk, chunkMetadata, subTaskId); lastTime[subTaskId] = chunkMetadata.getEndTime(); + lastTimeSet[subTaskId] = true; return true; } @@ -112,6 +113,7 @@ public boolean flushAlignedChunk(ChunkMetadataElement chunkMetadataElement, int fileWriter, timeChunk, timeChunkMetadata, valueChunks, valueChunkMetadatas, subTaskId); lastTime[subTaskId] = timeChunkMetadata.getEndTime(); + lastTimeSet[subTaskId] = true; return true; } @@ -142,6 +144,7 @@ public boolean flushBatchedValueChunk( fileWriter, null, timeChunkMetadata, valueChunks, valueChunkMetadatas, subTaskId); lastTime[subTaskId] = timeChunkMetadata.getEndTime(); + lastTimeSet[subTaskId] = true; return true; } @@ -180,6 +183,7 @@ public boolean flushAlignedPage(AlignedPageElement alignedPageElement, int subTa subTaskId); lastTime[subTaskId] = timePageHeader.getEndTime(); + lastTimeSet[subTaskId] = true; return true; } @@ -215,6 +219,7 @@ public boolean flushBatchedValuePage( subTaskId); lastTime[subTaskId] = timePageHeader.getEndTime(); + lastTimeSet[subTaskId] = true; return true; } @@ -240,6 +245,7 @@ public boolean flushNonAlignedPage( (ChunkWriterImpl) chunkWriters[subTaskId], compressedPageData, pageHeader, subTaskId); lastTime[subTaskId] = pageHeader.getEndTime(); + lastTimeSet[subTaskId] = true; return true; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointCrossCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointCrossCompactionWriter.java index 84ae142f2815..b92e3e66ff26 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointCrossCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointCrossCompactionWriter.java @@ -68,6 +68,7 @@ public void write(TsBlock tsBlock, int subTaskId) throws IOException { isDeviceExistedInTargetFiles[seqFileIndexArray[subTaskId]] = true; isEmptyFile[seqFileIndexArray[subTaskId]] = false; lastTime[subTaskId] = timestamps.getEndTime(); + lastTimeSet[subTaskId] = true; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java index 2d268761ecab..b19b54c4f740 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java @@ -67,12 +67,14 @@ import static org.apache.tsfile.read.reader.chunk.ChunkReader.decryptAndUncompressPageData; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class RepairDataFileScanUtil { private static final Logger logger = LoggerFactory.getLogger(RepairDataFileScanUtil.class); private final TsFileResource resource; private ArrayDeviceTimeIndex timeIndex; private boolean hasUnsortedDataOrWrongStatistics; private boolean isBrokenFile; + private boolean previousTimeSet; private long previousTime; private boolean printLog; @@ -83,7 +85,7 @@ public RepairDataFileScanUtil(TsFileResource resource) { public RepairDataFileScanUtil(TsFileResource resource, boolean printLog) { this.resource = resource; this.hasUnsortedDataOrWrongStatistics = false; - this.previousTime = Long.MIN_VALUE; + this.previousTimeSet = false; this.printLog = printLog; } @@ -109,32 +111,44 @@ public void scanTsFile(boolean checkTsFileResource) { resource.isSeq() ? CompactionType.INNER_SEQ_COMPACTION : CompactionType.INNER_UNSEQ_COMPACTION)) { - TsFileDeviceIterator deviceIterator = reader.getAllDevicesIteratorWithIsAligned(); + TsFileDeviceIterator deviceInFileIterator = reader.getAllDevicesIteratorWithIsAligned(); Set deviceIdsInTimeIndex = checkTsFileResource ? new HashSet<>(timeIndex.getDevices()) : Collections.emptySet(); - while (deviceIterator.hasNext()) { - Pair deviceIsAlignedPair = deviceIterator.next(); - IDeviceID device = deviceIsAlignedPair.getLeft(); + while (deviceInFileIterator.hasNext()) { + Pair deviceIsAlignedPair = deviceInFileIterator.next(); + IDeviceID deviceInfile = deviceIsAlignedPair.getLeft(); if (checkTsFileResource) { - if (!deviceIdsInTimeIndex.contains(device)) { + if (!deviceIdsInTimeIndex.contains(deviceInfile)) { throw new CompactionStatisticsCheckFailedException( - device + " does not exist in the resource file"); + deviceInfile + " does not exist in the resource file"); } - deviceIdsInTimeIndex.remove(device); + deviceIdsInTimeIndex.remove(deviceInfile); } MetadataIndexNode metadataIndexNode = - deviceIterator.getFirstMeasurementNodeOfCurrentDevice(); + deviceInFileIterator.getFirstMeasurementNodeOfCurrentDevice(); + + // presence checked above TimeRange deviceTimeRangeInResource = checkTsFileResource - ? new TimeRange(timeIndex.getStartTime(device), timeIndex.getEndTime(device)) + ? new TimeRange( + timeIndex.getStartTime(deviceInfile).get(), + timeIndex.getEndTime(deviceInfile).get()) : null; boolean isAligned = deviceIsAlignedPair.getRight(); if (isAligned) { checkAlignedDeviceSeries( - reader, device, metadataIndexNode, deviceTimeRangeInResource, checkTsFileResource); + reader, + deviceInfile, + metadataIndexNode, + deviceTimeRangeInResource, + checkTsFileResource); } else { checkNonAlignedDeviceSeries( - reader, device, metadataIndexNode, deviceTimeRangeInResource, checkTsFileResource); + reader, + deviceInfile, + metadataIndexNode, + deviceTimeRangeInResource, + checkTsFileResource); } } if (!deviceIdsInTimeIndex.isEmpty()) { @@ -204,7 +218,7 @@ private void checkAlignedDeviceSeries( } // reset previousTime - previousTime = Long.MIN_VALUE; + previousTimeSet = false; // check timeseries time range if (actualTimeseriesStartTime > actualTimeseriesEndTime) { @@ -277,7 +291,7 @@ private void checkNonAlignedDeviceSeries( actualDeviceEndTime = Math.max(actualDeviceEndTime, timeseriesMetadata.getStatistics().getEndTime()); checkSingleNonAlignedSeries(reader, device, timeseriesMetadata); - previousTime = Long.MIN_VALUE; + previousTimeSet = false; } if (!checkTsFileResource || actualDeviceStartTime > actualDeviceEndTime) { @@ -391,18 +405,20 @@ private void validateTimeData( } private void checkPreviousTimeAndUpdate(IDeviceID deviceID, String measurementId, long time) { - if (previousTime >= time) { + if (previousTimeSet && previousTime >= time) { throw new CompactionLastTimeCheckFailedException( deviceID.toString() + TsFileConstant.PATH_SEPARATOR + measurementId, time, previousTime); } previousTime = time; + previousTimeSet = true; } private void checkPreviousTimeAndUpdate(IDeviceID deviceID, long time) { - if (previousTime >= time) { + if (previousTimeSet && previousTime >= time) { throw new CompactionLastTimeCheckFailedException(deviceID.toString(), time, previousTime); } previousTime = time; + previousTimeSet = true; } public boolean hasUnsortedDataOrWrongStatistics() { @@ -413,6 +429,7 @@ public boolean isBrokenFile() { return isBrokenFile; } + @SuppressWarnings("OptionalGetWithoutIsPresent") public static List checkTimePartitionHasOverlap( List resources, boolean printOverlappedDevices) { List overlapResources = new ArrayList<>(); @@ -433,8 +450,9 @@ public static List checkTimePartitionHasOverlap( boolean fileHasOverlap = false; // check overlap for (IDeviceID device : devices) { - long deviceStartTimeInCurrentFile = deviceTimeIndex.getStartTime(device); - if (deviceStartTimeInCurrentFile > deviceTimeIndex.getEndTime(device)) { + // we are iterating the time index so the times are definitely present + long deviceStartTimeInCurrentFile = deviceTimeIndex.getStartTime(device).get(); + if (deviceStartTimeInCurrentFile > deviceTimeIndex.getEndTime(device).get()) { continue; } if (!deviceEndTimeMap.containsKey(device)) { @@ -457,7 +475,7 @@ public static List checkTimePartitionHasOverlap( // update end time map if (!fileHasOverlap) { for (IDeviceID device : devices) { - deviceEndTimeMap.put(device, deviceTimeIndex.getEndTime(device)); + deviceEndTimeMap.put(device, deviceTimeIndex.getEndTime(device).get()); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java index 4ec74c57d63e..3f46b7f79374 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java @@ -117,6 +117,7 @@ private FileInfo getFileInfoFromCache(TsFileResource resource) throws IOExceptio } } + @SuppressWarnings("OptionalGetWithoutIsPresent") protected int calculatingMaxOverlapFileNumInSubCompactionTask(List resources) throws IOException { Set devices = new HashSet<>(); @@ -128,10 +129,11 @@ protected int calculatingMaxOverlapFileNumInSubCompactionTask(List resourcesContainsCurrentDevice = resourceDevices.stream() .filter(resource -> !resource.definitelyNotContains(device)) - .sorted(Comparator.comparingLong(resource -> resource.getStartTime(device))) + .sorted(Comparator.comparingLong(resource -> resource.getStartTime(device).get())) .collect(Collectors.toList()); if (resourcesContainsCurrentDevice.size() < maxOverlapFileNumInSubCompactionTask) { continue; @@ -140,8 +142,9 @@ protected int calculatingMaxOverlapFileNumInSubCompactionTask(List modifications = resource.getAllModEntries(); @@ -218,23 +219,30 @@ private FileDirtyInfo selectFileBaseOnDirtyData(TsFileResource resource) throws ttl = DataNodeTTLCache.getInstance().getTTLForTable(storageGroupName, tableName); } boolean hasSetTTL = ttl != Long.MAX_VALUE; + + long endTime = timeIndex.getEndTime(device).get(); boolean isDeleted = !timeIndex.isDeviceAlive(device, ttl) || isDeviceDeletedByMods( - modifications, - device, - timeIndex.getStartTime(device), - timeIndex.getEndTime(device)); - + modifications, device, timeIndex.getStartTime(device).get(), endTime); if (hasSetTTL) { if (!isDeleted) { // For devices with TTL set, all data must expire in order to meet the conditions for // being selected. return new FileDirtyInfo(DirtyStatus.NOT_SATISFIED); } - long outdatedTimeDiff = currentTime - timeIndex.getEndTime(device); - hasExpiredTooLong = - hasExpiredTooLong || outdatedTimeDiff > Math.min(config.getMaxExpiredTime(), 3 * ttl); + + if (currentTime > endTime) { + long outdatedTimeDiff = currentTime - endTime; + if (endTime < 0 && outdatedTimeDiff < currentTime) { + // overflow, like 100 - Long.MIN + outdatedTimeDiff = Long.MAX_VALUE; + } + long ttlThreshold = 3 * ttl > ttl ? ttl : Long.MAX_VALUE; + hasExpiredTooLong = + hasExpiredTooLong + || outdatedTimeDiff > Math.min(config.getMaxExpiredTime(), ttlThreshold); + } // else hasExpiredTooLong unchanged } if (isDeleted) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java index 6213c7ee89e5..a2b0b2d40bf5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java @@ -33,6 +33,7 @@ import java.util.Iterator; import java.util.Set; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class TsFileResourceCandidate { @SuppressWarnings("squid:S1104") public TsFileResource resource; @@ -85,8 +86,8 @@ private void prepareDeviceInfos() throws IOException { } else { deviceTimeIndex = new ArrayDeviceTimeIndex(); for (IDeviceID device : ((ArrayDeviceTimeIndex) timeIndex).getDevices()) { - deviceTimeIndex.updateStartTime(device, timeIndex.getStartTime(device)); - deviceTimeIndex.updateEndTime(device, timeIndex.getEndTime(device)); + deviceTimeIndex.updateStartTime(device, timeIndex.getStartTime(device).get()); + deviceTimeIndex.updateEndTime(device, timeIndex.getEndTime(device).get()); } } } else { @@ -129,7 +130,9 @@ public boolean hasNext() { public DeviceInfo next() { IDeviceID deviceId = deviceIterator.next(); return new DeviceInfo( - deviceId, deviceTimeIndex.getStartTime(deviceId), deviceTimeIndex.getEndTime(deviceId)); + deviceId, + deviceTimeIndex.getStartTime(deviceId).get(), + deviceTimeIndex.getEndTime(deviceId).get()); } }; } @@ -142,7 +145,9 @@ public Set getDevices() throws IOException { public DeviceInfo getDeviceInfoById(IDeviceID deviceId) throws IOException { prepareDeviceInfos(); return new DeviceInfo( - deviceId, deviceTimeIndex.getStartTime(deviceId), deviceTimeIndex.getEndTime(deviceId)); + deviceId, + deviceTimeIndex.getStartTime(deviceId).get(), + deviceTimeIndex.getEndTime(deviceId).get()); } public boolean containsDevice(IDeviceID deviceId) throws IOException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index 027de99166ba..1eb707adabc2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -986,10 +986,8 @@ public void deserializeFromOldMemTableSnapshot(DataInputStream stream) throws IO public Map getMaxTime() { Map latestTimeForEachDevice = new HashMap<>(); for (Entry entry : memTableMap.entrySet()) { - // When insert null values in to IWritableMemChunkGroup, the maxTime will not be updated. - // In this scenario, the maxTime will be Long.MIN_VALUE. We shouldn't return this device. long maxTime = entry.getValue().getMaxTime(); - if (maxTime != Long.MIN_VALUE) { + if (entry.getValue().count() > 0) { latestTimeForEachDevice.put(entry.getKey(), maxTime); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index d173de421e63..f3fe8f9fc6f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -575,9 +575,9 @@ public long getTsFileSize() { } } - public long getStartTime(IDeviceID deviceId) { + public Optional getStartTime(IDeviceID deviceId) { try { - return deviceId == null ? getFileStartTime() : timeIndex.getStartTime(deviceId); + return deviceId == null ? Optional.of(getFileStartTime()) : timeIndex.getStartTime(deviceId); } catch (Exception e) { LOGGER.error( "meet error when getStartTime of {} in file {}", deviceId, file.getAbsolutePath(), e); @@ -589,9 +589,9 @@ public long getStartTime(IDeviceID deviceId) { } /** open file's end time is Long.MIN_VALUE */ - public long getEndTime(IDeviceID deviceId) { + public Optional getEndTime(IDeviceID deviceId) { try { - return deviceId == null ? getFileEndTime() : timeIndex.getEndTime(deviceId); + return deviceId == null ? Optional.of(getFileEndTime()) : timeIndex.getEndTime(deviceId); } catch (Exception e) { LOGGER.error( "meet error when getEndTime of {} in file {}", deviceId, file.getAbsolutePath(), e); @@ -604,8 +604,10 @@ public long getEndTime(IDeviceID deviceId) { // cannot use FileTimeIndex public long getOrderTimeForSeq(IDeviceID deviceId, boolean ascending) { - if (timeIndex instanceof ArrayDeviceTimeIndex) { - return ascending ? getStartTime(deviceId) : getEndTime(deviceId); + if (timeIndex instanceof ArrayDeviceTimeIndex && !definitelyNotContains(deviceId)) { + // checked above + //noinspection OptionalGetWithoutIsPresent + return ascending ? getStartTime(deviceId).get() : getEndTime(deviceId).get(); } else { return ascending ? Long.MIN_VALUE : Long.MAX_VALUE; } @@ -613,7 +615,13 @@ public long getOrderTimeForSeq(IDeviceID deviceId, boolean ascending) { // can use FileTimeIndex public long getOrderTimeForUnseq(IDeviceID deviceId, boolean ascending) { - return ascending ? getStartTime(deviceId) : getEndTime(deviceId); + if (!definitelyNotContains(deviceId)) { + // checked above + //noinspection OptionalGetWithoutIsPresent + return ascending ? getStartTime(deviceId).get() : getEndTime(deviceId).get(); + } else { + return ascending ? Long.MIN_VALUE : Long.MAX_VALUE; + } } @Override @@ -945,6 +953,7 @@ public boolean isDeviceIdExist(IDeviceID deviceId) { /** * @return true if the device is contained in the TsFile */ + @SuppressWarnings("OptionalGetWithoutIsPresent") public boolean isSatisfied(IDeviceID deviceId, Filter timeFilter, boolean isSeq, boolean debug) { if (deviceId != null && definitelyNotContains(deviceId)) { if (debug) { @@ -954,8 +963,9 @@ public boolean isSatisfied(IDeviceID deviceId, Filter timeFilter, boolean isSeq, return false; } - long startTime = getStartTime(deviceId); - long endTime = isClosed() || !isSeq ? getEndTime(deviceId) : Long.MAX_VALUE; + // check above + long startTime = getStartTime(deviceId).get(); + long endTime = isClosed() || !isSeq ? getEndTime(deviceId).get() : Long.MAX_VALUE; if (startTime > endTime) { // startTime > endTime indicates that there is something wrong with this TsFile. Return false // directly, or it may lead to infinite loop in GroupByMonthFilter#getTimePointPosition. @@ -1297,8 +1307,13 @@ private void generatePathToTimeSeriesMetadataMap() throws IOException { public void deleteRemovedDeviceAndUpdateEndTime(Map lastTimeForEachDevice) { ITimeIndex newTimeIndex = CONFIG.getTimeIndexLevel().getTimeIndex(); for (Map.Entry entry : lastTimeForEachDevice.entrySet()) { - newTimeIndex.updateStartTime(entry.getKey(), timeIndex.getStartTime(entry.getKey())); - newTimeIndex.updateEndTime(entry.getKey(), entry.getValue()); + timeIndex + .getStartTime(entry.getKey()) + .ifPresent( + startTime -> { + newTimeIndex.updateStartTime(entry.getKey(), startTime); + newTimeIndex.updateEndTime(entry.getKey(), entry.getValue()); + }); } timeIndex = newTimeIndex; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java index cf1cf1dd1db7..f6de24155aa9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java @@ -43,6 +43,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -318,9 +319,10 @@ private long getTimePartitionWithCheck() throws PartitionViolationException { @Override public void updateStartTime(IDeviceID deviceId, long time) { - long startTime = getStartTime(deviceId); + int index = getDeviceIndex(deviceId); + @SuppressWarnings("OptionalGetWithoutIsPresent") // must present after getDeviceIndex + long startTime = getStartTime(deviceId).get(); if (time < startTime) { - int index = getDeviceIndex(deviceId); startTimes[index] = time; } minStartTime = Math.min(minStartTime, time); @@ -328,9 +330,10 @@ public void updateStartTime(IDeviceID deviceId, long time) { @Override public void updateEndTime(IDeviceID deviceId, long time) { - long endTime = getEndTime(deviceId); + int index = getDeviceIndex(deviceId); + @SuppressWarnings("OptionalGetWithoutIsPresent") // must present after getDeviceIndex + long endTime = getEndTime(deviceId).get(); if (time > endTime) { - int index = getDeviceIndex(deviceId); endTimes[index] = time; } maxEndTime = Math.max(maxEndTime, time); @@ -351,19 +354,19 @@ public void putEndTime(IDeviceID deviceId, long time) { } @Override - public long getStartTime(IDeviceID deviceId) { + public Optional getStartTime(IDeviceID deviceId) { if (!deviceToIndex.containsKey(deviceId)) { - return Long.MAX_VALUE; + return Optional.empty(); } - return startTimes[deviceToIndex.get(deviceId)]; + return Optional.of(startTimes[deviceToIndex.get(deviceId)]); } @Override - public long getEndTime(IDeviceID deviceId) { + public Optional getEndTime(IDeviceID deviceId) { if (!deviceToIndex.containsKey(deviceId)) { - return Long.MIN_VALUE; + return Optional.empty(); } - return endTimes[deviceToIndex.get(deviceId)]; + return Optional.of(endTimes[deviceToIndex.get(deviceId)]); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java index 50d2cfc9e0ee..9ea286f0129a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java @@ -40,6 +40,7 @@ import java.nio.ByteBuffer; import java.nio.file.NoSuchFileException; import java.util.Collections; +import java.util.Optional; import java.util.Set; public class FileTimeIndex implements ITimeIndex { @@ -190,8 +191,8 @@ public void putEndTime(IDeviceID deviceId, long time) { } @Override - public long getStartTime(IDeviceID deviceId) { - return startTime; + public Optional getStartTime(IDeviceID deviceId) { + return Optional.of(startTime); } @Override @@ -200,8 +201,8 @@ public long getMinStartTime() { } @Override - public long getEndTime(IDeviceID deviceId) { - return endTime; + public Optional getEndTime(IDeviceID deviceId) { + return Optional.of(endTime); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java index 52035293a0cd..214ae53750c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java @@ -31,6 +31,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.Optional; import java.util.Set; public interface ITimeIndex { @@ -150,7 +151,7 @@ public interface ITimeIndex { * @param deviceId device name * @return start time */ - long getStartTime(IDeviceID deviceId); + Optional getStartTime(IDeviceID deviceId); /** * get end time of device @@ -158,7 +159,7 @@ public interface ITimeIndex { * @param deviceId device name * @return end time */ - long getEndTime(IDeviceID deviceId); + Optional getEndTime(IDeviceID deviceId); /** * check whether deviceId exists in TsFile diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileDeviceStartEndTimeIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileDeviceStartEndTimeIterator.java index fd0fff8cba9d..557d0764c1fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileDeviceStartEndTimeIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileDeviceStartEndTimeIterator.java @@ -46,7 +46,11 @@ public boolean hasNext() { public DeviceStartEndTime next() { IDeviceID deviceID = currentDevice.next(); + // iterating the index, must present + //noinspection OptionalGetWithoutIsPresent return new DeviceStartEndTime( - deviceID, deviceTimeIndex.getStartTime(deviceID), deviceTimeIndex.getEndTime(deviceID)); + deviceID, + deviceTimeIndex.getStartTime(deviceID).get(), + deviceTimeIndex.getEndTime(deviceID).get()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java index d8bc61db50a2..6ff2b605cf45 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java @@ -70,6 +70,7 @@ private TsFileResourceUtils() { // util class } + @SuppressWarnings("OptionalGetWithoutIsPresent") public static boolean validateTsFileResourceCorrectness(TsFileResource resource) { if (resource.isDeleted()) { return true; @@ -92,26 +93,9 @@ public static boolean validateTsFileResourceCorrectness(TsFileResource resource) return false; } for (IDeviceID device : devices) { - long startTime = timeIndex.getStartTime(device); - long endTime = timeIndex.getEndTime(device); - if (startTime == Long.MAX_VALUE) { - logger.error( - "{} {} the start time of {} is {}", - resource.getTsFilePath(), - VALIDATE_FAILED, - device, - Long.MAX_VALUE); - return false; - } - if (endTime == Long.MIN_VALUE) { - logger.error( - "{} {} the end time of {} is {}", - resource.getTsFilePath(), - VALIDATE_FAILED, - device, - Long.MIN_VALUE); - return false; - } + // iterating the index, must present + long startTime = timeIndex.getStartTime(device).get(); + long endTime = timeIndex.getEndTime(device).get(); if (startTime > endTime) { logger.error( "{} {} the start time of {} is greater than end time", @@ -375,6 +359,7 @@ public static Map getChunkMetadata(TsFileSequenceReader re return offset2ChunkMetadata; } + @SuppressWarnings("OptionalGetWithoutIsPresent") public static boolean validateTsFileResourcesHasNoOverlap(List resources) { // deviceID -> Map> lastEndTimeMap = new HashMap<>(); @@ -396,8 +381,9 @@ public static boolean validateTsFileResourcesHasNoOverlap(List r } Set devices = timeIndex.getDevices(); for (IDeviceID device : devices) { - long currentStartTime = timeIndex.getStartTime(device); - long currentEndTime = timeIndex.getEndTime(device); + // iterating the index, must present + long currentStartTime = timeIndex.getStartTime(device).get(); + long currentEndTime = timeIndex.getEndTime(device).get(); Pair lastDeviceInfo = lastEndTimeMap.computeIfAbsent(device, x -> new Pair<>(null, Long.MIN_VALUE)); long lastEndTime = lastDeviceInfo.right; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java index e3d560a01731..9274115bc47c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java @@ -42,6 +42,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; /** * This class helps redo wal logs into a TsFile. Notice: You should update time map in {@link @@ -88,14 +89,14 @@ void redoInsert(InsertNode node) throws WriteProcessException { if (tsFileResource != null) { // orders of insert node is guaranteed by storage engine, just check time in the file // the last chunk group may contain the same data with the logs, ignore such logs in seq file - long lastEndTime = tsFileResource.getEndTime(node.getDeviceID()); + Optional lastEndTime = tsFileResource.getEndTime(node.getDeviceID()); long minTimeInNode; if (node instanceof InsertRowNode) { minTimeInNode = ((InsertRowNode) node).getTime(); } else { minTimeInNode = ((InsertTabletNode) node).getTimes()[0]; } - if (lastEndTime != Long.MIN_VALUE && lastEndTime >= minTimeInNode) { + if (lastEndTime.isPresent() && lastEndTime.get() >= minTimeInNode) { return; } } @@ -131,10 +132,10 @@ void redoInsertRows(InsertRowsNode insertRowsNode) { // orders of insert node is guaranteed by storage engine, just check time in the file // the last chunk group may contain the same data with the logs, ignore such logs in seq // file - long lastEndTime = tsFileResource.getEndTime(node.getDeviceID()); + Optional lastEndTime = tsFileResource.getEndTime(node.getDeviceID()); long minTimeInNode; minTimeInNode = node.getTime(); - if (lastEndTime != Long.MIN_VALUE && lastEndTime >= minTimeInNode) { + if (lastEndTime.isPresent() && lastEndTime.get() >= minTimeInNode) { continue; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java index a0901f8772cc..c955f67c79c0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java @@ -161,6 +161,7 @@ private void constructResourceFromTsFile() { } /** Redo log. */ + @SuppressWarnings("OptionalGetWithoutIsPresent") public void redoLog(WALEntry walEntry) { // skip redo wal log when this TsFile is not crashed if (!hasCrashed()) { @@ -177,19 +178,20 @@ public void redoLog(WALEntry walEntry) { if (tsFileResource != null) { // delete data already flushed in the MemTable to avoid duplicates for (IDeviceID device : tsFileResource.getDevices()) { + // iterating the index, must present if (device.isTableModel()) { memTable.delete( new TableDeletionEntry( new DeletionPredicate(device.getTableName(), new FullExactMatch(device)), new TimeRange( - tsFileResource.getStartTime(device), - tsFileResource.getEndTime(device)))); + tsFileResource.getStartTime(device).get(), + tsFileResource.getEndTime(device).get()))); } else { memTable.delete( new TreeDeletionEntry( new MeasurementPath(device, "*"), - tsFileResource.getStartTime(device), - tsFileResource.getEndTime(device))); + tsFileResource.getStartTime(device).get(), + tsFileResource.getEndTime(device).get())); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java index 585421e3afe8..ca7493d6ca06 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java @@ -215,13 +215,17 @@ public void writeDecodePage(final long[] times, final Object[] values, final int pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1); satisfiedLengthQueue.offer(satisfiedLength); final long startTime = timePartitionSlot.getStartTime(); - final long endTime = startTime + TimePartitionUtils.getTimePartitionInterval(); + // beware of overflow + long endTime = startTime + TimePartitionUtils.getTimePartitionInterval() - 1; + if (endTime <= startTime) { + endTime = Long.MAX_VALUE; + } // serialize needDecode==true dataSize += ReadWriteIOUtils.write(true, stream); dataSize += ReadWriteIOUtils.write(satisfiedLength, stream); for (final long time : times) { - if (time >= endTime) { + if (time > endTime) { break; } if (time >= startTime) { @@ -235,7 +239,11 @@ public void writeDecodeValuePage( throws IOException { pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1); final long startTime = timePartitionSlot.getStartTime(); - final long endTime = startTime + TimePartitionUtils.getTimePartitionInterval(); + // beware of overflow + long endTime = startTime + TimePartitionUtils.getTimePartitionInterval() - 1; + if (endTime <= startTime) { + endTime = Long.MAX_VALUE; + } final int satisfiedLength = satisfiedLengthQueue.poll(); // serialize needDecode==true dataSize += ReadWriteIOUtils.write(true, stream); @@ -243,7 +251,7 @@ public void writeDecodeValuePage( satisfiedLengthQueue.offer(satisfiedLength); for (int i = 0; i < times.length; i++) { - if (times[i] >= endTime) { + if (times[i] > endTime) { break; } if (times[i] >= startTime) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java index a7b5a840ba66..5a8e22aa8e2a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java @@ -70,7 +70,11 @@ public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataT throws IOException { pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1); final long startTime = timePartitionSlot.getStartTime(); - final long endTime = startTime + TimePartitionUtils.getTimePartitionInterval(); + // beware of overflow + long endTime = startTime + TimePartitionUtils.getTimePartitionInterval() - 1; + if (endTime <= startTime) { + endTime = Long.MAX_VALUE; + } final int satisfiedLength = satisfiedLengthQueue.poll(); // serialize needDecode==true dataSize += ReadWriteIOUtils.write(true, stream); @@ -81,7 +85,7 @@ public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataT // its start time and end time. long pageStartTime = Long.MAX_VALUE, pageEndTime = Long.MIN_VALUE; for (int i = 0; i < times.length; i++) { - if (times[i] >= endTime) { + if (times[i] > endTime) { break; } if (times[i] >= startTime) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java index f2d17ab95fc3..ccf7970a1db2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java @@ -163,12 +163,16 @@ public void writeDecodePage(final long[] times, final Object[] values, final int throws IOException { pageNumber += 1; final long startTime = timePartitionSlot.getStartTime(); - final long endTime = startTime + TimePartitionUtils.getTimePartitionInterval(); + // beware of overflow + long endTime = startTime + TimePartitionUtils.getTimePartitionInterval() - 1; + if (endTime <= startTime) { + endTime = Long.MAX_VALUE; + } dataSize += ReadWriteIOUtils.write(true, stream); dataSize += ReadWriteIOUtils.write(satisfiedLength, stream); for (int i = 0; i < times.length; i++) { - if (times[i] >= endTime) { + if (times[i] > endTime) { break; } if (times[i] >= startTime) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java index 23ba30fde005..b6675b8cfd73 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java @@ -250,6 +250,10 @@ private void decodeAndWriteTimeChunkOrNonAlignedChunk( int satisfiedLength = 0; long endTime = timePartitionSlot.getStartTime() + TimePartitionUtils.getTimePartitionInterval(); + // beware of overflow + if (endTime <= timePartitionSlot.getStartTime()) { + endTime = Long.MAX_VALUE; + } for (int i = 0; i < times.length; i++) { if (times[i] >= endTime) { chunkData.writeDecodePage(times, values, satisfiedLength); @@ -265,6 +269,9 @@ private void decodeAndWriteTimeChunkOrNonAlignedChunk( satisfiedLength = 0; endTime = timePartitionSlot.getStartTime() + TimePartitionUtils.getTimePartitionInterval(); + if (endTime <= timePartitionSlot.getStartTime()) { + endTime = Long.MAX_VALUE; + } chunkData = ChunkData.createChunkData(isAligned, curDevice, header, timePartitionSlot); } satisfiedLength += 1; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/IoTDBDataDirViewer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/IoTDBDataDirViewer.java index 6713c51e05ec..4c174be196ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/IoTDBDataDirViewer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/IoTDBDataDirViewer.java @@ -164,15 +164,17 @@ private static void printResource(String filename, PrintWriter pw) throws IOExce // sort device strings SortedSet keys = new TreeSet<>(resource.getDevices()); for (IDeviceID device : keys) { + // iterating the index, must present + //noinspection OptionalGetWithoutIsPresent printlnBoth( pw, String.format( "| | | | | |--device %s, start time %d (%s), end time %d (%s)", device, resource.getStartTime(device), - DateTimeUtils.convertLongToDate(resource.getStartTime(device)), + DateTimeUtils.convertLongToDate(resource.getStartTime(device).get()), resource.getEndTime(device), - DateTimeUtils.convertLongToDate(resource.getEndTime(device)))); + DateTimeUtils.convertLongToDate(resource.getEndTime(device).get()))); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java index d3e5597ee3e5..7fe4ca1d01e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java @@ -77,13 +77,15 @@ public static void printResource(String filename) throws IOException { resource.getMinPlanIndex(), resource.getMaxPlanIndex()); for (IDeviceID device : resource.getDevices()) { + // iterating the index, must present + //noinspection OptionalGetWithoutIsPresent System.out.printf( "device %s, start time %d (%s), end time %d (%s)%n", device, - resource.getStartTime(device), - DateTimeUtils.convertLongToDate(resource.getStartTime(device)), - resource.getEndTime(device), - DateTimeUtils.convertLongToDate(resource.getEndTime(device))); + resource.getStartTime(device).get(), + DateTimeUtils.convertLongToDate(resource.getStartTime(device).get()), + resource.getEndTime(device).get(), + DateTimeUtils.convertLongToDate(resource.getEndTime(device).get())); } System.out.println(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileValidationScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileValidationScan.java index d4fd2356ea67..80cde572d8b7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileValidationScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileValidationScan.java @@ -42,6 +42,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; @SuppressWarnings("java:S2175") public class TsFileValidationScan extends TsFileSequenceScan { @@ -128,11 +129,11 @@ protected void onChunkGroup() throws IOException { FileLastTimeInfo fileNameLastTimePair = deviceEndTime.computeIfAbsent(currDeviceID, k -> new FileLastTimeInfo()); if (!currDeviceID.equals(EMPTY_DEVICE_ID)) { - long endTime = resource.getEndTime(currDeviceID); + Optional endTime = resource.getEndTime(currDeviceID); // record the end time of last device in current file - if (endTime > fileNameLastTimePair.lastTime) { + if (endTime.isPresent() && endTime.get() > fileNameLastTimePair.lastTime) { fileNameLastTimePair.lastFileName = file.getName(); - fileNameLastTimePair.endTimeInLastFile = endTime; + fileNameLastTimePair.endTimeInLastFile = endTime.get(); } } @@ -140,7 +141,8 @@ protected void onChunkGroup() throws IOException { fileNameLastTimePair = deviceEndTime.computeIfAbsent(currDeviceID, k -> new FileLastTimeInfo()); if (!Boolean.TRUE.equals(hasCheckedDeviceOverlap.getOrDefault(currDeviceID, false)) - && resource.getStartTime(currDeviceID) <= fileNameLastTimePair.endTimeInLastFile) { + && resource.getStartTime(currDeviceID).isPresent() + && resource.getStartTime(currDeviceID).get() <= fileNameLastTimePair.endTimeInLastFile) { // device overlap, find bad file recordDeviceOverlap(fileNameLastTimePair.lastFileName); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java index a383b99b462d..bf46e95e07b6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java @@ -199,6 +199,7 @@ private static String calculateTimePartitionKey( return storageGroup + "-" + dataRegion + "-" + timePartition; } + @SuppressWarnings("OptionalGetWithoutIsPresent") public static int checkTimePartitionHasOverlap(List resources) { int overlapTsFileNum = 0; Map deviceEndTimeMap = new HashMap<>(); @@ -208,8 +209,9 @@ public static int checkTimePartitionHasOverlap(List resources) { boolean fileHasOverlap = false; // check overlap for (IDeviceID device : devices) { - long deviceStartTimeInCurrentFile = resource.getStartTime(device); - if (deviceStartTimeInCurrentFile > resource.getEndTime(device)) { + // iterating the index, must present + long deviceStartTimeInCurrentFile = resource.getStartTime(device).get(); + if (deviceStartTimeInCurrentFile > resource.getEndTime(device).get()) { continue; } if (!deviceEndTimeMap.containsKey(device)) { @@ -234,7 +236,7 @@ public static int checkTimePartitionHasOverlap(List resources) { // update end time map if (!fileHasOverlap) { for (IDeviceID device : devices) { - deviceEndTimeMap.put(device, resource.getEndTime(device)); + deviceEndTimeMap.put(device, resource.getEndTime(device).get()); deviceLastExistTsFileMap.put(device, resource); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java index 599662076480..1267dcad61a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java @@ -19,11 +19,13 @@ package org.apache.iotdb.db.utils; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.impl.SettleSelectorImpl; import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -37,6 +39,7 @@ import java.util.Comparator; import java.util.List; import java.util.Objects; +import java.util.Optional; public class ModificationUtils { @@ -398,4 +401,18 @@ public static List sortAndMerge(List modifications) { } return result; } + + public static boolean isDeviceDeletedByMods( + Collection currentModifications, ITimeIndex currentTimeIndex, IDeviceID device) + throws IllegalPathException { + if (currentTimeIndex == null) { + return false; + } + Optional startTime = currentTimeIndex.getStartTime(device); + Optional endTime = currentTimeIndex.getEndTime(device); + if (startTime.isPresent() && endTime.isPresent()) { + return isAllDeletedByMods(currentModifications, device, startTime.get(), endTime.get()); + } + return false; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java index 768d0030e96d..44c8f117469e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java @@ -26,8 +26,6 @@ public class TimestampPrecisionUtils { public static String TIMESTAMP_PRECISION = CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); - private static final boolean isTimestampPrecisionCheckEnabled = - CommonDescriptor.getInstance().getConfig().isTimestampPrecisionCheckEnabled(); @FunctionalInterface private interface ConvertFunction { @@ -65,6 +63,8 @@ public static long convertToCurrPrecision(long sourceTime, TimeUnit sourceUnit) /** check whether the input timestamp match the current system timestamp precision. */ public static void checkTimestampPrecision(long time) { + final boolean isTimestampPrecisionCheckEnabled = + CommonDescriptor.getInstance().getConfig().isTimestampPrecisionCheckEnabled(); if (!isTimestampPrecisionCheckEnabled) { return; } @@ -73,8 +73,10 @@ public static void checkTimestampPrecision(long time) { if (time > 10_000_000_000_000L) { throw new SemanticException( String.format( - "Current system timestamp precision is %s, " - + "please check whether the timestamp %s is correct.", + "The timestamp is unexpectedly large, you may forget to set the timestamp precision." + + "Current system timestamp precision is %s, " + + "please check whether the timestamp %s is correct." + + "If you insist to insert this timestamp, please set timestamp_precision_check_enabled=false and restart the server.", TIMESTAMP_PRECISION, time)); } break; @@ -82,8 +84,10 @@ public static void checkTimestampPrecision(long time) { if (time > 10_000_000_000_000_000L) { throw new SemanticException( String.format( - "Current system timestamp precision is %s, " - + "please check whether the timestamp %s is correct.", + "The timestamp is unexpectedly large, you may forget to set the timestamp precision." + + "Current system timestamp precision is %s, " + + "please check whether the timestamp %s is correct." + + "If you insist to insert this timestamp, please set timestamp_precision_check_enabled=false and restart the server.", TIMESTAMP_PRECISION, time)); } break; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDataTypeNotMatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDataTypeNotMatchTest.java index 4e7810b65e4e..ff12a13c6dac 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDataTypeNotMatchTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDataTypeNotMatchTest.java @@ -48,6 +48,7 @@ import java.util.ArrayList; import java.util.List; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class CompactionDataTypeNotMatchTest extends AbstractCompactionTest { private final String oldThreadName = Thread.currentThread().getName(); private final IDeviceID device = @@ -75,7 +76,8 @@ public void testCompactNonAlignedSeriesWithReadChunkCompactionPerformer() 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); Assert.assertTrue(task.start()); TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).get(0).getStartTime(device)); + Assert.assertEquals( + 2, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); } @Test @@ -87,7 +89,8 @@ public void testCompactNonAlignedSeriesWithFastCompactionPerformer() 0, tsFileManager, seqResources, true, new FastCompactionPerformer(false), 0); Assert.assertTrue(task.start()); TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).get(0).getStartTime(device)); + Assert.assertEquals( + 2, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); } @Test @@ -99,7 +102,8 @@ public void testCompactNonAlignedSeriesWithReadPointCompactionPerformer() 0, tsFileManager, seqResources, true, new ReadPointCompactionPerformer(), 0); Assert.assertTrue(task.start()); TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).get(0).getStartTime(device)); + Assert.assertEquals( + 2, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); } @Test @@ -111,7 +115,8 @@ public void testCompactAlignedSeriesWithReadChunkCompactionPerformer() 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); Assert.assertTrue(task.start()); TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).get(0).getStartTime(device)); + Assert.assertEquals( + 2, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); } @Test @@ -123,7 +128,8 @@ public void testCompactAlignedSeriesWithFastCompactionPerformer() 0, tsFileManager, seqResources, true, new FastCompactionPerformer(false), 0); Assert.assertTrue(task.start()); TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).get(0).getStartTime(device)); + Assert.assertEquals( + 2, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); } @Test @@ -135,7 +141,8 @@ public void testCompactAlignedSeriesWithReadPointCompactionPerformer() 0, tsFileManager, seqResources, true, new ReadPointCompactionPerformer(), 0); Assert.assertTrue(task.start()); TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); - Assert.assertEquals(2, tsFileManager.getTsFileList(true).get(0).getStartTime(device)); + Assert.assertEquals( + 2, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); } private void generateDataTypeNotMatchFilesWithNonAlignedSeries() diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithEmptyPageTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithEmptyPageTest.java index 6a4b57f30801..7e9c25316a54 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithEmptyPageTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithEmptyPageTest.java @@ -46,6 +46,7 @@ import java.util.List; import java.util.Map; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class FastCompactionPerformerWithEmptyPageTest extends AbstractCompactionTest { int oldAlignedSeriesCompactionBatchSize; @@ -123,8 +124,8 @@ public void test1() throws IOException, IllegalPathException { } TsFileResource result = tsFileManager.getTsFileList(true).get(0); result.buildDeviceTimeIndex(); - Assert.assertEquals(20, result.getStartTime(device)); - Assert.assertEquals(50, result.getEndTime(device)); + Assert.assertEquals(20, ((long) result.getStartTime(device).get())); + Assert.assertEquals(50, ((long) result.getEndTime(device).get())); validateSeqFiles(true); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java index 8c9088f5cebc..a0d84c1e8104 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java @@ -43,6 +43,7 @@ import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TimeValuePair; @@ -75,6 +76,7 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR; import static org.junit.Assert.assertEquals; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class FastInnerCompactionPerformerTest extends AbstractCompactionTest { private boolean enableUnseqSpaceCompaction = @@ -243,47 +245,53 @@ public void testSeqInnerSpaceCompactionWithDifferentTimeseries() throws Exceptio assertEquals( 0, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d0"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d0")) + .get())); assertEquals( 0, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d1"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d1")) + .get())); assertEquals( 250, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d2"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d2")) + .get())); assertEquals( 600, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d3"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d3")) + .get())); assertEquals( 600, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d4"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d4")) + .get())); for (int i = 0; i < 5; i++) { assertEquals( 749, - targetResources - .get(0) - .getEndTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i))); + ((long) + targetResources + .get(0) + .getEndTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i)) + .get())); } for (int i = 0; i < 5; i++) { @@ -408,47 +416,53 @@ public void testSeqInnerSpaceCompactionWithFileTimeIndex() throws Exception { assertEquals( 0, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d0"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d0")) + .get())); assertEquals( 0, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d1"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d1")) + .get())); assertEquals( 250, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d2"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d2")) + .get())); assertEquals( 600, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d3"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d3")) + .get())); assertEquals( 600, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d4"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d4")) + .get())); for (int i = 0; i < 5; i++) { assertEquals( 749, - targetResources - .get(0) - .getEndTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i))); + ((long) + targetResources + .get(0) + .getEndTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i)) + .get())); } for (int i = 0; i < 5; i++) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointCompactionPerformerTest.java index 8f603332a123..65163221af4d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointCompactionPerformerTest.java @@ -43,6 +43,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.read.common.IBatchDataIterator; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.utils.Pair; @@ -66,6 +67,7 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR; import static org.junit.Assert.assertEquals; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class ReadPointCompactionPerformerTest extends AbstractCompactionTest { private final String oldThreadName = Thread.currentThread().getName(); @@ -228,47 +230,53 @@ public void testSeqInnerSpaceCompactionWithDifferentTimeseries() throws Exceptio targetResources, CompactionTaskType.INNER_SEQ, COMPACTION_TEST_SG); assertEquals( 0, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d0"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d0")) + .get())); assertEquals( 0, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d1"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d1")) + .get())); assertEquals( 250, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d2"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d2")) + .get())); assertEquals( 600, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d3"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d3")) + .get())); assertEquals( 600, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d4"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d4")) + .get())); for (int i = 0; i < 5; i++) { assertEquals( 749, - targetResources - .get(0) - .getEndTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i))); + ((long) + targetResources + .get(0) + .getEndTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i)) + .get())); } for (int i = 0; i < 5; i++) { @@ -390,47 +398,53 @@ public void testSeqInnerSpaceCompactionWithFileTimeIndex() throws Exception { targetResources, CompactionTaskType.INNER_SEQ, COMPACTION_TEST_SG); assertEquals( 0, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d0"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d0")) + .get())); assertEquals( 0, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d1"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d1")) + .get())); assertEquals( 250, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d2"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d2")) + .get())); assertEquals( 600, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d3"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d3")) + .get())); assertEquals( 600, - targetResources - .get(0) - .getStartTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d4"))); + ((long) + targetResources + .get(0) + .getStartTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d4")) + .get())); for (int i = 0; i < 5; i++) { assertEquals( 749, - targetResources - .get(0) - .getEndTime( - IDeviceID.Factory.DEFAULT_FACTORY.create( - COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i))); + ((long) + targetResources + .get(0) + .getEndTime( + Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i)) + .get())); } for (int i = 0; i < 5; i++) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/TsFileValidationCorrectnessTests.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/TsFileValidationCorrectnessTests.java index f47dbc839e90..0fe2f5b31be9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/TsFileValidationCorrectnessTests.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/TsFileValidationCorrectnessTests.java @@ -165,8 +165,9 @@ public void testAlignedTimestampHasOverlapBetweenPages() throws IOException { public void testAlignedTimestampTimeChunkOffsetEqualsMetadata() throws IOException { String path = dir + File.separator + "test6.tsfile"; TsFileResource tsFileResource = new TsFileResource(new File(path)); + IDeviceID deviceID; try (CompactionTestFileWriter writer = new CompactionTestFileWriter(tsFileResource)) { - writer.startChunkGroup("d1"); + deviceID = writer.startChunkGroup("d1"); VectorMeasurementSchema vectorMeasurementSchema = new VectorMeasurementSchema( "d1", new String[] {"s1"}, new TSDataType[] {TSDataType.INT32}); @@ -182,8 +183,8 @@ public void testAlignedTimestampTimeChunkOffsetEqualsMetadata() throws IOExcepti writer.endChunkGroup(); writer.endFile(); } - tsFileResource.updateStartTime(IDeviceID.Factory.DEFAULT_FACTORY.create("d1"), 1); - tsFileResource.updateEndTime(IDeviceID.Factory.DEFAULT_FACTORY.create("d1"), 3); + tsFileResource.updateStartTime(deviceID, 1); + tsFileResource.updateEndTime(deviceID, 3); tsFileResource.serialize(); boolean success = TsFileValidator.getInstance().validateTsFile(tsFileResource); Assert.assertTrue(success); @@ -272,8 +273,9 @@ public void testNonAlignedTimestampHasOverlapBetweenPages() throws IOException { public void testNonAlignedTimestampTimeChunkOffsetEqualsMetadata() throws IOException { String path = dir + File.separator + "test11.tsfile"; TsFileResource tsFileResource = new TsFileResource(new File(path)); + IDeviceID deviceID; try (CompactionTestFileWriter writer = new CompactionTestFileWriter(tsFileResource)) { - writer.startChunkGroup("d1"); + deviceID = writer.startChunkGroup("d1"); VectorMeasurementSchema vectorMeasurementSchema = new VectorMeasurementSchema( "d1", new String[] {"s1"}, new TSDataType[] {TSDataType.INT32}); @@ -289,8 +291,8 @@ public void testNonAlignedTimestampTimeChunkOffsetEqualsMetadata() throws IOExce writer.endChunkGroup(); writer.endFile(); } - tsFileResource.updateStartTime(IDeviceID.Factory.DEFAULT_FACTORY.create("d1"), 1); - tsFileResource.updateEndTime(IDeviceID.Factory.DEFAULT_FACTORY.create("d1"), 3); + tsFileResource.updateStartTime(deviceID, 1); + tsFileResource.updateEndTime(deviceID, 3); tsFileResource.serialize(); boolean success = TsFileValidator.getInstance().validateTsFile(tsFileResource); Assert.assertTrue(success); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java index b076755229ed..644ddf77e1f2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java @@ -76,6 +76,7 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR; import static org.apache.tsfile.utils.TsFileGeneratorUtils.alignDeviceOffset; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class CrossSpaceCompactionWithFastPerformerValidationTest extends AbstractCompactionTest { TsFileManager tsFileManager = new TsFileManager(COMPACTION_TEST_SG, "0", STORAGE_GROUP_DIR.getPath()); @@ -1849,7 +1850,7 @@ public void testWithUnclosedSeqFile() unclosedSeqResource.setStatusForTest(TsFileResourceStatus.UNCLOSED); TsFileResource lastSeqResource = seqResources.get(4); for (IDeviceID deviceID : lastSeqResource.getDevices()) { - unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID)); + unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID).get()); } seqResources.remove(4); seqResources.add(4, unclosedSeqResource); @@ -1902,7 +1903,7 @@ public void testWithUnclosedSeqFileAndNewSensorInUnseqFile() unclosedSeqResource.setStatusForTest(TsFileResourceStatus.UNCLOSED); TsFileResource lastSeqResource = seqResources.get(4); for (IDeviceID deviceID : lastSeqResource.getDevices()) { - unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID)); + unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID).get()); } seqResources.remove(4); seqResources.add(4, unclosedSeqResource); @@ -1948,7 +1949,7 @@ public void testUnseqFileOverlapWithUnclosedSeqFile() unclosedSeqResource.setStatusForTest(TsFileResourceStatus.UNCLOSED); TsFileResource lastSeqResource = seqResources.get(4); for (IDeviceID deviceID : lastSeqResource.getDevices()) { - unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID)); + unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID).get()); } seqResources.remove(4); seqResources.add(4, unclosedSeqResource); @@ -2000,7 +2001,7 @@ public void testUnseqFileOverlapWithUnclosedSeqFile2() unclosedSeqResource.setStatusForTest(TsFileResourceStatus.UNCLOSED); TsFileResource lastSeqResource = seqResources.get(4); for (IDeviceID deviceID : lastSeqResource.getDevices()) { - unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID)); + unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID).get()); } seqResources.remove(4); seqResources.add(4, unclosedSeqResource); @@ -2053,8 +2054,9 @@ public void testWithUnclosedUnSeqFile() unclosedUnSeqResource.setStatusForTest(TsFileResourceStatus.UNCLOSED); TsFileResource lastUnSeqResource = unseqResources.get(1); for (IDeviceID deviceID : lastUnSeqResource.getDevices()) { - unclosedUnSeqResource.updateStartTime(deviceID, lastUnSeqResource.getStartTime(deviceID)); - unclosedUnSeqResource.updateEndTime(deviceID, lastUnSeqResource.getEndTime(deviceID)); + unclosedUnSeqResource.updateStartTime( + deviceID, lastUnSeqResource.getStartTime(deviceID).get()); + unclosedUnSeqResource.updateEndTime(deviceID, lastUnSeqResource.getEndTime(deviceID).get()); } unseqResources.remove(1); unseqResources.add(1, unclosedUnSeqResource); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java index 77d5a66c301d..b05ab1e633e3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java @@ -74,6 +74,7 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR; import static org.apache.tsfile.utils.TsFileGeneratorUtils.alignDeviceOffset; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class CrossSpaceCompactionWithReadPointPerformerValidationTest extends AbstractCompactionTest { // TsFileManager tsFileManager = @@ -1848,7 +1849,7 @@ public void testWithUnclosedSeqFile() unclosedSeqResource.setStatusForTest(TsFileResourceStatus.UNCLOSED); TsFileResource lastSeqResource = seqResources.get(4); for (IDeviceID deviceID : lastSeqResource.getDevices()) { - unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID)); + unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID).get()); } seqResources.remove(4); seqResources.add(4, unclosedSeqResource); @@ -1901,7 +1902,7 @@ public void testWithUnclosedSeqFileAndNewSensorInUnseqFile() unclosedSeqResource.setStatusForTest(TsFileResourceStatus.UNCLOSED); TsFileResource lastSeqResource = seqResources.get(4); for (IDeviceID deviceID : lastSeqResource.getDevices()) { - unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID)); + unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID).get()); } seqResources.remove(4); seqResources.add(4, unclosedSeqResource); @@ -1947,7 +1948,7 @@ public void testUnseqFileOverlapWithUnclosedSeqFile() unclosedSeqResource.setStatusForTest(TsFileResourceStatus.UNCLOSED); TsFileResource lastSeqResource = seqResources.get(4); for (IDeviceID deviceID : lastSeqResource.getDevices()) { - unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID)); + unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID).get()); } seqResources.remove(4); seqResources.add(4, unclosedSeqResource); @@ -1999,7 +2000,7 @@ public void testUnseqFileOverlapWithUnclosedSeqFile2() unclosedSeqResource.setStatusForTest(TsFileResourceStatus.UNCLOSED); TsFileResource lastSeqResource = seqResources.get(4); for (IDeviceID deviceID : lastSeqResource.getDevices()) { - unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID)); + unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID).get()); } seqResources.remove(4); seqResources.add(4, unclosedSeqResource); @@ -2052,8 +2053,9 @@ public void testWithUnclosedUnSeqFile() unclosedUnSeqResource.setStatusForTest(TsFileResourceStatus.UNCLOSED); TsFileResource lastUnSeqResource = unseqResources.get(1); for (IDeviceID deviceID : lastUnSeqResource.getDevices()) { - unclosedUnSeqResource.updateStartTime(deviceID, lastUnSeqResource.getStartTime(deviceID)); - unclosedUnSeqResource.updateEndTime(deviceID, lastUnSeqResource.getEndTime(deviceID)); + unclosedUnSeqResource.updateStartTime( + deviceID, lastUnSeqResource.getStartTime(deviceID).get()); + unclosedUnSeqResource.updateEndTime(deviceID, lastUnSeqResource.getEndTime(deviceID).get()); } unseqResources.remove(1); unseqResources.add(1, unclosedUnSeqResource); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java index b89aef630bf1..75ce7382a4da 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java @@ -62,6 +62,7 @@ import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.writeNonAlignedChunk; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class InsertionCrossSpaceCompactionRecoverTest extends AbstractCompactionTest { @Before public void setUp() @@ -450,7 +451,8 @@ private void createTsFileByResource(TsFileResource resource) throws IOException new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN, CompressionType.SNAPPY); ChunkWriterImpl iChunkWriter = new ChunkWriterImpl(schema); List pages = new ArrayList<>(); - pages.add(new TimeRange(resource.getStartTime(device), resource.getEndTime(device))); + pages.add( + new TimeRange(resource.getStartTime(device).get(), resource.getEndTime(device).get())); writeNonAlignedChunk(iChunkWriter, tsFileIOWriter, pages, resource.isSeq()); tsFileIOWriter.endChunkGroup(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java index 2cb60ed22562..5478d05004df 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java @@ -57,6 +57,7 @@ import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.writeNonAlignedChunk; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class InsertionCrossSpaceCompactionSelectorTest extends AbstractCompactionTest { @Before @@ -2029,7 +2030,8 @@ private void createTsFileByResource(TsFileResource resource) throws IOException new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN, CompressionType.SNAPPY); ChunkWriterImpl iChunkWriter = new ChunkWriterImpl(schema); List pages = new ArrayList<>(); - pages.add(new TimeRange(resource.getStartTime(device), resource.getEndTime(device))); + pages.add( + new TimeRange(resource.getStartTime(device).get(), resource.getEndTime(device).get())); writeNonAlignedChunk(iChunkWriter, tsFileIOWriter, pages, resource.isSeq()); tsFileIOWriter.endChunkGroup(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java index e67ba538da93..3afaca0cf1b3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java @@ -178,7 +178,8 @@ public void testFileOpenSelection() ITimeIndex newTimeIndex = IoTDBDescriptor.getInstance().getConfig().getTimeIndexLevel().getTimeIndex(); for (IDeviceID device : devices) { - newTimeIndex.updateStartTime(device, timeIndex.getStartTime(device)); + //noinspection OptionalGetWithoutIsPresent + newTimeIndex.updateStartTime(device, timeIndex.getStartTime(device).get()); } secondTsFileResource.setTimeIndex(newTimeIndex); @@ -229,7 +230,8 @@ public void testFileOpenSelectionFromCompaction() ITimeIndex newTimeIndex = IoTDBDescriptor.getInstance().getConfig().getTimeIndexLevel().getTimeIndex(); for (IDeviceID device : devices) { - newTimeIndex.updateStartTime(device, timeIndex.getStartTime(device)); + //noinspection OptionalGetWithoutIsPresent + newTimeIndex.updateStartTime(device, timeIndex.getStartTime(device).get()); } secondTsFileResource.setTimeIndex(newTimeIndex); List newUnseqResources = new ArrayList<>(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java index 05c9b11f5b4e..6c3e692212a8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionCheckerUtils.java @@ -88,6 +88,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class CompactionCheckerUtils { public static void putOnePageChunks( @@ -353,11 +354,17 @@ private static void checkTsFileResource( long[] statistics = deviceCountEntry.getValue(); long startTime = Long.MAX_VALUE; for (TsFileResource mergedFile : mergedFiles) { - startTime = Math.min(startTime, mergedFile.getStartTime(device)); + if (mergedFile.definitelyNotContains(device)) { + continue; + } + startTime = Math.min(startTime, mergedFile.getStartTime(device).get()); } long endTime = Long.MIN_VALUE; for (TsFileResource mergedFile : mergedFiles) { - endTime = Math.max(endTime, mergedFile.getEndTime(device)); + if (mergedFile.definitelyNotContains(device)) { + continue; + } + endTime = Math.max(endTime, mergedFile.getEndTime(device).get()); } assertEquals(statistics[0], startTime); assertEquals(statistics[1], endTime); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java index 0393067f109f..6c41cb4ecb05 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java @@ -70,6 +70,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class TsFileProcessorTest { private TsFileProcessor processor; @@ -866,7 +867,7 @@ private void closeTsFileProcessor(TsFileProcessor unsealedTsFileProcessor) TsFileResource resource = unsealedTsFileProcessor.getTsFileResource(); synchronized (resource) { for (IDeviceID deviceId : resource.getDevices()) { - resource.updateEndTime(deviceId, resource.getStartTime(deviceId)); + resource.updateEndTime(deviceId, resource.getStartTime(deviceId).get()); } try { resource.close(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java index ba9fe65f6c15..fcf478f899bb 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java @@ -33,6 +33,7 @@ import org.apache.commons.io.FileUtils; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -56,6 +57,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class TsFileResourceTest { private final File file = new File( @@ -113,11 +115,13 @@ public void testDegradeAndFileTimeIndex() { Assert.assertEquals(deviceToIndex.keySet(), tsFileResource.getDevices()); for (int i = 0; i < DEVICE_NUM; i++) { Assert.assertEquals( - tsFileResource.getStartTime(IDeviceID.Factory.DEFAULT_FACTORY.create("root.sg1.d" + i)), - 0); + 0, + ((long) + tsFileResource.getStartTime(Factory.DEFAULT_FACTORY.create("root.sg1.d" + i)).get())); Assert.assertEquals( - tsFileResource.getEndTime(IDeviceID.Factory.DEFAULT_FACTORY.create("root.sg1.d" + i)), - DEVICE_NUM); + DEVICE_NUM, + ((long) + tsFileResource.getEndTime(Factory.DEFAULT_FACTORY.create("root.sg1.d" + i)).get())); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java index e4f82b5b367e..248269062837 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java @@ -89,6 +89,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class WALRecoverManagerTest { private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); private static final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); @@ -327,10 +328,10 @@ private void recoverAndCheck() throws Exception { assertEquals(15, chunk.getChunkStatistic().getEndTime()); reader.close(); // check .resource file in memory - assertEquals(1, tsFileWithWALResource.getStartTime(DEVICE1_NAME)); - assertEquals(2, tsFileWithWALResource.getEndTime(DEVICE1_NAME)); - assertEquals(3, tsFileWithWALResource.getStartTime(DEVICE2_NAME)); - assertEquals(15, tsFileWithWALResource.getEndTime(DEVICE2_NAME)); + assertEquals(1, ((long) tsFileWithWALResource.getStartTime(DEVICE1_NAME).get())); + assertEquals(2, ((long) tsFileWithWALResource.getEndTime(DEVICE1_NAME).get())); + assertEquals(3, ((long) tsFileWithWALResource.getStartTime(DEVICE2_NAME).get())); + assertEquals(15, ((long) tsFileWithWALResource.getEndTime(DEVICE2_NAME).get())); // check file existence assertTrue(new File(FILE_WITH_WAL_NAME).exists()); assertTrue(new File(FILE_WITH_WAL_NAME.concat(TsFileResource.RESOURCE_SUFFIX)).exists()); @@ -352,10 +353,10 @@ private void recoverAndCheck() throws Exception { assertEquals(3, chunk.getChunkStatistic().getEndTime()); reader.close(); // check .resource file in memory - assertEquals(1, tsFileWithoutWALResource.getStartTime(DEVICE1_NAME)); - assertEquals(2, tsFileWithoutWALResource.getEndTime(DEVICE1_NAME)); - assertEquals(3, tsFileWithoutWALResource.getStartTime(DEVICE2_NAME)); - assertEquals(3, tsFileWithoutWALResource.getEndTime(DEVICE2_NAME)); + assertEquals(1, ((long) tsFileWithoutWALResource.getStartTime(DEVICE1_NAME).get())); + assertEquals(2, ((long) tsFileWithoutWALResource.getEndTime(DEVICE1_NAME).get())); + assertEquals(3, ((long) tsFileWithoutWALResource.getStartTime(DEVICE2_NAME).get())); + assertEquals(3, ((long) tsFileWithoutWALResource.getEndTime(DEVICE2_NAME).get())); // check file existence assertTrue(new File(FILE_WITHOUT_WAL_NAME).exists()); assertTrue(new File(FILE_WITHOUT_WAL_NAME.concat(TsFileResource.RESOURCE_SUFFIX)).exists()); @@ -437,8 +438,8 @@ private void recoverFromOldWalAndCheck(boolean withBrokenTsFile) throws Exceptio assertEquals(15, chunk.getChunkStatistic().getEndTime()); reader.close(); // check .resource file in memory - assertEquals(4, tsFileWithWALResource.getStartTime(DEVICE2_NAME)); - assertEquals(15, tsFileWithWALResource.getEndTime(DEVICE2_NAME)); + assertEquals(4, ((long) tsFileWithWALResource.getStartTime(DEVICE2_NAME).get())); + assertEquals(15, ((long) tsFileWithWALResource.getEndTime(DEVICE2_NAME).get())); // check file existence assertTrue(fileWithWAL.exists()); assertTrue(new File(fileWithWAL.getPath().concat(TsFileResource.RESOURCE_SUFFIX)).exists()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformerTest.java index f1de11549e23..99563ec91761 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformerTest.java @@ -55,6 +55,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class SealedTsFileRecoverPerformerTest { private static final String SG_NAME = "root.recover_sg"; private static final IDeviceID DEVICE1_NAME = @@ -113,10 +114,10 @@ public void testCompleteFileWithResource() throws Exception { assertNotNull(chunkMetadataList); reader.close(); // check .resource file in memory - assertEquals(1, tsFileResource.getStartTime(DEVICE1_NAME)); - assertEquals(2, tsFileResource.getEndTime(DEVICE1_NAME)); - assertEquals(3, tsFileResource.getStartTime(DEVICE2_NAME)); - assertEquals(4, tsFileResource.getEndTime(DEVICE2_NAME)); + assertEquals(1, ((long) tsFileResource.getStartTime(DEVICE1_NAME).get())); + assertEquals(2, ((long) tsFileResource.getEndTime(DEVICE1_NAME).get())); + assertEquals(3, ((long) tsFileResource.getStartTime(DEVICE2_NAME).get())); + assertEquals(4, ((long) tsFileResource.getEndTime(DEVICE2_NAME).get())); // check file existence assertTrue(file.exists()); assertTrue(new File(FILE_NAME.concat(TsFileResource.RESOURCE_SUFFIX)).exists()); @@ -155,10 +156,10 @@ public void testCompleteFileWithBrokenResource() throws Exception { assertNotNull(chunkMetadataList); reader.close(); // check .resource file in memory - assertEquals(1, tsFileResource.getStartTime(DEVICE1_NAME)); - assertEquals(2, tsFileResource.getEndTime(DEVICE1_NAME)); - assertEquals(3, tsFileResource.getStartTime(DEVICE2_NAME)); - assertEquals(4, tsFileResource.getEndTime(DEVICE2_NAME)); + assertEquals(1, ((long) tsFileResource.getStartTime(DEVICE1_NAME).get())); + assertEquals(2, ((long) tsFileResource.getEndTime(DEVICE1_NAME).get())); + assertEquals(3, ((long) tsFileResource.getStartTime(DEVICE2_NAME).get())); + assertEquals(4, ((long) tsFileResource.getEndTime(DEVICE2_NAME).get())); // check file existence assertTrue(file.exists()); assertTrue(new File(FILE_NAME.concat(TsFileResource.RESOURCE_SUFFIX)).exists()); @@ -192,11 +193,11 @@ public void testCompleteFileWithoutResource() throws Exception { assertNotNull(chunkMetadataList); reader.close(); // check .resource file in memory - assertEquals(1, tsFileResource.getStartTime(DEVICE1_NAME)); + assertEquals(1, ((long) tsFileResource.getStartTime(DEVICE1_NAME).get())); - assertEquals(2, tsFileResource.getEndTime(DEVICE1_NAME)); - assertEquals(3, tsFileResource.getStartTime(DEVICE2_NAME)); - assertEquals(4, tsFileResource.getEndTime(DEVICE2_NAME)); + assertEquals(2, ((long) tsFileResource.getEndTime(DEVICE1_NAME).get())); + assertEquals(3, ((long) tsFileResource.getStartTime(DEVICE2_NAME).get())); + assertEquals(4, ((long) tsFileResource.getEndTime(DEVICE2_NAME).get())); // check file existence assertTrue(file.exists()); assertTrue(new File(FILE_NAME.concat(TsFileResource.RESOURCE_SUFFIX)).exists()); @@ -262,10 +263,10 @@ public void testCrashedFile() throws Exception { assertEquals(3, chunk.getChunkStatistic().getEndTime()); reader.close(); // check .resource file in memory - assertEquals(1, tsFileResource.getStartTime(DEVICE1_NAME)); - assertEquals(2, tsFileResource.getEndTime(DEVICE1_NAME)); - assertEquals(3, tsFileResource.getStartTime(DEVICE2_NAME)); - assertEquals(3, tsFileResource.getEndTime(DEVICE2_NAME)); + assertEquals(1, ((long) tsFileResource.getStartTime(DEVICE1_NAME).get())); + assertEquals(2, ((long) tsFileResource.getEndTime(DEVICE1_NAME).get())); + assertEquals(3, ((long) tsFileResource.getStartTime(DEVICE2_NAME).get())); + assertEquals(3, ((long) tsFileResource.getEndTime(DEVICE2_NAME).get())); // check file existence assertTrue(file.exists()); assertTrue(new File(FILE_NAME.concat(TsFileResource.RESOURCE_SUFFIX)).exists()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java index 391d27dcd32f..e191d905453c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java @@ -85,6 +85,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +@SuppressWarnings("OptionalGetWithoutIsPresent") public class UnsealedTsFileRecoverPerformerTest { private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); @@ -149,7 +150,7 @@ public void testRedoInsertPlan() throws Exception { recoverPerformer.startRecovery(); assertTrue(recoverPerformer.hasCrashed()); assertTrue(recoverPerformer.canWrite()); - assertEquals(3, tsFileResource.getEndTime(DEVICE2_NAME)); + assertEquals(3, ((long) tsFileResource.getEndTime(DEVICE2_NAME).get())); recoverPerformer.redoLog(walEntry); @@ -173,10 +174,10 @@ public void testRedoInsertPlan() throws Exception { assertEquals(4, chunk.getChunkStatistic().getEndTime()); reader.close(); // check .resource file in memory - assertEquals(1, tsFileResource.getStartTime(DEVICE1_NAME)); - assertEquals(2, tsFileResource.getEndTime(DEVICE1_NAME)); - assertEquals(3, tsFileResource.getStartTime(DEVICE2_NAME)); - assertEquals(4, tsFileResource.getEndTime(DEVICE2_NAME)); + assertEquals(1, ((long) tsFileResource.getStartTime(DEVICE1_NAME).get())); + assertEquals(2, ((long) tsFileResource.getEndTime(DEVICE1_NAME).get())); + assertEquals(3, ((long) tsFileResource.getStartTime(DEVICE2_NAME).get())); + assertEquals(4, ((long) tsFileResource.getEndTime(DEVICE2_NAME).get())); // check file existence assertTrue(file.exists()); assertTrue(new File(FILE_NAME.concat(TsFileResource.RESOURCE_SUFFIX)).exists()); @@ -245,7 +246,7 @@ public void testRedoDeletePlan() throws Exception { recoverPerformer.startRecovery(); assertTrue(recoverPerformer.hasCrashed()); assertTrue(recoverPerformer.canWrite()); - assertEquals(3, tsFileResource.getEndTime(DEVICE2_NAME)); + assertEquals(3, ((long) tsFileResource.getEndTime(DEVICE2_NAME).get())); recoverPerformer.redoLog(walEntry); @@ -267,10 +268,10 @@ public void testRedoDeletePlan() throws Exception { assertEquals(3, chunk.getChunkStatistic().getEndTime()); reader.close(); // check .resource file in memory - assertEquals(1, tsFileResource.getStartTime(DEVICE1_NAME)); - assertEquals(2, tsFileResource.getEndTime(DEVICE1_NAME)); - assertEquals(3, tsFileResource.getStartTime(DEVICE2_NAME)); - assertEquals(3, tsFileResource.getEndTime(DEVICE2_NAME)); + assertEquals(1, ((long) tsFileResource.getStartTime(DEVICE1_NAME).get())); + assertEquals(2, ((long) tsFileResource.getEndTime(DEVICE1_NAME).get())); + assertEquals(3, ((long) tsFileResource.getStartTime(DEVICE2_NAME).get())); + assertEquals(3, ((long) tsFileResource.getEndTime(DEVICE2_NAME).get())); // check file existence assertTrue(file.exists()); assertTrue(new File(FILE_NAME.concat(TsFileResource.RESOURCE_SUFFIX)).exists()); diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 0f5d5d2060ba..0e742be14f88 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1075,7 +1075,7 @@ timestamp_precision=ms # When the timestamp precision check is enabled, the timestamps those are over 13 digits for ms precision, or over 16 digits for us precision are not allowed to be inserted. # For all precisions, ms, us and ns, the timestamps cannot exceed the range of [-9223372036854775808, 9223372036854775807], regardless of whether the check is enabled or not. -# effectiveMode: first_start +# effectiveMode: hot_reload # Datatype: Boolean timestamp_precision_check_enabled=true