Skip to content

Commit bd5cdec

Browse files
CaideyipiCaideyipi
committed
Fix additional timestamp boundary overflows
1 parent b19e831 commit bd5cdec

14 files changed

Lines changed: 124 additions & 52 deletions

File tree

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
3030
import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType;
3131
import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils;
32-
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
3332
import org.apache.iotdb.commons.utils.TestOnly;
3433
import org.apache.iotdb.commons.utils.TimePartitionUtils;
3534
import org.apache.iotdb.db.exception.DataTypeInconsistentException;
@@ -77,6 +76,7 @@
7776
import java.util.Map.Entry;
7877
import java.util.Objects;
7978

79+
import static org.apache.iotdb.db.utils.CommonUtils.getTTLLowerBound;
8080
import static org.apache.iotdb.db.utils.CommonUtils.isAlive;
8181

8282
public class InsertTabletNode extends InsertNode implements WALEntryValue {
@@ -1391,7 +1391,7 @@ protected int checkTTLInternal(TSStatus[] results, long ttl, boolean breakOnFirs
13911391
String.format(
13921392
"Insertion time [%s] is less than ttl time bound [%s]",
13931393
DateTimeUtils.convertLongToDate(currTime),
1394-
DateTimeUtils.convertLongToDate(CommonDateTimeUtils.currentTime() - ttl)));
1394+
DateTimeUtils.convertLongToDate(getTTLLowerBound(ttl))));
13951395
} else {
13961396
if (firstAliveLoc == -1) {
13971397
firstAliveLoc = loc;
@@ -1405,8 +1405,7 @@ protected int checkTTLInternal(TSStatus[] results, long ttl, boolean breakOnFirs
14051405

14061406
if (firstAliveLoc == -1) {
14071407
// no alive data
1408-
throw new OutOfTTLException(
1409-
getTimes()[getTimes().length - 1], (CommonDateTimeUtils.currentTime() - ttl));
1408+
throw new OutOfTTLException(getTimes()[getTimes().length - 1], getTTLLowerBound(ttl));
14101409
}
14111410
return firstAliveLoc;
14121411
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@
2222
import org.apache.iotdb.commons.path.AlignedFullPath;
2323
import org.apache.iotdb.commons.path.IFullPath;
2424
import org.apache.iotdb.commons.path.NonAlignedFullPath;
25-
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
2625
import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TreeNonAlignedDeviceViewAggregationScanOperator;
26+
import org.apache.iotdb.db.utils.CommonUtils;
2727

2828
import org.apache.tsfile.read.filter.basic.Filter;
2929
import org.apache.tsfile.read.filter.factory.FilterFactory;
@@ -156,12 +156,11 @@ public void setTTLForTableView(long ttlForTableView) {
156156
*/
157157
public static Filter updateFilterUsingTTL(Filter filter, long dataTTL) {
158158
if (dataTTL != Long.MAX_VALUE) {
159+
long ttlLowerBound = CommonUtils.getTTLLowerBound(dataTTL);
159160
if (filter != null) {
160-
filter =
161-
FilterFactory.and(
162-
filter, TimeFilterApi.gtEq(CommonDateTimeUtils.currentTime() - dataTTL));
161+
filter = FilterFactory.and(filter, TimeFilterApi.gtEq(ttlLowerBound));
163162
} else {
164-
filter = TimeFilterApi.gtEq(CommonDateTimeUtils.currentTime() - dataTTL);
163+
filter = TimeFilterApi.gtEq(ttlLowerBound);
165164
}
166165
}
167166
return filter;

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@
4545
import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics;
4646
import org.apache.iotdb.commons.service.metric.enums.Metric;
4747
import org.apache.iotdb.commons.service.metric.enums.Tag;
48-
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
4948
import org.apache.iotdb.commons.utils.RetryUtils;
5049
import org.apache.iotdb.commons.utils.TestOnly;
5150
import org.apache.iotdb.commons.utils.TimePartitionUtils;
@@ -1186,8 +1185,7 @@ public void insert(InsertRowNode insertRowNode) throws WriteProcessException {
11861185
// reject insertions that are out of ttl
11871186
long ttl = getTTL(insertRowNode);
11881187
if (!CommonUtils.isAlive(insertRowNode.getTime(), ttl)) {
1189-
throw new OutOfTTLException(
1190-
insertRowNode.getTime(), (CommonDateTimeUtils.currentTime() - ttl));
1188+
throw new OutOfTTLException(insertRowNode.getTime(), CommonUtils.getTTLLowerBound(ttl));
11911189
}
11921190
StorageEngine.blockInsertionIfReject();
11931191
long startTime = System.nanoTime();
@@ -4669,8 +4667,7 @@ public void insert(InsertRowsOfOneDeviceNode insertRowsOfOneDeviceNode)
46694667
String.format(
46704668
"Insertion time [%s] is less than ttl time bound [%s]",
46714669
DateTimeUtils.convertLongToDate(insertRowNode.getTime()),
4672-
DateTimeUtils.convertLongToDate(
4673-
CommonDateTimeUtils.currentTime() - ttl))));
4670+
DateTimeUtils.convertLongToDate(CommonUtils.getTTLLowerBound(ttl)))));
46744671
continue;
46754672
}
46764673
// init map
@@ -4785,8 +4782,7 @@ public void insert(InsertRowsNode insertRowsNode)
47854782
String.format(
47864783
"Insertion time [%s] is less than ttl time bound [%s]",
47874784
DateTimeUtils.convertLongToDate(insertRowNode.getTime()),
4788-
DateTimeUtils.convertLongToDate(
4789-
CommonDateTimeUtils.currentTime() - ttl))));
4785+
DateTimeUtils.convertLongToDate(CommonUtils.getTTLLowerBound(ttl)))));
47904786
insertRowNode.setFailedMeasurementNumber(insertRowNode.getMeasurements().length);
47914787
insertRowNode.setMeasurements(null);
47924788
continue;

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import org.apache.iotdb.commons.path.PatternTreeMap;
2727
import org.apache.iotdb.commons.schema.table.TsTable;
2828
import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
29-
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
3029
import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache;
3130
import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
3231
import org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileReader;
@@ -35,6 +34,7 @@
3534
import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
3635
import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
3736
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
37+
import org.apache.iotdb.db.utils.CommonUtils;
3838
import org.apache.iotdb.db.utils.EncryptDBUtils;
3939
import org.apache.iotdb.db.utils.ModificationUtils;
4040
import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory;
@@ -245,7 +245,7 @@ public Pair<IDeviceID, Boolean> nextDevice() throws IllegalPathException, IOExce
245245
timeLowerBoundForCurrentDevice =
246246
ttlForCurrentDevice == Long.MAX_VALUE
247247
? Long.MIN_VALUE
248-
: CommonDateTimeUtils.currentTime() - ttlForCurrentDevice;
248+
: CommonUtils.getTTLLowerBound(ttlForCurrentDevice);
249249
return currentDevice;
250250
}
251251

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
4646
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex;
4747
import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
48+
import org.apache.iotdb.db.utils.CommonUtils;
4849

4950
import org.apache.tsfile.exception.StopReadTsFileByInterruptException;
5051
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -360,7 +361,7 @@ public List<CrossCompactionTaskResource> selectCrossSpaceTask(
360361
boolean isInsertionTask) {
361362
// TODO: (xingtanzjr) need to confirm what this ttl is used for
362363
long startTime = System.currentTimeMillis();
363-
long ttlLowerBound = System.currentTimeMillis() - Long.MAX_VALUE;
364+
long ttlLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE);
364365
// we record the variable `candidate` here is used for selecting more than one
365366
// CrossCompactionTaskResources in this method.
366367
// Add read lock for candidate source files to avoid being deleted during the selection.

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import org.apache.iotdb.commons.path.AlignedPath;
3030
import org.apache.iotdb.commons.path.IFullPath;
3131
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
32-
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
3332
import org.apache.iotdb.commons.utils.TestOnly;
3433
import org.apache.iotdb.db.conf.IoTDBConfig;
3534
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -2224,7 +2223,7 @@ private long getQueryTimeLowerBound(IDeviceID deviceID) {
22242223
DataNodeTTLCache.getInstance()
22252224
.getTTLForTable(this.dataRegionName, deviceID.getTableName());
22262225
}
2227-
return ttl != Long.MAX_VALUE ? CommonDateTimeUtils.currentTime() - ttl : Long.MIN_VALUE;
2226+
return ttl != Long.MAX_VALUE ? CommonUtils.getTTLLowerBound(ttl) : Long.MIN_VALUE;
22282227
}
22292228

22302229
public long getTimeRangeId() {

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java‎

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import org.apache.iotdb.commons.path.IFullPath;
2626
import org.apache.iotdb.commons.path.PartialPath;
2727
import org.apache.iotdb.commons.pipe.datastructure.resource.PersistentResource;
28-
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
2928
import org.apache.iotdb.commons.utils.TestOnly;
3029
import org.apache.iotdb.db.conf.IoTDBConfig;
3130
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -1055,13 +1054,6 @@ public boolean isSatisfied(IDeviceID deviceId, Filter timeFilter, boolean isSeq,
10551054
return true;
10561055
}
10571056

1058-
/**
1059-
* @return whether the given time falls in ttl
1060-
*/
1061-
private boolean isAlive(long time, long dataTTL) {
1062-
return dataTTL == Long.MAX_VALUE || (CommonDateTimeUtils.currentTime() - time) <= dataTTL;
1063-
}
1064-
10651057
/**
10661058
* Check whether the given device may still alive or not. Return false if the device does not
10671059
* exist or out of dated.
@@ -1293,15 +1285,11 @@ public static int compareFileName(TsFileResource o1, TsFileResource o2) {
12931285
public static int checkAndCompareFileName(String fileName1, String fileName2) throws IOException {
12941286
TsFileNameGenerator.TsFileName tsFileName1 = TsFileNameGenerator.getTsFileName(fileName1);
12951287
TsFileNameGenerator.TsFileName tsFileName2 = TsFileNameGenerator.getTsFileName(fileName2);
1296-
long timeDiff = tsFileName1.getTime() - tsFileName2.getTime();
1297-
if (timeDiff != 0) {
1298-
return timeDiff < 0 ? -1 : 1;
1288+
int timeCompare = Long.compare(tsFileName1.getTime(), tsFileName2.getTime());
1289+
if (timeCompare != 0) {
1290+
return timeCompare;
12991291
}
1300-
long versionDiff = tsFileName1.getVersion() - tsFileName2.getVersion();
1301-
if (versionDiff != 0) {
1302-
return versionDiff < 0 ? -1 : 1;
1303-
}
1304-
return 0;
1292+
return Long.compare(tsFileName1.getVersion(), tsFileName2.getVersion());
13051293
}
13061294

13071295
/**
@@ -1322,11 +1310,7 @@ public static int compareFileCreationOrderByDesc(TsFileResource o1, TsFileResour
13221310
TsFileNameGenerator.getTsFileName(o1.getTsFile().getName());
13231311
TsFileNameGenerator.TsFileName n2 =
13241312
TsFileNameGenerator.getTsFileName(o2.getTsFile().getName());
1325-
long versionDiff = n2.getVersion() - n1.getVersion();
1326-
if (versionDiff != 0) {
1327-
return versionDiff < 0 ? -1 : 1;
1328-
}
1329-
return 0;
1313+
return Long.compare(n2.getVersion(), n1.getVersion());
13301314
} catch (IOException e) {
13311315
LOGGER.error(StorageEngineMessages.FILE_NAME_NOT_STANDARD, e);
13321316
throw new RuntimeException(e.getMessage());

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,11 @@
2121

2222
import org.apache.iotdb.commons.exception.IllegalPathException;
2323
import org.apache.iotdb.commons.path.PartialPath;
24-
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
2524
import org.apache.iotdb.commons.utils.TimePartitionUtils;
2625
import org.apache.iotdb.db.exception.load.PartitionViolationException;
2726
import org.apache.iotdb.db.i18n.StorageEngineMessages;
2827
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
28+
import org.apache.iotdb.db.utils.CommonUtils;
2929

3030
import com.google.common.util.concurrent.RateLimiter;
3131
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -438,7 +438,7 @@ public boolean definitelyNotContains(IDeviceID device) {
438438
@Override
439439
public boolean isDeviceAlive(IDeviceID device, long ttl) {
440440
return ttl == Long.MAX_VALUE
441-
|| endTimes[deviceToIndex.get(device)] >= CommonDateTimeUtils.currentTime() - ttl;
441+
|| endTimes[deviceToIndex.get(device)] >= CommonUtils.getTTLLowerBound(ttl);
442442
}
443443

444444
@Override

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,12 @@
2020
package org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex;
2121

2222
import org.apache.iotdb.commons.path.PartialPath;
23-
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
2423
import org.apache.iotdb.commons.utils.IOUtils;
2524
import org.apache.iotdb.commons.utils.TimePartitionUtils;
2625
import org.apache.iotdb.db.exception.load.PartitionViolationException;
2726
import org.apache.iotdb.db.i18n.StorageEngineMessages;
2827
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
28+
import org.apache.iotdb.db.utils.CommonUtils;
2929

3030
import com.google.common.util.concurrent.RateLimiter;
3131
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -278,7 +278,7 @@ public boolean definitelyNotContains(IDeviceID device) {
278278

279279
@Override
280280
public boolean isDeviceAlive(IDeviceID device, long ttl) {
281-
return ttl == Long.MAX_VALUE || endTime >= CommonDateTimeUtils.currentTime() - ttl;
281+
return ttl == Long.MAX_VALUE || endTime >= CommonUtils.getTTLLowerBound(ttl);
282282
}
283283

284284
@Override

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,8 +122,17 @@ private static void moveSeqResourceToUnsequenceDir(TsFileResource resource) thro
122122
tsFileName.getInnerCompactionCnt(),
123123
0);
124124
targetFile = new File(targetDir.getAbsolutePath() + File.separator + fileNameStr);
125+
if (!targetFile.exists()) {
126+
break;
127+
}
128+
if (tsFileName.getTime() == Long.MAX_VALUE) {
129+
throw new IOException(
130+
"Cannot repair "
131+
+ tsfile.getAbsolutePath()
132+
+ " because the target file already exists and the file timestamp is Long.MAX_VALUE");
133+
}
125134
tsFileName.setTime(tsFileName.getTime() + 1);
126-
} while (targetFile.exists());
135+
} while (true);
127136

128137
moveFile(tsfile, targetFile);
129138
moveFile(

0 commit comments

Comments
 (0)