Skip to content

Commit a899c48

Browse files
shuwenweishuwenwei
authored
Optimize memtable region scan (#16883)
1 parent 13b0582 commit a899c48

File tree

7 files changed

+572
-87
lines changed

7 files changed

+572
-87
lines changed

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2435,7 +2435,8 @@ private List<IFileScanHandle> getFileHandleListForQuery(
24352435
} else {
24362436
tsFileResource
24372437
.getProcessor()
2438-
.queryForSeriesRegionScanWithoutLock(partialPaths, context, fileScanHandles);
2438+
.queryForSeriesRegionScanWithoutLock(
2439+
partialPaths, context, fileScanHandles, globalTimeFilter);
24392440
}
24402441
}
24412442
return fileScanHandles;
@@ -2512,7 +2513,8 @@ private List<IFileScanHandle> getFileHandleListForQuery(
25122513
} else {
25132514
tsFileResource
25142515
.getProcessor()
2515-
.queryForDeviceRegionScanWithoutLock(devicePathsToContext, context, fileScanHandles);
2516+
.queryForDeviceRegionScanWithoutLock(
2517+
devicePathsToContext, context, fileScanHandles, globalTimeFilter);
25162518
}
25172519
}
25182520
return fileScanHandles;

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java‎

Lines changed: 58 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070
import java.util.Map;
7171
import java.util.Map.Entry;
7272
import java.util.Objects;
73+
import java.util.Optional;
7374
import java.util.concurrent.atomic.AtomicLong;
7475
import java.util.stream.Collectors;
7576

@@ -488,7 +489,8 @@ public void queryForSeriesRegionScan(
488489
long ttlLowerBound,
489490
Map<String, List<IChunkMetadata>> chunkMetaDataMap,
490491
Map<String, List<IChunkHandle>> memChunkHandleMap,
491-
List<Pair<ModEntry, IMemTable>> modsToMemTabled) {
492+
List<Pair<ModEntry, IMemTable>> modsToMemTabled,
493+
Filter globalTimeFilter) {
492494

493495
IDeviceID deviceID = fullPath.getDeviceId();
494496
if (fullPath instanceof NonAlignedFullPath) {
@@ -506,7 +508,12 @@ public void queryForSeriesRegionScan(
506508
fullPath.getDeviceId(), measurementId, this, modsToMemTabled, ttlLowerBound);
507509
}
508510
getMemChunkHandleFromMemTable(
509-
deviceID, measurementId, chunkMetaDataMap, memChunkHandleMap, deletionList);
511+
deviceID,
512+
measurementId,
513+
chunkMetaDataMap,
514+
memChunkHandleMap,
515+
deletionList,
516+
globalTimeFilter);
510517
} else {
511518
// check If MemTable Contains this path
512519
if (!memTableMap.containsKey(deviceID)) {
@@ -528,7 +535,8 @@ public void queryForSeriesRegionScan(
528535
((AlignedFullPath) fullPath).getSchemaList(),
529536
chunkMetaDataMap,
530537
memChunkHandleMap,
531-
deletionList);
538+
deletionList,
539+
globalTimeFilter);
532540
}
533541
}
534542

@@ -539,7 +547,8 @@ public void queryForDeviceRegionScan(
539547
long ttlLowerBound,
540548
Map<String, List<IChunkMetadata>> chunkMetadataMap,
541549
Map<String, List<IChunkHandle>> memChunkHandleMap,
542-
List<Pair<ModEntry, IMemTable>> modsToMemTabled) {
550+
List<Pair<ModEntry, IMemTable>> modsToMemTabled,
551+
Filter globalTimeFilter) {
543552

544553
Map<IDeviceID, IWritableMemChunkGroup> memTableMap = getMemTableMap();
545554

@@ -556,15 +565,17 @@ public void queryForDeviceRegionScan(
556565
chunkMetadataMap,
557566
memChunkHandleMap,
558567
ttlLowerBound,
559-
modsToMemTabled);
568+
modsToMemTabled,
569+
globalTimeFilter);
560570
} else {
561571
getMemChunkHandleFromMemTable(
562572
deviceID,
563573
(WritableMemChunkGroup) writableMemChunkGroup,
564574
chunkMetadataMap,
565575
memChunkHandleMap,
566576
ttlLowerBound,
567-
modsToMemTabled);
577+
modsToMemTabled,
578+
globalTimeFilter);
568579
}
569580
}
570581

@@ -573,32 +584,39 @@ private void getMemChunkHandleFromMemTable(
573584
String measurementId,
574585
Map<String, List<IChunkMetadata>> chunkMetadataMap,
575586
Map<String, List<IChunkHandle>> memChunkHandleMap,
576-
List<TimeRange> deletionList) {
587+
List<TimeRange> deletionList,
588+
Filter globalTimeFilter) {
577589

578590
WritableMemChunk memChunk =
579591
(WritableMemChunk) memTableMap.get(deviceID).getMemChunkMap().get(measurementId);
580592

581-
long[] timestamps = memChunk.getFilteredTimestamp(deletionList);
593+
if (memChunk == null) {
594+
return;
595+
}
596+
Optional<Long> anySatisfiedTimestamp =
597+
memChunk.getAnySatisfiedTimestamp(deletionList, globalTimeFilter);
598+
if (!anySatisfiedTimestamp.isPresent()) {
599+
return;
600+
}
601+
long satisfiedTimestamp = anySatisfiedTimestamp.get();
582602

583603
chunkMetadataMap
584604
.computeIfAbsent(measurementId, k -> new ArrayList<>())
585605
.add(
586-
buildChunkMetaDataForMemoryChunk(
587-
measurementId,
588-
timestamps[0],
589-
timestamps[timestamps.length - 1],
590-
Collections.emptyList()));
606+
buildFakeChunkMetaDataForFakeMemoryChunk(
607+
measurementId, satisfiedTimestamp, satisfiedTimestamp, Collections.emptyList()));
591608
memChunkHandleMap
592609
.computeIfAbsent(measurementId, k -> new ArrayList<>())
593-
.add(new MemChunkHandleImpl(deviceID, measurementId, timestamps));
610+
.add(new MemChunkHandleImpl(deviceID, measurementId, new long[] {satisfiedTimestamp}));
594611
}
595612

596613
private void getMemAlignedChunkHandleFromMemTable(
597614
IDeviceID deviceID,
598615
List<IMeasurementSchema> schemaList,
599616
Map<String, List<IChunkMetadata>> chunkMetadataList,
600617
Map<String, List<IChunkHandle>> memChunkHandleMap,
601-
List<List<TimeRange>> deletionList) {
618+
List<List<TimeRange>> deletionList,
619+
Filter globalTimeFilter) {
602620

603621
AlignedWritableMemChunk alignedMemChunk =
604622
((AlignedWritableMemChunkGroup) memTableMap.get(deviceID)).getAlignedMemChunk();
@@ -615,7 +633,11 @@ private void getMemAlignedChunkHandleFromMemTable(
615633
}
616634

617635
List<BitMap> bitMaps = new ArrayList<>();
618-
long[] timestamps = alignedMemChunk.getFilteredTimestamp(deletionList, bitMaps, true);
636+
long[] timestamps =
637+
alignedMemChunk.getAnySatisfiedTimestamp(deletionList, bitMaps, true, globalTimeFilter);
638+
if (timestamps.length == 0) {
639+
return;
640+
}
619641

620642
buildAlignedMemChunkHandle(
621643
deviceID,
@@ -633,7 +655,8 @@ private void getMemAlignedChunkHandleFromMemTable(
633655
Map<String, List<IChunkMetadata>> chunkMetadataList,
634656
Map<String, List<IChunkHandle>> memChunkHandleMap,
635657
long ttlLowerBound,
636-
List<Pair<ModEntry, IMemTable>> modsToMemTabled) {
658+
List<Pair<ModEntry, IMemTable>> modsToMemTabled,
659+
Filter globalTimeFilter) {
637660

638661
AlignedWritableMemChunk memChunk = writableMemChunkGroup.getAlignedMemChunk();
639662
List<IMeasurementSchema> schemaList = memChunk.getSchemaList();
@@ -648,7 +671,11 @@ private void getMemAlignedChunkHandleFromMemTable(
648671
}
649672

650673
List<BitMap> bitMaps = new ArrayList<>();
651-
long[] timestamps = memChunk.getFilteredTimestamp(deletionList, bitMaps, true);
674+
long[] timestamps =
675+
memChunk.getAnySatisfiedTimestamp(deletionList, bitMaps, true, globalTimeFilter);
676+
if (timestamps.length == 0) {
677+
return;
678+
}
652679
buildAlignedMemChunkHandle(
653680
deviceID,
654681
timestamps,
@@ -665,7 +692,8 @@ private void getMemChunkHandleFromMemTable(
665692
Map<String, List<IChunkMetadata>> chunkMetadataMap,
666693
Map<String, List<IChunkHandle>> memChunkHandleMap,
667694
long ttlLowerBound,
668-
List<Pair<ModEntry, IMemTable>> modsToMemTabled) {
695+
List<Pair<ModEntry, IMemTable>> modsToMemTabled,
696+
Filter globalTimeFilter) {
669697

670698
for (Entry<String, IWritableMemChunk> entry :
671699
writableMemChunkGroup.getMemChunkMap().entrySet()) {
@@ -679,18 +707,20 @@ private void getMemChunkHandleFromMemTable(
679707
ModificationUtils.constructDeletionList(
680708
deviceID, measurementId, this, modsToMemTabled, ttlLowerBound);
681709
}
682-
long[] timestamps = writableMemChunk.getFilteredTimestamp(deletionList);
710+
Optional<Long> anySatisfiedTimestamp =
711+
writableMemChunk.getAnySatisfiedTimestamp(deletionList, globalTimeFilter);
712+
if (!anySatisfiedTimestamp.isPresent()) {
713+
return;
714+
}
715+
long satisfiedTimestamp = anySatisfiedTimestamp.get();
683716
chunkMetadataMap
684717
.computeIfAbsent(measurementId, k -> new ArrayList<>())
685718
.add(
686-
buildChunkMetaDataForMemoryChunk(
687-
measurementId,
688-
timestamps[0],
689-
timestamps[timestamps.length - 1],
690-
Collections.emptyList()));
719+
buildFakeChunkMetaDataForFakeMemoryChunk(
720+
measurementId, satisfiedTimestamp, satisfiedTimestamp, Collections.emptyList()));
691721
memChunkHandleMap
692722
.computeIfAbsent(measurementId, k -> new ArrayList<>())
693-
.add(new MemChunkHandleImpl(deviceID, measurementId, timestamps));
723+
.add(new MemChunkHandleImpl(deviceID, measurementId, new long[] {satisfiedTimestamp}));
694724
}
695725
}
696726

@@ -714,7 +744,7 @@ private void buildAlignedMemChunkHandle(
714744
chunkMetadataList
715745
.computeIfAbsent(measurement, k -> new ArrayList<>())
716746
.add(
717-
buildChunkMetaDataForMemoryChunk(
747+
buildFakeChunkMetaDataForFakeMemoryChunk(
718748
measurement, startEndTime[0], startEndTime[1], deletion));
719749
chunkHandleMap
720750
.computeIfAbsent(measurement, k -> new ArrayList<>())
@@ -745,7 +775,7 @@ private long[] calculateStartEndTime(long[] timestamps, List<BitMap> bitMaps, in
745775
return new long[] {startTime, endTime};
746776
}
747777

748-
private IChunkMetadata buildChunkMetaDataForMemoryChunk(
778+
private IChunkMetadata buildFakeChunkMetaDataForFakeMemoryChunk(
749779
String measurement, long startTime, long endTime, List<TimeRange> deletionList) {
750780
TimeStatistics timeStatistics = new TimeStatistics();
751781
timeStatistics.setStartTime(startTime);

0 commit comments

Comments
 (0)