Skip to content

Optimize memtable region scan #16883

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JackieTien97 merged 5 commits into from
Dec 9, 2025
Merged

Optimize memtable region scan #16883

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e60335c
optimize memtable region scan
shuwenwei Dec 9, 2025
adc719b
add ut
shuwenwei Dec 9, 2025
5c4c9cf
remove old test
shuwenwei Dec 9, 2025
ab0978d
spotless
shuwenwei Dec 9, 2025
92f4941
fix codecov
shuwenwei Dec 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions ...-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2253,7 +2253,7 @@
* lock if return true, the caller is responsible for unlocking all the already-acquiring lock
* in needToUnLockList
*/
private boolean tryGetFLushLock(

Check warning on line 2256 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

A "Brain Method" was detected. Refactor it to reduce at least one of the following metrics: LOC from 81 to 64, Complexity from 19 to 14, Nesting Level from 5 to 2, Number of Variables from 17 to 6. 


See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AZsBXCCCqBhanUnLmXXn&open=AZsBXCCCqBhanUnLmXXn&pullRequest=16883
long waitTimeInMs,
IDeviceID singleDeviceId,
Filter globalTimeFilter,
Expand Down Expand Up @@ -2435,7 +2435,8 @@
} else {
tsFileResource
.getProcessor()
.queryForSeriesRegionScanWithoutLock(partialPaths, context, fileScanHandles);
.queryForSeriesRegionScanWithoutLock(
partialPaths, context, fileScanHandles, globalTimeFilter);
}
}
return fileScanHandles;
Expand Down Expand Up @@ -2512,7 +2513,8 @@
} else {
tsFileResource
.getProcessor()
.queryForDeviceRegionScanWithoutLock(devicePathsToContext, context, fileScanHandles);
.queryForDeviceRegionScanWithoutLock(
devicePathsToContext, context, fileScanHandles, globalTimeFilter);
}
}
return fileScanHandles;
Expand Down Expand Up @@ -2982,7 +2984,7 @@
}
}

private void deleteDataInSealedFiles(Collection<TsFileResource> sealedTsFiles, ModEntry deletion)

Check warning on line 2987 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

A "Brain Method" was detected. Refactor it to reduce at least one of the following metrics: LOC from 130 to 64, Complexity from 24 to 14, Nesting Level from 5 to 2, Number of Variables from 23 to 6. 


See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AZsBXCCCqBhanUnLmXXo&open=AZsBXCCCqBhanUnLmXXo&pullRequest=16883
throws IOException {
Set<ModificationFile> involvedModificationFiles = new HashSet<>();
List<TsFileResource> deletedByMods = new ArrayList<>();
Expand Down
86 changes: 58 additions & 28 deletions ...src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -488,7 +489,8 @@ public void queryForSeriesRegionScan(
long ttlLowerBound,
Map<String, List<IChunkMetadata>> chunkMetaDataMap,
Map<String, List<IChunkHandle>> memChunkHandleMap,
List<Pair<ModEntry, IMemTable>> modsToMemTabled) {
List<Pair<ModEntry, IMemTable>> modsToMemTabled,
Filter globalTimeFilter) {

IDeviceID deviceID = fullPath.getDeviceId();
if (fullPath instanceof NonAlignedFullPath) {
Expand All @@ -506,7 +508,12 @@ public void queryForSeriesRegionScan(
fullPath.getDeviceId(), measurementId, this, modsToMemTabled, ttlLowerBound);
}
getMemChunkHandleFromMemTable(
deviceID, measurementId, chunkMetaDataMap, memChunkHandleMap, deletionList);
deviceID,
measurementId,
chunkMetaDataMap,
memChunkHandleMap,
deletionList,
globalTimeFilter);
} else {
// check If MemTable Contains this path
if (!memTableMap.containsKey(deviceID)) {
Expand All @@ -528,7 +535,8 @@ public void queryForSeriesRegionScan(
((AlignedFullPath) fullPath).getSchemaList(),
chunkMetaDataMap,
memChunkHandleMap,
deletionList);
deletionList,
globalTimeFilter);
}
}

Expand All @@ -539,7 +547,8 @@ public void queryForDeviceRegionScan(
long ttlLowerBound,
Map<String, List<IChunkMetadata>> chunkMetadataMap,
Map<String, List<IChunkHandle>> memChunkHandleMap,
List<Pair<ModEntry, IMemTable>> modsToMemTabled) {
List<Pair<ModEntry, IMemTable>> modsToMemTabled,
Filter globalTimeFilter) {

Map<IDeviceID, IWritableMemChunkGroup> memTableMap = getMemTableMap();

Expand All @@ -556,15 +565,17 @@ public void queryForDeviceRegionScan(
chunkMetadataMap,
memChunkHandleMap,
ttlLowerBound,
modsToMemTabled);
modsToMemTabled,
globalTimeFilter);
} else {
getMemChunkHandleFromMemTable(
deviceID,
(WritableMemChunkGroup) writableMemChunkGroup,
chunkMetadataMap,
memChunkHandleMap,
ttlLowerBound,
modsToMemTabled);
modsToMemTabled,
globalTimeFilter);
}
}

Expand All @@ -573,32 +584,39 @@ private void getMemChunkHandleFromMemTable(
String measurementId,
Map<String, List<IChunkMetadata>> chunkMetadataMap,
Map<String, List<IChunkHandle>> memChunkHandleMap,
List<TimeRange> deletionList) {
List<TimeRange> deletionList,
Filter globalTimeFilter) {

WritableMemChunk memChunk =
(WritableMemChunk) memTableMap.get(deviceID).getMemChunkMap().get(measurementId);

long[] timestamps = memChunk.getFilteredTimestamp(deletionList);
if (memChunk == null) {
return;
}
Optional<Long> anySatisfiedTimestamp =
memChunk.getAnySatisfiedTimestamp(deletionList, globalTimeFilter);
if (!anySatisfiedTimestamp.isPresent()) {
return;
}
long satisfiedTimestamp = anySatisfiedTimestamp.get();

chunkMetadataMap
.computeIfAbsent(measurementId, k -> new ArrayList<>())
.add(
buildChunkMetaDataForMemoryChunk(
measurementId,
timestamps[0],
timestamps[timestamps.length - 1],
Collections.emptyList()));
buildFakeChunkMetaDataForFakeMemoryChunk(
measurementId, satisfiedTimestamp, satisfiedTimestamp, Collections.emptyList()));
memChunkHandleMap
.computeIfAbsent(measurementId, k -> new ArrayList<>())
.add(new MemChunkHandleImpl(deviceID, measurementId, timestamps));
.add(new MemChunkHandleImpl(deviceID, measurementId, new long[] {satisfiedTimestamp}));
}

private void getMemAlignedChunkHandleFromMemTable(
IDeviceID deviceID,
List<IMeasurementSchema> schemaList,
Map<String, List<IChunkMetadata>> chunkMetadataList,
Map<String, List<IChunkHandle>> memChunkHandleMap,
List<List<TimeRange>> deletionList) {
List<List<TimeRange>> deletionList,
Filter globalTimeFilter) {

AlignedWritableMemChunk alignedMemChunk =
((AlignedWritableMemChunkGroup) memTableMap.get(deviceID)).getAlignedMemChunk();
Expand All @@ -615,7 +633,11 @@ private void getMemAlignedChunkHandleFromMemTable(
}

List<BitMap> bitMaps = new ArrayList<>();
long[] timestamps = alignedMemChunk.getFilteredTimestamp(deletionList, bitMaps, true);
long[] timestamps =
alignedMemChunk.getAnySatisfiedTimestamp(deletionList, bitMaps, true, globalTimeFilter);
if (timestamps.length == 0) {
return;
}

buildAlignedMemChunkHandle(
deviceID,
Expand All @@ -633,7 +655,8 @@ private void getMemAlignedChunkHandleFromMemTable(
Map<String, List<IChunkMetadata>> chunkMetadataList,
Map<String, List<IChunkHandle>> memChunkHandleMap,
long ttlLowerBound,
List<Pair<ModEntry, IMemTable>> modsToMemTabled) {
List<Pair<ModEntry, IMemTable>> modsToMemTabled,
Filter globalTimeFilter) {

AlignedWritableMemChunk memChunk = writableMemChunkGroup.getAlignedMemChunk();
List<IMeasurementSchema> schemaList = memChunk.getSchemaList();
Expand All @@ -648,7 +671,11 @@ private void getMemAlignedChunkHandleFromMemTable(
}

List<BitMap> bitMaps = new ArrayList<>();
long[] timestamps = memChunk.getFilteredTimestamp(deletionList, bitMaps, true);
long[] timestamps =
memChunk.getAnySatisfiedTimestamp(deletionList, bitMaps, true, globalTimeFilter);
if (timestamps.length == 0) {
return;
}
buildAlignedMemChunkHandle(
deviceID,
timestamps,
Expand All @@ -665,7 +692,8 @@ private void getMemChunkHandleFromMemTable(
Map<String, List<IChunkMetadata>> chunkMetadataMap,
Map<String, List<IChunkHandle>> memChunkHandleMap,
long ttlLowerBound,
List<Pair<ModEntry, IMemTable>> modsToMemTabled) {
List<Pair<ModEntry, IMemTable>> modsToMemTabled,
Filter globalTimeFilter) {

for (Entry<String, IWritableMemChunk> entry :
writableMemChunkGroup.getMemChunkMap().entrySet()) {
Expand All @@ -679,18 +707,20 @@ private void getMemChunkHandleFromMemTable(
ModificationUtils.constructDeletionList(
deviceID, measurementId, this, modsToMemTabled, ttlLowerBound);
}
long[] timestamps = writableMemChunk.getFilteredTimestamp(deletionList);
Optional<Long> anySatisfiedTimestamp =
writableMemChunk.getAnySatisfiedTimestamp(deletionList, globalTimeFilter);
if (!anySatisfiedTimestamp.isPresent()) {
return;
}
long satisfiedTimestamp = anySatisfiedTimestamp.get();
chunkMetadataMap
.computeIfAbsent(measurementId, k -> new ArrayList<>())
.add(
buildChunkMetaDataForMemoryChunk(
measurementId,
timestamps[0],
timestamps[timestamps.length - 1],
Collections.emptyList()));
buildFakeChunkMetaDataForFakeMemoryChunk(
measurementId, satisfiedTimestamp, satisfiedTimestamp, Collections.emptyList()));
memChunkHandleMap
.computeIfAbsent(measurementId, k -> new ArrayList<>())
.add(new MemChunkHandleImpl(deviceID, measurementId, timestamps));
.add(new MemChunkHandleImpl(deviceID, measurementId, new long[] {satisfiedTimestamp}));
}
}

Expand All @@ -714,7 +744,7 @@ private void buildAlignedMemChunkHandle(
chunkMetadataList
.computeIfAbsent(measurement, k -> new ArrayList<>())
.add(
buildChunkMetaDataForMemoryChunk(
buildFakeChunkMetaDataForFakeMemoryChunk(
measurement, startEndTime[0], startEndTime[1], deletion));
chunkHandleMap
.computeIfAbsent(measurement, k -> new ArrayList<>())
Expand Down Expand Up @@ -745,7 +775,7 @@ private long[] calculateStartEndTime(long[] timestamps, List<BitMap> bitMaps, in
return new long[] {startTime, endTime};
}

private IChunkMetadata buildChunkMetaDataForMemoryChunk(
private IChunkMetadata buildFakeChunkMetaDataForFakeMemoryChunk(
String measurement, long startTime, long endTime, List<TimeRange> deletionList) {
TimeStatistics timeStatistics = new TimeStatistics();
timeStatistics.setStartTime(startTime);
Expand Down
Loading
Loading