Skip to content

Commit 18f93be

Browse files
zhouliizhoulii
authored
[core] Improve partition display format for partitions table (#6633)
1 parent 5949538 commit 18f93be

File tree

5 files changed

+70
-32
lines changed

5 files changed

+70
-32
lines changed

‎paimon-core/src/main/java/org/apache/paimon/table/system/PartitionsTable.java‎

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,10 @@
4040
import org.apache.paimon.table.source.TableRead;
4141
import org.apache.paimon.types.BigIntType;
4242
import org.apache.paimon.types.DataField;
43+
import org.apache.paimon.types.DataType;
4344
import org.apache.paimon.types.DataTypes;
4445
import org.apache.paimon.types.RowType;
46+
import org.apache.paimon.utils.InternalRowUtils;
4547
import org.apache.paimon.utils.IteratorRecordReader;
4648
import org.apache.paimon.utils.ProjectedRow;
4749
import org.apache.paimon.utils.SerializationUtils;
@@ -58,6 +60,7 @@
5860
import java.util.Iterator;
5961
import java.util.List;
6062
import java.util.Map;
63+
import java.util.stream.Collectors;
6164

6265
import static org.apache.paimon.catalog.Identifier.SYSTEM_TABLE_SPLITTER;
6366

@@ -185,16 +188,28 @@ public RecordReader<InternalRow> createReader(Split split) throws IOException {
185188
List<PartitionEntry> partitions =
186189
fileStoreTable.newScan().withLevelFilter(level -> true).listPartitionEntries();
187190

188-
@SuppressWarnings("unchecked")
189-
CastExecutor<InternalRow, BinaryString> partitionCastExecutor =
190-
(CastExecutor<InternalRow, BinaryString>)
191-
CastExecutors.resolveToString(
192-
fileStoreTable.schema().logicalPartitionType());
191+
List<DataType> fieldTypes =
192+
fileStoreTable.schema().logicalPartitionType().getFieldTypes();
193+
InternalRow.FieldGetter[] fieldGetters =
194+
InternalRowUtils.createFieldGetters(fieldTypes);
195+
List<CastExecutor> castExecutors =
196+
fieldTypes.stream()
197+
.map(CastExecutors::resolveToString)
198+
.collect(Collectors.toList());
193199

194200
// sorted by partition
195201
Iterator<InternalRow> iterator =
196202
partitions.stream()
197-
.map(partitionEntry -> toRow(partitionEntry, partitionCastExecutor))
203+
.map(
204+
partitionEntry ->
205+
toRow(
206+
partitionEntry,
207+
fileStoreTable.partitionKeys(),
208+
castExecutors,
209+
fieldGetters,
210+
fileStoreTable
211+
.coreOptions()
212+
.partitionDefaultName()))
198213
.sorted(Comparator.comparing(row -> row.getString(0)))
199214
.iterator();
200215

@@ -211,9 +226,32 @@ public RecordReader<InternalRow> createReader(Split split) throws IOException {
211226

212227
private InternalRow toRow(
213228
PartitionEntry entry,
214-
CastExecutor<InternalRow, BinaryString> partitionCastExecutor) {
229+
List<String> partitionKeys,
230+
List<CastExecutor> castExecutors,
231+
InternalRow.FieldGetter[] fieldGetters,
232+
String defaultPartitionName) {
233+
StringBuilder partitionStringBuilder = new StringBuilder();
234+
235+
for (int i = 0; i < partitionKeys.size(); i++) {
236+
if (i > 0) {
237+
partitionStringBuilder.append("/");
238+
}
239+
Object partitionValue = fieldGetters[i].getFieldOrNull(entry.partition());
240+
String partitionValueString =
241+
partitionValue == null
242+
? defaultPartitionName
243+
: castExecutors
244+
.get(i)
245+
.cast(fieldGetters[i].getFieldOrNull(entry.partition()))
246+
.toString();
247+
partitionStringBuilder
248+
.append(partitionKeys.get(i))
249+
.append("=")
250+
.append(partitionValueString);
251+
}
252+
215253
return GenericRow.of(
216-
partitionCastExecutor.cast(entry.partition()),
254+
BinaryString.fromString(partitionStringBuilder.toString()),
217255
entry.recordCount(),
218256
entry.fileSizeInBytes(),
219257
entry.fileCount(),

‎paimon-core/src/test/java/org/apache/paimon/table/system/PartitionsTableTest.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -85,9 +85,9 @@ public void before() throws Exception {
8585
@Test
8686
public void testPartitionRecordCount() throws Exception {
8787
List<InternalRow> expectedRow = new ArrayList<>();
88-
expectedRow.add(GenericRow.of(BinaryString.fromString("{1}"), 2L));
89-
expectedRow.add(GenericRow.of(BinaryString.fromString("{2}"), 1L));
90-
expectedRow.add(GenericRow.of(BinaryString.fromString("{3}"), 1L));
88+
expectedRow.add(GenericRow.of(BinaryString.fromString("pt=1"), 2L));
89+
expectedRow.add(GenericRow.of(BinaryString.fromString("pt=2"), 1L));
90+
expectedRow.add(GenericRow.of(BinaryString.fromString("pt=3"), 1L));
9191

9292
// Only read partition and record count, record size may not stable.
9393
List<InternalRow> result = read(partitionsTable, new int[] {0, 1});
@@ -97,8 +97,8 @@ public void testPartitionRecordCount() throws Exception {
9797
@Test
9898
public void testPartitionTimeTravel() throws Exception {
9999
List<InternalRow> expectedRow = new ArrayList<>();
100-
expectedRow.add(GenericRow.of(BinaryString.fromString("{1}"), 1L));
101-
expectedRow.add(GenericRow.of(BinaryString.fromString("{3}"), 1L));
100+
expectedRow.add(GenericRow.of(BinaryString.fromString("pt=1"), 1L));
101+
expectedRow.add(GenericRow.of(BinaryString.fromString("pt=3"), 1L));
102102

103103
// Only read partition and record count, record size may not stable.
104104
List<InternalRow> result =
@@ -113,9 +113,9 @@ public void testPartitionTimeTravel() throws Exception {
113113
public void testPartitionValue() throws Exception {
114114
write(table, GenericRow.of(2, 1, 3), GenericRow.of(3, 1, 4));
115115
List<InternalRow> expectedRow = new ArrayList<>();
116-
expectedRow.add(GenericRow.of(BinaryString.fromString("{1}"), 4L, 3L));
117-
expectedRow.add(GenericRow.of(BinaryString.fromString("{2}"), 1L, 1L));
118-
expectedRow.add(GenericRow.of(BinaryString.fromString("{3}"), 1L, 1L));
116+
expectedRow.add(GenericRow.of(BinaryString.fromString("pt=1"), 4L, 3L));
117+
expectedRow.add(GenericRow.of(BinaryString.fromString("pt=2"), 1L, 1L));
118+
expectedRow.add(GenericRow.of(BinaryString.fromString("pt=3"), 1L, 1L));
119119

120120
List<InternalRow> result = read(partitionsTable, new int[] {0, 1, 3});
121121
assertThat(result).containsExactlyInAnyOrderElementsOf(expectedRow);

‎paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -746,15 +746,15 @@ public void testBranchPartitionsTable() throws Exception {
746746
sql("INSERT INTO t$branch_b1 VALUES (1, 4, 'S3'), (2, 2, 'S4')");
747747

748748
assertThat(collectResult("SELECT `partition`, record_count, file_count FROM t$partitions"))
749-
.containsExactlyInAnyOrder("+I[{1}, 3, 3]", "+I[{2}, 3, 2]");
749+
.containsExactlyInAnyOrder("+I[a=1, 3, 3]", "+I[a=2, 3, 2]");
750750
assertThat(
751751
collectResult(
752752
"SELECT `partition`, record_count, file_count FROM t$branch_b1$partitions"))
753-
.containsExactlyInAnyOrder("+I[{1}, 2, 2]", "+I[{2}, 3, 2]");
753+
.containsExactlyInAnyOrder("+I[a=1, 2, 2]", "+I[a=2, 3, 2]");
754754
assertThat(
755755
collectResult(
756756
"SELECT `partition`, record_count, file_count FROM t$partitions /*+ OPTIONS('branch'='b1') */"))
757-
.containsExactlyInAnyOrder("+I[{1}, 2, 2]", "+I[{2}, 3, 2]");
757+
.containsExactlyInAnyOrder("+I[a=1, 2, 2]", "+I[a=2, 3, 2]");
758758
}
759759

760760
@Test

‎paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1023,7 +1023,7 @@ public void testPartitionsTable() {
10231023
sql("INSERT INTO %s VALUES (3, 1, 4, 'S3'), (1, 2, 2, 'S4')", table);
10241024
List<Row> result =
10251025
sql("SELECT `partition`, record_count, file_count FROM %s$partitions", table);
1026-
assertThat(result).containsExactlyInAnyOrder(Row.of("{1}", 2L, 2L), Row.of("{2}", 3L, 2L));
1026+
assertThat(result).containsExactlyInAnyOrder(Row.of("p=1", 2L, 2L), Row.of("p=2", 3L, 2L));
10271027

10281028
// assert new files in partition
10291029
sql("INSERT INTO %s VALUES (3, 4, 4, 'S3'), (1, 3, 2, 'S4')", table);
@@ -1035,10 +1035,10 @@ public void testPartitionsTable() {
10351035
table));
10361036
assertThat(result)
10371037
.containsExactlyInAnyOrder(
1038-
Row.of("{1}", 3L, 3L),
1039-
Row.of("{2}", 4L, 3L),
1040-
Row.of("{3}", 1L, 1L),
1041-
Row.of("{4}", 1L, 1L));
1038+
Row.of("p=1", 3L, 3L),
1039+
Row.of("p=2", 4L, 3L),
1040+
Row.of("p=3", 1L, 1L),
1041+
Row.of("p=4", 1L, 1L));
10421042

10431043
// assert delete partitions
10441044
sql("ALTER TABLE %s DROP PARTITION (p = 2)", table);
@@ -1049,7 +1049,7 @@ public void testPartitionsTable() {
10491049
table));
10501050
assertThat(result)
10511051
.containsExactlyInAnyOrder(
1052-
Row.of("{1}", 3L, 3L), Row.of("{3}", 1L, 1L), Row.of("{4}", 1L, 1L));
1052+
Row.of("p=1", 3L, 3L), Row.of("p=3", 1L, 1L), Row.of("p=4", 1L, 1L));
10531053

10541054
// add new file to p 2
10551055
sql("INSERT INTO %s VALUES (1, 2, 2, 'S1')", table);
@@ -1060,10 +1060,10 @@ public void testPartitionsTable() {
10601060
table));
10611061
assertThat(result)
10621062
.containsExactlyInAnyOrder(
1063-
Row.of("{1}", 3L, 3L),
1064-
Row.of("{2}", 1L, 1L),
1065-
Row.of("{3}", 1L, 1L),
1066-
Row.of("{4}", 1L, 1L));
1063+
Row.of("p=1", 3L, 3L),
1064+
Row.of("p=2", 1L, 1L),
1065+
Row.of("p=3", 1L, 1L),
1066+
Row.of("p=4", 1L, 1L));
10671067
}
10681068

10691069
@Test

‎paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonSystemTableTest.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,8 @@ class PaimonSystemTableTest extends PaimonSparkTestBase {
5858
checkAnswer(spark.sql("select count(*) from `T$partitions`"), Row(5) :: Nil)
5959
checkAnswer(
6060
spark.sql("select partition from `T$partitions`"),
61-
Row("{2024-10-09, 01}") :: Row("{2024-10-09, 02}") :: Row("{2024-10-10, 01}") :: Row(
62-
"{2024-10-10, 12}") :: Row("{2024-10-10, 23}") :: Nil
61+
Row("dt=2024-10-09/hh=01") :: Row("dt=2024-10-09/hh=02") :: Row("dt=2024-10-10/hh=01") :: Row(
62+
"dt=2024-10-10/hh=12") :: Row("dt=2024-10-10/hh=23") :: Nil
6363
)
6464
}
6565

@@ -93,7 +93,7 @@ class PaimonSystemTableTest extends PaimonSparkTestBase {
9393

9494
checkAnswer(
9595
sql("SELECT partition FROM `T$partitions`"),
96-
Seq(Row("{2024-10-10, 1}"), Row("{null, 1}")))
96+
Seq(Row("p1=2024-10-10/p2=1"), Row("p1=__DEFAULT_PARTITION__/p2=1")))
9797

9898
checkAnswer(
9999
sql("SELECT partition, bucket FROM `T$buckets`"),

0 commit comments

Comments
 (0)