diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java index 81b527529..59d2da32b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java @@ -185,7 +185,7 @@ public static int write(Map map, ByteBuffer buffer) { if (entry.getKey() == null) { buffer.putInt(-1); } else { - bytes = entry.getKey().getBytes(); + bytes = entry.getKey().getBytes(TSFileConfig.STRING_CHARSET); buffer.putInt(bytes.length); buffer.put(bytes); length += bytes.length; @@ -194,7 +194,7 @@ public static int write(Map map, ByteBuffer buffer) { if (entry.getValue() == null) { buffer.putInt(-1); } else { - bytes = entry.getValue().getBytes(); + bytes = entry.getValue().getBytes(TSFileConfig.STRING_CHARSET); buffer.putInt(bytes.length); buffer.put(bytes); length += bytes.length; @@ -509,7 +509,7 @@ public static int sizeToWrite(String s) { if (s == null) { return INT_LEN; } - return INT_LEN + s.getBytes().length; + return INT_LEN + s.getBytes(TSFileConfig.STRING_CHARSET).length; } /** read a byte var from inputStream. */ @@ -1202,7 +1202,7 @@ public static void writeObject(Object value, DataOutputStream outputStream) { outputStream.write(NONE.ordinal()); } else { outputStream.write(STRING.ordinal()); - byte[] bytes = value.toString().getBytes(); + byte[] bytes = value.toString().getBytes(TSFileConfig.STRING_CHARSET); outputStream.writeInt(bytes.length); outputStream.write(bytes); } @@ -1238,7 +1238,7 @@ public static void writeObject(Object value, ByteBuffer byteBuffer) { byteBuffer.putInt(NONE.ordinal()); } else { byteBuffer.putInt(STRING.ordinal()); - byte[] bytes = value.toString().getBytes(); + byte[] bytes = value.toString().getBytes(TSFileConfig.STRING_CHARSET); byteBuffer.putInt(bytes.length); byteBuffer.put(bytes); } @@ -1271,7 +1271,7 @@ public static Object readObject(ByteBuffer buffer) { length = buffer.getInt(); bytes = new byte[length]; buffer.get(bytes); - return new String(bytes); + return new String(bytes, TSFileConfig.STRING_CHARSET); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 6093350e2..2bad6c953 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -748,13 +748,26 @@ private Object createValueColumnOfDataType(TSDataType dataType, int capacity) { /** Serialize {@link Tablet} */ public ByteBuffer serialize() throws IOException { - try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final int serializedSize = serializedSize(); + try (PublicBAOS byteArrayOutputStream = new PublicBAOS(serializedSize); DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { serialize(outputStream); return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); } } + /** Return the exact serialized byte size of this tablet. */ + public int serializedSize() { + int size = 0; + size = Math.addExact(size, ReadWriteIOUtils.sizeToWrite(insertTargetName)); + size = Math.addExact(size, Integer.BYTES); + size = Math.addExact(size, serializedSizeOfMeasurementSchemas()); + size = Math.addExact(size, serializedSizeOfTimes()); + size = Math.addExact(size, serializedSizeOfBitMaps()); + size = Math.addExact(size, serializedSizeOfValues()); + return size; + } + public void serialize(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(insertTargetName, stream); ReadWriteIOUtils.write(rowSize, stream); @@ -764,6 +777,104 @@ public void serialize(DataOutputStream stream) throws IOException { writeValues(stream); } + private int serializedSizeOfMeasurementSchemas() { + int size = Byte.BYTES; + if (schemas != null) { + size = Math.addExact(size, Integer.BYTES); + for (int i = 0; i < schemas.size(); i++) { + size = Math.addExact(size, Byte.BYTES); + final IMeasurementSchema schema = schemas.get(i); + if (schema != null) { + size = Math.addExact(size, schema.serializedSize()); + size = Math.addExact(size, Byte.BYTES); + } + } + } + return size; + } + + private int serializedSizeOfTimes() { + int size = Byte.BYTES; + if (timestamps != null) { + size = Math.addExact(size, Math.multiplyExact(Long.BYTES, rowSize)); + } + return size; + } + + private int serializedSizeOfBitMaps() { + int size = Byte.BYTES; + if (bitMaps != null) { + final int columnCount = schemas == null ? 0 : schemas.size(); + for (int i = 0; i < columnCount; i++) { + if (bitMaps[i] == null || bitMaps[i].isAllUnmarked(rowSize)) { + size = Math.addExact(size, Byte.BYTES); + } else { + size = Math.addExact(size, Byte.BYTES); + size = Math.addExact(size, Integer.BYTES); + size = Math.addExact(size, Integer.BYTES); + size = Math.addExact(size, BitMap.getSizeOfBytes(rowSize)); + } + } + } + return size; + } + + private int serializedSizeOfValues() { + int size = Byte.BYTES; + if (values != null) { + final int columnCount = schemas == null ? 0 : schemas.size(); + for (int i = 0; i < columnCount; i++) { + size = Math.addExact(size, serializedSizeOfColumn(schemas.get(i).getType(), values[i])); + } + } + return size; + } + + private int serializedSizeOfColumn(final TSDataType dataType, final Object column) { + int size = Byte.BYTES; + if (column == null) { + return size; + } + switch (dataType) { + case INT32: + return Math.addExact(size, Math.multiplyExact(Integer.BYTES, rowSize)); + case DATE: + return Math.addExact(size, Math.multiplyExact(Integer.BYTES, rowSize)); + case INT64: + case TIMESTAMP: + return Math.addExact(size, Math.multiplyExact(Long.BYTES, rowSize)); + case FLOAT: + return Math.addExact(size, Math.multiplyExact(Float.BYTES, rowSize)); + case DOUBLE: + return Math.addExact(size, Math.multiplyExact(Double.BYTES, rowSize)); + case BOOLEAN: + return Math.addExact(size, rowSize); + case TEXT: + case STRING: + case BLOB: + case OBJECT: + return Math.addExact(size, serializedSizeOfBinaryValues((Binary[]) column)); + default: + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", dataType)); + } + } + + private static int serializedSizeOfBinaryValues(final Binary[] binaryValues, final int rowSize) { + int size = 0; + for (int j = 0; j < rowSize; j++) { + size = Math.addExact(size, Byte.BYTES); + if (binaryValues[j] != null) { + size = Math.addExact(size, ReadWriteIOUtils.sizeToWrite(binaryValues[j])); + } + } + return size; + } + + private int serializedSizeOfBinaryValues(final Binary[] binaryValues) { + return serializedSizeOfBinaryValues(binaryValues, rowSize); + } + /** Serialize {@link MeasurementSchema}s */ private void writeMeasurementSchemas(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(BytesUtils.boolToByte(schemas != null), stream); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchema.java b/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchema.java index aaaf7d841..16dab7789 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchema.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/schema/MeasurementSchema.java @@ -319,15 +319,15 @@ public int serializeTo(OutputStream outputStream) throws IOException { @Override public int serializedSize() { int byteLen = 0; - byteLen += ReadWriteIOUtils.sizeToWrite(measurementName); - byteLen += 3 * Byte.BYTES; + byteLen = Math.addExact(byteLen, ReadWriteIOUtils.sizeToWrite(measurementName)); + byteLen = Math.addExact(byteLen, 3 * Byte.BYTES); if (props == null) { - byteLen += Integer.BYTES; + byteLen = Math.addExact(byteLen, Integer.BYTES); } else { - byteLen += Integer.BYTES; + byteLen = Math.addExact(byteLen, Integer.BYTES); for (Map.Entry entry : props.entrySet()) { - byteLen += ReadWriteIOUtils.sizeToWrite(entry.getKey()); - byteLen += ReadWriteIOUtils.sizeToWrite(entry.getValue()); + byteLen = Math.addExact(byteLen, ReadWriteIOUtils.sizeToWrite(entry.getKey())); + byteLen = Math.addExact(byteLen, ReadWriteIOUtils.sizeToWrite(entry.getValue())); } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java index a0cb9a0a0..3b0b20a24 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java @@ -184,6 +184,13 @@ public void mapSerdeTest() { Assert.assertNotNull(result); Assert.assertEquals(map, result); + ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_BUFFER_SIZE); + ReadWriteIOUtils.write(map, buffer); + buffer.flip(); + result = ReadWriteIOUtils.readMap(buffer); + Assert.assertNotNull(result); + Assert.assertEquals(map, result); + // 7. null map = null; byteArrayOutputStream = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java index 65911c18a..ab4bf377b 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java @@ -22,26 +22,34 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; import org.junit.Test; +import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; +import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -147,6 +155,7 @@ public void testSerializationAndDeSerializationWithMoreData() { measurementSchemas.add(new MeasurementSchema("s7", TSDataType.BLOB, TSEncoding.PLAIN)); measurementSchemas.add(new MeasurementSchema("s8", TSDataType.TIMESTAMP, TSEncoding.PLAIN)); measurementSchemas.add(new MeasurementSchema("s9", TSDataType.DATE, TSEncoding.PLAIN)); + measurementSchemas.add(new MeasurementSchema("s10", TSDataType.OBJECT, TSEncoding.PLAIN)); final int rowSize = 1000; final Tablet tablet = new Tablet(deviceId, measurementSchemas); @@ -170,6 +179,7 @@ public void testSerializationAndDeSerializationWithMoreData() { measurementSchemas.get(9).getMeasurementName(), i, LocalDate.of(2000 + i, i / 100 + 1, i / 100 + 1)); + tablet.addValue(i, 10, i % 2 == 0, (long) i, new byte[] {(byte) i, (byte) (i + 1)}); tablet.getBitMaps()[i % measurementSchemas.size()].mark(i); } @@ -186,9 +196,11 @@ public void testSerializationAndDeSerializationWithMoreData() { tablet.addValue(measurementSchemas.get(7).getMeasurementName(), rowSize - 1, null); tablet.addValue(measurementSchemas.get(8).getMeasurementName(), rowSize - 1, null); tablet.addValue(measurementSchemas.get(9).getMeasurementName(), rowSize - 1, null); + tablet.addValue(measurementSchemas.get(10).getMeasurementName(), rowSize - 1, null); try { final ByteBuffer byteBuffer = tablet.serialize(); + assertEquals(tablet.serializedSize(), byteBuffer.remaining()); final Tablet newTablet = Tablet.deserialize(byteBuffer); assertEquals(tablet, newTablet); for (int i = 0; i < rowSize; i++) { @@ -357,6 +369,390 @@ public void testSerializeDateColumnWithNullValue() throws IOException { Assert.assertTrue(deserializeTablet.isNull(1, 0)); } + private static final Set NON_SERIALIZABLE_DATA_TYPES = + EnumSet.of(TSDataType.VECTOR, TSDataType.UNKNOWN); + + private static final List SERIALIZABLE_DATA_TYPES = + Arrays.stream(TSDataType.values()) + .filter(dataType -> !NON_SERIALIZABLE_DATA_TYPES.contains(dataType)) + .collect(Collectors.toList()); + + private static final int[] ROW_COUNTS_FOR_SIZE_TEST = {0, 1, 7, 50}; + + @Test + public void testSerializedSizeMatchesActualSize() throws IOException { + // tree model: single column per type + for (final TSDataType type : SERIALIZABLE_DATA_TYPES) { + for (final int rowCount : ROW_COUNTS_FOR_SIZE_TEST) { + assertSerializedSizeMatches( + createAndFillTreeTablet( + "root.sg.d1", + columnNamesForType(type), + Arrays.asList(type), + rowCount, + 0, + false, + false), + "tree single column " + type + " rows=" + rowCount); + } + } + + // table model: single column per type + for (final TSDataType type : SERIALIZABLE_DATA_TYPES) { + for (final int rowCount : ROW_COUNTS_FOR_SIZE_TEST) { + assertSerializedSizeMatches( + createAndFillTableTablet( + "table1", + columnNamesForType(type), + Arrays.asList(type), + ColumnCategory.nCopy(ColumnCategory.FIELD, 1), + rowCount, + 0, + false, + false), + "table single column " + type + " rows=" + rowCount); + } + } + + // all types combined + final List treeTypes = SERIALIZABLE_DATA_TYPES; + final List tableTypes = new ArrayList<>(); + tableTypes.add(TSDataType.STRING); + tableTypes.addAll(treeTypes); + for (final int rowCount : new int[] {1, 25, 100}) { + assertSerializedSizeMatches( + createAndFillTreeTablet( + "root.sg.d1", buildColumnNames(treeTypes), treeTypes, rowCount, 100, false, false), + "tree all types combined rows=" + rowCount); + assertSerializedSizeMatches( + createAndFillTableTablet( + "table1", + buildColumnNames(tableTypes), + tableTypes, + buildTableColumnCategories(tableTypes.size()), + rowCount, + 100, + false, + false), + "table all types combined rows=" + rowCount); + } + + // variable-length binary columns + final List binaryTypes = + Arrays.asList(TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT); + assertSerializedSizeMatches( + createAndFillTreeTablet( + "root.sg.d1", buildColumnNames(binaryTypes), binaryTypes, 30, 0, false, true), + "tree variable binary lengths"); + assertSerializedSizeMatches( + createAndFillTableTablet( + "table1", + buildColumnNames(binaryTypes), + binaryTypes, + ColumnCategory.nCopy(ColumnCategory.FIELD, binaryTypes.size()), + 30, + 0, + false, + true), + "table variable binary lengths"); + + // sparse null values + assertSerializedSizeMatches( + createAndFillTreeTablet( + "root.sg.d1", buildColumnNames(treeTypes), treeTypes, 40, 0, true, false), + "tree with null values"); + assertSerializedSizeMatches( + createAndFillTableTablet( + "table1", + buildColumnNames(tableTypes), + tableTypes, + buildTableColumnCategories(tableTypes.size()), + 40, + 0, + true, + false), + "table with null values"); + + // table model with TAG columns + final List tagColumnNames = new ArrayList<>(); + final List tagDataTypes = new ArrayList<>(); + final List tagCategories = new ArrayList<>(); + tagColumnNames.add("region"); + tagDataTypes.add(TSDataType.STRING); + tagCategories.add(ColumnCategory.TAG); + for (int i = 0; i < SERIALIZABLE_DATA_TYPES.size(); i++) { + tagColumnNames.add("m" + i); + tagDataTypes.add(SERIALIZABLE_DATA_TYPES.get(i)); + tagCategories.add(ColumnCategory.FIELD); + } + assertSerializedSizeMatches( + createAndFillTableTablet( + "metrics_table", tagColumnNames, tagDataTypes, tagCategories, 20, 0, false, true), + "table model with TAG columns"); + + // mixed fixed-length and variable-length columns + final List mixedTypes = + Arrays.asList( + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.STRING, + TSDataType.BLOB, + TSDataType.DOUBLE); + assertSerializedSizeMatches( + createAndFillTreeTablet( + "root.sg.d1", buildColumnNames(mixedTypes), mixedTypes, 15, 5, false, true), + "tree mixed column payload lengths"); + assertSerializedSizeMatches( + createAndFillTableTablet( + "table1", + buildColumnNames(mixedTypes), + mixedTypes, + ColumnCategory.nCopy(ColumnCategory.FIELD, mixedTypes.size()), + 15, + 5, + false, + true), + "table mixed column payload lengths"); + + // OBJECT column via dedicated write API + final List objectSchemas = + Arrays.asList(new MeasurementSchema("obj", TSDataType.OBJECT, TSEncoding.PLAIN)); + final Tablet objectTablet = new Tablet("root.sg.d1", objectSchemas, 5); + for (int i = 0; i < 5; i++) { + objectTablet.addTimestamp(i, i); + objectTablet.addValue(i, 0, i % 2 == 0, i * 10L, new byte[] {(byte) i, (byte) (i + 1)}); + } + assertSerializedSizeMatches(objectTablet, "tree OBJECT column"); + final Tablet deserializedObject = Tablet.deserialize(objectTablet.serialize()); + assertEquals(objectTablet, deserializedObject); + for (int i = 0; i < 5; i++) { + assertEquals(objectTablet.getValue(i, 0), deserializedObject.getValue(i, 0)); + } + + final Map propsWithNonAscii = new HashMap<>(); + propsWithNonAscii.put("编码", "字典"); + final Tablet nonAsciiTreeTablet = + new Tablet( + "root.测试.设备1", + Arrays.asList( + new MeasurementSchema( + "温度", + TSDataType.TEXT, + TSEncoding.PLAIN, + CompressionType.UNCOMPRESSED, + propsWithNonAscii)), + 3); + for (int i = 0; i < 3; i++) { + nonAsciiTreeTablet.addTimestamp(i, i); + nonAsciiTreeTablet.addValue("温度", i, "值" + i); + } + assertSerializedSizeMatches(nonAsciiTreeTablet, "tree non-ASCII names and schema props"); + + final Tablet nonAsciiTableTablet = + createAndFillTableTablet( + "表一", + Arrays.asList("标签", "数值"), + Arrays.asList(TSDataType.STRING, TSDataType.DOUBLE), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD), + 3, + 0, + false, + true); + assertSerializedSizeMatches(nonAsciiTableTablet, "table non-ASCII names"); + } + + private static List buildTableColumnCategories(int columnCount) { + final List categories = new ArrayList<>(columnCount); + categories.add(ColumnCategory.TAG); + for (int i = 1; i < columnCount; i++) { + categories.add(ColumnCategory.FIELD); + } + return categories; + } + + private static List buildColumnNames(List dataTypes) { + final List names = new ArrayList<>(dataTypes.size()); + for (int i = 0; i < dataTypes.size(); i++) { + if (i == 0 && dataTypes.size() > 1) { + names.add("tag"); + } else { + names.add("m_" + dataTypes.get(i).name() + "_" + i); + } + } + return names; + } + + private static List columnNamesForType(TSDataType type) { + return Arrays.asList("m_" + type.name() + "_0"); + } + + private Tablet createAndFillTreeTablet( + String deviceId, + List columnNames, + List dataTypes, + int rowCount, + int valueOffset, + boolean withNulls, + boolean variableBinaryLength) + throws IOException { + validateTabletSchema(columnNames, dataTypes, null); + final List schemas = new ArrayList<>(dataTypes.size()); + for (int i = 0; i < dataTypes.size(); i++) { + schemas.add(new MeasurementSchema(columnNames.get(i), dataTypes.get(i), TSEncoding.PLAIN)); + } + final Tablet tablet = new Tablet(deviceId, schemas, Math.max(1024, rowCount + 1)); + fillTabletRows(tablet, rowCount, valueOffset, withNulls, variableBinaryLength); + return tablet; + } + + private Tablet createAndFillTableTablet( + String tableName, + List columnNames, + List dataTypes, + List columnCategories, + int rowCount, + int valueOffset, + boolean withNulls, + boolean variableBinaryLength) + throws IOException { + validateTabletSchema(columnNames, dataTypes, columnCategories); + final Tablet tablet = + new Tablet( + tableName, columnNames, dataTypes, columnCategories, Math.max(1024, rowCount + 1)); + fillTabletRows(tablet, rowCount, valueOffset, withNulls, variableBinaryLength); + return tablet; + } + + private static void validateTabletSchema( + List columnNames, List dataTypes, List columnCategories) { + if (columnNames.size() != dataTypes.size()) { + throw new IllegalArgumentException( + "columnNames size " + + columnNames.size() + + " must match dataTypes size " + + dataTypes.size()); + } + if (columnCategories != null && columnCategories.size() != dataTypes.size()) { + throw new IllegalArgumentException( + "columnCategories size " + + columnCategories.size() + + " must match dataTypes size " + + dataTypes.size()); + } + } + + private void fillTabletRows( + Tablet tablet, + int rowCount, + int valueOffset, + boolean withNulls, + boolean variableBinaryLength) { + if (rowCount > 0) { + fillTabletForSerializedSizeTest( + tablet, valueOffset, rowCount, withNulls, variableBinaryLength); + } + } + + private void fillTabletForSerializedSizeTest( + Tablet tablet, + int valueOffset, + int rowCount, + boolean withNulls, + boolean variableBinaryLength) { + for (int row = 0; row < rowCount; row++) { + tablet.addTimestamp(row, valueOffset + row); + for (int col = 0; col < tablet.getSchemas().size(); col++) { + final TSDataType type = tablet.getSchemas().get(col).getType(); + if (isNullCell(withNulls, row, col)) { + tablet.addValue(tablet.getSchemas().get(col).getMeasurementName(), row, null); + } else if (type == TSDataType.OBJECT) { + tablet.addValue( + row, + col, + (row + col) % 2 == 0, + valueOffset + row * 1000L + col, + payloadBytes(binaryPayloadLength(variableBinaryLength, row, col))); + } else { + tablet.addValue( + tablet.getSchemas().get(col).getMeasurementName(), + row, + sampleValue(type, row, col, variableBinaryLength)); + } + } + } + } + + private static boolean isNullCell(boolean withNulls, int row, int col) { + return withNulls && (row + col) % 3 == 0; + } + + private static int binaryPayloadLength(boolean variableBinaryLength, int row, int col) { + if (variableBinaryLength) { + return (col + 1) * 17 + row * 3 + 1; + } + return 8 + row % 11; + } + + private Object sampleValue(TSDataType type, int row, int col, boolean variableBinaryLength) { + switch (type) { + case BOOLEAN: + return (row + col) % 2 == 0; + case INT32: + return row + col * 100; + case INT64: + case TIMESTAMP: + return (long) (valueOffset(row, col) * 1_000_000L); + case FLOAT: + return (row + col) * 1.5f; + case DOUBLE: + return (row + col) * 2.5; + case TEXT: + case STRING: + return stringOfLength(binaryPayloadLength(variableBinaryLength, row, col)); + case BLOB: + return binaryOfLength(binaryPayloadLength(variableBinaryLength, row, col)); + case DATE: + return LocalDate.of(2000 + (row % 20), (col % 12) + 1, (row % 28) + 1); + default: + throw new IllegalArgumentException("Unsupported type in test: " + type); + } + } + + private static int valueOffset(int row, int col) { + return row + col + 1; + } + + private static String stringOfLength(int length) { + final char[] chars = new char[length]; + Arrays.fill(chars, 'x'); + return new String(chars); + } + + private static Binary binaryOfLength(int length) { + final byte[] bytes = new byte[length]; + Arrays.fill(bytes, (byte) 'b'); + return new Binary(bytes); + } + + private static byte[] payloadBytes(int length) { + final byte[] bytes = new byte[length]; + Arrays.fill(bytes, (byte) 'p'); + return bytes; + } + + private void assertSerializedSizeMatches(Tablet tablet, String scenario) throws IOException { + final int expectedSize = tablet.serializedSize(); + final ByteBuffer buffer = tablet.serialize(); + assertEquals(scenario + ": serialize() buffer size", expectedSize, buffer.remaining()); + try (PublicBAOS baos = new PublicBAOS(); + DataOutputStream outputStream = new DataOutputStream(baos)) { + tablet.serialize(outputStream); + assertEquals(scenario + ": serialize(stream) size", expectedSize, baos.size()); + } + buffer.rewind(); + assertEquals(scenario + ": deserialize roundtrip", tablet, Tablet.deserialize(buffer)); + } + @Test public void testAppendInconsistent() { Tablet t1 = @@ -425,6 +821,9 @@ private void fillTablet(Tablet t, int valueOffset, int length) { case BLOB: t.addValue(i, j, String.valueOf(i + valueOffset)); break; + case OBJECT: + t.addValue(i, j, (i + valueOffset) % 2 == 0, i + valueOffset, new byte[] {(byte) i}); + break; case DATE: t.addValue(i, j, LocalDate.of(i + valueOffset, 1, 1)); break; @@ -655,6 +1054,16 @@ private void checkAppendedTablet( new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8)), result.getValue(i, j)); break; + case OBJECT: + { + byte[] content = new byte[] {(byte) i}; + byte[] expected = new byte[content.length + 9]; + expected[0] = (byte) (i % 2); + System.arraycopy(BytesUtils.longToBytes(i), 0, expected, 1, 8); + System.arraycopy(content, 0, expected, 9, content.length); + assertEquals(new Binary(expected), result.getValue(i, j)); + } + break; case DATE: assertEquals(LocalDate.of(i, 1, 1), result.getValue(i, j)); break;