From 4e8d3bfa97d6d27cb23f3c4a51b4f97e2d4373fe Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Thu, 19 Nov 2026 20:27:44 +0800 Subject: [PATCH 1/2] Fix BinaryColumn.getSizeInBytes for over-allocated and shared arrays getSizeInBytes() is documented as the size of the column as if it was compacted. BinaryColumn derived it from the retained size of the whole array, scaled by positionCount / values.length. The unused slots of an array hold no value, so that scaling divides the size of the values the column does hold: - A column built by BinaryColumnBuilder keeps the over-allocated array of the builder. With a null and a 1024-byte value in 8 slots it reported 294 bytes instead of 1152. A block with a single row, built by TsBlockBuilder, reports about one eighth of its value this way. - A region shares the array of the column it was taken from and got an equal share per position, whatever its own positions hold. Of values of {4096, 16, 16, 16} bytes, the region with the first one reported 1094 bytes instead of 4224. - The size was fixed at construction, so it was stale after setPositionCount(): after shrinking two 1024-byte values to one position, the column still reported 2208 bytes instead of 1152. TsBlock.buildTsBlock() creates its columns with zero positions for readers that fill them in place, so those kept reporting 0. The size is now the sum over the positions of the column, calculated on the first call and again after setPositionCount(). A column that fills its array reports the same value as before. getRegion() stays O(1), and getRetainedSizeInBytes() is unchanged. TsBlock.getSizeInBytes() sums the sizes of its columns, so blocks with TEXT, STRING or BLOB columns report larger and correct sizes. TsBlockSerde uses that value for the initial capacity of its output buffer and for the "TsBlock too large" check. Callers that reserve memory by this size reserve more than before for such blocks. --- .../common/block/column/BinaryColumn.java | 30 +++++++++-- .../tsfile/common/block/TsBlockTest.java | 13 +++++ .../apache/tsfile/read/common/ColumnTest.java | 50 +++++++++++++++++++ 3 files changed, 89 insertions(+), 4 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java index 6b67f87fb0..1f3b227329 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java @@ -41,6 +41,7 @@ import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF; import static org.apache.tsfile.utils.RamUsageEstimator.sizeOf; import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray; +import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfObjectArray; public class BinaryColumn implements Column { @@ -55,7 +56,8 @@ public class BinaryColumn implements Column { private final Binary[] values; private final long retainedSizeInBytes; - private final long sizeInBytes; + // calculated on demand, -1 means that it has to be calculated + private volatile long sizeInBytes = -1; public BinaryColumn(int initialCapacity) { this(0, 0, null, new Binary[initialCapacity]); @@ -86,7 +88,6 @@ public BinaryColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, B this.valueIsNull = valueIsNull; retainedSizeInBytes = INSTANCE_SIZE + sizeOfBooleanArray(positionCount) + sizeOf(values); - sizeInBytes = values.length > 0 ? retainedSizeInBytes * positionCount / values.length : 0L; } // called by getRegion which already knows the underlying retainedSizeInBytes @@ -115,7 +116,6 @@ private BinaryColumn( } this.valueIsNull = valueIsNull; this.retainedSizeInBytes = retainedSizeInBytes; - this.sizeInBytes = values.length > 0 ? retainedSizeInBytes * positionCount / values.length : 0L; } @Override @@ -233,7 +233,28 @@ public long getRetainedSizeInBytes() { @Override public long getSizeInBytes() { - return sizeInBytes; + long size = sizeInBytes; + if (size < 0) { + size = calculateSizeInBytes(); + sizeInBytes = size; + } + return size; + } + + /** + * Only the positions of this column count. The array can be longer than that: a builder + * over-allocates it, and a region shares the array of the column it was taken from. + */ + private long calculateSizeInBytes() { + long size = + INSTANCE_SIZE + sizeOfBooleanArray(positionCount) + sizeOfObjectArray(positionCount); + for (int i = arrayOffset, end = arrayOffset + positionCount; i < end; i++) { + Binary value = values[i]; + if (value != null) { + size += value.ramBytesUsed(); + } + } + return size; } @Override @@ -332,6 +353,7 @@ public int getInstanceSize() { @Override public void setPositionCount(int count) { positionCount = count; + sizeInBytes = -1; } @Override diff --git a/java/tsfile/src/test/java/org/apache/tsfile/common/block/TsBlockTest.java b/java/tsfile/src/test/java/org/apache/tsfile/common/block/TsBlockTest.java index 47db65626c..7f05d9b5ad 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/common/block/TsBlockTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/common/block/TsBlockTest.java @@ -181,6 +181,19 @@ public void testBinaryTsBlock() { } } + @Test + public void testBinaryTsBlockSize() { + byte[] bytes = new byte[1024 * 1024]; + TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.TEXT)); + builder.getTimeColumnBuilder().writeLong(1L); + builder.getColumnBuilder(0).writeBinary(new Binary(bytes)); + builder.declarePosition(); + TsBlock tsBlock = builder.build(); + // the builder allocates more slots than the only row needs + assertTrue(tsBlock.getColumn(0).getBinaries().length > tsBlock.getPositionCount()); + assertTrue(tsBlock.getSizeInBytes() >= bytes.length); + } + @Test public void testIntTsBlockWithNull() { long[] timeArray = {1L, 2L, 3L, 4L, 5L}; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java index aebf795a39..25e5b8ebbb 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java @@ -488,6 +488,56 @@ public void binaryColumnFilterPositionsTest() { Assert.assertEquals("5", columnByCopyPositions.getBinary(1).toString()); } + @Test + public void binaryColumnSizeIgnoresUnusedCapacityTest() { + Binary value = new Binary(new byte[1024]); + // the builder allocates 8 slots for the two positions + BinaryColumnBuilder columnBuilder = new BinaryColumnBuilder(null, 8); + columnBuilder.appendNull(); + columnBuilder.writeBinary(value); + Column overAllocatedColumn = columnBuilder.build(); + Assert.assertEquals(8, overAllocatedColumn.getBinaries().length); + + Column compactColumn = + new BinaryColumn(2, Optional.of(new boolean[] {true, false}), new Binary[] {null, value}); + Assert.assertEquals(compactColumn.getSizeInBytes(), overAllocatedColumn.getSizeInBytes()); + Assert.assertTrue(overAllocatedColumn.getSizeInBytes() >= 1024); + } + + @Test + public void binaryColumnSizeOfRegionTest() { + Binary largeValue = new Binary(new byte[4096]); + Binary smallValue = new Binary(new byte[16]); + Column column = + new BinaryColumn( + 4, Optional.empty(), new Binary[] {largeValue, smallValue, smallValue, smallValue}); + + // a region only counts its own positions, whatever else the shared array holds + Column largeRegion = column.getRegion(0, 1); + Assert.assertSame(column.getBinaries(), largeRegion.getBinaries()); + Assert.assertEquals(column.getRegionCopy(0, 1).getSizeInBytes(), largeRegion.getSizeInBytes()); + Assert.assertTrue(largeRegion.getSizeInBytes() >= 4096); + + Column smallRegion = column.getRegion(1, 3); + Assert.assertEquals(column.getRegionCopy(1, 3).getSizeInBytes(), smallRegion.getSizeInBytes()); + Assert.assertTrue(smallRegion.getSizeInBytes() < 4096); + + Column subColumn = column.subColumn(1); + Assert.assertSame(column.getBinaries(), subColumn.getBinaries()); + Assert.assertEquals(column.subColumnCopy(1).getSizeInBytes(), subColumn.getSizeInBytes()); + } + + @Test + public void binaryColumnSizeAfterSetPositionCountTest() { + Binary value = new Binary(new byte[1024]); + Column column = new BinaryColumn(2, Optional.empty(), new Binary[] {value, value}); + Assert.assertTrue(column.getSizeInBytes() >= 2048); + + column.setPositionCount(1); + Column compactColumn = new BinaryColumn(1, Optional.empty(), new Binary[] {value}); + Assert.assertEquals(compactColumn.getSizeInBytes(), column.getSizeInBytes()); + } + @Test public void booleanColumnSubColumnTest() { BooleanColumnBuilder columnBuilder = new BooleanColumnBuilder(null, 10); From 5ed43e266f19b431d575701fd2c89dc51d2bfd14 Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Fri, 20 Nov 2026 14:43:12 +0800 Subject: [PATCH 2/2] Reuse measured BinaryColumn value sizes for region estimates --- .../tsfile/read/common/block/TsBlock.java | 5 +- .../common/block/column/BinaryColumn.java | 93 ++++++++++--------- .../tsfile/common/block/TsBlockTest.java | 27 ++++++ .../apache/tsfile/read/common/ColumnTest.java | 93 +++++++++++++++---- skills/tsfile/references/performance.md | 5 + 5 files changed, 162 insertions(+), 61 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java index de61584096..072df41923 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java @@ -103,6 +103,9 @@ public int getPositionCount() { public void setPositionCount(int positionCount) { this.positionCount = positionCount; + if (sizeInBytes >= 0) { + sizeInBytes = -1; + } } public long getStartTime() { @@ -270,7 +273,7 @@ public void reset() { if (positionCount == 0) { return; } - positionCount = 0; + setPositionCount(0); timeColumn.reset(); for (Column valueColumn : valueColumns) { valueColumn.reset(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java index 1f3b227329..757e859a59 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java @@ -39,7 +39,6 @@ import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkReadablePosition; import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkValidRegion; import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF; -import static org.apache.tsfile.utils.RamUsageEstimator.sizeOf; import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray; import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfObjectArray; @@ -56,8 +55,9 @@ public class BinaryColumn implements Column { private final Binary[] values; private final long retainedSizeInBytes; - // calculated on demand, -1 means that it has to be calculated - private volatile long sizeInBytes = -1; + // A view uses the same value bytes and row count as the column they were measured on. + private long measuredValueSizeInBytes; + private int measuredPositionCount; public BinaryColumn(int initialCapacity) { this(0, 0, null, new Binary[initialCapacity]); @@ -68,35 +68,15 @@ public BinaryColumn(int positionCount, Optional valueIsNull, Binary[] } public BinaryColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, Binary[] values) { - if (arrayOffset < 0) { - throw new IllegalArgumentException(Messages.get("error.read.col_array_offset_negative")); - } - this.arrayOffset = arrayOffset; - if (positionCount < 0) { - throw new IllegalArgumentException(Messages.get("error.read.col_position_count_negative")); - } - this.positionCount = positionCount; - - if (values.length - arrayOffset < positionCount) { - throw new IllegalArgumentException(Messages.get("error.read.col_values_length_lt_position")); - } - this.values = values; - - if (valueIsNull != null && valueIsNull.length - arrayOffset < positionCount) { - throw new IllegalArgumentException(Messages.get("error.read.col_isnull_length_lt_position")); - } - this.valueIsNull = valueIsNull; - - retainedSizeInBytes = INSTANCE_SIZE + sizeOfBooleanArray(positionCount) + sizeOf(values); + this(arrayOffset, positionCount, valueIsNull, values, null); } - // called by getRegion which already knows the underlying retainedSizeInBytes private BinaryColumn( int arrayOffset, int positionCount, boolean[] valueIsNull, Binary[] values, - long retainedSizeInBytes) { + BinaryColumn parent) { if (arrayOffset < 0) { throw new IllegalArgumentException(Messages.get("error.read.col_array_offset_negative")); } @@ -115,7 +95,22 @@ private BinaryColumn( throw new IllegalArgumentException(Messages.get("error.read.col_isnull_length_lt_position")); } this.valueIsNull = valueIsNull; - this.retainedSizeInBytes = retainedSizeInBytes; + if (parent == null) { + int end = arrayOffset + positionCount; + measuredPositionCount = positionCount; + measuredValueSizeInBytes = sizeOfValues(arrayOffset, end); + retainedSizeInBytes = + INSTANCE_SIZE + + sizeOfBooleanArray(positionCount) + + sizeOfObjectArray(values.length) + + measuredValueSizeInBytes + + sizeOfValues(0, arrayOffset) + + sizeOfValues(end, values.length); + } else { + retainedSizeInBytes = parent.retainedSizeInBytes; + measuredPositionCount = parent.measuredPositionCount; + measuredValueSizeInBytes = parent.measuredValueSizeInBytes; + } } @Override @@ -231,24 +226,25 @@ public long getRetainedSizeInBytes() { return retainedSizeInBytes; } + /** Shared regions estimate their value bytes using the average over the measured positions. */ @Override public long getSizeInBytes() { - long size = sizeInBytes; - if (size < 0) { - size = calculateSizeInBytes(); - sizeInBytes = size; + long shallowSize = + INSTANCE_SIZE + sizeOfBooleanArray(positionCount) + sizeOfObjectArray(positionCount); + if (positionCount == measuredPositionCount) { + return shallowSize + measuredValueSizeInBytes; } - return size; + // Scale only the values. Dividing first avoids overflowing when the column is large. + long bytesPerPosition = measuredValueSizeInBytes / measuredPositionCount; + long remainder = measuredValueSizeInBytes % measuredPositionCount; + return shallowSize + + bytesPerPosition * positionCount + + remainder * positionCount / measuredPositionCount; } - /** - * Only the positions of this column count. The array can be longer than that: a builder - * over-allocates it, and a region shares the array of the column it was taken from. - */ - private long calculateSizeInBytes() { - long size = - INSTANCE_SIZE + sizeOfBooleanArray(positionCount) + sizeOfObjectArray(positionCount); - for (int i = arrayOffset, end = arrayOffset + positionCount; i < end; i++) { + private long sizeOfValues(int from, int end) { + long size = 0; + for (int i = from; i < end; i++) { Binary value = values[i]; if (value != null) { size += value.ramBytesUsed(); @@ -260,8 +256,7 @@ private long calculateSizeInBytes() { @Override public Column getRegion(int positionOffset, int length) { checkValidRegion(getPositionCount(), positionOffset, length); - return new BinaryColumn( - positionOffset + arrayOffset, length, valueIsNull, values, getRetainedSizeInBytes()); + return new BinaryColumn(positionOffset + arrayOffset, length, valueIsNull, values, this); } @Override @@ -282,8 +277,7 @@ public Column subColumn(int fromIndex) { if (fromIndex > positionCount) { throw new IllegalArgumentException(Messages.get("error.read.col_from_index_invalid")); } - return new BinaryColumn( - arrayOffset + fromIndex, positionCount - fromIndex, valueIsNull, values); + return getRegion(fromIndex, positionCount - fromIndex); } @Override @@ -352,8 +346,19 @@ public int getInstanceSize() { @Override public void setPositionCount(int count) { + int capacity = + valueIsNull == null ? values.length : Math.min(values.length, valueIsNull.length); + checkValidRegion(capacity - arrayOffset, 0, count); + if (count > positionCount && positionCount == measuredPositionCount) { + // Readers fill the backing array in place and publish the newly written positions here. + measuredValueSizeInBytes += sizeOfValues(arrayOffset + positionCount, arrayOffset + count); + measuredPositionCount = count; + } else if (count >= positionCount && count > 0) { + // The previous range was estimated, or existing positions have been filled again. + measuredValueSizeInBytes = sizeOfValues(arrayOffset, arrayOffset + count); + measuredPositionCount = count; + } positionCount = count; - sizeInBytes = -1; } @Override diff --git a/java/tsfile/src/test/java/org/apache/tsfile/common/block/TsBlockTest.java b/java/tsfile/src/test/java/org/apache/tsfile/common/block/TsBlockTest.java index 7f05d9b5ad..d72d175eb5 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/common/block/TsBlockTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/common/block/TsBlockTest.java @@ -31,6 +31,7 @@ import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.utils.Binary; import org.junit.Assert; @@ -194,6 +195,32 @@ public void testBinaryTsBlockSize() { assertTrue(tsBlock.getSizeInBytes() >= bytes.length); } + @Test + public void testBinaryTsBlockSizeAfterRefill() { + Column time = new TimeColumn(1); + Column column = new BinaryColumn(1); + TsBlock block = new TsBlock(time, column); + long emptySize = block.getSizeInBytes(); + Binary small = new Binary(new byte[16]); + Binary large = new Binary(new byte[4096]); + + column.getBinaries()[0] = small; + column.setPositionCount(1); + time.setPositionCount(1); + block.setPositionCount(1); + long smallBlockSize = block.getSizeInBytes(); + assertTrue(smallBlockSize >= emptySize + small.ramBytesUsed()); + + block.reset(); + assertEquals(emptySize, block.getSizeInBytes()); + column.getBinaries()[0] = large; + column.setPositionCount(1); + time.setPositionCount(1); + block.setPositionCount(1); + assertEquals( + smallBlockSize - small.ramBytesUsed() + large.ramBytesUsed(), block.getSizeInBytes()); + } + @Test public void testIntTsBlockWithNull() { long[] timeArray = {1L, 2L, 3L, 4L, 5L}; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java index 25e5b8ebbb..54988f9654 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java @@ -46,6 +46,7 @@ import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; @@ -502,29 +503,61 @@ public void binaryColumnSizeIgnoresUnusedCapacityTest() { new BinaryColumn(2, Optional.of(new boolean[] {true, false}), new Binary[] {null, value}); Assert.assertEquals(compactColumn.getSizeInBytes(), overAllocatedColumn.getSizeInBytes()); Assert.assertTrue(overAllocatedColumn.getSizeInBytes() >= 1024); + Assert.assertEquals( + compactColumn.getRegion(1, 1).getSizeInBytes(), + overAllocatedColumn.getRegion(1, 1).getSizeInBytes()); } @Test public void binaryColumnSizeOfRegionTest() { Binary largeValue = new Binary(new byte[4096]); Binary smallValue = new Binary(new byte[16]); - Column column = - new BinaryColumn( - 4, Optional.empty(), new Binary[] {largeValue, smallValue, smallValue, smallValue}); - - // a region only counts its own positions, whatever else the shared array holds - Column largeRegion = column.getRegion(0, 1); - Assert.assertSame(column.getBinaries(), largeRegion.getBinaries()); - Assert.assertEquals(column.getRegionCopy(0, 1).getSizeInBytes(), largeRegion.getSizeInBytes()); - Assert.assertTrue(largeRegion.getSizeInBytes() >= 4096); - - Column smallRegion = column.getRegion(1, 3); - Assert.assertEquals(column.getRegionCopy(1, 3).getSizeInBytes(), smallRegion.getSizeInBytes()); - Assert.assertTrue(smallRegion.getSizeInBytes() < 4096); + Binary[] values = new Binary[8]; + Arrays.fill(values, smallValue); + values[2] = largeValue; + Column column = new BinaryColumn(8, Optional.empty(), values); + long smallBytes = smallValue.ramBytesUsed(); + long largeBytes = largeValue.ramBytesUsed(); + long threeRows = new BinaryColumn(3, Optional.empty(), new Binary[3]).getSizeInBytes(); + + // A directly constructed range measures its own values during the retained-size scan. + Column direct = new BinaryColumn(2, 3, null, values); + Assert.assertEquals(threeRows + largeBytes + 2 * smallBytes, direct.getSizeInBytes()); + Assert.assertEquals(column.getRegionCopy(2, 3).getSizeInBytes(), direct.getSizeInBytes()); + long retainedArrays = new BinaryColumn(2, 3, null, new Binary[8]).getRetainedSizeInBytes(); + Assert.assertEquals( + retainedArrays + largeBytes + 7 * smallBytes, direct.getRetainedSizeInBytes()); + + // A shared view of the same positions takes three eighths of its parent's value bytes. + Column region = column.getRegion(2, 3); + long estimate = threeRows + (largeBytes + 7 * smallBytes) * 3 / 8; + Assert.assertSame(values, region.getBinaries()); + Assert.assertEquals(column.getRetainedSizeInBytes(), region.getRetainedSizeInBytes()); + Assert.assertEquals(estimate, region.getSizeInBytes()); + Assert.assertEquals(estimate, column.getRegion(1, 4).subColumn(1).getSizeInBytes()); + Assert.assertEquals(estimate, column.subColumn(5).getSizeInBytes()); + + long oneRow = new BinaryColumn(1, Optional.empty(), new Binary[1]).getSizeInBytes(); + Assert.assertEquals( + oneRow + (largeBytes + 2 * smallBytes) / 3, direct.subColumn(2).getSizeInBytes()); + Assert.assertEquals(new BinaryColumn(0).getSizeInBytes(), direct.subColumn(3).getSizeInBytes()); + } - Column subColumn = column.subColumn(1); - Assert.assertSame(column.getBinaries(), subColumn.getBinaries()); - Assert.assertEquals(column.subColumnCopy(1).getSizeInBytes(), subColumn.getSizeInBytes()); + @Test + public void binaryColumnSizeReadsEachValueOnceTest() { + Binary value = Mockito.mock(Binary.class); + // The sum fits in a long, but multiplying it by the region length would overflow. + long bytes = Long.MAX_VALUE / 10; + Mockito.when(value.ramBytesUsed()).thenReturn(bytes); + Binary[] values = new Binary[8]; + Arrays.fill(values, value); + Column column = new BinaryColumn(8, Optional.empty(), values); + long threeRows = new BinaryColumn(3, Optional.empty(), new Binary[3]).getSizeInBytes(); + + Assert.assertEquals(column.getRetainedSizeInBytes(), column.getSizeInBytes()); + Assert.assertEquals(threeRows + 3 * bytes, column.getRegion(2, 3).getSizeInBytes()); + Assert.assertEquals(threeRows + 3 * bytes, column.subColumn(5).getSizeInBytes()); + Mockito.verify(value, Mockito.times(values.length)).ramBytesUsed(); } @Test @@ -538,6 +571,34 @@ public void binaryColumnSizeAfterSetPositionCountTest() { Assert.assertEquals(compactColumn.getSizeInBytes(), column.getSizeInBytes()); } + @Test + public void binaryColumnSizeAfterInPlaceWritesTest() { + Binary small = new Binary(new byte[16]); + Binary large = new Binary(new byte[4096]); + Column column = new BinaryColumn(2); + long emptySize = column.getSizeInBytes(); + column.getBinaries()[0] = small; + column.setPositionCount(1); + Assert.assertEquals( + new BinaryColumn(1, Optional.empty(), new Binary[] {small}).getSizeInBytes(), + column.getSizeInBytes()); + column.getBinaries()[1] = large; + column.setPositionCount(2); + long fullSize = + new BinaryColumn(2, Optional.empty(), new Binary[] {small, large}).getSizeInBytes(); + Assert.assertEquals(fullSize, column.getSizeInBytes()); + + column.setPositionCount(1); + column.setPositionCount(2); + Assert.assertEquals(fullSize, column.getSizeInBytes()); + column.getBinaries()[0] = large; + column.setPositionCount(2); + Assert.assertEquals( + fullSize - small.ramBytesUsed() + large.ramBytesUsed(), column.getSizeInBytes()); + column.reset(); + Assert.assertEquals(emptySize, column.getSizeInBytes()); + } + @Test public void booleanColumnSubColumnTest() { BooleanColumnBuilder columnBuilder = new BooleanColumnBuilder(null, 10); diff --git a/skills/tsfile/references/performance.md b/skills/tsfile/references/performance.md index 6a745844e8..1a5b657129 100644 --- a/skills/tsfile/references/performance.md +++ b/skills/tsfile/references/performance.md @@ -41,6 +41,11 @@ than a universal setting. result set. - Separate metadata inspection from page decoding when the available tool or API supports it. +- Java `BinaryColumn.getSizeInBytes()` counts the values in the column's + logical range at construction. Shared views from `getRegion()` and + `subColumn()` estimate value bytes in proportion to their row count. + Equal-length views of the same column get the same estimate even when + their strings or blobs differ in length. A copied region counts its own values. ## Encoding and Compression