diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java index de6158409..072df4192 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java @@ -103,6 +103,9 @@ public int getPositionCount() { public void setPositionCount(int positionCount) { this.positionCount = positionCount; + if (sizeInBytes >= 0) { + sizeInBytes = -1; + } } public long getStartTime() { @@ -270,7 +273,7 @@ public void reset() { if (positionCount == 0) { return; } - positionCount = 0; + setPositionCount(0); timeColumn.reset(); for (Column valueColumn : valueColumns) { valueColumn.reset(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java index 6b67f87fb..757e859a5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java @@ -39,8 +39,8 @@ import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkReadablePosition; import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkValidRegion; import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF; -import static org.apache.tsfile.utils.RamUsageEstimator.sizeOf; import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray; +import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfObjectArray; public class BinaryColumn implements Column { @@ -55,7 +55,9 @@ public class BinaryColumn implements Column { private final Binary[] values; private final long retainedSizeInBytes; - private final long sizeInBytes; + // A view uses the same value bytes and row count as the column they were measured on. + private long measuredValueSizeInBytes; + private int measuredPositionCount; public BinaryColumn(int initialCapacity) { this(0, 0, null, new Binary[initialCapacity]); @@ -66,36 +68,15 @@ public BinaryColumn(int positionCount, Optional valueIsNull, Binary[] } public BinaryColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, Binary[] values) { - if (arrayOffset < 0) { - throw new IllegalArgumentException(Messages.get("error.read.col_array_offset_negative")); - } - this.arrayOffset = arrayOffset; - if (positionCount < 0) { - throw new IllegalArgumentException(Messages.get("error.read.col_position_count_negative")); - } - this.positionCount = positionCount; - - if (values.length - arrayOffset < positionCount) { - throw new IllegalArgumentException(Messages.get("error.read.col_values_length_lt_position")); - } - this.values = values; - - if (valueIsNull != null && valueIsNull.length - arrayOffset < positionCount) { - throw new IllegalArgumentException(Messages.get("error.read.col_isnull_length_lt_position")); - } - this.valueIsNull = valueIsNull; - - retainedSizeInBytes = INSTANCE_SIZE + sizeOfBooleanArray(positionCount) + sizeOf(values); - sizeInBytes = values.length > 0 ? retainedSizeInBytes * positionCount / values.length : 0L; + this(arrayOffset, positionCount, valueIsNull, values, null); } - // called by getRegion which already knows the underlying retainedSizeInBytes private BinaryColumn( int arrayOffset, int positionCount, boolean[] valueIsNull, Binary[] values, - long retainedSizeInBytes) { + BinaryColumn parent) { if (arrayOffset < 0) { throw new IllegalArgumentException(Messages.get("error.read.col_array_offset_negative")); } @@ -114,8 +95,22 @@ private BinaryColumn( throw new IllegalArgumentException(Messages.get("error.read.col_isnull_length_lt_position")); } this.valueIsNull = valueIsNull; - this.retainedSizeInBytes = retainedSizeInBytes; - this.sizeInBytes = values.length > 0 ? retainedSizeInBytes * positionCount / values.length : 0L; + if (parent == null) { + int end = arrayOffset + positionCount; + measuredPositionCount = positionCount; + measuredValueSizeInBytes = sizeOfValues(arrayOffset, end); + retainedSizeInBytes = + INSTANCE_SIZE + + sizeOfBooleanArray(positionCount) + + sizeOfObjectArray(values.length) + + measuredValueSizeInBytes + + sizeOfValues(0, arrayOffset) + + sizeOfValues(end, values.length); + } else { + retainedSizeInBytes = parent.retainedSizeInBytes; + measuredPositionCount = parent.measuredPositionCount; + measuredValueSizeInBytes = parent.measuredValueSizeInBytes; + } } @Override @@ -231,16 +226,37 @@ public long getRetainedSizeInBytes() { return retainedSizeInBytes; } + /** Shared regions estimate their value bytes using the average over the measured positions. */ @Override public long getSizeInBytes() { - return sizeInBytes; + long shallowSize = + INSTANCE_SIZE + sizeOfBooleanArray(positionCount) + sizeOfObjectArray(positionCount); + if (positionCount == measuredPositionCount) { + return shallowSize + measuredValueSizeInBytes; + } + // Scale only the values. Dividing first avoids overflowing when the column is large. + long bytesPerPosition = measuredValueSizeInBytes / measuredPositionCount; + long remainder = measuredValueSizeInBytes % measuredPositionCount; + return shallowSize + + bytesPerPosition * positionCount + + remainder * positionCount / measuredPositionCount; + } + + private long sizeOfValues(int from, int end) { + long size = 0; + for (int i = from; i < end; i++) { + Binary value = values[i]; + if (value != null) { + size += value.ramBytesUsed(); + } + } + return size; } @Override public Column getRegion(int positionOffset, int length) { checkValidRegion(getPositionCount(), positionOffset, length); - return new BinaryColumn( - positionOffset + arrayOffset, length, valueIsNull, values, getRetainedSizeInBytes()); + return new BinaryColumn(positionOffset + arrayOffset, length, valueIsNull, values, this); } @Override @@ -261,8 +277,7 @@ public Column subColumn(int fromIndex) { if (fromIndex > positionCount) { throw new IllegalArgumentException(Messages.get("error.read.col_from_index_invalid")); } - return new BinaryColumn( - arrayOffset + fromIndex, positionCount - fromIndex, valueIsNull, values); + return getRegion(fromIndex, positionCount - fromIndex); } @Override @@ -331,6 +346,18 @@ public int getInstanceSize() { @Override public void setPositionCount(int count) { + int capacity = + valueIsNull == null ? values.length : Math.min(values.length, valueIsNull.length); + checkValidRegion(capacity - arrayOffset, 0, count); + if (count > positionCount && positionCount == measuredPositionCount) { + // Readers fill the backing array in place and publish the newly written positions here. + measuredValueSizeInBytes += sizeOfValues(arrayOffset + positionCount, arrayOffset + count); + measuredPositionCount = count; + } else if (count >= positionCount && count > 0) { + // The previous range was estimated, or existing positions have been filled again. + measuredValueSizeInBytes = sizeOfValues(arrayOffset, arrayOffset + count); + measuredPositionCount = count; + } positionCount = count; } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/common/block/TsBlockTest.java b/java/tsfile/src/test/java/org/apache/tsfile/common/block/TsBlockTest.java index 47db65626..d72d175eb 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/common/block/TsBlockTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/common/block/TsBlockTest.java @@ -31,6 +31,7 @@ import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.utils.Binary; import org.junit.Assert; @@ -181,6 +182,45 @@ public void testBinaryTsBlock() { } } + @Test + public void testBinaryTsBlockSize() { + byte[] bytes = new byte[1024 * 1024]; + TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.TEXT)); + builder.getTimeColumnBuilder().writeLong(1L); + builder.getColumnBuilder(0).writeBinary(new Binary(bytes)); + builder.declarePosition(); + TsBlock tsBlock = builder.build(); + // the builder allocates more slots than the only row needs + assertTrue(tsBlock.getColumn(0).getBinaries().length > tsBlock.getPositionCount()); + assertTrue(tsBlock.getSizeInBytes() >= bytes.length); + } + + @Test + public void testBinaryTsBlockSizeAfterRefill() { + Column time = new TimeColumn(1); + Column column = new BinaryColumn(1); + TsBlock block = new TsBlock(time, column); + long emptySize = block.getSizeInBytes(); + Binary small = new Binary(new byte[16]); + Binary large = new Binary(new byte[4096]); + + column.getBinaries()[0] = small; + column.setPositionCount(1); + time.setPositionCount(1); + block.setPositionCount(1); + long smallBlockSize = block.getSizeInBytes(); + assertTrue(smallBlockSize >= emptySize + small.ramBytesUsed()); + + block.reset(); + assertEquals(emptySize, block.getSizeInBytes()); + column.getBinaries()[0] = large; + column.setPositionCount(1); + time.setPositionCount(1); + block.setPositionCount(1); + assertEquals( + smallBlockSize - small.ramBytesUsed() + large.ramBytesUsed(), block.getSizeInBytes()); + } + @Test public void testIntTsBlockWithNull() { long[] timeArray = {1L, 2L, 3L, 4L, 5L}; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java index aebf795a3..54988f965 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java @@ -46,6 +46,7 @@ import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; @@ -488,6 +489,116 @@ public void binaryColumnFilterPositionsTest() { Assert.assertEquals("5", columnByCopyPositions.getBinary(1).toString()); } + @Test + public void binaryColumnSizeIgnoresUnusedCapacityTest() { + Binary value = new Binary(new byte[1024]); + // the builder allocates 8 slots for the two positions + BinaryColumnBuilder columnBuilder = new BinaryColumnBuilder(null, 8); + columnBuilder.appendNull(); + columnBuilder.writeBinary(value); + Column overAllocatedColumn = columnBuilder.build(); + Assert.assertEquals(8, overAllocatedColumn.getBinaries().length); + + Column compactColumn = + new BinaryColumn(2, Optional.of(new boolean[] {true, false}), new Binary[] {null, value}); + Assert.assertEquals(compactColumn.getSizeInBytes(), overAllocatedColumn.getSizeInBytes()); + Assert.assertTrue(overAllocatedColumn.getSizeInBytes() >= 1024); + Assert.assertEquals( + compactColumn.getRegion(1, 1).getSizeInBytes(), + overAllocatedColumn.getRegion(1, 1).getSizeInBytes()); + } + + @Test + public void binaryColumnSizeOfRegionTest() { + Binary largeValue = new Binary(new byte[4096]); + Binary smallValue = new Binary(new byte[16]); + Binary[] values = new Binary[8]; + Arrays.fill(values, smallValue); + values[2] = largeValue; + Column column = new BinaryColumn(8, Optional.empty(), values); + long smallBytes = smallValue.ramBytesUsed(); + long largeBytes = largeValue.ramBytesUsed(); + long threeRows = new BinaryColumn(3, Optional.empty(), new Binary[3]).getSizeInBytes(); + + // A directly constructed range measures its own values during the retained-size scan. + Column direct = new BinaryColumn(2, 3, null, values); + Assert.assertEquals(threeRows + largeBytes + 2 * smallBytes, direct.getSizeInBytes()); + Assert.assertEquals(column.getRegionCopy(2, 3).getSizeInBytes(), direct.getSizeInBytes()); + long retainedArrays = new BinaryColumn(2, 3, null, new Binary[8]).getRetainedSizeInBytes(); + Assert.assertEquals( + retainedArrays + largeBytes + 7 * smallBytes, direct.getRetainedSizeInBytes()); + + // A shared view of the same positions takes three eighths of its parent's value bytes. + Column region = column.getRegion(2, 3); + long estimate = threeRows + (largeBytes + 7 * smallBytes) * 3 / 8; + Assert.assertSame(values, region.getBinaries()); + Assert.assertEquals(column.getRetainedSizeInBytes(), region.getRetainedSizeInBytes()); + Assert.assertEquals(estimate, region.getSizeInBytes()); + Assert.assertEquals(estimate, column.getRegion(1, 4).subColumn(1).getSizeInBytes()); + Assert.assertEquals(estimate, column.subColumn(5).getSizeInBytes()); + + long oneRow = new BinaryColumn(1, Optional.empty(), new Binary[1]).getSizeInBytes(); + Assert.assertEquals( + oneRow + (largeBytes + 2 * smallBytes) / 3, direct.subColumn(2).getSizeInBytes()); + Assert.assertEquals(new BinaryColumn(0).getSizeInBytes(), direct.subColumn(3).getSizeInBytes()); + } + + @Test + public void binaryColumnSizeReadsEachValueOnceTest() { + Binary value = Mockito.mock(Binary.class); + // The sum fits in a long, but multiplying it by the region length would overflow. + long bytes = Long.MAX_VALUE / 10; + Mockito.when(value.ramBytesUsed()).thenReturn(bytes); + Binary[] values = new Binary[8]; + Arrays.fill(values, value); + Column column = new BinaryColumn(8, Optional.empty(), values); + long threeRows = new BinaryColumn(3, Optional.empty(), new Binary[3]).getSizeInBytes(); + + Assert.assertEquals(column.getRetainedSizeInBytes(), column.getSizeInBytes()); + Assert.assertEquals(threeRows + 3 * bytes, column.getRegion(2, 3).getSizeInBytes()); + Assert.assertEquals(threeRows + 3 * bytes, column.subColumn(5).getSizeInBytes()); + Mockito.verify(value, Mockito.times(values.length)).ramBytesUsed(); + } + + @Test + public void binaryColumnSizeAfterSetPositionCountTest() { + Binary value = new Binary(new byte[1024]); + Column column = new BinaryColumn(2, Optional.empty(), new Binary[] {value, value}); + Assert.assertTrue(column.getSizeInBytes() >= 2048); + + column.setPositionCount(1); + Column compactColumn = new BinaryColumn(1, Optional.empty(), new Binary[] {value}); + Assert.assertEquals(compactColumn.getSizeInBytes(), column.getSizeInBytes()); + } + + @Test + public void binaryColumnSizeAfterInPlaceWritesTest() { + Binary small = new Binary(new byte[16]); + Binary large = new Binary(new byte[4096]); + Column column = new BinaryColumn(2); + long emptySize = column.getSizeInBytes(); + column.getBinaries()[0] = small; + column.setPositionCount(1); + Assert.assertEquals( + new BinaryColumn(1, Optional.empty(), new Binary[] {small}).getSizeInBytes(), + column.getSizeInBytes()); + column.getBinaries()[1] = large; + column.setPositionCount(2); + long fullSize = + new BinaryColumn(2, Optional.empty(), new Binary[] {small, large}).getSizeInBytes(); + Assert.assertEquals(fullSize, column.getSizeInBytes()); + + column.setPositionCount(1); + column.setPositionCount(2); + Assert.assertEquals(fullSize, column.getSizeInBytes()); + column.getBinaries()[0] = large; + column.setPositionCount(2); + Assert.assertEquals( + fullSize - small.ramBytesUsed() + large.ramBytesUsed(), column.getSizeInBytes()); + column.reset(); + Assert.assertEquals(emptySize, column.getSizeInBytes()); + } + @Test public void booleanColumnSubColumnTest() { BooleanColumnBuilder columnBuilder = new BooleanColumnBuilder(null, 10); diff --git a/skills/tsfile/references/performance.md b/skills/tsfile/references/performance.md index 6a745844e..1a5b65712 100644 --- a/skills/tsfile/references/performance.md +++ b/skills/tsfile/references/performance.md @@ -41,6 +41,11 @@ than a universal setting. result set. - Separate metadata inspection from page decoding when the available tool or API supports it. +- Java `BinaryColumn.getSizeInBytes()` counts the values in the column's + logical range at construction. Shared views from `getRegion()` and + `subColumn()` estimate value bytes in proportion to their row count. + Equal-length views of the same column get the same estimate even when + their strings or blobs differ in length. A copied region counts its own values. ## Encoding and Compression