Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,9 @@ public int getPositionCount() {

public void setPositionCount(int positionCount) {
this.positionCount = positionCount;
if (sizeInBytes >= 0) {
sizeInBytes = -1;
}
}

public long getStartTime() {
Expand Down Expand Up @@ -270,7 +273,7 @@ public void reset() {
if (positionCount == 0) {
return;
}
positionCount = 0;
setPositionCount(0);
timeColumn.reset();
for (Column valueColumn : valueColumns) {
valueColumn.reset();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@
import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkReadablePosition;
import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkValidRegion;
import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
import static org.apache.tsfile.utils.RamUsageEstimator.sizeOf;
import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray;
import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfObjectArray;

public class BinaryColumn implements Column {

Expand All @@ -55,7 +55,9 @@ public class BinaryColumn implements Column {
private final Binary[] values;

private final long retainedSizeInBytes;
private final long sizeInBytes;
// A view uses the same value bytes and row count as the column they were measured on.
private long measuredValueSizeInBytes;
private int measuredPositionCount;

public BinaryColumn(int initialCapacity) {
this(0, 0, null, new Binary[initialCapacity]);
Expand All @@ -66,36 +68,15 @@ public BinaryColumn(int positionCount, Optional<boolean[]> valueIsNull, Binary[]
}

public BinaryColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, Binary[] values) {
if (arrayOffset < 0) {
throw new IllegalArgumentException(Messages.get("error.read.col_array_offset_negative"));
}
this.arrayOffset = arrayOffset;
if (positionCount < 0) {
throw new IllegalArgumentException(Messages.get("error.read.col_position_count_negative"));
}
this.positionCount = positionCount;

if (values.length - arrayOffset < positionCount) {
throw new IllegalArgumentException(Messages.get("error.read.col_values_length_lt_position"));
}
this.values = values;

if (valueIsNull != null && valueIsNull.length - arrayOffset < positionCount) {
throw new IllegalArgumentException(Messages.get("error.read.col_isnull_length_lt_position"));
}
this.valueIsNull = valueIsNull;

retainedSizeInBytes = INSTANCE_SIZE + sizeOfBooleanArray(positionCount) + sizeOf(values);
sizeInBytes = values.length > 0 ? retainedSizeInBytes * positionCount / values.length : 0L;
this(arrayOffset, positionCount, valueIsNull, values, null);
}

// called by getRegion which already knows the underlying retainedSizeInBytes
private BinaryColumn(
int arrayOffset,
int positionCount,
boolean[] valueIsNull,
Binary[] values,
long retainedSizeInBytes) {
BinaryColumn parent) {
if (arrayOffset < 0) {
throw new IllegalArgumentException(Messages.get("error.read.col_array_offset_negative"));
}
Expand All @@ -114,8 +95,22 @@ private BinaryColumn(
throw new IllegalArgumentException(Messages.get("error.read.col_isnull_length_lt_position"));
}
this.valueIsNull = valueIsNull;
this.retainedSizeInBytes = retainedSizeInBytes;
this.sizeInBytes = values.length > 0 ? retainedSizeInBytes * positionCount / values.length : 0L;
if (parent == null) {
int end = arrayOffset + positionCount;
measuredPositionCount = positionCount;
measuredValueSizeInBytes = sizeOfValues(arrayOffset, end);
retainedSizeInBytes =
INSTANCE_SIZE
+ sizeOfBooleanArray(positionCount)
+ sizeOfObjectArray(values.length)
+ measuredValueSizeInBytes
+ sizeOfValues(0, arrayOffset)
+ sizeOfValues(end, values.length);
} else {
retainedSizeInBytes = parent.retainedSizeInBytes;
measuredPositionCount = parent.measuredPositionCount;
measuredValueSizeInBytes = parent.measuredValueSizeInBytes;
}
}

@Override
Expand Down Expand Up @@ -231,16 +226,37 @@ public long getRetainedSizeInBytes() {
return retainedSizeInBytes;
}

/** Shared regions estimate their value bytes using the average over the measured positions. */
@Override
public long getSizeInBytes() {
return sizeInBytes;
long shallowSize =
INSTANCE_SIZE + sizeOfBooleanArray(positionCount) + sizeOfObjectArray(positionCount);
if (positionCount == measuredPositionCount) {
return shallowSize + measuredValueSizeInBytes;
}
// Scale only the values. Dividing first avoids overflowing when the column is large.
long bytesPerPosition = measuredValueSizeInBytes / measuredPositionCount;
long remainder = measuredValueSizeInBytes % measuredPositionCount;
return shallowSize
+ bytesPerPosition * positionCount
+ remainder * positionCount / measuredPositionCount;
}

private long sizeOfValues(int from, int end) {
long size = 0;
for (int i = from; i < end; i++) {
Binary value = values[i];
if (value != null) {
size += value.ramBytesUsed();
}
}
return size;
}

@Override
public Column getRegion(int positionOffset, int length) {
checkValidRegion(getPositionCount(), positionOffset, length);
return new BinaryColumn(
positionOffset + arrayOffset, length, valueIsNull, values, getRetainedSizeInBytes());
return new BinaryColumn(positionOffset + arrayOffset, length, valueIsNull, values, this);
}

@Override
Expand All @@ -261,8 +277,7 @@ public Column subColumn(int fromIndex) {
if (fromIndex > positionCount) {
throw new IllegalArgumentException(Messages.get("error.read.col_from_index_invalid"));
}
return new BinaryColumn(
arrayOffset + fromIndex, positionCount - fromIndex, valueIsNull, values);
return getRegion(fromIndex, positionCount - fromIndex);
}

@Override
Expand Down Expand Up @@ -331,6 +346,18 @@ public int getInstanceSize() {

@Override
public void setPositionCount(int count) {
int capacity =
valueIsNull == null ? values.length : Math.min(values.length, valueIsNull.length);
checkValidRegion(capacity - arrayOffset, 0, count);
if (count > positionCount && positionCount == measuredPositionCount) {
// Readers fill the backing array in place and publish the newly written positions here.
measuredValueSizeInBytes += sizeOfValues(arrayOffset + positionCount, arrayOffset + count);
measuredPositionCount = count;
} else if (count >= positionCount && count > 0) {
// The previous range was estimated, or existing positions have been filled again.
measuredValueSizeInBytes = sizeOfValues(arrayOffset, arrayOffset + count);
measuredPositionCount = count;
}
positionCount = count;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.apache.tsfile.read.common.block.column.IntColumn;
import org.apache.tsfile.read.common.block.column.LongColumn;
import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn;
import org.apache.tsfile.read.common.block.column.TimeColumn;
import org.apache.tsfile.utils.Binary;

import org.junit.Assert;
Expand Down Expand Up @@ -181,6 +182,45 @@ public void testBinaryTsBlock() {
}
}

@Test
public void testBinaryTsBlockSize() {
byte[] bytes = new byte[1024 * 1024];
TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.TEXT));
builder.getTimeColumnBuilder().writeLong(1L);
builder.getColumnBuilder(0).writeBinary(new Binary(bytes));
builder.declarePosition();
TsBlock tsBlock = builder.build();
// the builder allocates more slots than the only row needs
assertTrue(tsBlock.getColumn(0).getBinaries().length > tsBlock.getPositionCount());
assertTrue(tsBlock.getSizeInBytes() >= bytes.length);
}

@Test
public void testBinaryTsBlockSizeAfterRefill() {
Column time = new TimeColumn(1);
Column column = new BinaryColumn(1);
TsBlock block = new TsBlock(time, column);
long emptySize = block.getSizeInBytes();
Binary small = new Binary(new byte[16]);
Binary large = new Binary(new byte[4096]);

column.getBinaries()[0] = small;
column.setPositionCount(1);
time.setPositionCount(1);
block.setPositionCount(1);
long smallBlockSize = block.getSizeInBytes();
assertTrue(smallBlockSize >= emptySize + small.ramBytesUsed());

block.reset();
assertEquals(emptySize, block.getSizeInBytes());
column.getBinaries()[0] = large;
column.setPositionCount(1);
time.setPositionCount(1);
block.setPositionCount(1);
assertEquals(
smallBlockSize - small.ramBytesUsed() + large.ramBytesUsed(), block.getSizeInBytes());
}

@Test
public void testIntTsBlockWithNull() {
long[] timeArray = {1L, 2L, 3L, 4L, 5L};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@

import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;

import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
Expand Down Expand Up @@ -488,6 +489,116 @@ public void binaryColumnFilterPositionsTest() {
Assert.assertEquals("5", columnByCopyPositions.getBinary(1).toString());
}

@Test
public void binaryColumnSizeIgnoresUnusedCapacityTest() {
Binary value = new Binary(new byte[1024]);
// the builder allocates 8 slots for the two positions
BinaryColumnBuilder columnBuilder = new BinaryColumnBuilder(null, 8);
columnBuilder.appendNull();
columnBuilder.writeBinary(value);
Column overAllocatedColumn = columnBuilder.build();
Assert.assertEquals(8, overAllocatedColumn.getBinaries().length);

Column compactColumn =
new BinaryColumn(2, Optional.of(new boolean[] {true, false}), new Binary[] {null, value});
Assert.assertEquals(compactColumn.getSizeInBytes(), overAllocatedColumn.getSizeInBytes());
Assert.assertTrue(overAllocatedColumn.getSizeInBytes() >= 1024);
Assert.assertEquals(
compactColumn.getRegion(1, 1).getSizeInBytes(),
overAllocatedColumn.getRegion(1, 1).getSizeInBytes());
}

@Test
public void binaryColumnSizeOfRegionTest() {
Binary largeValue = new Binary(new byte[4096]);
Binary smallValue = new Binary(new byte[16]);
Binary[] values = new Binary[8];
Arrays.fill(values, smallValue);
values[2] = largeValue;
Column column = new BinaryColumn(8, Optional.empty(), values);
long smallBytes = smallValue.ramBytesUsed();
long largeBytes = largeValue.ramBytesUsed();
long threeRows = new BinaryColumn(3, Optional.empty(), new Binary[3]).getSizeInBytes();

// A directly constructed range measures its own values during the retained-size scan.
Column direct = new BinaryColumn(2, 3, null, values);
Assert.assertEquals(threeRows + largeBytes + 2 * smallBytes, direct.getSizeInBytes());
Assert.assertEquals(column.getRegionCopy(2, 3).getSizeInBytes(), direct.getSizeInBytes());
long retainedArrays = new BinaryColumn(2, 3, null, new Binary[8]).getRetainedSizeInBytes();
Assert.assertEquals(
retainedArrays + largeBytes + 7 * smallBytes, direct.getRetainedSizeInBytes());

// A shared view of the same positions takes three eighths of its parent's value bytes.
Column region = column.getRegion(2, 3);
long estimate = threeRows + (largeBytes + 7 * smallBytes) * 3 / 8;
Assert.assertSame(values, region.getBinaries());
Assert.assertEquals(column.getRetainedSizeInBytes(), region.getRetainedSizeInBytes());
Assert.assertEquals(estimate, region.getSizeInBytes());
Assert.assertEquals(estimate, column.getRegion(1, 4).subColumn(1).getSizeInBytes());
Assert.assertEquals(estimate, column.subColumn(5).getSizeInBytes());

long oneRow = new BinaryColumn(1, Optional.empty(), new Binary[1]).getSizeInBytes();
Assert.assertEquals(
oneRow + (largeBytes + 2 * smallBytes) / 3, direct.subColumn(2).getSizeInBytes());
Assert.assertEquals(new BinaryColumn(0).getSizeInBytes(), direct.subColumn(3).getSizeInBytes());
}

@Test
public void binaryColumnSizeReadsEachValueOnceTest() {
Binary value = Mockito.mock(Binary.class);
// The sum fits in a long, but multiplying it by the region length would overflow.
long bytes = Long.MAX_VALUE / 10;
Mockito.when(value.ramBytesUsed()).thenReturn(bytes);
Binary[] values = new Binary[8];
Arrays.fill(values, value);
Column column = new BinaryColumn(8, Optional.empty(), values);
long threeRows = new BinaryColumn(3, Optional.empty(), new Binary[3]).getSizeInBytes();

Assert.assertEquals(column.getRetainedSizeInBytes(), column.getSizeInBytes());
Assert.assertEquals(threeRows + 3 * bytes, column.getRegion(2, 3).getSizeInBytes());
Assert.assertEquals(threeRows + 3 * bytes, column.subColumn(5).getSizeInBytes());
Mockito.verify(value, Mockito.times(values.length)).ramBytesUsed();
}

@Test
public void binaryColumnSizeAfterSetPositionCountTest() {
Binary value = new Binary(new byte[1024]);
Column column = new BinaryColumn(2, Optional.empty(), new Binary[] {value, value});
Assert.assertTrue(column.getSizeInBytes() >= 2048);

column.setPositionCount(1);
Column compactColumn = new BinaryColumn(1, Optional.empty(), new Binary[] {value});
Assert.assertEquals(compactColumn.getSizeInBytes(), column.getSizeInBytes());
}

@Test
public void binaryColumnSizeAfterInPlaceWritesTest() {
Binary small = new Binary(new byte[16]);
Binary large = new Binary(new byte[4096]);
Column column = new BinaryColumn(2);
long emptySize = column.getSizeInBytes();
column.getBinaries()[0] = small;
column.setPositionCount(1);
Assert.assertEquals(
new BinaryColumn(1, Optional.empty(), new Binary[] {small}).getSizeInBytes(),
column.getSizeInBytes());
column.getBinaries()[1] = large;
column.setPositionCount(2);
long fullSize =
new BinaryColumn(2, Optional.empty(), new Binary[] {small, large}).getSizeInBytes();
Assert.assertEquals(fullSize, column.getSizeInBytes());

column.setPositionCount(1);
column.setPositionCount(2);
Assert.assertEquals(fullSize, column.getSizeInBytes());
column.getBinaries()[0] = large;
column.setPositionCount(2);
Assert.assertEquals(
fullSize - small.ramBytesUsed() + large.ramBytesUsed(), column.getSizeInBytes());
column.reset();
Assert.assertEquals(emptySize, column.getSizeInBytes());
}

@Test
public void booleanColumnSubColumnTest() {
BooleanColumnBuilder columnBuilder = new BooleanColumnBuilder(null, 10);
Expand Down
5 changes: 5 additions & 0 deletions skills/tsfile/references/performance.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ than a universal setting.
result set.
- Separate metadata inspection from page decoding when the available tool or
API supports it.
- Java `BinaryColumn.getSizeInBytes()` counts the values in the column's
logical range at construction. Shared views from `getRegion()` and
`subColumn()` estimate value bytes in proportion to their row count.
Equal-length views of the same column get the same estimate even when
their strings or blobs differ in length. A copied region counts its own values.

## Encoding and Compression

Expand Down
Loading