Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -201,13 +201,11 @@

// get min/max value of cached Long class instances:
long longCacheMinValue = 0;
while (longCacheMinValue > Long.MIN_VALUE
&& Long.valueOf(longCacheMinValue - 1) == Long.valueOf(longCacheMinValue - 1)) {
while (longCacheMinValue > Long.MIN_VALUE && isCachedLong(longCacheMinValue - 1)) {
longCacheMinValue -= 1;
}
long longCacheMaxValue = -1;
while (longCacheMaxValue < Long.MAX_VALUE
&& Long.valueOf(longCacheMaxValue + 1) == Long.valueOf(longCacheMaxValue + 1)) {
while (longCacheMaxValue < Long.MAX_VALUE && isCachedLong(longCacheMaxValue + 1)) {
longCacheMaxValue += 1;
}
LONG_CACHE_MIN_VALUE = longCacheMinValue;
Expand All @@ -231,6 +229,18 @@
return (size + ALIGN_MASK) & ~ALIGN_MASK;
}

/**
* Returns true when {@link Long#valueOf(long)} hands out the same cached instance for {@code
* value}. The two references are compared on purpose — identity is exactly what identifies a
* value inside the JVM's {@code Long} cache — and they are typed as {@link Object} so this is not
* a boxed-value comparison.
*/
private static boolean isCachedLong(long value) {
final Object first = Long.valueOf(value);
final Object second = Long.valueOf(value);
return first == second;
}

/**
* Return the size of the provided {@link Long} object, returning 0 if it is cached by the JVM and
* its shallow size otherwise.
Expand Down Expand Up @@ -576,7 +586,7 @@
// Walk type hierarchy
for (; clazz != null; clazz = clazz.getSuperclass()) {
final Field[] fields =
AccessController.doPrivileged((PrivilegedAction<Field[]>) clazz::getDeclaredFields);

Check warning on line 589 in java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java

View workflow job for this annotation

GitHub Actions / Generate java fixtures (shard 0)

java.security.AccessController in java.security has been deprecated and marked for removal

Check warning on line 589 in java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java

View workflow job for this annotation

GitHub Actions / Generate java fixtures (shard 1)

java.security.AccessController in java.security has been deprecated and marked for removal

Check warning on line 589 in java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java

View workflow job for this annotation

GitHub Actions / Generate java fixtures (shard 2)

java.security.AccessController in java.security has been deprecated and marked for removal

Check warning on line 589 in java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java

View workflow job for this annotation

GitHub Actions / unit-test (17, macos-latest)

java.security.AccessController in java.security has been deprecated and marked for removal

Check warning on line 589 in java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java

View workflow job for this annotation

GitHub Actions / unit-test (17, ubuntu-latest)

java.security.AccessController in java.security has been deprecated and marked for removal

Check warning on line 589 in java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java

View workflow job for this annotation

GitHub Actions / unit-test (21, macos-latest)

java.security.AccessController in java.security has been deprecated and marked for removal

Check warning on line 589 in java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java

View workflow job for this annotation

GitHub Actions / unit-test (21, ubuntu-latest)

java.security.AccessController in java.security has been deprecated and marked for removal

Check warning on line 589 in java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java

View workflow job for this annotation

GitHub Actions / unit-test (25, macos-latest)

java.security.AccessController in java.security has been deprecated and marked for removal

Check warning on line 589 in java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java

View workflow job for this annotation

GitHub Actions / unit-test (25, ubuntu-latest)

java.security.AccessController in java.security has been deprecated and marked for removal

Check warning on line 589 in java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java

View workflow job for this annotation

GitHub Actions / Validate cpp shard 2 with java

java.security.AccessController in java.security has been deprecated and marked for removal

Check warning on line 589 in java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java

View workflow job for this annotation

GitHub Actions / Validate cpp shard 0 with java

java.security.AccessController in java.security has been deprecated and marked for removal

Check warning on line 589 in java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java

View workflow job for this annotation

GitHub Actions / Validate cpp shard 1 with java

java.security.AccessController in java.security has been deprecated and marked for removal
for (Field f : fields) {
if (!Modifier.isStatic(f.getModifiers())) {
size = adjustForField(size, f);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,6 @@ public int getOneItemMaxSize() {
@Override
public long getMaxByteSize() {
// byteCacheSize + byteDictSize + (byte array + array length) * byteDictSize
return (long) 4 + 4 + ((values.size() + 7) / 8 + 4) * values.size();
return (long) 4 + 4 + ((values.size() + 7L) / 8 + 4) * values.size();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ public int getOneItemMaxSize() {
@Override
public long getMaxByteSize() {
// The meaning of 24 is: index(4)+width(4)+minDeltaBase(4)+firstValue(4)
return (long) 24 + writeIndex * 4;
return (long) 24 + (long) writeIndex * 4;
}
}

Expand Down Expand Up @@ -294,7 +294,7 @@ public int getOneItemMaxSize() {
@Override
public long getMaxByteSize() {
// The meaning of 24 is: index(4)+width(4)+minDeltaBase(8)+firstValue(8)
return (long) 24 + writeIndex * 8;
return (long) 24 + (long) writeIndex * 8;
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,6 @@ public long getMaxByteSize() {
}
// try to calculate max value
int groupNum = (values.size() / 8 + 1) / 63 + 1;
return (long) 8 + groupNum * 5 + values.size() * 4;
return (long) 8 + groupNum * 5L + (long) values.size() * 4;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,6 @@ public long getMaxByteSize() {
return 0;
}
// try to calculate max value
return (long) 8 + values.size() * 4;
return (long) 8 + (long) values.size() * 4;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,6 @@ public long getMaxByteSize() {
}
// try to calculate max value
int groupNum = (values.size() / 8 + 1) / 63 + 1;
return (long) 8 + groupNum * 5 + values.size() * 8;
return (long) 8 + groupNum * 5L + (long) values.size() * 8;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,6 @@ public long getMaxByteSize() {
return 0;
}
// try to calculate max value
return (long) 8 + values.size() * 4;
return (long) 8 + (long) values.size() * 4;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ public int getOneItemMaxSize() {
public long getMaxByteSize() {
// The meaning of 20 is:
// identifier(4)+bitmapLength(4)+index(4)+minDeltaBase(4)+firstValue(4)
return (long) 20 + (writeIndex * 2 / 8) + (writeIndex * 4);
return (long) 20 + ((long) writeIndex * 2 / 8) + ((long) writeIndex * 4);
}

@Override
Expand Down Expand Up @@ -291,7 +291,7 @@ public int getOneItemMaxSize() {
public long getMaxByteSize() {
// The meaning of 20 is:
// identifier(4)+bitmapLength(4)+index(4)+minDeltaBase(8)+firstValue(8)
return (long) 28 + (writeIndex * 2 / 8) + (writeIndex * 8);
return (long) 28 + ((long) writeIndex * 2 / 8) + ((long) writeIndex * 8);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -784,11 +784,17 @@ public static void copyDirectory(
requireDirectory(srcDir, "srcDir");
requireCanonicalPathsNotEquals(srcDir, destDir);

// Cater for destination being directory within the source directory (see IO-141)
// Cater for destination being directory within the source directory (see IO-141).
// Compare against a path prefix that ends with the separator, so a sibling
// such as "/a/bc" is not mistaken for a child of "/a/b".
List<String> exclusionList = null;
final String srcDirCanonicalPath = srcDir.getCanonicalPath();
final String destDirCanonicalPath = destDir.getCanonicalPath();
if (destDirCanonicalPath.startsWith(srcDirCanonicalPath)) {
final String srcDirPrefix =
srcDirCanonicalPath.endsWith(File.separator)
? srcDirCanonicalPath
: srcDirCanonicalPath + File.separator;
if (destDirCanonicalPath.startsWith(srcDirPrefix)) {
final File[] srcFiles = listFiles(srcDir, fileFilter);
if (srcFiles.length > 0) {
exclusionList = new ArrayList<>(srcFiles.length);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,15 +96,17 @@ public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
if (!(o instanceof ExtractValueCompareFilter)) {
return false;
}
if (getClass() != o.getClass()) {
return false;
}
if (!super.equals(o)) {
return false;
}
ExtractTimeFilterOperators.ExtractTimeCompareFilter thatDelegate =
((ExtractValueCompareFilter) o).delegate;
return delegate.equals(thatDelegate);
ExtractValueCompareFilter that = (ExtractValueCompareFilter) o;
return delegate.equals(that.delegate);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,13 @@ public boolean containsMonth() {
* @return the total duration of this timeDuration in current precision
*/
public long getTotalDuration(TimeUnit currPrecision) {
return currPrecision.convert(monthDuration * 30 * 86400_000L, TimeUnit.MILLISECONDS)
return currPrecision.convert(monthDuration * 30L * 86400_000L, TimeUnit.MILLISECONDS)
+ nonMonthDuration;
}

/** Think month as 31 days. */
public long getMaxTotalDuration(TimeUnit currPrecision) {
return currPrecision.convert(monthDuration * 31 * 86400_000L, TimeUnit.MILLISECONDS)
return currPrecision.convert(monthDuration * 31L * 86400_000L, TimeUnit.MILLISECONDS)
+ nonMonthDuration;
}

Expand All @@ -84,7 +84,7 @@ public TimeDuration multiple(long times) {

/** Think month as 28 days. */
public long getMinTotalDuration(TimeUnit currPrecision) {
return currPrecision.convert(monthDuration * 28 * 86400_000L, TimeUnit.MILLISECONDS)
return currPrecision.convert(monthDuration * 28L * 86400_000L, TimeUnit.MILLISECONDS)
+ nonMonthDuration;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -986,6 +986,15 @@ public boolean equals(Object o) {
return true;
}

@Override
public int hashCode() {
// Must stay consistent with equals(): `schemas` is left out so the hash
// does not depend on the List's element hash implementations; the
// remaining partial fields keep equal tablets on the same hash, and
// unequal tablets may still collide, which is allowed.
return Objects.hash(rowSize, insertTargetName, columnCategories, measurementIndex);
}

private boolean isTimestampsEqual(long[] thisTimestamps, long[] thatTimestamps, int rowSize) {
if (thisTimestamps == thatTimestamps) {
return true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -483,7 +483,15 @@ public boolean equals(Object o) {

@Override
public int hashCode() {
return Objects.hash(deviceId, types, encodings, unifiedCompressor, compressors);
// The arrays are compared by content in equals(), so hash their contents
// as well: array identity hashing would give equal schemas different
// hashes.
return Objects.hash(
deviceId,
Arrays.hashCode(types),
Arrays.hashCode(encodings),
unifiedCompressor,
Arrays.hashCode(compressors));
}

/** compare by vector name */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ public class BitmapDecoderTest {
private static final Logger logger = LoggerFactory.getLogger(BitmapDecoderTest.class);

private List<Integer> intList;
private List<Boolean> booleanList;

@Before
public void setUp() {
Expand All @@ -52,14 +51,6 @@ public void setUp() {
for (int i = 0; i < int_num; i++) {
intList.add(int_array[i % int_len]);
}

booleanList = new ArrayList<Boolean>();
boolean[] boolean_array = {true, false, true, true, false, true, false, false};
int boolean_len = boolean_array.length;
int boolean_num = 100000;
for (int i = 0; i < boolean_num; i++) {
booleanList.add(boolean_array[i % boolean_len]);
}
}

@After
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ public class ReadInPartitionTest {
private static final String FILE_PATH = TsFileGeneratorForTest.outputDataFile;
private static TsFileReader roTsFile = null;
private ArrayList<TimeRange> d1s6timeRangeList = new ArrayList<>();
private ArrayList<TimeRange> d2s1timeRangeList = new ArrayList<>();
private ArrayList<long[]> d1chunkGroupMetaDataOffsetList = new ArrayList<>();

@Before
Expand Down Expand Up @@ -91,12 +90,6 @@ public void before() throws IOException {
+ chunkMetaData.getStatistics().getSerializedSize();
d1chunkGroupMetaDataOffsetList.add(startEndOffsets);
}

List<ChunkMetadata> d2s1List = reader.getChunkMetadataList(new Path("d2", "s1", true));
for (ChunkMetadata chunkMetaData : d2s1List) {
d2s1timeRangeList.add(
new TimeRange(chunkMetaData.getStartTime(), chunkMetaData.getEndTime()));
}
}

@After
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.StringArrayDeviceID;
import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.read.v4.DeviceTableModelReader;
import org.apache.tsfile.read.v4.ITsFileReader;
Expand All @@ -45,7 +44,6 @@
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

Expand Down Expand Up @@ -207,7 +205,6 @@ public void testGetTableDeviceMethods() throws Exception {
startValue,
chunkGroupSize,
pageSize);
List<IDeviceID> deviceIDList = new ArrayList<>();
TableSchema tableSchema =
new TableSchema(
"t1",
Expand Down Expand Up @@ -245,8 +242,6 @@ public void testGetTableDeviceMethods() throws Exception {
tablet.addValue("id1", i, ids[i][0]);
tablet.addValue("id2", i, ids[i][1]);
tablet.addValue("id3", i, ids[i][2]);
deviceIDList.add(
new StringArrayDeviceID(tableSchema.getTableName(), ids[i][0], ids[i][1], ids[i][2]));
tablet.addValue("s1", i, i);
}
tablet.setRowSize(ids.length);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.TsFileSequenceReader;
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.read.common.RowRecord;
import org.apache.tsfile.read.common.block.TsBlock;
import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.read.common.type.service.TypeService;
Expand Down Expand Up @@ -497,9 +496,8 @@ public void testHybridWrite() throws Exception {
final QueryExpression queryExpression = QueryExpression.create(selectedSeries, null);
final QueryDataSet queryDataSet = queryExecutor.execute(queryExpression);
cnt = 0;
List<RowRecord> rowRecords = new ArrayList<>();
while (queryDataSet.hasNext()) {
rowRecords.add(queryDataSet.next());
queryDataSet.next();
cnt++;
}
assertEquals(10, cnt);
Expand Down
25 changes: 5 additions & 20 deletions python/tests/test_batch_arrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,7 @@ def test_batch_read_arrow_basic():
tablet.add_value_by_name("value1", i, i * 10)
tablet.add_value_by_name("value2", i, i * 1.5)
writer.write_table(tablet)
try:
import pyarrow as pa
except ImportError:
pytest.skip("pyarrow is not installed")
pa = pytest.importorskip("pyarrow")

reader = TsFileReader(file_path)
result_set = reader.query_table(
Expand Down Expand Up @@ -134,10 +131,7 @@ def test_batch_read_arrow_compare_with_dataframe():
tablet.add_value_by_name("value3", i, i % 2 == 0)
writer.write_table(tablet)

try:
import pyarrow as pa
except ImportError:
pytest.skip("pyarrow is not installed")
pa = pytest.importorskip("pyarrow")

reader1 = TsFileReader(file_path)
result_set1 = reader1.query_table(
Expand Down Expand Up @@ -242,10 +236,7 @@ def test_batch_read_arrow_empty_result():
tablet.add_value_by_name("value", i, i)
writer.write_table(tablet)

try:
import pyarrow as pa
except ImportError:
pytest.skip("pyarrow is not installed")
pa = pytest.importorskip("pyarrow")

reader = TsFileReader(file_path)
result_set = reader.query_table(
Expand Down Expand Up @@ -294,10 +285,7 @@ def test_batch_read_arrow_time_range():
tablet.add_value_by_name("value", i, i)
writer.write_table(tablet)

try:
import pyarrow as pa
except ImportError:
pytest.skip("pyarrow is not installed")
pa = pytest.importorskip("pyarrow")

reader = TsFileReader(file_path)
result_set = reader.query_table(
Expand Down Expand Up @@ -384,10 +372,7 @@ def test_batch_read_arrow_all_datatypes():
tablet.add_value_by_name("date_val", i, date(2025, 1, (i % 28) + 1))
writer.write_table(tablet)

try:
import pyarrow as pa
except ImportError:
pytest.skip("pyarrow is not installed")
pa = pytest.importorskip("pyarrow")

reader = TsFileReader(file_path)
result_set = reader.query_table(
Expand Down
Loading