From d678307bf6aee926132049c4c42c507718c22728 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 11:55:06 +0800 Subject: [PATCH 1/2] test(cpp): cover time operator filters Add unit tests for TimeBetween, TimeIn, TimeEq, TimeNotEq, TimeGt, TimeGtEq, TimeLt and TimeLtEq: statistic pruning, row-level predicates, range introspection and the SIMD/scalar batch paths. The tests exposed a precedence bug in TimeBetween::satisfy(): `^` binds tighter than `&&`, so NOT BETWEEN rejected timestamps below the lower bound instead of accepting everything outside the interval. Parenthesize the range test to match satisfy(Statistic*) and satisfy_start_end_time(). --- cpp/src/reader/filter/time_operator.cc | 7 +- cpp/test/reader/filter/time_operator_test.cc | 323 +++++++++++++++++++ 2 files changed, 328 insertions(+), 2 deletions(-) create mode 100644 cpp/test/reader/filter/time_operator_test.cc diff --git a/cpp/src/reader/filter/time_operator.cc b/cpp/src/reader/filter/time_operator.cc index 0bb12e4ec..9799ab0f1 100644 --- a/cpp/src/reader/filter/time_operator.cc +++ b/cpp/src/reader/filter/time_operator.cc @@ -56,11 +56,14 @@ bool TimeBetween::satisfy(Statistic* statistic) { } bool TimeBetween::satisfy(int64_t time, int64_t value) { - return (value1_ <= time) && (time <= value2_) ^ not_; + // Parenthesize the range test: `^` binds tighter than `&&`, so without + // the inner parentheses NOT BETWEEN rejects times below value1_ instead + // of accepting every time outside the interval. + return ((value1_ <= time) && (time <= value2_)) ^ not_; } bool TimeBetween::satisfy(int64_t time, common::String value) { - return (value1_ <= time) && (time <= value2_) ^ not_; + return ((value1_ <= time) && (time <= value2_)) ^ not_; } bool TimeBetween::satisfy_start_end_time(int64_t start_time, int64_t end_time) { diff --git a/cpp/test/reader/filter/time_operator_test.cc b/cpp/test/reader/filter/time_operator_test.cc new file mode 100644 index 000000000..d16c54b93 --- /dev/null +++ b/cpp/test/reader/filter/time_operator_test.cc @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "reader/filter/time_operator.h" + +#include + +#include +#include +#include +#include +#include + +#include "common/statistic.h" + +namespace storage { +namespace { + +using Range = std::pair; + +Statistic make_stat(int64_t start_time, int64_t end_time) { + Statistic stat; + stat.start_time_ = start_time; + stat.end_time_ = end_time; + return stat; +} + +bool satisfy_stat(Filter& filter, int64_t start_time, int64_t end_time) { + Statistic stat = make_stat(start_time, end_time); + return filter.satisfy(&stat); +} + +std::vector collect_time_ranges(Filter& filter) { + std::unique_ptr> raw(filter.get_time_ranges()); + std::vector ranges; + for (TimeRange* range : *raw) { + ranges.emplace_back(range->start_time_, range->end_time_); + delete range; + } + return ranges; +} + +std::vector run_batch(Filter& filter, const std::vector& times) { + std::unique_ptr mask = + times.empty() ? nullptr + : std::unique_ptr(new bool[times.size()]); + const int pass = filter.satisfy_batch_time( + times.data(), static_cast(times.size()), mask.get()); + std::vector result(times.size()); + int counted = 0; + for (size_t i = 0; i < times.size(); ++i) { + result[i] = mask[i]; + if (mask[i]) { + ++counted; + } + } + EXPECT_EQ(pass, counted); + return result; +} + +// Six and five elements exercise both the vector width and the scalar tail on +// every platform (NEON processes pairs, the SSE/AVX2 kernel processes quads). +const std::vector kSixTimes = {1, 2, 3, 4, 5, 6}; + +TEST(TimeOperatorTest, TimeBetweenBoundarySemantics) { + TimeBetween between(3, 5, /*not_between=*/false); + Statistic overlap = make_stat(4, 8); + Statistic below = make_stat(1, 2); + Statistic above = make_stat(6, 9); + EXPECT_TRUE(between.satisfy(&overlap)); + EXPECT_FALSE(between.satisfy(&below)); + EXPECT_FALSE(between.satisfy(&above)); + + EXPECT_TRUE(between.satisfy(3, 0)); + EXPECT_TRUE(between.satisfy(5, 0)); + EXPECT_FALSE(between.satisfy(2, 0)); + EXPECT_FALSE(between.satisfy(6, 0)); + EXPECT_TRUE(between.satisfy(4, common::String("ignored"))); + EXPECT_FALSE(between.satisfy(6, common::String("ignored"))); + + EXPECT_TRUE(between.satisfy_start_end_time(2, 4)); + EXPECT_FALSE(between.satisfy_start_end_time(6, 8)); + EXPECT_TRUE(between.contain_start_end_time(3, 5)); + EXPECT_FALSE(between.contain_start_end_time(2, 5)); + + EXPECT_EQ(collect_time_ranges(between), (std::vector{{3, 5}})); + EXPECT_FALSE(between.is_unbounded_time_filter()); + EXPECT_TRUE(between.is_contiguous_time_filter()); + EXPECT_EQ(between.get_filter_type(), TIME_FILTER); + + EXPECT_EQ(run_batch(between, kSixTimes), + (std::vector{false, false, true, true, true, false})); +} + +TEST(TimeOperatorTest, TimeBetweenNotBetweenSemantics) { + TimeBetween not_between(3, 5, /*not_between=*/true); + Statistic inside = make_stat(4, 4); + Statistic outside = make_stat(6, 9); + EXPECT_FALSE(not_between.satisfy(&inside)); + EXPECT_TRUE(not_between.satisfy(&outside)); + + EXPECT_FALSE(not_between.satisfy(4, 0)); + EXPECT_TRUE(not_between.satisfy(6, 0)); + EXPECT_TRUE(not_between.satisfy(2, common::String("ignored"))); + + EXPECT_TRUE(not_between.satisfy_start_end_time(0, 2)); + EXPECT_TRUE(not_between.contain_start_end_time(6, 8)); + EXPECT_FALSE(not_between.contain_start_end_time(4, 5)); + + EXPECT_EQ(collect_time_ranges(not_between), + (std::vector{{std::numeric_limits::min(), 2}, + {6, std::numeric_limits::max()}})); + EXPECT_FALSE(not_between.is_contiguous_time_filter()); + EXPECT_EQ(run_batch(not_between, kSixTimes), + (std::vector{true, true, false, false, false, true})); +} + +TEST(TimeOperatorTest, TimeBetweenInvertedRange) { + TimeBetween inverted(5, 3, /*not_between=*/false); + Statistic any = make_stat(0, 100); + EXPECT_FALSE(inverted.satisfy(&any)); + EXPECT_FALSE(inverted.satisfy(4, 0)); + EXPECT_FALSE(inverted.satisfy_start_end_time(0, 100)); + EXPECT_FALSE(inverted.contain_start_end_time(0, 100)); + EXPECT_TRUE(collect_time_ranges(inverted).empty()); + + TimeBetween inverted_not(5, 3, /*not_between=*/true); + EXPECT_TRUE(inverted_not.satisfy(&any)); + EXPECT_TRUE(inverted_not.satisfy(4, 0)); + EXPECT_TRUE(inverted_not.satisfy_start_end_time(0, 100)); + EXPECT_EQ(collect_time_ranges(inverted_not), + (std::vector{{std::numeric_limits::min(), + std::numeric_limits::max()}})); + + TimeBetween unbounded(std::numeric_limits::min(), + std::numeric_limits::max(), + /*not_between=*/false); + EXPECT_TRUE(unbounded.is_unbounded_time_filter()); +} + +TEST(TimeOperatorTest, TimeInSemantics) { + TimeIn in({2, 8}, /*not_in=*/false); + Statistic any = make_stat(10, 20); + EXPECT_TRUE(in.satisfy(&any)); + EXPECT_TRUE(in.satisfy(2, 0)); + EXPECT_FALSE(in.satisfy(5, 0)); + EXPECT_TRUE(in.satisfy(8, common::String("ignored"))); + EXPECT_FALSE(in.satisfy(9, common::String("ignored"))); + EXPECT_FALSE(in.satisfy_start_end_time(3, 7)); + EXPECT_EQ(collect_time_ranges(in), (std::vector{{2, 2}, {8, 8}})); + EXPECT_EQ(run_batch(in, kSixTimes), + (std::vector{false, true, false, false, false, false})); + + TimeIn not_in({2, 8}, /*not_in=*/true); + EXPECT_FALSE(not_in.satisfy(2, 0)); + EXPECT_TRUE(not_in.satisfy(5, 0)); + EXPECT_FALSE(not_in.satisfy(8, common::String("ignored"))); + EXPECT_EQ(run_batch(not_in, kSixTimes), + (std::vector{true, false, true, true, true, true})); +} + +TEST(TimeOperatorTest, TimeEqSemantics) { + TimeEq eq(3); + Statistic containing = make_stat(1, 5); + Statistic excluding = make_stat(4, 6); + EXPECT_TRUE(eq.satisfy(&containing)); + EXPECT_FALSE(eq.satisfy(&excluding)); + EXPECT_TRUE(eq.satisfy(3, 0)); + EXPECT_FALSE(eq.satisfy(4, 0)); + EXPECT_TRUE(eq.satisfy(3, common::String("ignored"))); + EXPECT_FALSE(eq.satisfy(2, common::String("ignored"))); + EXPECT_TRUE(eq.satisfy_start_end_time(2, 4)); + EXPECT_FALSE(eq.satisfy_start_end_time(4, 6)); + EXPECT_TRUE(eq.contain_start_end_time(3, 3)); + EXPECT_FALSE(eq.contain_start_end_time(2, 3)); + EXPECT_EQ(collect_time_ranges(eq), (std::vector{{3, 3}})); + EXPECT_EQ(run_batch(eq, kSixTimes), + (std::vector{false, false, true, false, false, false})); +} + +TEST(TimeOperatorTest, TimeNotEqSemantics) { + TimeNotEq neq(3); + Statistic single = make_stat(3, 3); + Statistic range = make_stat(3, 4); + EXPECT_FALSE(neq.satisfy(&single)); + EXPECT_TRUE(neq.satisfy(&range)); + EXPECT_FALSE(neq.satisfy(3, 0)); + EXPECT_TRUE(neq.satisfy(4, 0)); + EXPECT_FALSE(neq.satisfy(3, common::String("ignored"))); + EXPECT_TRUE(neq.satisfy_start_end_time(1, 2)); + EXPECT_FALSE(neq.satisfy_start_end_time(3, 5)); + EXPECT_FALSE(neq.satisfy_start_end_time(1, 3)); + EXPECT_TRUE(neq.contain_start_end_time(4, 5)); + EXPECT_TRUE(neq.contain_start_end_time(1, 2)); + EXPECT_FALSE(neq.contain_start_end_time(1, 4)); + EXPECT_EQ(collect_time_ranges(neq), + (std::vector{{std::numeric_limits::min(), 2}, + {4, std::numeric_limits::max()}})); + EXPECT_EQ(run_batch(neq, kSixTimes), + (std::vector{true, true, false, true, true, true})); + + TimeNotEq not_min(std::numeric_limits::min()); + EXPECT_EQ(collect_time_ranges(not_min), + (std::vector{{std::numeric_limits::min() + 1, + std::numeric_limits::max()}})); + TimeNotEq not_max(std::numeric_limits::max()); + EXPECT_EQ(collect_time_ranges(not_max), + (std::vector{{std::numeric_limits::min(), + std::numeric_limits::max() - 1}})); +} + +TEST(TimeOperatorTest, TimeGtSemantics) { + TimeGt gt(2); + EXPECT_TRUE(satisfy_stat(gt, 1, 3)); + EXPECT_FALSE(satisfy_stat(gt, 1, 2)); + EXPECT_TRUE(gt.satisfy(3, 0)); + EXPECT_FALSE(gt.satisfy(2, 0)); + EXPECT_TRUE(gt.satisfy(3, common::String("ignored"))); + EXPECT_FALSE(gt.satisfy(1, common::String("ignored"))); + EXPECT_TRUE(gt.satisfy_start_end_time(1, 3)); + EXPECT_FALSE(gt.satisfy_start_end_time(1, 2)); + EXPECT_TRUE(gt.contain_start_end_time(3, 5)); + EXPECT_FALSE(gt.contain_start_end_time(1, 5)); + EXPECT_EQ(collect_time_ranges(gt), + (std::vector{{3, std::numeric_limits::max()}})); + EXPECT_EQ(run_batch(gt, kSixTimes), + (std::vector{false, false, true, true, true, true})); + + TimeGt gt_max(std::numeric_limits::max()); + EXPECT_TRUE(collect_time_ranges(gt_max).empty()); +} + +TEST(TimeOperatorTest, TimeGtEqSemantics) { + TimeGtEq gt_eq(2); + EXPECT_TRUE(satisfy_stat(gt_eq, 2, 3)); + EXPECT_TRUE(satisfy_stat(gt_eq, 1, 2)); + EXPECT_FALSE(satisfy_stat(gt_eq, 1, 1)); + EXPECT_TRUE(gt_eq.satisfy(2, 0)); + EXPECT_FALSE(gt_eq.satisfy(1, 0)); + EXPECT_TRUE(gt_eq.satisfy(2, common::String("ignored"))); + EXPECT_FALSE(gt_eq.satisfy(0, common::String("ignored"))); + EXPECT_TRUE(gt_eq.satisfy_start_end_time(1, 2)); + EXPECT_FALSE(gt_eq.satisfy_start_end_time(1, 1)); + EXPECT_TRUE(gt_eq.contain_start_end_time(2, 5)); + EXPECT_TRUE(gt_eq.contain_start_end_time(3, 5)); + EXPECT_FALSE(gt_eq.contain_start_end_time(1, 5)); + EXPECT_EQ(collect_time_ranges(gt_eq), + (std::vector{{2, std::numeric_limits::max()}})); + gt_eq.reset_value(4); + EXPECT_EQ(run_batch(gt_eq, kSixTimes), + (std::vector{false, false, false, true, true, true})); +} + +TEST(TimeOperatorTest, TimeLtSemantics) { + TimeLt lt(5); + EXPECT_TRUE(satisfy_stat(lt, 4, 9)); + EXPECT_FALSE(satisfy_stat(lt, 5, 9)); + EXPECT_TRUE(lt.satisfy(4, 0)); + EXPECT_FALSE(lt.satisfy(5, 0)); + EXPECT_TRUE(lt.satisfy(4, common::String("ignored"))); + EXPECT_FALSE(lt.satisfy(6, common::String("ignored"))); + EXPECT_TRUE(lt.satisfy_start_end_time(4, 9)); + EXPECT_FALSE(lt.satisfy_start_end_time(5, 9)); + EXPECT_TRUE(lt.contain_start_end_time(0, 4)); + EXPECT_FALSE(lt.contain_start_end_time(0, 6)); + EXPECT_EQ(collect_time_ranges(lt), + (std::vector{{std::numeric_limits::min(), 4}})); + EXPECT_EQ(run_batch(lt, kSixTimes), + (std::vector{true, true, true, true, false, false})); + + TimeLt lt_min(std::numeric_limits::min()); + EXPECT_TRUE(collect_time_ranges(lt_min).empty()); +} + +TEST(TimeOperatorTest, TimeLtEqSemantics) { + TimeLtEq lt_eq(5); + EXPECT_TRUE(satisfy_stat(lt_eq, 5, 9)); + EXPECT_FALSE(satisfy_stat(lt_eq, 6, 9)); + EXPECT_TRUE(lt_eq.satisfy(5, 0)); + EXPECT_FALSE(lt_eq.satisfy(6, 0)); + EXPECT_TRUE(lt_eq.satisfy(5, common::String("ignored"))); + EXPECT_FALSE(lt_eq.satisfy(7, common::String("ignored"))); + EXPECT_TRUE(lt_eq.satisfy_start_end_time(5, 9)); + EXPECT_FALSE(lt_eq.satisfy_start_end_time(6, 9)); + EXPECT_TRUE(lt_eq.contain_start_end_time(0, 4)); + EXPECT_TRUE(lt_eq.contain_start_end_time(0, 5)); + EXPECT_FALSE(lt_eq.contain_start_end_time(0, 6)); + EXPECT_EQ(collect_time_ranges(lt_eq), + (std::vector{{std::numeric_limits::min(), 5}})); + EXPECT_EQ(run_batch(lt_eq, kSixTimes), + (std::vector{true, true, true, true, true, false})); +} + +TEST(TimeOperatorTest, BatchHandlesVectorWidthAndTail) { + TimeGt gt(3); + const std::vector five_times = {1, 3, 4, 5, 9}; + EXPECT_EQ(run_batch(gt, five_times), + (std::vector{false, false, true, true, true})); + EXPECT_TRUE(run_batch(gt, {}).empty()); + + TimeBetween between(2, 4, /*not_between=*/false); + EXPECT_EQ(run_batch(between, five_times), + (std::vector{false, true, true, false, false})); +} + +} // namespace +} // namespace storage From 4541a7c203c893182fc1d7e17622cf4b61607a36 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 12:23:24 +0800 Subject: [PATCH 2/2] fix(cpp): do not prune blocks for NOT EQ time filters TimeNotEq::satisfy_start_end_time() returned false whenever the excluded value matched either end of the block range. Chunk readers skip a block when it returns false, so `time != 3` dropped every row of a block starting or ending at 3. Only a single-point range can be fully excluded. Expect true for ranges that straddle the excluded value and add an end-to-end query regression that reads 299 of 300 rows. --- cpp/src/reader/filter/time_operator.cc | 6 +- cpp/test/reader/filter/time_operator_test.cc | 93 +++++++++++++++++++- 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/cpp/src/reader/filter/time_operator.cc b/cpp/src/reader/filter/time_operator.cc index 9799ab0f1..fd3b4d4f0 100644 --- a/cpp/src/reader/filter/time_operator.cc +++ b/cpp/src/reader/filter/time_operator.cc @@ -233,7 +233,11 @@ bool TimeNotEq::satisfy(int64_t time, common::String value) { } bool TimeNotEq::satisfy_start_end_time(int64_t start_time, int64_t end_time) { - return value_ != end_time && value_ != start_time; + // "Could any time in [start, end] satisfy the filter?" Only a + // single-point range can be excluded completely; any wider range contains + // a timestamp different from value_, and chunk readers skip the whole + // block when this returns false. + return start_time != end_time || value_ != start_time; } bool TimeNotEq::contain_start_end_time(int64_t start_time, int64_t end_time) { diff --git a/cpp/test/reader/filter/time_operator_test.cc b/cpp/test/reader/filter/time_operator_test.cc index d16c54b93..c2ee5b1c6 100644 --- a/cpp/test/reader/filter/time_operator_test.cc +++ b/cpp/test/reader/filter/time_operator_test.cc @@ -21,17 +21,36 @@ #include +#include #include +#include #include #include +#include #include #include +#include "common/global.h" +#include "common/record.h" +#include "common/schema.h" #include "common/statistic.h" +#include "reader/expression.h" +#include "reader/tsfile_reader.h" +#include "writer/tsfile_writer.h" + +#if defined(_WIN32) +#include +#define TIME_OP_GETPID _getpid +#else +#include +#define TIME_OP_GETPID getpid +#endif namespace storage { namespace { +using namespace common; + using Range = std::pair; Statistic make_stat(int64_t start_time, int64_t end_time) { @@ -204,8 +223,9 @@ TEST(TimeOperatorTest, TimeNotEqSemantics) { EXPECT_TRUE(neq.satisfy(4, 0)); EXPECT_FALSE(neq.satisfy(3, common::String("ignored"))); EXPECT_TRUE(neq.satisfy_start_end_time(1, 2)); - EXPECT_FALSE(neq.satisfy_start_end_time(3, 5)); - EXPECT_FALSE(neq.satisfy_start_end_time(1, 3)); + EXPECT_FALSE(neq.satisfy_start_end_time(3, 3)); + EXPECT_TRUE(neq.satisfy_start_end_time(3, 5)); + EXPECT_TRUE(neq.satisfy_start_end_time(1, 3)); EXPECT_TRUE(neq.contain_start_end_time(4, 5)); EXPECT_TRUE(neq.contain_start_end_time(1, 2)); EXPECT_FALSE(neq.contain_start_end_time(1, 4)); @@ -319,5 +339,74 @@ TEST(TimeOperatorTest, BatchHandlesVectorWidthAndTail) { (std::vector{false, true, true, false, false})); } +std::string unique_time_operator_file() { + static std::atomic counter{0}; + return "time_operator_query_" + std::to_string(TIME_OP_GETPID()) + "_" + + std::to_string(counter.fetch_add(1)) + ".tsfile"; +} + +// Regression: TimeNotEq::satisfy_start_end_time() used to return false when +// the excluded value was either end of the block range. Chunk readers skip a +// block on false, so `time != 3` dropped every row of the block starting at 3. +TEST(TimeOperatorTest, TimeNotEqDoesNotSkipOverlappingBlocks) { + ASSERT_EQ(libtsfile_init(), E_OK); + const std::string file = unique_time_operator_file(); + const std::string device = "root.time_operator"; + const std::string measurement = "v"; + std::remove(file.c_str()); + + { + TsFileWriter writer; + ASSERT_EQ(writer.open(file), E_OK); + MeasurementSchema schema(measurement, INT64); + ASSERT_EQ(writer.register_timeseries(device, schema), E_OK); + for (int64_t t = 3; t <= 302; ++t) { + TsRecord record(t, device); + record.add_point(measurement, static_cast(t)); + ASSERT_EQ(writer.write_record(record), E_OK); + } + ASSERT_EQ(writer.flush(), E_OK); + ASSERT_EQ(writer.close(), E_OK); + } + + { + TsFileReader reader; + ASSERT_EQ(reader.open(file), E_OK); + Path path(device + "." + measurement); + std::vector paths{path}; + ResultSet* result = nullptr; + ASSERT_EQ(reader.query(QueryExpression::create( + paths, new Expression(SERIES_EXPR, path, + new TimeNotEq(3))), + result), + E_OK); + ASSERT_NE(result, nullptr); + + int64_t expected_time = 4; + int rows = 0; + bool has_next = false; + while (true) { + ASSERT_EQ(result->next(has_next), E_OK); + if (!has_next) { + break; + } + RowRecord* record = result->get_row_record(); + ASSERT_NE(record, nullptr); + ASSERT_EQ(record->get_timestamp(), expected_time); + ASSERT_EQ(record->get_field(1)->value_.lval_, expected_time); + ++expected_time; + ++rows; + } + EXPECT_EQ(rows, 299); + EXPECT_EQ(expected_time, 303); + + reader.destroy_query_data_set(result); + ASSERT_EQ(reader.close(), E_OK); + } + + std::remove(file.c_str()); + libtsfile_destroy(); +} + } // namespace } // namespace storage