diff --git a/cpp/src/reader/expression.cc b/cpp/src/reader/expression.cc index 3623b390e..00764ce0f 100644 --- a/cpp/src/reader/expression.cc +++ b/cpp/src/reader/expression.cc @@ -4,7 +4,7 @@ * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance + * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 @@ -24,180 +24,283 @@ namespace storage { -void QueryExpression::add_time_filter_to_series_filter( +namespace { + +// Takes ownership of both filters and returns the combined filter, or +// nullptr after releasing them when either side is missing. +Filter* make_binary_filter(Filter* left, Filter* right, ExpressionType type) { + if (left == nullptr || right == nullptr) { + delete left; + delete right; + return nullptr; + } + if (type == AND_EXPR) { + return new AndFilter(left, right); + } + return new OrFilter(left, right); +} + +} // namespace + +bool QueryExpression::add_time_filter_to_series_filter( Filter* time_filter, Expression* single_series_expr) { - Filter* filter = new AndFilter(single_series_expr->filter_, time_filter); - single_series_expr->filter_ = filter; - my_filters_.push_back(filter); + if (time_filter == nullptr) { + return false; + } + // Each leaf owns its own copy: a single Filter instance must not be + // attached to several expressions, or destruction would free it twice. + Filter* copy = time_filter->clone(); + if (copy == nullptr) { + return false; + } + if (single_series_expr->filter_ == nullptr) { + single_series_expr->filter_ = copy; + } else { + single_series_expr->filter_ = + new AndFilter(single_series_expr->filter_, copy); + } + return true; } -void QueryExpression::add_time_filter_to_query_filter(Filter* time_filter, +bool QueryExpression::add_time_filter_to_query_filter(Filter* time_filter, Expression* expression) { if (expression->type_ == SERIES_EXPR) { - add_time_filter_to_series_filter(time_filter, expression); + return add_time_filter_to_series_filter(time_filter, expression); } else if ((expression->type_ == AND_EXPR) || (expression->type_ == OR_EXPR)) { - add_time_filter_to_query_filter(time_filter, expression->left_); - add_time_filter_to_query_filter(time_filter, expression->right_); - } else { - std::cout << "Expression should contains only SingleSeriesExpression " - "but other type is found" - << std::endl; + return add_time_filter_to_query_filter(time_filter, + expression->left_) && + add_time_filter_to_query_filter(time_filter, expression->right_); } + std::cout << "Expression should contains only SingleSeriesExpression " + "but other type is found" + << std::endl; + return false; } Expression* QueryExpression::combine_two_global_time_filter( - Expression* left, Expression* right, ExpressionType type) { - if (type == AND_EXPR) { - Filter* filter = new AndFilter(left->filter_, right->filter_); - Expression* expr = new Expression(GLOBALTIME_EXPR, filter); - my_filters_.push_back(filter); - my_exprs_.push_back(expr); - return expr; - } else if (type == OR_EXPR) { - Filter* filter = new OrFilter(left->filter_, right->filter_); - Expression* expr = new Expression(GLOBALTIME_EXPR, filter); - my_filters_.push_back(filter); - my_exprs_.push_back(expr); - return expr; + Expression* parent, ExpressionType type) { + Expression* left = parent->left_; + Expression* right = parent->right_; + parent->left_ = nullptr; + parent->right_ = nullptr; + + Filter* combined = nullptr; + if (left != nullptr && right != nullptr) { + Filter* left_filter = left->filter_; + Filter* right_filter = right->filter_; + left->filter_ = nullptr; + right->filter_ = nullptr; + delete left; + delete right; + combined = make_binary_filter(left_filter, right_filter, type); } - std::cout << "unrecognized QueryFilterOperatorType :" << type << std::endl; - return nullptr; + if (combined == nullptr) { + std::cout << "unrecognized QueryFilterOperatorType :" << type + << std::endl; + delete parent; + return nullptr; + } + parent->type_ = GLOBALTIME_EXPR; + parent->filter_ = combined; + return parent; } bool QueryExpression::update_filter_with_or(Expression* expression, - Filter* filter, Path& path) { + Filter* filter, const Path& path) { if (expression->type_ == SERIES_EXPR && expression->series_path_ == path) { - Filter* node_filter = expression->filter_; - node_filter = new OrFilter(node_filter, filter); - my_filters_.push_back(node_filter); - expression->filter_ = node_filter; + if (expression->filter_ == nullptr) { + expression->filter_ = filter; + } else { + expression->filter_ = new OrFilter(expression->filter_, filter); + } return true; } else if (expression->type_ == OR_EXPR) { - Expression* left = expression->left_; - Expression* right = expression->right_; - return update_filter_with_or(left, filter, path) || - update_filter_with_or(right, filter, path); - } else { - return false; + return update_filter_with_or(expression->left_, filter, path) || + update_filter_with_or(expression->right_, filter, path); } + return false; } Expression* QueryExpression::merge_second_tree_to_first_tree( Expression* left_expression, Expression* right_expression) { + if (right_expression == nullptr) { + return left_expression; + } + if (left_expression == nullptr) { + delete right_expression; + return nullptr; + } if (right_expression->type_ == SERIES_EXPR) { - Expression* leaf = right_expression; - update_filter_with_or(left_expression, leaf->filter_, - leaf->series_path_); + Filter* filter = right_expression->filter_; + right_expression->filter_ = nullptr; + const Path path = right_expression->series_path_; + delete right_expression; + if (!update_filter_with_or(left_expression, filter, path)) { + delete filter; + } return left_expression; } else if (right_expression->type_ == OR_EXPR) { - Expression* left_child = right_expression->left_; - Expression* right_child = right_expression->right_; - left_expression = - merge_second_tree_to_first_tree(left_expression, left_child); + Expression* right_left = right_expression->left_; + Expression* right_right = right_expression->right_; + right_expression->left_ = nullptr; + right_expression->right_ = nullptr; + delete right_expression; left_expression = - merge_second_tree_to_first_tree(left_expression, right_child); - return left_expression; - } else { - Expression* expr = - new Expression(OR_EXPR, left_expression, right_expression); - my_exprs_.push_back(expr); - return expr; + merge_second_tree_to_first_tree(left_expression, right_left); + return merge_second_tree_to_first_tree(left_expression, right_right); } + return new Expression(OR_EXPR, left_expression, right_expression); } Expression* QueryExpression::push_global_time_filter_to_all_series( - Expression* time_filter, std::vector& selected_series) { - if (selected_series.empty()) { + Filter* time_filter, std::vector& selected_series) { + if (time_filter == nullptr || selected_series.empty()) { std::cout << "size of selectSeries could not be 0" << std::endl; return nullptr; } - Expression* expression = new Expression(SERIES_EXPR, selected_series.at(0), - time_filter->filter_); - my_exprs_.push_back(expression); - for (uint32_t i = 1; i < selected_series.size(); i++) { - Expression* r = new Expression(SERIES_EXPR, selected_series.at(i), - time_filter->filter_); - expression = new Expression(OR_EXPR, expression, r); - my_exprs_.push_back(r); - my_exprs_.push_back(expression); + Filter* first_copy = time_filter->clone(); + if (first_copy == nullptr) { + return nullptr; + } + Expression* expression = + new Expression(SERIES_EXPR, selected_series.at(0), first_copy); + for (size_t i = 1; i < selected_series.size(); i++) { + Filter* copy = time_filter->clone(); + if (copy == nullptr) { + delete expression; + return nullptr; + } + Expression* right = + new Expression(SERIES_EXPR, selected_series.at(i), copy); + expression = new Expression(OR_EXPR, expression, right); } return expression; } Expression* QueryExpression::handle_one_global_time_filter( - Expression* left, Expression* expression, + Expression* parent, Expression* global, Expression* rest, std::vector& selected_series, ExpressionType type) { - Expression* expr = optimize(expression, selected_series); - if (expr == nullptr) { + // Detach the parent's children: they are consumed below and deleting the + // parent at the end must not free them again. + parent->left_ = nullptr; + parent->right_ = nullptr; + + Filter* time_filter = global->filter_; + global->filter_ = nullptr; + delete global; + + Expression* optimized_rest = optimize(rest, selected_series); + if (optimized_rest == nullptr) { + delete time_filter; + delete parent; return nullptr; } - if (expr->type_ == GLOBALTIME_EXPR) { - return combine_two_global_time_filter(left, expr, type); + if (optimized_rest->type_ == GLOBALTIME_EXPR) { + Filter* other = optimized_rest->filter_; + optimized_rest->filter_ = nullptr; + delete optimized_rest; + Filter* combined = make_binary_filter(time_filter, other, type); + if (combined == nullptr) { + delete parent; + return nullptr; + } + parent->type_ = GLOBALTIME_EXPR; + parent->filter_ = combined; + return parent; } if (type == AND_EXPR) { - add_time_filter_to_query_filter(left->filter_, expr); - return expr; + const bool applied = + add_time_filter_to_query_filter(time_filter, optimized_rest); + delete time_filter; + delete parent; + if (!applied) { + delete optimized_rest; + return nullptr; + } + return optimized_rest; } else if (type == OR_EXPR) { Expression* after_transform = - push_global_time_filter_to_all_series(left, selected_series); + push_global_time_filter_to_all_series(time_filter, selected_series); + delete time_filter; + delete parent; if (after_transform == nullptr) { + delete optimized_rest; return nullptr; } - return merge_second_tree_to_first_tree(after_transform, expr); + return merge_second_tree_to_first_tree(after_transform, optimized_rest); } std::cout << "unknown relation in Expression:" << type << std::endl; + delete time_filter; + delete optimized_rest; + delete parent; return nullptr; } +// Contract: consumes @expression. The returned tree is owned by the caller; +// on failure the input tree has been released and nullptr is returned. Expression* QueryExpression::optimize(Expression* expression, std::vector& series_paths) { - ExpressionType type = expression->type_; + if (expression == nullptr) { + return nullptr; + } + const ExpressionType type = expression->type_; if (type == GLOBALTIME_EXPR || type == SERIES_EXPR) { return expression; - } else if (type == AND_EXPR || type == OR_EXPR) { - Expression* left = expression->left_; - Expression* right = expression->right_; - if (left->type_ == GLOBALTIME_EXPR && right->type_ == GLOBALTIME_EXPR) { - return combine_two_global_time_filter(left, right, type); - } else if (left->type_ == GLOBALTIME_EXPR && - right->type_ != GLOBALTIME_EXPR) { - return handle_one_global_time_filter(left, right, series_paths, - type); - } else if (left->type_ != GLOBALTIME_EXPR && - right->type_ == GLOBALTIME_EXPR) { - return handle_one_global_time_filter(right, left, series_paths, - type); - } else if (left->type_ != GLOBALTIME_EXPR && - right->type_ != GLOBALTIME_EXPR) { - Expression* regular_left = optimize(left, series_paths); - Expression* regular_right = optimize(right, series_paths); - if (regular_left == nullptr || regular_right == nullptr) { - return nullptr; - } - // Reuse the node being optimized instead of allocating a second - // node wrapping the same children: the result becomes - // QueryExpression::expression_ and must be owned exactly once. - expression->left_ = regular_left; - expression->right_ = regular_right; - - if (regular_left->type_ == GLOBALTIME_EXPR || - regular_right->type_ == GLOBALTIME_EXPR) { - return optimize(expression, series_paths); - } else { - return expression; - } - } - } else { + } + if (type != AND_EXPR && type != OR_EXPR) { std::cout << "unknown Expression type:" << type << std::endl; + delete expression; + return nullptr; } - return nullptr; + + Expression* left = expression->left_; + Expression* right = expression->right_; + const bool left_global = left != nullptr && left->type_ == GLOBALTIME_EXPR; + const bool right_global = + right != nullptr && right->type_ == GLOBALTIME_EXPR; + + if (left_global && right_global) { + return combine_two_global_time_filter(expression, type); + } else if (left_global || right_global) { + Expression* global = left_global ? left : right; + Expression* rest = left_global ? right : left; + return handle_one_global_time_filter(expression, global, rest, + series_paths, type); + } + + // Reuse this node: optimize the children and keep exactly one owner for + // the result tree. + expression->left_ = nullptr; + expression->right_ = nullptr; + Expression* regular_left = optimize(left, series_paths); + if (regular_left == nullptr) { + delete right; + delete expression; + return nullptr; + } + Expression* regular_right = optimize(right, series_paths); + if (regular_right == nullptr) { + delete regular_left; + delete expression; + return nullptr; + } + expression->left_ = regular_left; + expression->right_ = regular_right; + + if (regular_left->type_ == GLOBALTIME_EXPR || + regular_right->type_ == GLOBALTIME_EXPR) { + return optimize(expression, series_paths); + } + return expression; } void QueryExpression::destory() { + // Kept for compatibility: optimization no longer registers detached + // nodes/filters here, so these lists stay empty. for (size_t i = 0; i < my_exprs_.size(); i++) { delete my_exprs_[i]; my_exprs_[i] = nullptr; @@ -207,11 +310,11 @@ void QueryExpression::destory() { delete my_filters_[i]; my_filters_[i] = nullptr; } + my_filters_.clear(); if (expression_ != nullptr) { delete expression_; expression_ = nullptr; } - my_filters_.clear(); } } // namespace storage diff --git a/cpp/src/reader/expression.h b/cpp/src/reader/expression.h index 1cf72b6d2..88520417d 100644 --- a/cpp/src/reader/expression.h +++ b/cpp/src/reader/expression.h @@ -118,22 +118,21 @@ class QueryExpression { void destory(); private: - Expression* combine_two_global_time_filter(Expression* left, - Expression* right, + Expression* combine_two_global_time_filter(Expression* parent, ExpressionType type); Expression* handle_one_global_time_filter( - Expression* left, Expression* expression, + Expression* parent, Expression* global, Expression* rest, std::vector& selected_series, ExpressionType type); - void add_time_filter_to_query_filter(Filter* time_filter, + bool add_time_filter_to_query_filter(Filter* time_filter, Expression* expression); - void add_time_filter_to_series_filter(Filter* time_filter, + bool add_time_filter_to_series_filter(Filter* time_filter, Expression* single_series_expr); Expression* push_global_time_filter_to_all_series( - Expression* time_filter, std::vector& selected_series); + Filter* time_filter, std::vector& selected_series); Expression* merge_second_tree_to_first_tree(Expression* left_expression, Expression* right_expression); bool update_filter_with_or(Expression* expression, Filter* filter, - Path& path); + const Path& path); public: bool has_filter_; diff --git a/cpp/src/reader/filter/and_filter.h b/cpp/src/reader/filter/and_filter.h index 289115baf..10b77a532 100644 --- a/cpp/src/reader/filter/and_filter.h +++ b/cpp/src/reader/filter/and_filter.h @@ -101,8 +101,27 @@ class AndFilter : public BinaryFilter { } } } + for (TimeRange* range : *left_time_ranges) { + delete range; + } + for (TimeRange* range : *right_time_ranges) { + delete range; + } + delete left_time_ranges; + delete right_time_ranges; return result; } + + Filter* clone() const override { + Filter* left = left_->clone(); + Filter* right = right_->clone(); + if (left == nullptr || right == nullptr) { + delete left; + delete right; + return nullptr; + } + return new AndFilter(left, right); + } }; } // namespace storage diff --git a/cpp/src/reader/filter/binary_filter.h b/cpp/src/reader/filter/binary_filter.h index 586e94f6a..cedf959a6 100644 --- a/cpp/src/reader/filter/binary_filter.h +++ b/cpp/src/reader/filter/binary_filter.h @@ -28,7 +28,12 @@ class BinaryFilter : public Filter { BinaryFilter() : Filter(), left_(nullptr), right_(nullptr) {} BinaryFilter(Filter* left, Filter* right) : Filter(), left_(left), right_(right) {} - virtual ~BinaryFilter() {} + // A binary filter owns its operands: callers hand over freshly created + // filters (or filters detached from the expression tree they wrapped). + virtual ~BinaryFilter() { + delete left_; + delete right_; + } void set_left(Filter* left) { left_ = left; } void set_right(Filter* right) { right_ = right; } diff --git a/cpp/src/reader/filter/filter.h b/cpp/src/reader/filter/filter.h index fc2494fac..ac7bb4f67 100644 --- a/cpp/src/reader/filter/filter.h +++ b/cpp/src/reader/filter/filter.h @@ -64,6 +64,12 @@ class Filter { return nullptr; } + // Returns an independent copy owned by the caller, or nullptr when the + // filter cannot be cloned. Callers that must attach the same predicate to + // several owners (e.g. pushing a global time filter into every series + // leaf) treat nullptr as a failure rather than sharing ownership. + virtual Filter* clone() const { return nullptr; } + // True when the predicate accepts every timestamp (e.g. TimeBetween over // the full int64 range). Callers may then skip per-row evaluation. virtual bool is_unbounded_time_filter() const { return false; } diff --git a/cpp/src/reader/filter/or_filter.h b/cpp/src/reader/filter/or_filter.h index 518308982..16daaaaa2 100644 --- a/cpp/src/reader/filter/or_filter.h +++ b/cpp/src/reader/filter/or_filter.h @@ -79,6 +79,11 @@ class OrFilter : public BinaryFilter { int left_index = 0, right_index = 0; int left_size = left_time_ranges->size(); int right_size = right_time_ranges->size(); + if (left_size == 0 && right_size == 0) { + delete left_time_ranges; + delete right_time_ranges; + return result; + } TimeRange* range = choose_next_range( left_time_ranges, right_time_ranges, left_index, right_index); while (left_index < left_size || right_index < right_size) { @@ -94,9 +99,28 @@ class OrFilter : public BinaryFilter { } } result->push_back(new TimeRange(range->start_time_, range->end_time_)); + for (TimeRange* child_range : *left_time_ranges) { + delete child_range; + } + for (TimeRange* child_range : *right_time_ranges) { + delete child_range; + } + delete left_time_ranges; + delete right_time_ranges; return result; } + Filter* clone() const override { + Filter* left = left_->clone(); + Filter* right = right_->clone(); + if (left == nullptr || right == nullptr) { + delete left; + delete right; + return nullptr; + } + return new OrFilter(left, right); + } + private: TimeRange* choose_next_range(std::vector* left_time_ranges, std::vector* right_time_ranges, diff --git a/cpp/src/reader/filter/time_operator.h b/cpp/src/reader/filter/time_operator.h index 3a4502670..374674e0c 100644 --- a/cpp/src/reader/filter/time_operator.h +++ b/cpp/src/reader/filter/time_operator.h @@ -59,6 +59,10 @@ class TimeBetween : public Filter { bool is_contiguous_time_filter() const override { return !not_; } + Filter* clone() const override { + return new TimeBetween(value1_, value2_, not_); + } + private: int64_t value1_; int64_t value2_; @@ -86,6 +90,8 @@ class TimeIn : public Filter { FilterType get_filter_type() { return type_; } + Filter* clone() const override { return new TimeIn(values_, not_); } + private: std::vector values_; FilterType type_; @@ -113,6 +119,8 @@ class TimeEq : public Filter { FilterType get_filter_type() { return type_; } + Filter* clone() const override { return new TimeEq(value_); } + private: int64_t value_; FilterType type_; @@ -139,6 +147,8 @@ class TimeNotEq : public Filter { FilterType get_filter_type() { return type_; } + Filter* clone() const override { return new TimeNotEq(value_); } + private: int64_t value_; FilterType type_; @@ -165,6 +175,8 @@ class TimeGt : public Filter { FilterType get_filter_type() { return type_; } + Filter* clone() const override { return new TimeGt(value_); } + private: int64_t value_; FilterType type_; @@ -192,6 +204,8 @@ class TimeGtEq : public Filter { void reset_value(int64_t val) { value_ = val; } FilterType get_filter_type() { return type_; } + Filter* clone() const override { return new TimeGtEq(value_); } + private: int64_t value_; FilterType type_; @@ -218,6 +232,8 @@ class TimeLt : public Filter { FilterType get_filter_type() { return type_; } + Filter* clone() const override { return new TimeLt(value_); } + private: int64_t value_; FilterType type_; @@ -244,6 +260,8 @@ class TimeLtEq : public Filter { FilterType get_filter_type() { return type_; } + Filter* clone() const override { return new TimeLtEq(value_); } + private: int64_t value_; FilterType type_; diff --git a/cpp/src/reader/tsfile_executor.cc b/cpp/src/reader/tsfile_executor.cc index c2aa8d8cd..833ce1ce9 100644 --- a/cpp/src/reader/tsfile_executor.cc +++ b/cpp/src/reader/tsfile_executor.cc @@ -71,6 +71,8 @@ int TsFileExecutor::execute(QueryExpression* query_expr, ResultSet*& ret_qds) { if (query_exprs_->has_filter_) { regular_expr = query_exprs_->optimize(origin_expr, paths); if (regular_expr == nullptr) { + // optimize() consumes the input tree, including on failure. + query_exprs_->expression_ = nullptr; return E_INVALID_ARG; } query_exprs_->set_expression(regular_expr); diff --git a/cpp/test/reader/expression_test.cc b/cpp/test/reader/expression_test.cc index f4f9fbacc..b8e0fd38f 100644 --- a/cpp/test/reader/expression_test.cc +++ b/cpp/test/reader/expression_test.cc @@ -23,18 +23,52 @@ #include +#include "reader/filter/time_operator.h" + namespace storage { +namespace { + +// A filter without clone(): pushing it onto several owners must fail instead +// of sharing a single instance (and double-freeing it later). +class StatelessFilter : public Filter { + public: + bool satisfy(Statistic*) override { return true; } + bool satisfy(int64_t, int64_t) override { return true; } +}; + +} // namespace + +TEST(QueryExpressionTest, OptimizeFailsWhenAndCannotCloneTimeFilter) { + QueryExpression query_expression; + std::vector paths{Path("root.sg.d1.s1")}; + Expression* expression = new Expression( + AND_EXPR, new Expression(GLOBALTIME_EXPR, new StatelessFilter()), + new Expression(SERIES_EXPR, paths.at(0), new TimeGtEq(0))); + + EXPECT_EQ(query_expression.optimize(expression, paths), nullptr); +} + +TEST(QueryExpressionTest, OptimizeFailsWhenOrCannotCloneTimeFilter) { + QueryExpression query_expression; + std::vector paths{Path("root.sg.d1.s1")}; + Expression* expression = new Expression( + OR_EXPR, new Expression(GLOBALTIME_EXPR, new StatelessFilter()), + new Expression(SERIES_EXPR, paths.at(0), new TimeGtEq(0))); + + EXPECT_EQ(query_expression.optimize(expression, paths), nullptr); +} + TEST(QueryExpressionTest, EmptySelectedSeriesReturnsNull) { QueryExpression query_expression; std::vector selected_series; + // optimize() consumes the input tree, including on failure: the empty + // selected-series list makes the global time filter impossible to push. Expression* expression = new Expression( OR_EXPR, new Expression(GLOBALTIME_EXPR, static_cast(nullptr)), new Expression(SERIES_EXPR, Path(), static_cast(nullptr))); EXPECT_EQ(query_expression.optimize(expression, selected_series), nullptr); - - delete expression; } } // namespace storage diff --git a/cpp/test/reader/filter/logical_filter_test.cc b/cpp/test/reader/filter/logical_filter_test.cc new file mode 100644 index 000000000..71ae68c7e --- /dev/null +++ b/cpp/test/reader/filter/logical_filter_test.cc @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include +#include +#include + +#include "common/statistic.h" +#include "reader/filter/and_filter.h" +#include "reader/filter/or_filter.h" +#include "reader/filter/time_operator.h" + +namespace storage { +namespace { + +using Range = std::pair; + +Statistic make_stat(int64_t start_time, int64_t end_time) { + Statistic stat; + stat.start_time_ = start_time; + stat.end_time_ = end_time; + return stat; +} + +std::vector collect_time_ranges(Filter& filter) { + std::unique_ptr> raw(filter.get_time_ranges()); + std::vector ranges; + for (TimeRange* range : *raw) { + ranges.emplace_back(range->start_time_, range->end_time_); + delete range; + } + return ranges; +} + +std::vector run_batch(Filter& filter, const std::vector& times) { + std::unique_ptr mask = + times.empty() ? nullptr + : std::unique_ptr(new bool[times.size()]); + const int pass = filter.satisfy_batch_time( + times.data(), static_cast(times.size()), mask.get()); + std::vector result(times.size()); + int counted = 0; + for (size_t i = 0; i < times.size(); ++i) { + result[i] = mask[i]; + if (mask[i]) { + ++counted; + } + } + EXPECT_EQ(pass, counted); + return result; +} + +TEST(LogicalFilterTest, AndCombinesPruningAndRowPredicates) { + TimeBetween* between = new TimeBetween(2, 4, /*not_between=*/false); + TimeEq* eq = new TimeEq(3); + AndFilter filter(between, eq); + + Statistic spanning = make_stat(2, 5); + Statistic without_point = make_stat(4, 6); + EXPECT_TRUE(filter.satisfy(&spanning)); + EXPECT_FALSE(filter.satisfy(&without_point)); + + EXPECT_TRUE(filter.satisfy(3, 0)); + EXPECT_FALSE(filter.satisfy(4, 0)); + + EXPECT_TRUE(filter.satisfy_start_end_time(2, 5)); + EXPECT_FALSE(filter.satisfy_start_end_time(4, 6)); + EXPECT_TRUE(filter.contain_start_end_time(3, 3)); + EXPECT_FALSE(filter.contain_start_end_time(2, 4)); + EXPECT_FALSE(filter.contain_start_end_time(2, 2)); + + EXPECT_EQ(collect_time_ranges(filter), (std::vector{{3, 3}})); + EXPECT_EQ(run_batch(filter, {1, 2, 3, 4, 5, 6}), + (std::vector{false, false, true, false, false, false})); +} + +TEST(LogicalFilterTest, OrUnionsPruningAndRowPredicates) { + TimeBetween* between = new TimeBetween(2, 4, /*not_between=*/false); + TimeGt* gt = new TimeGt(5); + OrFilter filter(between, gt); + + Statistic low = make_stat(1, 2); + Statistic high = make_stat(6, 8); + Statistic miss = make_stat(5, 5); + EXPECT_TRUE(filter.satisfy(&low)); + EXPECT_TRUE(filter.satisfy(&high)); + EXPECT_FALSE(filter.satisfy(&miss)); + + EXPECT_TRUE(filter.satisfy(3, 0)); + EXPECT_TRUE(filter.satisfy(6, 0)); + EXPECT_FALSE(filter.satisfy(1, 0)); + + EXPECT_TRUE(filter.satisfy_start_end_time(1, 2)); + EXPECT_TRUE(filter.satisfy_start_end_time(6, 9)); + EXPECT_FALSE(filter.satisfy_start_end_time(5, 5)); + EXPECT_TRUE(filter.contain_start_end_time(2, 4)); + EXPECT_FALSE(filter.contain_start_end_time(1, 9)); + + EXPECT_EQ( + collect_time_ranges(filter), + (std::vector{{2, 4}, {6, std::numeric_limits::max()}})); + EXPECT_EQ(run_batch(filter, {1, 2, 3, 4, 5, 6}), + (std::vector{false, true, true, true, false, true})); +} + +TEST(LogicalFilterTest, NestedLogicalFiltersPropagate) { + TimeBetween* between = new TimeBetween(1, 3, /*not_between=*/false); + TimeGt* gt = new TimeGt(6); + OrFilter* outer_or = new OrFilter(between, gt); + TimeEq* eq = new TimeEq(2); + AndFilter nested(outer_or, eq); + + EXPECT_TRUE(nested.satisfy(2, 0)); + EXPECT_FALSE(nested.satisfy(3, 0)); + EXPECT_FALSE(nested.satisfy(7, 0)); + EXPECT_EQ(collect_time_ranges(nested), (std::vector{{2, 2}})); + EXPECT_EQ(run_batch(nested, {1, 2, 3, 7}), + (std::vector{false, true, false, false})); +} + +} // namespace +} // namespace storage diff --git a/cpp/test/reader/global_time_filter_test.cc b/cpp/test/reader/global_time_filter_test.cc new file mode 100644 index 000000000..3c8e9da41 --- /dev/null +++ b/cpp/test/reader/global_time_filter_test.cc @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +#include +#include +#include +#include + +#include "common/global.h" +#include "common/record.h" +#include "common/schema.h" +#include "reader/expression.h" +#include "reader/filter/time_filter.h" +#include "reader/filter/time_operator.h" +#include "reader/tsfile_reader.h" +#include "writer/tsfile_writer.h" + +namespace storage { +namespace { + +using namespace common; + +std::string unique_global_filter_file() { + static std::atomic counter{0}; +#ifdef _WIN32 + const auto pid = static_cast(_getpid()); +#else + const auto pid = static_cast(getpid()); +#endif + return "global_time_filter_" + std::to_string(pid) + "_" + + std::to_string(counter.fetch_add(1)) + ".tsfile"; +} + +Path series_path(const std::string& device, const std::string& measurement) { + return Path(device + "." + measurement); +} + +Expression* series_expr(const std::string& device, + const std::string& measurement, Filter* filter) { + return new Expression(SERIES_EXPR, series_path(device, measurement), + filter); +} + +Expression* global_expr(Filter* filter) { + return new Expression(GLOBALTIME_EXPR, filter); +} + +std::vector collect_timestamps(ResultSet* result) { + std::vector timestamps; + bool has_next = false; + while (true) { + EXPECT_EQ(result->next(has_next), E_OK); + if (!has_next) { + break; + } + timestamps.push_back(result->get_row_record()->get_timestamp()); + } + return timestamps; +} + +class GlobalTimeFilterTest : public ::testing::Test { + protected: + void SetUp() override { + ASSERT_EQ(libtsfile_init(), E_OK); + file_ = unique_global_filter_file(); + std::remove(file_.c_str()); + write_file(); + ASSERT_EQ(reader_.open(file_), E_OK); + } + + void TearDown() override { + reader_.close(); + std::remove(file_.c_str()); + libtsfile_destroy(); + } + + // s1: [1, 8] -> 10 * t; s2: {2, 4, 6} -> 100 * t. + void write_file() { + TsFileWriter writer; + ASSERT_EQ(writer.open(file_), E_OK); + MeasurementSchema s1_schema("s1", INT64); + MeasurementSchema s2_schema("s2", INT64); + ASSERT_EQ(writer.register_timeseries(device_, s1_schema), E_OK); + ASSERT_EQ(writer.register_timeseries(device_, s2_schema), E_OK); + for (int64_t t = 1; t <= 8; ++t) { + TsRecord record(t, device_); + record.add_point("s1", static_cast(10 * t)); + ASSERT_EQ(writer.write_record(record), E_OK); + } + for (int64_t t : {2, 4, 6}) { + TsRecord record(t, device_); + record.add_point("s2", static_cast(100 * t)); + ASSERT_EQ(writer.write_record(record), E_OK); + } + ASSERT_EQ(writer.flush(), E_OK); + ASSERT_EQ(writer.close(), E_OK); + } + + ResultSet* query(Expression* expression, + const std::vector& measurements) { + std::vector paths; + for (const auto& measurement : measurements) { + paths.push_back(series_path(device_, measurement)); + } + ResultSet* result = nullptr; + EXPECT_EQ( + reader_.query(QueryExpression::create(paths, expression), result), + E_OK); + return result; + } + + const std::string device_ = "root.global_time_filter"; + std::string file_; + TsFileReader reader_; +}; + +TEST_F(GlobalTimeFilterTest, OrGlobalTimeWithSeriesFilter) { + Expression* expression = + new Expression(OR_EXPR, global_expr(new TimeBetween(3, 5, false)), + series_expr(device_, "s1", new TimeGt(6))); + ResultSet* result = query(expression, {"s1"}); + ASSERT_NE(result, nullptr); + + EXPECT_EQ(collect_timestamps(result), + (std::vector{3, 4, 5, 7, 8})); + reader_.destroy_query_data_set(result); +} + +TEST_F(GlobalTimeFilterTest, AndGlobalTimeWithSeriesFilter) { + Expression* expression = + new Expression(AND_EXPR, global_expr(new TimeBetween(3, 6, false)), + series_expr(device_, "s1", new TimeGt(4))); + ResultSet* result = query(expression, {"s1"}); + ASSERT_NE(result, nullptr); + + EXPECT_EQ(collect_timestamps(result), (std::vector{5, 6})); + reader_.destroy_query_data_set(result); +} + +TEST_F(GlobalTimeFilterTest, OrOfTwoGlobalFilters) { + Expression* expression = + new Expression(OR_EXPR, global_expr(new TimeBetween(1, 2, false)), + global_expr(new TimeGt(7))); + ResultSet* result = query(expression, {"s1"}); + ASSERT_NE(result, nullptr); + + EXPECT_EQ(collect_timestamps(result), (std::vector{1, 2, 8})); + reader_.destroy_query_data_set(result); +} + +TEST_F(GlobalTimeFilterTest, AndOfTwoGlobalFilters) { + Expression* expression = + new Expression(AND_EXPR, global_expr(new TimeBetween(2, 6, false)), + global_expr(new TimeGt(4))); + ResultSet* result = query(expression, {"s1"}); + ASSERT_NE(result, nullptr); + + EXPECT_EQ(collect_timestamps(result), (std::vector{5, 6})); + reader_.destroy_query_data_set(result); +} + +TEST_F(GlobalTimeFilterTest, OrGlobalTimeWithMultiSeriesExpression) { + Expression* series_tree = + new Expression(OR_EXPR, series_expr(device_, "s1", new TimeGt(7)), + series_expr(device_, "s2", new TimeEq(6))); + Expression* expression = new Expression( + OR_EXPR, global_expr(new TimeBetween(2, 3, false)), series_tree); + ResultSet* result = query(expression, {"s1", "s2"}); + ASSERT_NE(result, nullptr); + + // s1: {2, 3} from the global filter and {8} from its own filter; + // s2: {2} from the global filter and {6} from its own filter. + EXPECT_EQ(collect_timestamps(result), (std::vector{2, 3, 6, 8})); + reader_.destroy_query_data_set(result); +} + +TEST_F(GlobalTimeFilterTest, AndGlobalTimeWithAndOfTwoSeries) { + Expression* series_tree = + new Expression(AND_EXPR, series_expr(device_, "s1", new TimeGtEq(3)), + series_expr(device_, "s2", new TimeGtEq(0))); + Expression* expression = new Expression( + AND_EXPR, global_expr(new TimeBetween(2, 7, false)), series_tree); + ResultSet* result = query(expression, {"s1", "s2"}); + ASSERT_NE(result, nullptr); + + EXPECT_EQ(collect_timestamps(result), (std::vector{4, 6})); + reader_.destroy_query_data_set(result); +} + +TEST_F(GlobalTimeFilterTest, AndGlobalTimeWithEmptySeriesBranch) { + Expression* series_tree = + new Expression(OR_EXPR, series_expr(device_, "s1", new TimeEq(1)), + series_expr(device_, "s2", new TimeEq(6))); + Expression* expression = new Expression( + AND_EXPR, global_expr(new TimeBetween(2, 5, false)), series_tree); + ResultSet* result = query(expression, {"s1", "s2"}); + ASSERT_NE(result, nullptr); + + EXPECT_TRUE(collect_timestamps(result).empty()); + reader_.destroy_query_data_set(result); +} + +} // namespace +} // namespace storage