diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3be4a7fd6..539ecbea1 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -303,7 +303,7 @@ endif () option(BUILD_TOOLS "Build the tsfile command-line tools" ON) message("cmake using: BUILD_TOOLS=${BUILD_TOOLS}") -option(BUILD_BENCHMARK "Build the read-backend benchmark" OFF) +option(BUILD_BENCHMARK "Build the file-backend benchmarks" OFF) message("cmake using: BUILD_BENCHMARK=${BUILD_BENCHMARK}") option(ENABLE_ANTLR4 "Enable ANTLR4 runtime" ON) @@ -472,6 +472,12 @@ if (BUILD_BENCHMARK) ${PROJECT_SRC_DIR}) target_link_libraries(read_backend_benchmark PRIVATE tsfile Threads::Threads) + add_executable(write_backend_benchmark + bench_mark/bench_mark_src/write_backend_benchmark.cc) + target_include_directories(write_backend_benchmark PRIVATE + ${PROJECT_SRC_DIR}) + target_link_libraries(write_backend_benchmark PRIVATE + tsfile Threads::Threads) endif () if (BUILD_TOOLS) add_subdirectory(tools) diff --git a/cpp/bench_mark/bench_mark_src/write_backend_benchmark.cc b/cpp/bench_mark/bench_mark_src/write_backend_benchmark.cc new file mode 100644 index 000000000..db1b70d57 --- /dev/null +++ b/cpp/bench_mark/bench_mark_src/write_backend_benchmark.cc @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Compares the local-file and in-memory RandomAccessWriteFile backends on the +// same table-model write workload, and measures the cost of reaching the +// local backend through the abstract interface instead of the concrete +// WriteFile class. +// +// Build with -DBUILD_BENCHMARK=ON, run: +// ./build/Release/write_backend_benchmark [--points=N] [--repeat=N] + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/global.h" +#include "common/schema.h" +#include "common/tablet.h" +#include "file/memory_random_access_write_file.h" +#include "file/write_file.h" +#include "writer/tsfile_table_writer.h" + +namespace { + +const uint32_t kBatchedRows = 50000; +const size_t kDispatchChunkBytes = 64 * 1024; +const size_t kDispatchTotalBytes = static_cast(1) << 28; +const uint32_t kFieldCount = 4; + +struct RunResult { + double seconds; + int64_t bytes; +}; + +double now_seconds() { + return std::chrono::duration( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} + +// Deterministic pseudo-random values so the page encoders/compressors see +// realistic, poorly compressible data instead of arithmetic sequences. +uint64_t next_random(uint64_t& state) { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + return state; +} + +storage::TableSchema make_table_schema() { + std::vector columns; + columns.emplace_back("device", common::TSDataType::STRING, + common::ColumnCategory::TAG); + columns.emplace_back("v0", common::TSDataType::INT64, + common::ColumnCategory::FIELD); + columns.emplace_back("v1", common::TSDataType::INT64, + common::ColumnCategory::FIELD); + columns.emplace_back("v2", common::TSDataType::DOUBLE, + common::ColumnCategory::FIELD); + columns.emplace_back("v3", common::TSDataType::INT32, + common::ColumnCategory::FIELD); + return storage::TableSchema("bench_table", columns); +} + +std::vector make_batches(int64_t points) { + std::vector names = {"device", "v0", "v1", "v2", "v3"}; + std::vector types = { + common::TSDataType::STRING, common::TSDataType::INT64, + common::TSDataType::INT64, common::TSDataType::DOUBLE, + common::TSDataType::INT32}; + std::vector batches; + uint64_t random_state = 0x9e3779b97f4a7c15ULL; + int64_t written = 0; + int64_t timestamp = 1600000000000LL; + int64_t batch_index = 0; + while (written < points) { + const uint32_t rows = static_cast( + std::min(kBatchedRows, points - written)); + batches.emplace_back(names, types, rows); + storage::Tablet& tablet = batches.back(); + const std::string device = "device_" + std::to_string(batch_index % 8); + for (uint32_t row = 0; row < rows; ++row) { + timestamp += 1 + static_cast( + (next_random(random_state) >> 33) % 100); + tablet.add_timestamp(row, timestamp); + tablet.add_value(row, "device", device.c_str()); + tablet.add_value(row, "v0", + static_cast(next_random(random_state))); + tablet.add_value(row, "v1", + static_cast(next_random(random_state))); + tablet.add_value( + row, "v2", + static_cast(next_random(random_state) >> 11) / + static_cast(1ULL << 53)); + tablet.add_value(row, "v3", + static_cast(next_random(random_state))); + } + written += rows; + ++batch_index; + } + return batches; +} + +template +RunResult run_table_workload(int64_t points, storage::TableSchema& schema, + SinkFactory make_sink) { + std::vector batches = make_batches(points); + std::unique_ptr sink = make_sink(); + + const double start = now_seconds(); + { + storage::TsFileTableWriter writer(sink.get(), &schema); + if (writer.get_error() != common::E_OK) { + std::cerr << "table writer init failed: " << writer.get_error() + << std::endl; + return {0, 0}; + } + for (storage::Tablet& tablet : batches) { + const int ret = writer.write_table(tablet); + if (ret != common::E_OK) { + std::cerr << "write_table failed: " << ret << std::endl; + return {0, 0}; + } + } + if (writer.close() != common::E_OK) { + std::cerr << "table writer close failed" << std::endl; + return {0, 0}; + } + } + const double seconds = now_seconds() - start; + // The writer's close() closes the sink, so read the size back from the + // backend itself: the in-memory sink keeps its bytes, the local backend + // is measured through the file it produced. + int64_t bytes = 0; + auto* memory_sink = + dynamic_cast(sink.get()); + if (memory_sink != nullptr) { + bytes = memory_sink->size(); + } else { + std::ifstream in(sink->file_path(), std::ios::binary | std::ios::ate); + if (in) { + bytes = static_cast(in.tellg()); + } + } + return {seconds, bytes}; +} + +double median(std::vector values) { + std::sort(values.begin(), values.end()); + return values[values.size() / 2]; +} + +void print_table_workload(const std::string& name, + const std::vector& runs, int64_t points) { + std::vector seconds; + for (const RunResult& run : runs) { + seconds.push_back(run.seconds); + } + const double med = median(seconds); + const double values = + static_cast(points) * static_cast(kFieldCount); + int64_t bytes = 0; + for (const RunResult& run : runs) { + bytes = run.bytes; + } + std::cout << std::left << std::setw(10) << name << std::right << std::fixed + << std::setprecision(3); + for (double s : seconds) { + std::cout << std::setw(9) << s; + } + std::cout << std::setw(11) << med << std::setw(11) + << (static_cast(bytes) / med / 1e6) << std::setw(12) + << (values / med / 1e6) << std::endl; +} + +// Measures the one indirect call the abstraction adds by writing the same +// buffer to a pre-allocated in-memory sink through the concrete class and +// through RandomAccessWriteFile*. The memory backend is used so filesystem +// and page-cache effects do not swamp the ~1 ns dispatch cost. +RunResult run_memory_dispatch_bench(bool interface_call) { + std::vector buffer(kDispatchChunkBytes); + for (size_t i = 0; i < buffer.size(); ++i) { + buffer[i] = static_cast(i * 131); + } + storage::MemoryRandomAccessWriteFile sink; + storage::RandomAccessWriteFile* sink_iface = &sink; + const size_t calls = kDispatchTotalBytes / kDispatchChunkBytes; + const uint32_t chunk = static_cast(buffer.size()); + + const double start = now_seconds(); + int ret = common::E_OK; + if (interface_call) { + for (size_t i = 0; i < calls && ret == common::E_OK; ++i) { + ret = sink_iface->write(buffer.data(), chunk); + } + } else { + for (size_t i = 0; i < calls && ret == common::E_OK; ++i) { + ret = sink.write(buffer.data(), chunk); + } + } + if (ret != common::E_OK) { + return {0, 0}; + } + return {now_seconds() - start, sink.size()}; +} + +void print_dispatch_workload(const std::string& name, double seconds, + int64_t bytes) { + const double calls = + static_cast(kDispatchTotalBytes / kDispatchChunkBytes); + std::cout << std::left << std::setw(26) << name << std::right << std::fixed + << std::setprecision(3) << std::setw(9) << seconds + << std::setw(11) << (static_cast(bytes) / seconds / 1e9) + << std::setw(12) << (seconds * 1e9 / calls) << std::endl; +} + +int64_t parse_int_option(const char* arg, const char* prefix, + int64_t fallback) { + const std::string value(arg); + const std::string name(prefix); + if (value.rfind(name, 0) != 0) { + return fallback; + } + std::istringstream stream(value.substr(name.size())); + int64_t parsed = fallback; + stream >> parsed; + return stream.fail() ? fallback : parsed; +} + +} // namespace + +int main(int argc, char** argv) { + int64_t points = 1000000; + int repeat = 3; + for (int i = 1; i < argc; ++i) { + points = parse_int_option(argv[i], "--points=", points); + const int64_t repeat_arg = parse_int_option(argv[i], "--repeat=", 0); + if (repeat_arg > 0) { + repeat = static_cast(repeat_arg); + } + } + if (points <= 0 || repeat <= 0) { + std::cerr << "usage: write_backend_benchmark [--points=N] [--repeat=N]" + << std::endl; + return 2; + } + + const int init_ret = storage::libtsfile_init(); + if (init_ret != common::E_OK) { + std::cerr << "failed to initialize TsFile: error " << init_ret + << std::endl; + return 1; + } + + storage::TableSchema schema = make_table_schema(); + const std::string local_path = "write_backend_benchmark.tsfile"; + + std::vector local_runs; + std::vector memory_runs; + for (int run = 0; run < repeat; ++run) { + local_runs.push_back(run_table_workload(points, schema, [&]() { + std::remove(local_path.c_str()); + std::unique_ptr file(new storage::WriteFile()); + int flags = O_RDWR | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + if (file->create(local_path, flags, 0666) != common::E_OK) { + std::cerr << "failed to create " << local_path << std::endl; + } + return std::unique_ptr( + file.release()); + })); + memory_runs.push_back(run_table_workload(points, schema, []() { + return std::unique_ptr( + new storage::MemoryRandomAccessWriteFile( + "memory://bench.tsfile")); + })); + } + std::remove(local_path.c_str()); + + std::cout << "table-model target: " << points << " rows x " << kFieldCount + << " fields, " << kBatchedRows << " rows per tablet, " << repeat + << " runs" << std::endl; + std::cout << std::left << std::setw(10) << "sink" << std::right; + for (int run = 1; run <= repeat; ++run) { + std::cout << std::setw(9) << ("run" + std::to_string(run)); + } + std::cout << std::setw(11) << "median" << std::setw(11) << "MB/s" + << std::setw(12) << "Mval/s" << std::endl; + print_table_workload("local", local_runs, points); + print_table_workload("memory", memory_runs, points); + + std::cout << "\nsequential " << (kDispatchTotalBytes >> 20) << " MiB in " + << (kDispatchChunkBytes >> 10) + << " KiB writes to the in-memory sink (median of 5, alternated)" + << std::endl; + std::cout << std::left << std::setw(26) << "sink" << std::right + << std::setw(9) << "median" << std::setw(11) << "GiB/s" + << std::setw(12) << "ns/call" << std::endl; + std::vector concrete; + std::vector through_interface; + int64_t bytes = 0; + for (int run = 0; run < 5; ++run) { + RunResult direct{0, 0}; + RunResult indirect{0, 0}; + if ((run % 2) == 0) { + direct = run_memory_dispatch_bench(false); + indirect = run_memory_dispatch_bench(true); + } else { + indirect = run_memory_dispatch_bench(true); + direct = run_memory_dispatch_bench(false); + } + concrete.push_back(direct.seconds); + through_interface.push_back(indirect.seconds); + bytes = direct.bytes; + } + print_dispatch_workload("memory (concrete)", median(concrete), bytes); + print_dispatch_workload("memory (interface ptr)", median(through_interface), + bytes); + + storage::libtsfile_destroy(); + return 0; +} diff --git a/cpp/src/file/memory_random_access_read_file.cc b/cpp/src/file/memory_random_access_read_file.cc new file mode 100644 index 000000000..9f2df72ae --- /dev/null +++ b/cpp/src/file/memory_random_access_read_file.cc @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "file/memory_random_access_read_file.h" + +#include +#include + +#include "utils/errno_define.h" + +namespace storage { + +namespace { +// Same FNV-1a construction the local backend uses for size+mtime, but over +// the bytes themselves: an in-memory source has no stat data and the content +// is immutable, so the content digest is the generation. +uint64_t content_fingerprint(const std::vector& data) { + uint64_t hash = 1469598103934665603ULL; + const uint64_t size = static_cast(data.size()); + for (size_t byte = 0; byte < 8; ++byte) { + hash ^= static_cast(size >> (byte * 8)); + hash *= 1099511628211ULL; + } + for (size_t i = 0; i < data.size(); ++i) { + hash ^= static_cast(data[i]); + hash *= 1099511628211ULL; + } + return hash; +} +} // namespace + +MemoryRandomAccessReadFile::MemoryRandomAccessReadFile(std::vector data, + const std::string& name) + : path_(name), data_(std::move(data)), fingerprint_(0), opened_(true) { + init_fingerprint(); +} + +MemoryRandomAccessReadFile::MemoryRandomAccessReadFile(const char* data, + size_t size, + const std::string& name) + : path_(name), data_(), fingerprint_(0), opened_(true) { + if (data != nullptr && size > 0) { + data_.assign(data, data + size); + } + init_fingerprint(); +} + +void MemoryRandomAccessReadFile::init_fingerprint() { + fingerprint_ = content_fingerprint(data_); +} + +int64_t MemoryRandomAccessReadFile::file_size() const { + return opened_ ? static_cast(data_.size()) : -1; +} + +int MemoryRandomAccessReadFile::generation(uint64_t& size, + uint64_t& fingerprint) const { + if (!opened_) { + return common::E_FILE_READ_ERR; + } + size = static_cast(data_.size()); + fingerprint = fingerprint_; + return common::E_OK; +} + +int MemoryRandomAccessReadFile::read(int64_t offset, char* buffer, int32_t size, + int32_t& read_size) { + read_size = 0; + if (offset < 0 || size < 0 || (buffer == nullptr && size > 0)) { + return common::E_INVALID_ARG; + } + if (!opened_) { + return common::E_FILE_READ_ERR; + } + if (size == 0 || offset >= static_cast(data_.size())) { + return common::E_OK; + } + const int64_t available = static_cast(data_.size()) - offset; + const int32_t target_size = available < static_cast(size) + ? static_cast(available) + : size; + std::memcpy(buffer, data_.data() + offset, + static_cast(target_size)); + read_size = target_size; + return common::E_OK; +} + +void MemoryRandomAccessReadFile::close() { + opened_ = false; + std::vector().swap(data_); +} + +} // namespace storage diff --git a/cpp/src/file/memory_random_access_read_file.h b/cpp/src/file/memory_random_access_read_file.h new file mode 100644 index 000000000..c2b046bff --- /dev/null +++ b/cpp/src/file/memory_random_access_read_file.h @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef FILE_MEMORY_RANDOM_ACCESS_READ_FILE_H +#define FILE_MEMORY_RANDOM_ACCESS_READ_FILE_H + +#include + +#include +#include +#include + +#include "file/random_access_read_file.h" + +namespace storage { + +/** + * A RandomAccessReadFile over an in-memory byte sequence. + * + * Takes ownership of the bytes (move the result of + * MemoryRandomAccessWriteFile::release_data() in) and serves reads from them + * until close(). The content is immutable, so the generation fingerprint is + * computed once at construction. + */ +class MemoryRandomAccessReadFile : public RandomAccessReadFile { + public: + explicit MemoryRandomAccessReadFile( + std::vector data, const std::string& name = std::string()); + MemoryRandomAccessReadFile(const char* data, size_t size, + const std::string& name = std::string()); + ~MemoryRandomAccessReadFile() override = default; + + MemoryRandomAccessReadFile(const MemoryRandomAccessReadFile&) = delete; + MemoryRandomAccessReadFile& operator=(const MemoryRandomAccessReadFile&) = + delete; + + bool is_opened() const override { return opened_; } + int64_t file_size() const override; + const std::string& file_path() const override { return path_; } + int generation(uint64_t& size, uint64_t& fingerprint) const override; + + int read(int64_t offset, char* buffer, int32_t size, + int32_t& read_size) override; + /** Drops the backing bytes; read() afterwards returns E_FILE_READ_ERR. */ + void close() override; + + private: + void init_fingerprint(); + + std::string path_; + std::vector data_; + uint64_t fingerprint_; + bool opened_; +}; + +} // namespace storage + +#endif // FILE_MEMORY_RANDOM_ACCESS_READ_FILE_H diff --git a/cpp/src/file/memory_random_access_write_file.cc b/cpp/src/file/memory_random_access_write_file.cc new file mode 100644 index 000000000..34b6a78d8 --- /dev/null +++ b/cpp/src/file/memory_random_access_write_file.cc @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "file/memory_random_access_write_file.h" + +#include +#include + +#include "utils/errno_define.h" + +namespace storage { + +MemoryRandomAccessWriteFile::MemoryRandomAccessWriteFile( + const std::string& name) + : path_(name), data_(), position_(0), opened_(true) {} + +int MemoryRandomAccessWriteFile::write(const char* buf, uint32_t len) { + if (!opened_) { + return common::E_FILE_WRITE_ERR; + } + if (buf == nullptr) { + return len == 0 ? common::E_OK : common::E_INVALID_ARG; + } + if (len == 0) { + return common::E_OK; + } + const int64_t end = position_ + static_cast(len); + try { + // A previous truncate() may have left the cursor past the end (the + // same hole POSIX ftruncate() allows); extend with zeros first. + if (end > static_cast(data_.size())) { + data_.resize(static_cast(end), 0); + } + } catch (const std::bad_alloc&) { + return common::E_OOM; + } + std::memcpy(data_.data() + position_, buf, len); + position_ = end; + return common::E_OK; +} + +int64_t MemoryRandomAccessWriteFile::get_position() { + return opened_ ? position_ : 0; +} + +int MemoryRandomAccessWriteFile::sync() { + return opened_ ? common::E_OK : common::E_FILE_WRITE_ERR; +} + +int MemoryRandomAccessWriteFile::truncate(int64_t size) { + if (!opened_) { + return common::E_FILE_WRITE_ERR; + } + if (size < 0) { + return common::E_INVALID_ARG; + } + try { + data_.resize(static_cast(size), 0); + } catch (const std::bad_alloc&) { + return common::E_OOM; + } + // ftruncate() does not move the write cursor, so neither do we. + return common::E_OK; +} + +int MemoryRandomAccessWriteFile::seek_to_end() { + if (!opened_) { + return common::E_FILE_WRITE_ERR; + } + position_ = static_cast(data_.size()); + return common::E_OK; +} + +int MemoryRandomAccessWriteFile::close() { + opened_ = false; + return common::E_OK; +} + +std::vector MemoryRandomAccessWriteFile::release_data() { + std::vector data; + data.swap(data_); + position_ = 0; + return data; +} + +} // namespace storage diff --git a/cpp/src/file/memory_random_access_write_file.h b/cpp/src/file/memory_random_access_write_file.h new file mode 100644 index 000000000..b630be486 --- /dev/null +++ b/cpp/src/file/memory_random_access_write_file.h @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef FILE_MEMORY_RANDOM_ACCESS_WRITE_FILE_H +#define FILE_MEMORY_RANDOM_ACCESS_WRITE_FILE_H + +#include + +#include +#include +#include + +#include "file/random_access_write_file.h" + +namespace storage { + +/** + * A RandomAccessWriteFile that keeps the written bytes in a growable buffer. + * + * A default-constructed sink is open and empty, so it can be handed to + * TsFileIOWriter/TsFileWriter directly. close() marks it closed (further + * writes fail) but keeps the bytes: read data()/size() afterwards to obtain + * the serialized TsFile, or hand release_data() to a + * MemoryRandomAccessReadFile to read the file back. + * + * The sink cannot be opened for append: it never reads an existing file and + * has no path, so recovery (RestorableTsFileIOWriter) stays local-file only. + */ +class MemoryRandomAccessWriteFile : public RandomAccessWriteFile { + public: + explicit MemoryRandomAccessWriteFile( + const std::string& name = std::string()); + ~MemoryRandomAccessWriteFile() override = default; + + MemoryRandomAccessWriteFile(const MemoryRandomAccessWriteFile&) = delete; + MemoryRandomAccessWriteFile& operator=(const MemoryRandomAccessWriteFile&) = + delete; + + bool is_opened() const override { return opened_; } + const std::string& file_path() const override { return path_; } + + int write(const char* buf, uint32_t len) override; + /** Cursor position; 0 after close(), matching the local backend. */ + int64_t get_position() override; + /** No-op: the bytes are already in memory. Returns E_OK while open. */ + int sync() override; + int truncate(int64_t size) override; + int seek_to_end() override; + /** Marks the sink closed; the accumulated bytes remain available. */ + int close() override; + + /** Bytes written so far, regardless of the open/closed state. */ + const std::vector& data() const { return data_; } + int64_t size() const { return static_cast(data_.size()); } + /** Move the accumulated bytes out and reset the cursor; the sink becomes + * empty but stays in its current open/closed state. */ + std::vector release_data(); + + private: + std::string path_; + std::vector data_; + int64_t position_; + bool opened_; +}; + +} // namespace storage + +#endif // FILE_MEMORY_RANDOM_ACCESS_WRITE_FILE_H diff --git a/cpp/src/file/random_access_write_file.h b/cpp/src/file/random_access_write_file.h new file mode 100644 index 000000000..d92afd1fe --- /dev/null +++ b/cpp/src/file/random_access_write_file.h @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef FILE_RANDOM_ACCESS_WRITE_FILE_H +#define FILE_RANDOM_ACCESS_WRITE_FILE_H + +#include + +#include + +namespace storage { + +/** + * A writable byte sink used by the TsFile writer stack. + * + * The writer only ever appends at the current write cursor, truncates a + * partially written file during recovery, and seeks to the end to resume + * an existing file, so a backend only has to support those operations. + * All methods return common error codes (see utils/errno_define.h) rather + * than throwing; buffers are owned by the caller. + * + * Contract: + * - After close(), every mutating call (write/truncate/seek_to_end/sync) + * returns common::E_FILE_WRITE_ERR and is a no-op. close() is + * idempotent and returns an error code for backends whose close can fail. + * - write() appends @p len bytes at the current position and advances the + * position by @p len on success. + * - truncate() shrinks or extends the byte sequence to @p size bytes, + * zero-filling when extending. Like POSIX ftruncate() it does not move + * the write cursor, so a later write may land past the current end and + * leave a zero-filled hole. + * - seek_to_end() places the cursor at the current end of the byte + * sequence; get_position() reports the cursor. + * - file_path() is a human-readable identifier. Backends that have no real + * path return the name they were constructed with (possibly empty). + * + * A TsFileReader cannot read from this interface; a backend that must be + * read back (e.g. to verify a written file) exposes the bytes through a + * matching RandomAccessReadFile implementation instead. + * + * Implementations are handed to the writers already open. Reopening an + * existing file for append/recovery is a local-file capability + * (RestorableTsFileIOWriter reads the old bytes back by path), so a + * non-local sink cannot be recovered in place. + */ +class RandomAccessWriteFile { + public: + virtual ~RandomAccessWriteFile() = default; + + virtual bool is_opened() const = 0; + virtual const std::string& file_path() const = 0; + + virtual int write(const char* buf, uint32_t len) = 0; + virtual int64_t get_position() = 0; + virtual int sync() = 0; + virtual int truncate(int64_t size) = 0; + virtual int seek_to_end() = 0; + virtual int close() = 0; +}; + +} // namespace storage + +#endif // FILE_RANDOM_ACCESS_WRITE_FILE_H diff --git a/cpp/src/file/restorable_tsfile_io_writer.cc b/cpp/src/file/restorable_tsfile_io_writer.cc index 9846f1ff9..605ff6e03 100644 --- a/cpp/src/file/restorable_tsfile_io_writer.cc +++ b/cpp/src/file/restorable_tsfile_io_writer.cc @@ -879,7 +879,7 @@ TsFileIOWriter* RestorableTsFileIOWriter::get_tsfile_io_writer() { return can_write_ ? this : nullptr; } -WriteFile* RestorableTsFileIOWriter::get_write_file() { +RandomAccessWriteFile* RestorableTsFileIOWriter::get_write_file() { return can_write_ ? write_file_ : nullptr; } diff --git a/cpp/src/file/restorable_tsfile_io_writer.h b/cpp/src/file/restorable_tsfile_io_writer.h index 051bf7d8c..8c06c4907 100644 --- a/cpp/src/file/restorable_tsfile_io_writer.h +++ b/cpp/src/file/restorable_tsfile_io_writer.h @@ -101,10 +101,12 @@ class RestorableTsFileIOWriter : public TsFileIOWriter { TsFileIOWriter* get_tsfile_io_writer(); /** - * Get the WriteFile for TsFileWriter::init(). Only valid when can_write(). - * Caller must not destroy the returned pointer. + * Get the sink for TsFileWriter::init(). Only valid when can_write(). + * Caller must not destroy the returned pointer. Recovery always uses a + * local file: self_check reads the existing bytes back by path, so the + * sink cannot be a non-local RandomAccessWriteFile backend. */ - WriteFile* get_write_file(); + RandomAccessWriteFile* get_write_file(); std::string get_file_path() const; diff --git a/cpp/src/file/tsfile_io_writer.cc b/cpp/src/file/tsfile_io_writer.cc index 7e2fc52d5..00aa23118 100644 --- a/cpp/src/file/tsfile_io_writer.cc +++ b/cpp/src/file/tsfile_io_writer.cc @@ -48,7 +48,7 @@ int64_t TsFileIOWriter::get_meta_size() const { return meta_allocator_.get_total_used_bytes(); } -int TsFileIOWriter::init(WriteFile* write_file) { +int TsFileIOWriter::init(RandomAccessWriteFile* write_file) { int ret = E_OK; const uint32_t page_size = 1024; meta_allocator_.init(page_size, MOD_TSFILE_WRITER_META); @@ -119,7 +119,7 @@ void TsFileIOWriter::destroy() { int TsFileIOWriter::add_tsfile_property(const std::string& key, const uint8_t* value, uint32_t value_len) { - if (file_ == nullptr || file_->get_fd() < 0) { + if (file_ == nullptr || !file_->is_opened()) { return common::E_FILE_WRITE_ERR; } if (value_len > 0 && value == nullptr) { @@ -136,7 +136,7 @@ int TsFileIOWriter::add_tsfile_property(const std::string& key, int TsFileIOWriter::add_tsfile_property(const std::string& key, const std::vector& value) { - if (file_ == nullptr || file_->get_fd() < 0) { + if (file_ == nullptr || !file_->is_opened()) { return common::E_FILE_WRITE_ERR; } if (key.size() > static_cast(std::numeric_limits::max()) || @@ -308,7 +308,7 @@ int TsFileIOWriter::end_flush_chunk_group(bool is_aligned) { int TsFileIOWriter::end_file() { int ret = E_OK; - if (file_->get_fd() == -1) { + if (file_ == nullptr || !file_->is_opened()) { return E_OK; } OFFSET_DEBUG("before end file"); diff --git a/cpp/src/file/tsfile_io_writer.h b/cpp/src/file/tsfile_io_writer.h index bbb1e4988..8a683f826 100644 --- a/cpp/src/file/tsfile_io_writer.h +++ b/cpp/src/file/tsfile_io_writer.h @@ -28,8 +28,8 @@ #include "common/global.h" #include "common/schema.h" #include "common/tsfile_common.h" +#include "random_access_write_file.h" #include "reader/bloom_filter.h" -#include "write_file.h" namespace storage { @@ -85,7 +85,7 @@ class TsFileIOWriter { } } ~TsFileIOWriter() { destroy(); } - int init(WriteFile* write_file); + int init(RandomAccessWriteFile* write_file); void destroy(); void set_generate_table_schema(bool generate_table_schema); @@ -110,7 +110,7 @@ class TsFileIOWriter { int end_flush_chunk_group(bool is_aligned = false); int end_file(); - FORCE_INLINE std::string get_file_path() { return file_->get_file_path(); } + FORCE_INLINE std::string get_file_path() { return file_->file_path(); } FORCE_INLINE std::shared_ptr get_schema() { return schema_; } int64_t get_meta_size() const; @@ -239,7 +239,7 @@ class TsFileIOWriter { chunk_group_meta_index_; bool use_prev_alloc_cgm_; // chunk group meta std::shared_ptr cur_device_name_; - WriteFile* file_; + RandomAccessWriteFile* file_; bool write_file_created_; bool generate_table_schema_; std::shared_ptr schema_; diff --git a/cpp/src/file/write_file.cc b/cpp/src/file/write_file.cc index b55af6f54..bb5c0a88f 100644 --- a/cpp/src/file/write_file.cc +++ b/cpp/src/file/write_file.cc @@ -42,7 +42,7 @@ using namespace common; namespace storage { int WriteFile::create(const std::string& file_path, int flags, mode_t mode) { - if (fd_ > 0) { + if (is_opened()) { // log_err("file already opened, fd=%d", fd_); ASSERT(false); return E_ALREADY_EXIST; @@ -70,7 +70,9 @@ int WriteFile::do_create(int flags, mode_t mode) { } int WriteFile::write(const char* buf, uint32_t len) { - ASSERT(fd_ > 0); + if (fd_ < 0) { + return E_FILE_WRITE_ERR; + } #if 0 // DEBUG_SE struct stat statbuf; @@ -104,7 +106,9 @@ int WriteFile::write(const char* buf, uint32_t len) { } int WriteFile::sync() { - ASSERT(fd_ > 0); + if (fd_ < 0) { + return E_FILE_WRITE_ERR; + } if (::fsync(fd_) < 0) { // log_err("file fsync error, path=%s, errno=%d", path_.c_str(), errno); return E_FILE_SYNC_ERR; @@ -136,7 +140,9 @@ int WriteFile::close() { } int WriteFile::truncate(int64_t size) { - ASSERT(fd_ > 0); + if (fd_ < 0) { + return E_FILE_WRITE_ERR; + } #ifdef _WIN32 if (_chsize_s(fd_, size) != 0) { return E_FILE_WRITE_ERR; @@ -150,7 +156,9 @@ int WriteFile::truncate(int64_t size) { } int WriteFile::seek_to_end() { - ASSERT(fd_ > 0); + if (fd_ < 0) { + return E_FILE_WRITE_ERR; + } #ifdef _WIN32 if (_lseeki64(fd_, 0, SEEK_END) < 0) { return E_FILE_READ_ERR; diff --git a/cpp/src/file/write_file.h b/cpp/src/file/write_file.h index 84358dcdf..d62c7e21e 100644 --- a/cpp/src/file/write_file.h +++ b/cpp/src/file/write_file.h @@ -22,12 +22,19 @@ #include +#include "file/random_access_write_file.h" #include "utils/storage_utils.h" #include "utils/util_define.h" namespace storage { -class WriteFile { +/** + * The local-file backend: RandomAccessWriteFile implemented over a POSIX + * file descriptor. create() opens the path; every other operation is the + * fd-based implementation below. See RandomAccessWriteFile for the method + * contracts. + */ +class WriteFile : public RandomAccessWriteFile { public: WriteFile() : path_(), fd_(-1) {} // Release the OS file handle on destruction. Without this, a writer left @@ -35,21 +42,24 @@ class WriteFile { // the fd — harmless on POSIX (unlink works on open files) but on Windows it // keeps the file locked so a subsequent remove() fails. close() is // idempotent, so this is a no-op when the file was already closed normally. - ~WriteFile() { close(); } + ~WriteFile() override { close(); } int create(const std::string& file_name, int flags, mode_t mode); - bool file_opened() const { return fd_ > 0; } - int write(const char* buf, uint32_t len); + /** Kept for source compatibility; use is_opened(). */ + bool file_opened() const { return is_opened(); } + bool is_opened() const override { return fd_ >= 0; } + int write(const char* buf, uint32_t len) override; FORCE_INLINE int get_fd() const { return fd_; } - int sync(); - int close(); + int sync() override; + int close() override; /** Truncate file to the given size (bytes). File must be open. */ - int truncate(int64_t size); + int truncate(int64_t size) override; /** Seek to end of file. Used after open to position for append. */ - int seek_to_end(); + int seek_to_end() override; FORCE_INLINE const std::string& get_file_path() const { return path_; } + FORCE_INLINE const std::string& file_path() const override { return path_; } /** Current file offset. After seek_to_end(), equals file size (for * recovery). */ - int64_t get_position(); + int64_t get_position() override; private: int do_create(int flags, mode_t mode); diff --git a/cpp/src/writer/tsfile_table_writer.h b/cpp/src/writer/tsfile_table_writer.h index d0b0bca1a..0e9b715e3 100644 --- a/cpp/src/writer/tsfile_table_writer.h +++ b/cpp/src/writer/tsfile_table_writer.h @@ -42,7 +42,7 @@ class TsFileTableWriter { * TsFileTableWriter is used to write table data into a target file with the * given schema, optionally limiting the memory usage. * - * @param writer_file Target file where the table data will be written. Must + * @param writer_file Target sink where the table data will be written. Must * not be null. * @param table_schema Used to construct table structures. Defines the * schema of the table being written. @@ -51,7 +51,8 @@ class TsFileTableWriter { * disk. Default value is 128MB. */ template - explicit TsFileTableWriter(storage::WriteFile* writer_file, T* table_schema, + explicit TsFileTableWriter(storage::RandomAccessWriteFile* writer_file, + T* table_schema, uint64_t memory_threshold = 128 * 1024 * 1024) { static_assert(!std::is_same::value, "table_schema cannot be nullptr"); diff --git a/cpp/src/writer/tsfile_tree_writer.cc b/cpp/src/writer/tsfile_tree_writer.cc index 28913c2bd..e3f1a79bb 100644 --- a/cpp/src/writer/tsfile_tree_writer.cc +++ b/cpp/src/writer/tsfile_tree_writer.cc @@ -23,7 +23,7 @@ namespace storage { -TsFileTreeWriter::TsFileTreeWriter(storage::WriteFile* writer_file, +TsFileTreeWriter::TsFileTreeWriter(storage::RandomAccessWriteFile* writer_file, uint64_t memory_threshold) { tsfile_writer_ = std::make_shared(); tsfile_writer_->init(writer_file); diff --git a/cpp/src/writer/tsfile_tree_writer.h b/cpp/src/writer/tsfile_tree_writer.h index f07046838..170c06ddc 100644 --- a/cpp/src/writer/tsfile_tree_writer.h +++ b/cpp/src/writer/tsfile_tree_writer.h @@ -48,13 +48,13 @@ class TsFileTreeWriter { * data into the specified file, with an optional memory threshold for * buffered data. * - * @param writer_file Target file where the TsFile data will be written. + * @param writer_file Target sink where the TsFile data will be written. * Must not be null. * @param memory_threshold Optional parameter specifying the memory usage * threshold for buffered data before automatic flush occurs. Default is * 128MB. */ - explicit TsFileTreeWriter(storage::WriteFile* writer_file, + explicit TsFileTreeWriter(storage::RandomAccessWriteFile* writer_file, uint64_t memory_threshold = 128 * 1024 * 1024); /** diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc index 6fbc806d7..d1e06a0ba 100644 --- a/cpp/src/writer/tsfile_writer.cc +++ b/cpp/src/writer/tsfile_writer.cc @@ -117,10 +117,10 @@ void TsFileWriter::destroy() { record_count_since_last_flush_ = 0; } -int TsFileWriter::init(WriteFile* write_file) { +int TsFileWriter::init(RandomAccessWriteFile* write_file) { if (write_file == nullptr) { return E_INVALID_ARG; - } else if (!write_file->file_opened()) { + } else if (!write_file->is_opened()) { return E_INVALID_ARG; } write_file_ = write_file; diff --git a/cpp/src/writer/tsfile_writer.h b/cpp/src/writer/tsfile_writer.h index b5fb86702..39645f2d3 100644 --- a/cpp/src/writer/tsfile_writer.h +++ b/cpp/src/writer/tsfile_writer.h @@ -35,7 +35,7 @@ #include "common/tablet.h" namespace storage { -class WriteFile; +class RandomAccessWriteFile; class ChunkWriter; class TsFileIOWriter; class RestorableTsFileIOWriter; @@ -60,7 +60,7 @@ class TsFileWriter { int open(const std::string& file_path, int flags, mode_t mode); int open(const std::string& file_path); - int init(storage::WriteFile* write_file); + int init(storage::RandomAccessWriteFile* write_file); int init(storage::RestorableTsFileIOWriter* rw); void set_generate_table_schema(bool generate_table_schema); @@ -198,7 +198,7 @@ class TsFileWriter { split_tablet_by_device(const Tablet& tablet); private: - storage::WriteFile* write_file_; + storage::RandomAccessWriteFile* write_file_; storage::TsFileIOWriter* io_writer_; // device_id -> MeasurementSchemaGroup DeviceSchemasMap schemas_; diff --git a/cpp/test/file/memory_random_access_write_file_test.cc b/cpp/test/file/memory_random_access_write_file_test.cc new file mode 100644 index 000000000..32f5c084f --- /dev/null +++ b/cpp/test/file/memory_random_access_write_file_test.cc @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License a + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "file/memory_random_access_write_file.h" + +#include +#include + +#include +#include +#include +#include + +#include "file/write_file.h" + +using namespace storage; +using namespace common; + +namespace { + +std::string data_as_string(const MemoryRandomAccessWriteFile& sink) { + return std::string(sink.data().begin(), sink.data().end()); +} + +} // namespace + +TEST(MemoryRandomAccessWriteFileTest, StartsOpenAndEmpty) { + MemoryRandomAccessWriteFile sink("memory://test.tsfile"); + EXPECT_TRUE(sink.is_opened()); + EXPECT_EQ(sink.get_position(), 0); + EXPECT_EQ(sink.size(), 0); + EXPECT_TRUE(sink.data().empty()); + EXPECT_EQ(sink.file_path(), "memory://test.tsfile"); + EXPECT_EQ(sink.sync(), E_OK); +} + +TEST(MemoryRandomAccessWriteFileTest, SequentialWritesAdvancePosition) { + MemoryRandomAccessWriteFile sink; + ASSERT_EQ(sink.write("hello", 5), E_OK); + EXPECT_EQ(sink.get_position(), 5); + ASSERT_EQ(sink.write(" world", 6), E_OK); + EXPECT_EQ(sink.get_position(), 11); + EXPECT_EQ(sink.size(), 11); + EXPECT_EQ(data_as_string(sink), "hello world"); +} + +TEST(MemoryRandomAccessWriteFileTest, WriteAfterCloseFailsAndKeepsBytes) { + MemoryRandomAccessWriteFile sink; + ASSERT_EQ(sink.write("abc", 3), E_OK); + sink.close(); + EXPECT_FALSE(sink.is_opened()); + EXPECT_EQ(sink.write("more", 4), E_FILE_WRITE_ERR); + EXPECT_EQ(sink.sync(), E_FILE_WRITE_ERR); + EXPECT_EQ(sink.truncate(1), E_FILE_WRITE_ERR); + EXPECT_EQ(sink.seek_to_end(), E_FILE_WRITE_ERR); + EXPECT_EQ(sink.get_position(), 0); + // The bytes survive close() so the caller can still obtain the TsFile. + EXPECT_EQ(data_as_string(sink), "abc"); + // close() is idempotent. + sink.close(); + EXPECT_FALSE(sink.is_opened()); +} + +TEST(MemoryRandomAccessWriteFileTest, TruncateShrinksAndKeepsCursor) { + MemoryRandomAccessWriteFile sink; + const char* content = "Hello, Truncate World!"; + const uint32_t len = static_cast(std::strlen(content)); + ASSERT_EQ(sink.write(content, len), E_OK); + ASSERT_EQ(sink.truncate(7), E_OK); + EXPECT_EQ(sink.size(), 7); + EXPECT_EQ(data_as_string(sink), "Hello, "); + // ftruncate() does not move the cursor; neither does the sink. + EXPECT_EQ(sink.get_position(), static_cast(len)); + ASSERT_EQ(sink.seek_to_end(), E_OK); + EXPECT_EQ(sink.get_position(), 7); + ASSERT_EQ(sink.write("X", 1), E_OK); + EXPECT_EQ(data_as_string(sink), "Hello, X"); +} + +TEST(MemoryRandomAccessWriteFileTest, TruncateBeyondEndZeroFills) { + MemoryRandomAccessWriteFile sink; + ASSERT_EQ(sink.write("abc", 3), E_OK); + ASSERT_EQ(sink.truncate(6), E_OK); + EXPECT_EQ(sink.size(), 6); + const std::vector& bytes = sink.data(); + EXPECT_EQ(bytes[0], 'a'); + EXPECT_EQ(bytes[1], 'b'); + EXPECT_EQ(bytes[2], 'c'); + EXPECT_EQ(bytes[3], 0); + EXPECT_EQ(bytes[4], 0); + EXPECT_EQ(bytes[5], 0); + + // A write through the hole keeps the zero bytes in between. + ASSERT_EQ(sink.truncate(0), E_OK); + ASSERT_EQ(sink.seek_to_end(), E_OK); + ASSERT_EQ(sink.write("a", 1), E_OK); + ASSERT_EQ(sink.truncate(3), E_OK); + ASSERT_EQ(sink.seek_to_end(), E_OK); + ASSERT_EQ(sink.write("d", 1), E_OK); + EXPECT_EQ(sink.size(), 4); + EXPECT_EQ(sink.data()[0], 'a'); + EXPECT_EQ(sink.data()[1], 0); + EXPECT_EQ(sink.data()[2], 0); + EXPECT_EQ(sink.data()[3], 'd'); +} + +TEST(MemoryRandomAccessWriteFileTest, RejectsInvalidArguments) { + MemoryRandomAccessWriteFile sink; + EXPECT_EQ(sink.write(nullptr, 1), E_INVALID_ARG); + EXPECT_EQ(sink.write(nullptr, 0), E_OK); + EXPECT_EQ(sink.write("", 0), E_OK); + EXPECT_EQ(sink.truncate(-1), E_INVALID_ARG); + EXPECT_EQ(sink.get_position(), 0); +} + +TEST(MemoryRandomAccessWriteFileTest, ReleaseDataMovesBytesOut) { + MemoryRandomAccessWriteFile sink; + ASSERT_EQ(sink.write("payload", 7), E_OK); + std::vector bytes = sink.release_data(); + EXPECT_EQ(std::string(bytes.begin(), bytes.end()), "payload"); + EXPECT_EQ(sink.size(), 0); + EXPECT_EQ(sink.get_position(), 0); + EXPECT_TRUE(sink.is_opened()); + ASSERT_EQ(sink.write("next", 4), E_OK); + EXPECT_EQ(data_as_string(sink), "next"); +} + +// The fd-backed local backend must remain usable through the abstract +// interface, since that is how the writers now reach it. +TEST(RandomAccessWriteFileTest, LocalWriteFileThroughInterface) { + const std::string file_name = "random_access_write_file_interface.dat"; + std::remove(file_name.c_str()); + + WriteFile write_file; + RandomAccessWriteFile* sink = &write_file; + + int flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + ASSERT_EQ(write_file.create(file_name, flags, 0666), E_OK); + ASSERT_EQ(sink->write("abc", 3), E_OK); + EXPECT_EQ(sink->get_position(), 3); + EXPECT_EQ(sink->sync(), E_OK); + ASSERT_EQ(sink->seek_to_end(), E_OK); + ASSERT_EQ(sink->write("def", 3), E_OK); + ASSERT_EQ(sink->truncate(4), E_OK); + sink->close(); + EXPECT_FALSE(sink->is_opened()); + EXPECT_EQ(sink->write("x", 1), E_FILE_WRITE_ERR); + + std::ifstream in(file_name, std::ios::binary); + std::string content((std::istreambuf_iterator(in)), + std::istreambuf_iterator()); + EXPECT_EQ(content, "abcd"); + + std::remove(file_name.c_str()); +} diff --git a/cpp/test/file/write_sink_e2e_test.cc b/cpp/test/file/write_sink_e2e_test.cc new file mode 100644 index 000000000..9ab2cc931 --- /dev/null +++ b/cpp/test/file/write_sink_e2e_test.cc @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License a + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/record.h" +#include "common/schema.h" +#include "common/tablet.h" +#include "file/memory_random_access_read_file.h" +#include "file/memory_random_access_write_file.h" +#include "file/restorable_tsfile_io_writer.h" +#include "file/write_file.h" +#include "reader/qds_without_timegenerator.h" +#include "reader/result_set.h" +#include "reader/table_result_set.h" +#include "reader/tsfile_reader.h" +#include "writer/tsfile_table_writer.h" +#include "writer/tsfile_writer.h" + +using namespace storage; +using namespace common; + +namespace { + +const int64_t kTimestampBase = 1000000; + +TableSchema make_table_schema() { + std::vector columns; + columns.emplace_back("device", TSDataType::STRING, ColumnCategory::TAG); + columns.emplace_back("value", TSDataType::INT64, ColumnCategory::FIELD); + return TableSchema("memory_sink_table", columns); +} + +Tablet make_tablet(int row_count) { + std::vector names = {"device", "value"}; + std::vector types = {TSDataType::STRING, TSDataType::INT64}; + Tablet tablet(names, types, static_cast(row_count)); + for (int i = 0; i < row_count; ++i) { + tablet.add_timestamp(i, kTimestampBase + i); + tablet.add_value(i, "device", "d0"); + tablet.add_value(i, "value", static_cast(i) * 10); + } + return tablet; +} + +std::unique_ptr as_read_file( + MemoryRandomAccessWriteFile& sink) { + return std::unique_ptr(new MemoryRandomAccessReadFile( + sink.release_data(), "memory://written.tsfile")); +} + +} // namespace + +class WriteSinkEndToEndTest : public ::testing::Test { + protected: + void SetUp() override { libtsfile_init(); } + void TearDown() override { libtsfile_destroy(); } +}; + +TEST_F(WriteSinkEndToEndTest, TableModelRoundTrip) { + const int kRows = 200; + TableSchema schema = make_table_schema(); + Tablet tablet = make_tablet(kRows); + + MemoryRandomAccessWriteFile sink("memory://table.tsfile"); + TsFileTableWriter writer(&sink, &schema); + ASSERT_EQ(writer.get_error(), E_OK); + ASSERT_EQ(writer.write_table(tablet), E_OK); + ASSERT_EQ(writer.flush(), E_OK); + ASSERT_EQ(writer.close(), E_OK); + EXPECT_FALSE(sink.is_opened()); + EXPECT_GT(sink.size(), 0); + + TsFileReader reader; + ASSERT_EQ(reader.open(as_read_file(sink)), E_OK); + ResultSet* result = nullptr; + ASSERT_EQ( + reader.query("memory_sink_table", {"value"}, 0, INT64_MAX, result), + E_OK); + ASSERT_NE(result, nullptr); + auto* table_result = static_cast(result); + bool has_next = false; + int rows = 0; + while (table_result->next(has_next) == E_OK && has_next) { + EXPECT_EQ(table_result->get_value(1), kTimestampBase + rows); + EXPECT_EQ(table_result->get_value(2), + static_cast(rows) * 10); + ++rows; + } + EXPECT_EQ(rows, kRows); + reader.destroy_query_data_set(result); + reader.close(); +} + +TEST_F(WriteSinkEndToEndTest, TreeModelRoundTrip) { + const int kRows = 100; + const std::string device = "root.memory.d0"; + + MemoryRandomAccessWriteFile sink; + { + TsFileWriter writer; + ASSERT_EQ(writer.init(&sink), E_OK); + ASSERT_EQ( + writer.register_timeseries( + device, MeasurementSchema("s0", INT32, PLAIN, UNCOMPRESSED)), + E_OK); + for (int i = 0; i < kRows; ++i) { + TsRecord record(kTimestampBase + i, device); + ASSERT_EQ(record.add_point("s0", static_cast(i)), E_OK); + ASSERT_EQ(writer.write_record(record), E_OK); + } + ASSERT_EQ(writer.close(), E_OK); + } + EXPECT_FALSE(sink.is_opened()); + + TsFileReader reader; + ASSERT_EQ(reader.open(as_read_file(sink)), E_OK); + std::vector paths = {device + ".s0"}; + ResultSet* result = nullptr; + ASSERT_EQ(reader.query(paths, 0, INT64_MAX, result), E_OK); + ASSERT_NE(result, nullptr); + auto* qds = static_cast(result); + bool has_next = false; + int rows = 0; + while (qds->next(has_next) == E_OK && has_next) { + EXPECT_EQ(qds->get_value(1), kTimestampBase + rows); + EXPECT_EQ(qds->get_value(2), rows); + ++rows; + } + EXPECT_EQ(rows, kRows); + reader.destroy_query_data_set(result); + reader.close(); +} + +TEST_F(WriteSinkEndToEndTest, AlignedTreeModelRoundTrip) { + const int kRows = 64; + const std::string device = "root.memory.aligned"; + + MemoryRandomAccessWriteFile sink; + { + TsFileWriter writer; + ASSERT_EQ(writer.init(&sink), E_OK); + std::vector schemas; + schemas.push_back( + new MeasurementSchema("s0", INT64, PLAIN, UNCOMPRESSED)); + schemas.push_back( + new MeasurementSchema("s1", DOUBLE, PLAIN, UNCOMPRESSED)); + ASSERT_EQ(writer.register_aligned_timeseries(device, schemas), E_OK); + for (int i = 0; i < kRows; ++i) { + TsRecord record(kTimestampBase + i, device); + ASSERT_EQ(record.add_point("s0", static_cast(i)), E_OK); + ASSERT_EQ(record.add_point("s1", static_cast(i) + 0.5), + E_OK); + ASSERT_EQ(writer.write_record_aligned(record), E_OK); + } + ASSERT_EQ(writer.close(), E_OK); + } + EXPECT_FALSE(sink.is_opened()); + + TsFileReader reader; + ASSERT_EQ(reader.open(as_read_file(sink)), E_OK); + std::vector paths = {device + ".s0", device + ".s1"}; + ResultSet* result = nullptr; + ASSERT_EQ(reader.query(paths, 0, INT64_MAX, result), E_OK); + ASSERT_NE(result, nullptr); + auto* qds = static_cast(result); + bool has_next = false; + int rows = 0; + while (qds->next(has_next) == E_OK && has_next) { + EXPECT_EQ(qds->get_value(1), kTimestampBase + rows); + EXPECT_EQ(qds->get_value(2), static_cast(rows)); + EXPECT_DOUBLE_EQ(qds->get_value(3), + static_cast(rows) + 0.5); + ++rows; + } + EXPECT_EQ(rows, kRows); + reader.destroy_query_data_set(result); + reader.close(); +} + +// The two backends must produce the same bytes for the same workload; this is +// what makes an in-memory sink a faithful stand-in for a local file. +TEST_F(WriteSinkEndToEndTest, MemorySinkMatchesLocalFileByteForByte) { + const int kRows = 500; + TableSchema schema = make_table_schema(); + Tablet memory_tablet = make_tablet(kRows); + Tablet local_tablet = make_tablet(kRows); + + MemoryRandomAccessWriteFile sink; + { + TsFileTableWriter writer(&sink, &schema); + ASSERT_EQ(writer.get_error(), E_OK); + ASSERT_EQ(writer.write_table(memory_tablet), E_OK); + ASSERT_EQ(writer.close(), E_OK); + } + + const std::string path = "write_sink_e2e_local.tsfile"; + std::remove(path.c_str()); + WriteFile local_file; + const int flags = O_RDWR | O_CREAT | O_TRUNC; + ASSERT_EQ(local_file.create(path, flags, 0666), E_OK); + { + TsFileTableWriter writer(&local_file, &schema); + ASSERT_EQ(writer.get_error(), E_OK); + ASSERT_EQ(writer.write_table(local_tablet), E_OK); + ASSERT_EQ(writer.close(), E_OK); + } + + std::ifstream in(path, std::ios::binary); + std::vector local_bytes((std::istreambuf_iterator(in)), + std::istreambuf_iterator()); + in.close(); + std::remove(path.c_str()); + + ASSERT_GT(local_bytes.size(), 0u); + ASSERT_EQ(local_bytes.size(), sink.data().size()); + EXPECT_TRUE(std::equal(local_bytes.begin(), local_bytes.end(), + sink.data().begin())); +} + +// The memory sink cannot back the recovery path (self-check reads the +// existing bytes by path), so recovery must keep working on a local file and +// expose its sink through the interface. +TEST_F(WriteSinkEndToEndTest, RestorableWriterStillLocalOnly) { + const std::string path = "write_sink_e2e_restorable.tsfile"; + std::remove(path.c_str()); + + { + RestorableTsFileIOWriter restorable; + ASSERT_EQ(restorable.open(path), E_OK); + ASSERT_TRUE(restorable.can_write()); + RandomAccessWriteFile* sink = restorable.get_write_file(); + ASSERT_NE(sink, nullptr); + ASSERT_TRUE(sink->is_opened()); + EXPECT_EQ(sink->file_path(), path); + + TsFileWriter writer; + ASSERT_EQ(writer.init(&restorable), E_OK); + ASSERT_EQ(writer.register_timeseries( + "root.memory.restore", + MeasurementSchema("s0", INT64, PLAIN, UNCOMPRESSED)), + E_OK); + TsRecord record(kTimestampBase, "root.memory.restore"); + ASSERT_EQ(record.add_point("s0", static_cast(7)), E_OK); + ASSERT_EQ(writer.write_record(record), E_OK); + ASSERT_EQ(writer.close(), E_OK); + restorable.close(); + } + + TsFileReader reader; + ASSERT_EQ(reader.open(path), E_OK); + reader.close(); + std::remove(path.c_str()); +}