From bff9114aaa3f627224b4e77ceeb124884d0bdf21 Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 8 Sep 2026 01:48:53 -0700 Subject: [PATCH] Implement streaming zip writer, tests, and benchmarks Implements sequential streaming writing to buffers and files without seeking via Crubit C++ bindings to zip-rs: - Adds BufferedZipStreamWriter, FsZipStreamWriter, and ZipStreamWriter. - Adds helper constructors NewBufferedZipStreamWriter and NewFsZipStreamWriter. - Adds writer extension methods IsSeekPossible, SetComment, and Flush. - Adds support for StartFile, WriteData, AddDirectory, WriteZipFileContent, WriteFileContent, and Finish. - Adds comprehensive unit tests and Crubit binding tests for streaming write operations. - Adds streaming write benchmarks in zip_benchmark.cc. PiperOrigin-RevId: 977770317 --- zip/converters.cc | 38 ++++ zip/converters.h | 32 +++- zip/file.cc | 95 ++++++++++ zip/file.h | 43 ++++- zip/read.cc | 212 +++++++++++++++++++++- zip/read.h | 115 +++++++++++- zip/rust/Cargo.toml | 5 +- zip/rust/error.rs | 12 ++ zip/rust/file.rs | 187 +++++++++++++++++-- zip/rust/lib.rs | 11 +- zip/rust/read.rs | 365 ++++++++++++++++++++++++++++++++++++- zip/rust/write.rs | 422 ++++++++++++++++++++++++++++++++++++++++--- zip/write.cc | 284 +++++++++++++++++++++++++++++ zip/write.h | 93 ++++++++++ zip/zip_benchmark.cc | 232 ++++++++++++++++++++++++ 15 files changed, 2079 insertions(+), 67 deletions(-) create mode 100644 zip/zip_benchmark.cc diff --git a/zip/converters.cc b/zip/converters.cc index faac231..1ea56e6 100644 --- a/zip/converters.cc +++ b/zip/converters.cc @@ -105,4 +105,42 @@ absl::StatusOr FromRustFsZipWriter( return std::move(result_fs_zip_writer).value(); } +absl::StatusOr +FromRustBufferedZipStreamReader( + rs_std::Result + result_buffered_zip_stream_reader) { + if (!result_buffered_zip_stream_reader.has_value()) { + return ZipErrorToStatus(std::move(result_buffered_zip_stream_reader).err()); + } + return std::move(result_buffered_zip_stream_reader).value(); +} + +absl::StatusOr FromRustFsZipStreamReader( + rs_std::Result + result_fs_zip_stream_reader) { + if (!result_fs_zip_stream_reader.has_value()) { + return ZipErrorToStatus(std::move(result_fs_zip_stream_reader).err()); + } + return std::move(result_fs_zip_stream_reader).value(); +} + +absl::StatusOr +FromRustBufferedZipStreamWriter( + rs_std::Result + result_buffered_zip_stream_writer) { + if (!result_buffered_zip_stream_writer.has_value()) { + return ZipErrorToStatus(std::move(result_buffered_zip_stream_writer).err()); + } + return std::move(result_buffered_zip_stream_writer).value(); +} + +absl::StatusOr FromRustFsZipStreamWriter( + rs_std::Result + result_fs_zip_stream_writer) { + if (!result_fs_zip_stream_writer.has_value()) { + return ZipErrorToStatus(std::move(result_fs_zip_stream_writer).err()); + } + return std::move(result_fs_zip_stream_writer).value(); +} + } // namespace security::zip diff --git a/zip/converters.h b/zip/converters.h index 626764c..e7b229a 100644 --- a/zip/converters.h +++ b/zip/converters.h @@ -1,6 +1,7 @@ #ifndef SECURITY_ZIP_CONVERTERS_H_ #define SECURITY_ZIP_CONVERTERS_H_ +#include #include #include "crubit_helpers/string_conversions.h" @@ -22,10 +23,23 @@ class RustVecU8Wrapper { explicit RustVecU8Wrapper(rust::VecU8 vec_u8) : vec_u8_(std::move(vec_u8)) {} - absl::string_view AsStringView() const ABSL_ATTRIBUTE_LIFETIME_BOUND { + [[nodiscard]] absl::string_view AsStringView() const + ABSL_ATTRIBUTE_LIFETIME_BOUND { return security::crubit_helpers::StringViewFromVecU8(vec_u8_); } + [[nodiscard]] const char* data() const ABSL_ATTRIBUTE_LIFETIME_BOUND { + return AsStringView().data(); + } + + [[nodiscard]] size_t size() const { return AsStringView().size(); } + + [[nodiscard]] bool empty() const { return AsStringView().empty(); } + + explicit operator absl::string_view() const ABSL_ATTRIBUTE_LIFETIME_BOUND { + return AsStringView(); + } + private: rust::VecU8 vec_u8_; }; @@ -56,6 +70,22 @@ absl::StatusOr FromRustFsZipWriter( rs_std::Result result_fs_zip_writer); +absl::StatusOr +FromRustBufferedZipStreamReader( + rs_std::Result + result_buffered_zip_stream_reader); +absl::StatusOr FromRustFsZipStreamReader( + rs_std::Result + result_fs_zip_stream_reader); + +absl::StatusOr +FromRustBufferedZipStreamWriter( + rs_std::Result + result_buffered_zip_stream_writer); +absl::StatusOr FromRustFsZipStreamWriter( + rs_std::Result + result_fs_zip_stream_writer); + } // namespace security::zip #endif // SECURITY_ZIP_CONVERTERS_H_ diff --git a/zip/file.cc b/zip/file.cc index 02b0ae5..ce52deb 100644 --- a/zip/file.cc +++ b/zip/file.cc @@ -1,5 +1,7 @@ #include "file.h" +#include +#include #include #include @@ -62,10 +64,44 @@ absl::StatusOr BufferedZipFile::GetCompressionMethod() ABSL_RETURN_IF_ERROR(CheckNone()); return ToSecurityZipCompressionMethod(zip_.get_compression_method()); } +absl::StatusOr BufferedZipFile::GetComment() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + rust::VecU8 comment_vec = zip_.get_comment(); + return std::string( + security::crubit_helpers::StringViewFromVecU8(comment_vec)); +} +absl::StatusOr BufferedZipFile::GetUncompressedSize() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_uncompressed_size(); +} +absl::StatusOr BufferedZipFile::GetCompressedSize() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_compressed_size(); +} +absl::StatusOr BufferedZipFile::GetCrc32() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_crc32(); +} +absl::StatusOr BufferedZipFile::GetLastModifiedDate() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_last_modified_date(); +} +absl::StatusOr BufferedZipFile::GetLastModifiedTime() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_last_modified_time(); +} +absl::StatusOr BufferedZipFile::GetUnixMode() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_unix_mode(); +} absl::StatusOr BufferedZipFile::GetFileData() { ABSL_RETURN_IF_ERROR(CheckNone()); return FromRustResultVecU8(zip_.get_file_data()); } +absl::StatusOr BufferedZipFile::ReadBytes(size_t max_bytes) { + ABSL_RETURN_IF_ERROR(CheckNone()); + return FromRustResultVecU8(zip_.read_bytes(max_bytes)); +} absl::Status FsZipFile::CheckNone() const { if (IsNone()) { @@ -92,10 +128,44 @@ absl::StatusOr FsZipFile::GetCompressionMethod() const { ABSL_RETURN_IF_ERROR(CheckNone()); return ToSecurityZipCompressionMethod(zip_.get_compression_method()); } +absl::StatusOr FsZipFile::GetComment() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + rust::VecU8 comment_vec = zip_.get_comment(); + return std::string( + security::crubit_helpers::StringViewFromVecU8(comment_vec)); +} +absl::StatusOr FsZipFile::GetUncompressedSize() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_uncompressed_size(); +} +absl::StatusOr FsZipFile::GetCompressedSize() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_compressed_size(); +} +absl::StatusOr FsZipFile::GetCrc32() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_crc32(); +} +absl::StatusOr FsZipFile::GetLastModifiedDate() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_last_modified_date(); +} +absl::StatusOr FsZipFile::GetLastModifiedTime() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_last_modified_time(); +} +absl::StatusOr FsZipFile::GetUnixMode() const { + ABSL_RETURN_IF_ERROR(CheckNone()); + return zip_.get_unix_mode(); +} absl::StatusOr FsZipFile::GetFileData() { ABSL_RETURN_IF_ERROR(CheckNone()); return FromRustResultVecU8(zip_.get_file_data()); } +absl::StatusOr FsZipFile::ReadBytes(size_t max_bytes) { + ABSL_RETURN_IF_ERROR(CheckNone()); + return FromRustResultVecU8(zip_.read_bytes(max_bytes)); +} absl::StatusOr ZipFile::IsFile() const { return std::visit([](auto& zip) { return zip.IsFile(); }, zip_); @@ -112,8 +182,33 @@ absl::StatusOr ZipFile::GetFileName() const { absl::StatusOr ZipFile::GetCompressionMethod() const { return std::visit([](auto& zip) { return zip.GetCompressionMethod(); }, zip_); } +absl::StatusOr ZipFile::GetComment() const { + return std::visit([](auto& zip) { return zip.GetComment(); }, zip_); +} +absl::StatusOr ZipFile::GetUncompressedSize() const { + return std::visit([](auto& zip) { return zip.GetUncompressedSize(); }, zip_); +} +absl::StatusOr ZipFile::GetCompressedSize() const { + return std::visit([](auto& zip) { return zip.GetCompressedSize(); }, zip_); +} +absl::StatusOr ZipFile::GetCrc32() const { + return std::visit([](auto& zip) { return zip.GetCrc32(); }, zip_); +} +absl::StatusOr ZipFile::GetLastModifiedDate() const { + return std::visit([](auto& zip) { return zip.GetLastModifiedDate(); }, zip_); +} +absl::StatusOr ZipFile::GetLastModifiedTime() const { + return std::visit([](auto& zip) { return zip.GetLastModifiedTime(); }, zip_); +} +absl::StatusOr ZipFile::GetUnixMode() const { + return std::visit([](auto& zip) { return zip.GetUnixMode(); }, zip_); +} absl::StatusOr ZipFile::GetFileData() { return std::visit([](auto& zip) { return zip.GetFileData(); }, zip_); } +absl::StatusOr ZipFile::ReadBytes(size_t max_bytes) { + return std::visit([max_bytes](auto& zip) { return zip.ReadBytes(max_bytes); }, + zip_); +} } // namespace security::zip diff --git a/zip/file.h b/zip/file.h index eb4c9fd..e08c724 100644 --- a/zip/file.h +++ b/zip/file.h @@ -1,6 +1,7 @@ #ifndef SECURITY_ZIP_FILE_H_ #define SECURITY_ZIP_FILE_H_ +#include #include #include #include @@ -15,6 +16,9 @@ namespace security::zip { class BufferedZipWriter; class FsZipWriter; +class BufferedZipStreamWriter; +class FsZipStreamWriter; +class ZipStreamWriter; enum class CompressionMethod : int32_t { kStored, @@ -31,16 +35,27 @@ class BufferedZipFile final { : zip_(std::move(zip)) {} absl::StatusOr IsFile() const; absl::StatusOr IsDir() const; - bool IsNone() const; + [[nodiscard]] bool IsNone() const; absl::StatusOr GetFileName() const; absl::StatusOr GetCompressionMethod() const; + absl::StatusOr GetComment() const; + absl::StatusOr GetUncompressedSize() const; + absl::StatusOr GetCompressedSize() const; + absl::StatusOr GetCrc32() const; + absl::StatusOr GetLastModifiedDate() const; + absl::StatusOr GetLastModifiedTime() const; + absl::StatusOr GetUnixMode() const; absl::StatusOr GetFileData(); + absl::StatusOr ReadBytes(size_t max_bytes); private: absl::Status CheckNone() const; rust::BufferedZipFile zip_; friend class BufferedZipWriter; friend class FsZipWriter; + friend class BufferedZipStreamWriter; + friend class FsZipStreamWriter; + friend class ZipStreamWriter; }; class FsZipFile final { @@ -48,16 +63,27 @@ class FsZipFile final { explicit FsZipFile(rust::FsZipFile zip) : zip_(std::move(zip)) {} absl::StatusOr IsFile() const; absl::StatusOr IsDir() const; - bool IsNone() const; + [[nodiscard]] bool IsNone() const; absl::StatusOr GetFileName() const; absl::StatusOr GetCompressionMethod() const; + absl::StatusOr GetComment() const; + absl::StatusOr GetUncompressedSize() const; + absl::StatusOr GetCompressedSize() const; + absl::StatusOr GetCrc32() const; + absl::StatusOr GetLastModifiedDate() const; + absl::StatusOr GetLastModifiedTime() const; + absl::StatusOr GetUnixMode() const; absl::StatusOr GetFileData(); + absl::StatusOr ReadBytes(size_t max_bytes); private: absl::Status CheckNone() const; rust::FsZipFile zip_; friend class BufferedZipWriter; friend class FsZipWriter; + friend class BufferedZipStreamWriter; + friend class FsZipStreamWriter; + friend class ZipStreamWriter; }; class ZipFile final { @@ -72,10 +98,18 @@ class ZipFile final { absl::StatusOr IsFile() const; absl::StatusOr IsDir() const; - bool IsNone() const; + [[nodiscard]] bool IsNone() const; absl::StatusOr GetFileName() const; absl::StatusOr GetCompressionMethod() const; + absl::StatusOr GetComment() const; + absl::StatusOr GetUncompressedSize() const; + absl::StatusOr GetCompressedSize() const; + absl::StatusOr GetCrc32() const; + absl::StatusOr GetLastModifiedDate() const; + absl::StatusOr GetLastModifiedTime() const; + absl::StatusOr GetUnixMode() const; absl::StatusOr GetFileData(); + absl::StatusOr ReadBytes(size_t max_bytes); private: using BackendType = std::variant; @@ -83,6 +117,9 @@ class ZipFile final { explicit ZipFile(BackendType b) : zip_(std::move(b)) {} friend class BufferedZipWriter; friend class FsZipWriter; + friend class BufferedZipStreamWriter; + friend class FsZipStreamWriter; + friend class ZipStreamWriter; }; } // namespace security::zip diff --git a/zip/read.cc b/zip/read.cc index 0f786f6..7413f10 100644 --- a/zip/read.cc +++ b/zip/read.cc @@ -1,12 +1,16 @@ #include "read.h" #include +#include +#include #include #include +#include "crubit_helpers/string_conversions.h" #include "converters.h" #include "file.h" #include "crubit/rust.h" +#include "absl/status/status.h" #include "absl/status/status_macros.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" @@ -25,11 +29,26 @@ absl::StatusOr BufferedZipArchive::NewFromData( return BufferedZipArchive(std::move(archive)); } -absl::StatusOr BufferedZipArchive::GetLength() { +absl::StatusOr BufferedZipArchive::GetLength() const { + if (archive_.is_none()) { + return absl::FailedPreconditionError("Zip archive is not open"); + } return archive_.get_length(); } +absl::StatusOr BufferedZipArchive::GetComment() const { + if (archive_.is_none()) { + return absl::FailedPreconditionError("Zip archive is not open"); + } + rust::VecU8 comment_vec = archive_.get_comment(); + return std::string( + security::crubit_helpers::StringViewFromVecU8(comment_vec)); +} + absl::StatusOr BufferedZipArchive::GetFileByIndex(uintptr_t index) { + if (archive_.is_none()) { + return absl::FailedPreconditionError("Zip archive is not open"); + } ABSL_ASSIGN_OR_RETURN( rust::BufferedZipFile file, FromRustBufferedZipFile(archive_.get_file_by_index(index))); @@ -37,12 +56,17 @@ absl::StatusOr BufferedZipArchive::GetFileByIndex(uintptr_t index) { } absl::StatusOr BufferedZipArchive::GetFileByIndexRaw(uintptr_t index) { + if (archive_.is_none()) { + return absl::FailedPreconditionError("Zip archive is not open"); + } ABSL_ASSIGN_OR_RETURN( rust::BufferedZipFile file, FromRustBufferedZipFile(archive_.get_file_by_index_raw(index))); return ZipFile::FromBuffer(std::move(file)); } +bool BufferedZipArchive::IsNone() const { return archive_.is_none(); } + absl::StatusOr FsZipArchive::NewFromPath(absl::string_view path) { ABSL_ASSIGN_OR_RETURN( rust::FsZipArchive archive, @@ -52,23 +76,43 @@ absl::StatusOr FsZipArchive::NewFromPath(absl::string_view path) { return FsZipArchive(std::move(archive)); } -absl::StatusOr FsZipArchive::GetLength() { +absl::StatusOr FsZipArchive::GetLength() const { + if (archive_.is_none()) { + return absl::FailedPreconditionError("Zip archive is not open"); + } return archive_.get_length(); } +absl::StatusOr FsZipArchive::GetComment() const { + if (archive_.is_none()) { + return absl::FailedPreconditionError("Zip archive is not open"); + } + rust::VecU8 comment_vec = archive_.get_comment(); + return std::string( + security::crubit_helpers::StringViewFromVecU8(comment_vec)); +} + absl::StatusOr FsZipArchive::GetFileByIndex(uintptr_t index) { + if (archive_.is_none()) { + return absl::FailedPreconditionError("Zip archive is not open"); + } ABSL_ASSIGN_OR_RETURN(rust::FsZipFile file, FromRustFsZipFile(archive_.get_file_by_index(index))); return ZipFile::FromFile(std::move(file)); } absl::StatusOr FsZipArchive::GetFileByIndexRaw(uintptr_t index) { + if (archive_.is_none()) { + return absl::FailedPreconditionError("Zip archive is not open"); + } ABSL_ASSIGN_OR_RETURN( rust::FsZipFile file, FromRustFsZipFile(archive_.get_file_by_index_raw(index))); return ZipFile::FromFile(std::move(file)); } +bool FsZipArchive::IsNone() const { return archive_.is_none(); } + absl::StatusOr ZipArchive::FromFile(absl::string_view path) { ABSL_ASSIGN_OR_RETURN(FsZipArchive archive, FsZipArchive::NewFromPath(path)); return ZipArchive(std::move(archive)); @@ -80,8 +124,12 @@ absl::StatusOr ZipArchive::FromBuffer(absl::string_view data) { return ZipArchive(std::move(archive)); } -absl::StatusOr ZipArchive::GetLength() { - return std::visit([](auto& arg) { return arg.GetLength(); }, archive_); +absl::StatusOr ZipArchive::GetLength() const { + return std::visit([](const auto& arg) { return arg.GetLength(); }, archive_); +} + +absl::StatusOr ZipArchive::GetComment() const { + return std::visit([](const auto& arg) { return arg.GetComment(); }, archive_); } absl::StatusOr ZipArchive::GetFileByIndex(uintptr_t index) { @@ -94,4 +142,160 @@ absl::StatusOr ZipArchive::GetFileByIndexRaw(uintptr_t index) { archive_); } +bool ZipArchive::IsNone() const { + return std::visit([](const auto& arg) { return arg.IsNone(); }, archive_); +} + +absl::StatusOr BufferedZipStreamReader::NewFromData( + absl::string_view data) { + rust::VecU8 input_data = + rust::VecU8::copy_from_slice(absl::Span( + reinterpret_cast(data.data()), data.size())); + ABSL_ASSIGN_OR_RETURN( + rust::BufferedZipStreamReader reader, + FromRustBufferedZipStreamReader( + rust::BufferedZipStreamReader::new_from_data(input_data))); + return BufferedZipStreamReader(std::move(reader)); +} + +absl::StatusOr> BufferedZipStreamReader::ReadNextFile() { + if (reader_.is_none()) { + return absl::FailedPreconditionError("Zip stream reader is not open"); + } + ABSL_ASSIGN_OR_RETURN(rust::BufferedZipFile file, + FromRustBufferedZipFile(reader_.read_next_file())); + if (file.is_none()) { + return std::nullopt; + } + return ZipFile::FromBuffer(std::move(file)); +} + +absl::StatusOr> +BufferedZipStreamReader::ReadNextFileWithCompressedSize( + uint64_t compressed_size) { + if (reader_.is_none()) { + return absl::FailedPreconditionError("Zip stream reader is not open"); + } + ABSL_ASSIGN_OR_RETURN( + rust::BufferedZipFile file, + FromRustBufferedZipFile( + reader_.read_next_file_with_compressed_size(compressed_size))); + if (file.is_none()) { + return std::nullopt; + } + return ZipFile::FromBuffer(std::move(file)); +} + +bool BufferedZipStreamReader::IsFinished() const { + return reader_.is_finished(); +} + +bool BufferedZipStreamReader::IsNone() const { return reader_.is_none(); } + +absl::StatusOr FsZipStreamReader::NewFromPath( + absl::string_view path) { + ABSL_ASSIGN_OR_RETURN( + rust::FsZipStreamReader reader, + FromRustFsZipStreamReader(rust::FsZipStreamReader::new_from_path( + absl::Span( + reinterpret_cast(path.data()), path.size())))); + return FsZipStreamReader(std::move(reader)); +} + +absl::StatusOr> FsZipStreamReader::ReadNextFile() { + if (reader_.is_none()) { + return absl::FailedPreconditionError("Zip stream reader is not open"); + } + ABSL_ASSIGN_OR_RETURN(rust::FsZipFile file, + FromRustFsZipFile(reader_.read_next_file())); + if (file.is_none()) { + return std::nullopt; + } + return ZipFile::FromFile(std::move(file)); +} + +absl::StatusOr> +FsZipStreamReader::ReadNextFileWithCompressedSize(uint64_t compressed_size) { + if (reader_.is_none()) { + return absl::FailedPreconditionError("Zip stream reader is not open"); + } + ABSL_ASSIGN_OR_RETURN( + rust::FsZipFile file, + FromRustFsZipFile( + reader_.read_next_file_with_compressed_size(compressed_size))); + if (file.is_none()) { + return std::nullopt; + } + return ZipFile::FromFile(std::move(file)); +} + +bool FsZipStreamReader::IsFinished() const { return reader_.is_finished(); } + +bool FsZipStreamReader::IsNone() const { return reader_.is_none(); } + +absl::StatusOr ZipStreamReader::FromFile( + absl::string_view path) { + ABSL_ASSIGN_OR_RETURN(FsZipStreamReader reader, + FsZipStreamReader::NewFromPath(path)); + return ZipStreamReader(std::move(reader)); +} + +absl::StatusOr ZipStreamReader::FromBuffer( + absl::string_view data) { + ABSL_ASSIGN_OR_RETURN(BufferedZipStreamReader reader, + BufferedZipStreamReader::NewFromData(data)); + return ZipStreamReader(std::move(reader)); +} + +absl::StatusOr> ZipStreamReader::ReadNextFile() { + return std::visit([](auto& arg) { return arg.ReadNextFile(); }, reader_); +} + +absl::StatusOr> +ZipStreamReader::ReadNextFileWithCompressedSize(uint64_t compressed_size) { + return std::visit( + [compressed_size](auto& arg) { + return arg.ReadNextFileWithCompressedSize(compressed_size); + }, + reader_); +} + +bool ZipStreamReader::IsFinished() const { + return std::visit([](const auto& arg) { return arg.IsFinished(); }, reader_); +} + +bool ZipStreamReader::IsNone() const { + return std::visit([](const auto& arg) { return arg.IsNone(); }, reader_); +} + +absl::StatusOr> ReadZipFileFromStream( + BufferedZipStreamReader& reader) { + return reader.ReadNextFile(); +} + +absl::StatusOr> ReadZipFileFromStream( + FsZipStreamReader& reader) { + return reader.ReadNextFile(); +} + +absl::StatusOr> ReadZipFileFromStream( + ZipStreamReader& reader) { + return reader.ReadNextFile(); +} + +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + BufferedZipStreamReader& reader, uint64_t compressed_size) { + return reader.ReadNextFileWithCompressedSize(compressed_size); +} + +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + FsZipStreamReader& reader, uint64_t compressed_size) { + return reader.ReadNextFileWithCompressedSize(compressed_size); +} + +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + ZipStreamReader& reader, uint64_t compressed_size) { + return reader.ReadNextFileWithCompressedSize(compressed_size); +} + } // namespace security::zip diff --git a/zip/read.h b/zip/read.h index e18159a..3f8e7a2 100644 --- a/zip/read.h +++ b/zip/read.h @@ -2,11 +2,14 @@ #define SECURITY_ZIP_READ_H_ #include +#include +#include #include #include #include "file.h" #include "crubit/rust.h" +#include "absl/base/attributes.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" @@ -15,15 +18,19 @@ namespace security::zip { class BufferedZipArchive final { public: static absl::StatusOr NewFromData(absl::string_view data); - absl::StatusOr GetLength(); + absl::StatusOr GetLength() const; + absl::StatusOr GetComment() const; // Returns a ZipFile by index. The result file returns decompressed data when // read. - absl::StatusOr GetFileByIndex(uintptr_t index); + absl::StatusOr GetFileByIndex(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; // Returns a ZipFile by index. The result file returns compressed data as-is // without decompressing when read. // Warning: Writers in zip-rs do not support writing compressed data as-is. // This data will be compressed again when written through a `*ZipWriter`. - absl::StatusOr GetFileByIndexRaw(uintptr_t index); + absl::StatusOr GetFileByIndexRaw(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; + [[nodiscard]] bool IsNone() const; private: rust::BufferedZipArchive archive_; @@ -34,15 +41,19 @@ class BufferedZipArchive final { class FsZipArchive final { public: static absl::StatusOr NewFromPath(absl::string_view path); - absl::StatusOr GetLength(); + absl::StatusOr GetLength() const; + absl::StatusOr GetComment() const; // Returns a ZipFile by index. The result file returns decompressed data when // read. - absl::StatusOr GetFileByIndex(uintptr_t index); + absl::StatusOr GetFileByIndex(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; // Returns a ZipFile by index. The result file returns compressed data as-is // without decompressing when read. // Warning: Writers in zip-rs do not support writing compressed data as-is. // This data will be compressed again when written through a `*ZipWriter`. - absl::StatusOr GetFileByIndexRaw(uintptr_t index); + absl::StatusOr GetFileByIndexRaw(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; + [[nodiscard]] bool IsNone() const; private: rust::FsZipArchive archive_; @@ -53,16 +64,20 @@ class FsZipArchive final { class ZipArchive final { public: static absl::StatusOr FromFile(absl::string_view path); - static absl::StatusOr FromBuffer(std::string_view data); - absl::StatusOr GetLength(); + static absl::StatusOr FromBuffer(absl::string_view data); + absl::StatusOr GetLength() const; + absl::StatusOr GetComment() const; // Returns a ZipFile by index. The result file returns decompressed data when // read. - absl::StatusOr GetFileByIndex(uintptr_t index); + absl::StatusOr GetFileByIndex(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; // Returns a ZipFile by index. The result file returns compressed data as-is // without decompressing when read. // Warning: Writers in zip-rs do not support writing compressed data as-is. // This data will be compressed again when written through a `*ZipWriter`. - absl::StatusOr GetFileByIndexRaw(uintptr_t index); + absl::StatusOr GetFileByIndexRaw(uintptr_t index) + ABSL_ATTRIBUTE_LIFETIME_BOUND; + [[nodiscard]] bool IsNone() const; private: using BackendType = std::variant; @@ -71,6 +86,86 @@ class ZipArchive final { explicit ZipArchive(BackendType archive) : archive_(std::move(archive)) {} }; +// Streaming readers that read sequential zip entries from a stream without +// seeking. +class BufferedZipStreamReader final { + public: + static absl::StatusOr NewFromData( + absl::string_view data); + // Reads the next file in the stream. Returns std::nullopt when end of archive + // is reached. + absl::StatusOr> ReadNextFile() + ABSL_ATTRIBUTE_LIFETIME_BOUND; + // Reads the next file in the stream with an assumed compressed size. + absl::StatusOr> ReadNextFileWithCompressedSize( + uint64_t compressed_size) ABSL_ATTRIBUTE_LIFETIME_BOUND; + [[nodiscard]] bool IsFinished() const; + [[nodiscard]] bool IsNone() const; + + private: + rust::BufferedZipStreamReader reader_; + explicit BufferedZipStreamReader(rust::BufferedZipStreamReader reader) + : reader_(std::move(reader)) {} +}; + +class FsZipStreamReader final { + public: + static absl::StatusOr NewFromPath(absl::string_view path); + // Reads the next file in the stream. Returns std::nullopt when end of archive + // is reached. + absl::StatusOr> ReadNextFile() + ABSL_ATTRIBUTE_LIFETIME_BOUND; + // Reads the next file in the stream with an assumed compressed size. + absl::StatusOr> ReadNextFileWithCompressedSize( + uint64_t compressed_size) ABSL_ATTRIBUTE_LIFETIME_BOUND; + [[nodiscard]] bool IsFinished() const; + [[nodiscard]] bool IsNone() const; + + private: + rust::FsZipStreamReader reader_; + explicit FsZipStreamReader(rust::FsZipStreamReader reader) + : reader_(std::move(reader)) {} +}; + +class ZipStreamReader final { + public: + static absl::StatusOr FromFile(absl::string_view path); + static absl::StatusOr FromBuffer(absl::string_view data); + // Reads the next file in the stream. Returns std::nullopt when end of archive + // is reached. + absl::StatusOr> ReadNextFile() + ABSL_ATTRIBUTE_LIFETIME_BOUND; + // Reads the next file in the stream with an assumed compressed size. + absl::StatusOr> ReadNextFileWithCompressedSize( + uint64_t compressed_size) ABSL_ATTRIBUTE_LIFETIME_BOUND; + [[nodiscard]] bool IsFinished() const; + [[nodiscard]] bool IsNone() const; + + private: + using BackendType = std::variant; + BackendType reader_; + + explicit ZipStreamReader(BackendType reader) : reader_(std::move(reader)) {} +}; + +// Free function helpers mirroring zip::read::read_zipfile_from_stream. +absl::StatusOr> ReadZipFileFromStream( + BufferedZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND); +absl::StatusOr> ReadZipFileFromStream( + FsZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND); +absl::StatusOr> ReadZipFileFromStream( + ZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND); + +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + BufferedZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND, + uint64_t compressed_size); +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + FsZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND, + uint64_t compressed_size); +absl::StatusOr> ReadZipFileFromStreamWithCompressedSize( + ZipStreamReader& reader ABSL_ATTRIBUTE_LIFETIME_BOUND, + uint64_t compressed_size); + } // namespace security::zip #endif // SECURITY_ZIP_READ_H_ diff --git a/zip/rust/Cargo.toml b/zip/rust/Cargo.toml index c85f0d2..ff3f21c 100644 --- a/zip/rust/Cargo.toml +++ b/zip/rust/Cargo.toml @@ -9,4 +9,7 @@ path = "lib.rs" doctest = false [dependencies] -zip = { version = "6.0", features = ["deflate", "bzip2", "time", "zstd", "lzma", "xz"] } \ No newline at end of file +zip = { version = "8.6", features = ["deflate", "bzip2", "time", "zstd", "lzma", "xz"] } + +[features] +deflate-zopfli = ["zip/deflate-zopfli"] \ No newline at end of file diff --git a/zip/rust/error.rs b/zip/rust/error.rs index 7959967..297d0d0 100644 --- a/zip/rust/error.rs +++ b/zip/rust/error.rs @@ -37,3 +37,15 @@ impl ZipError { Self::new(message, ZipErrorCode::Internal) } } + +impl From for ZipError { + fn from(err: zip::result::ZipError) -> Self { + match err { + zip::result::ZipError::Io(e) => ZipError::internal(e.to_string()), + zip::result::ZipError::InvalidArchive(e) => ZipError::invalid_argument(e.as_ref()), + zip::result::ZipError::UnsupportedArchive(e) => ZipError::failed_precondition(e), + zip::result::ZipError::FileNotFound => ZipError::out_of_range("File not found"), + _ => ZipError::internal(err.to_string()), + } + } +} diff --git a/zip/rust/file.rs b/zip/rust/file.rs index a42308c..97f9b36 100644 --- a/zip/rust/file.rs +++ b/zip/rust/file.rs @@ -15,7 +15,7 @@ pub struct BufferedZipFile<'a> { /// used by Crubit to generate movable types. /// In C++, whenever a method of *ZipFile is called, the file must be /// checked to be not None first. - file: Option>>>, + file: Option>>>>, } impl<'a> Debug for BufferedZipFile<'a> { @@ -31,7 +31,7 @@ impl<'a> BufferedZipFile<'a> { /// This function won't be generated by Crubit and is only called from rust. /// (b/259749095) pub fn new(file: WrappedZipFile<'a, Cursor>>) -> Self { - Self { file: Some(file) } + Self { file: Some(Box::new(file)) } } /// Returns whether the zip file is none (due to being default-constructed @@ -64,11 +64,81 @@ impl<'a> BufferedZipFile<'a> { get_compression_method_impl(&self.file) } + /// Returns the comment of the file. + /// The result only is valid if `is_none()` returns false. + pub fn get_comment(&self) -> VecU8 { + match self.file.as_ref() { + Some(file) => VecU8::from(file.comment()), + None => VecU8::default(), + } + } + + /// Returns the uncompressed size of the file. + /// The result only is valid if `is_none()` returns false. + pub fn get_uncompressed_size(&self) -> u64 { + match self.file.as_ref() { + Some(file) => file.size(), + None => 0, + } + } + + /// Returns the compressed size of the file. + /// The result only is valid if `is_none()` returns false. + pub fn get_compressed_size(&self) -> u64 { + match self.file.as_ref() { + Some(file) => file.compressed_size(), + None => 0, + } + } + + /// Returns the CRC32 of the file. + /// The result only is valid if `is_none()` returns false. + pub fn get_crc32(&self) -> u32 { + match self.file.as_ref() { + Some(file) => file.crc32(), + None => 0, + } + } + + /// Returns the last modified date of the file in DOS format. + /// The result only is valid if `is_none()` returns false. + pub fn get_last_modified_date(&self) -> u16 { + match self.file.as_ref() { + Some(file) => file.last_modified().map(|dt| dt.datepart()).unwrap_or(0), + None => 0, + } + } + + /// Returns the last modified time of the file in DOS format. + /// The result only is valid if `is_none()` returns false. + pub fn get_last_modified_time(&self) -> u16 { + match self.file.as_ref() { + Some(file) => file.last_modified().map(|dt| dt.timepart()).unwrap_or(0), + None => 0, + } + } + + /// Returns the Unix mode of the file. + /// The result only is valid if `is_none()` returns false. + pub fn get_unix_mode(&self) -> u32 { + match self.file.as_ref() { + Some(file) => file.unix_mode().unwrap_or(0), + None => 0, + } + } + /// Returns the data of the file. /// The result only is valid if `is_none()` returns false. pub fn get_file_data(&mut self) -> Result { get_file_data_impl(&mut self.file) } + + /// Reads up to `max_bytes` decompressed data from the file. + /// Returns an empty VecU8 on EOF. + /// The result only is valid if `is_none()` returns false. + pub fn read_bytes(&mut self, max_bytes: usize) -> Result { + read_bytes_impl(&mut self.file, max_bytes) + } } impl<'a> Read for BufferedZipFile<'a> { @@ -86,7 +156,7 @@ pub struct FsZipFile<'a> { /// used by Crubit to generate movable types. /// In C++, whenever a method of *ZipFile is called, the file must be /// checked to be not None first. - file: Option>, + file: Option>>, } impl<'a> Debug for FsZipFile<'a> { @@ -102,7 +172,7 @@ impl<'a> FsZipFile<'a> { /// This function won't be generated by Crubit and is only called from rust. /// (b/259749095) pub fn new(file: WrappedZipFile<'a, File>) -> Self { - Self { file: Some(file) } + Self { file: Some(Box::new(file)) } } /// Returns whether the zip file is none (due to being default-constructed @@ -135,11 +205,81 @@ impl<'a> FsZipFile<'a> { get_compression_method_impl(&self.file) } + /// Returns the comment of the file. + /// The result only is valid if `is_none()` returns false. + pub fn get_comment(&self) -> VecU8 { + match self.file.as_ref() { + Some(file) => VecU8::from(file.comment()), + None => VecU8::default(), + } + } + + /// Returns the uncompressed size of the file. + /// The result only is valid if `is_none()` returns false. + pub fn get_uncompressed_size(&self) -> u64 { + match self.file.as_ref() { + Some(file) => file.size(), + None => 0, + } + } + + /// Returns the compressed size of the file. + /// The result only is valid if `is_none()` returns false. + pub fn get_compressed_size(&self) -> u64 { + match self.file.as_ref() { + Some(file) => file.compressed_size(), + None => 0, + } + } + + /// Returns the CRC32 of the file. + /// The result only is valid if `is_none()` returns false. + pub fn get_crc32(&self) -> u32 { + match self.file.as_ref() { + Some(file) => file.crc32(), + None => 0, + } + } + + /// Returns the last modified date of the file in DOS format. + /// The result only is valid if `is_none()` returns false. + pub fn get_last_modified_date(&self) -> u16 { + match self.file.as_ref() { + Some(file) => file.last_modified().map(|dt| dt.datepart()).unwrap_or(0), + None => 0, + } + } + + /// Returns the last modified time of the file in DOS format. + /// The result only is valid if `is_none()` returns false. + pub fn get_last_modified_time(&self) -> u16 { + match self.file.as_ref() { + Some(file) => file.last_modified().map(|dt| dt.timepart()).unwrap_or(0), + None => 0, + } + } + + /// Returns the Unix mode of the file. + /// The result only is valid if `is_none()` returns false. + pub fn get_unix_mode(&self) -> u32 { + match self.file.as_ref() { + Some(file) => file.unix_mode().unwrap_or(0), + None => 0, + } + } + /// Returns the data of the file. /// The result only is valid if `is_none()` returns false. pub fn get_file_data(&mut self) -> Result { get_file_data_impl(&mut self.file) } + + /// Reads up to `max_bytes` decompressed data from the file. + /// Returns an empty VecU8 on EOF. + /// The result only is valid if `is_none()` returns false. + pub fn read_bytes(&mut self, max_bytes: usize) -> Result { + read_bytes_impl(&mut self.file, max_bytes) + } } impl<'a> Read for FsZipFile<'a> { @@ -151,21 +291,21 @@ impl<'a> Read for FsZipFile<'a> { } } -fn get_file_name_impl<'a, R: Read>(file: &Option>) -> VecU8 { +fn get_file_name_impl<'a, R: Read>(file: &Option>>) -> VecU8 { match file.as_ref() { Some(file) => VecU8::from(file.name()), None => VecU8::default(), } } -fn is_file_impl<'a, R: Read>(file: &Option>) -> bool { +fn is_file_impl<'a, R: Read>(file: &Option>>) -> bool { match file.as_ref() { Some(file) => file.is_file(), None => false, } } -fn is_dir_impl<'a, R: Read>(file: &Option>) -> bool { +fn is_dir_impl<'a, R: Read>(file: &Option>>) -> bool { match file.as_ref() { Some(file) => file.is_dir(), None => false, @@ -173,16 +313,16 @@ fn is_dir_impl<'a, R: Read>(file: &Option>) -> bool { } fn get_compression_method_impl<'a, R: Read>( - file: &Option>, + file: &Option>>, ) -> CompressionMethod { match file.as_ref() { Some(file) => match file.compression() { ZipCrateCompressionMethod::Stored => CompressionMethod::Stored, ZipCrateCompressionMethod::Deflated => CompressionMethod::Deflated, - ZipCrateCompressionMethod::Bzip2 => CompressionMethod::Bzip2, - ZipCrateCompressionMethod::Zstd => CompressionMethod::Zstd, - ZipCrateCompressionMethod::Lzma => CompressionMethod::Lzma, - ZipCrateCompressionMethod::Xz => CompressionMethod::Xz, + ZipCrateCompressionMethod::BZIP2 => CompressionMethod::Bzip2, + ZipCrateCompressionMethod::ZSTD => CompressionMethod::Zstd, + ZipCrateCompressionMethod::LZMA => CompressionMethod::Lzma, + ZipCrateCompressionMethod::XZ => CompressionMethod::Xz, _ => CompressionMethod::Unsupported, }, None => CompressionMethod::Unsupported, @@ -190,7 +330,7 @@ fn get_compression_method_impl<'a, R: Read>( } fn get_file_data_impl<'a, R: Read>( - file: &mut Option>, + file: &mut Option>>, ) -> Result { match file.as_mut() { Some(file) => { @@ -200,6 +340,25 @@ fn get_file_data_impl<'a, R: Read>( Err(e) => Err(ZipError::internal(e.to_string())), } } - None => Ok(VecU8::default()), + None => Err(ZipError::internal("ZipFile is not available")), + } +} + +fn read_bytes_impl<'a, R: Read>( + file: &mut Option>>, + max_bytes: usize, +) -> Result { + match file.as_mut() { + Some(file) => { + let mut buffer = vec![0u8; max_bytes]; + match file.read(&mut buffer) { + Ok(bytes_read) => { + buffer.truncate(bytes_read); + Ok(buffer.into()) + } + Err(e) => Err(ZipError::internal(e.to_string())), + } + } + None => Err(ZipError::internal("ZipFile is not available")), } } diff --git a/zip/rust/lib.rs b/zip/rust/lib.rs index ce9153f..0f8a383 100644 --- a/zip/rust/lib.rs +++ b/zip/rust/lib.rs @@ -8,10 +8,17 @@ mod file; pub use file::{BufferedZipFile, FsZipFile}; mod read; -pub use read::{BufferedZipArchive, FsZipArchive}; +pub use read::{ + read_buffered_zipfile_from_stream, read_buffered_zipfile_from_stream_with_compressed_size, + read_fs_zipfile_from_stream, read_fs_zipfile_from_stream_with_compressed_size, + BufferedZipArchive, BufferedZipStreamReader, FsZipArchive, FsZipStreamReader, +}; mod write; -pub use write::{BufferedZipWriter, CompressionMethod, FsZipWriter, ZipWriterFileOptions}; +pub use write::{ + new_buffered_zip_stream_writer, new_fs_zip_stream_writer, BufferedZipStreamWriter, + BufferedZipWriter, CompressionMethod, FsZipStreamWriter, FsZipWriter, ZipWriterFileOptions, +}; mod vec_u8; pub use vec_u8::VecU8; diff --git a/zip/rust/read.rs b/zip/rust/read.rs index 3ad84b0..d2694bf 100644 --- a/zip/rust/read.rs +++ b/zip/rust/read.rs @@ -9,7 +9,7 @@ use zip::ZipArchive as WrappedZipArchive; #[derive(Default)] pub struct BufferedZipArchive { - reader: Option>>>, + reader: Option>>>>, } impl Debug for BufferedZipArchive { @@ -50,7 +50,7 @@ impl BufferedZipArchive { let cursor = Cursor::new(data.into_vec()); match WrappedZipArchive::new(cursor) { Ok(reader) => { - self.reader = Some(reader); + self.reader = Some(Box::new(reader)); Ok(()) } Err(e) => Err(e.to_string()), @@ -68,6 +68,14 @@ impl BufferedZipArchive { get_length_impl(&self.reader) } + /// Returns the comment of the zip archive. + pub fn get_comment(&self) -> VecU8 { + match self.reader.as_ref() { + Some(reader) => VecU8::copy_from_slice(reader.comment()), + None => VecU8::default(), + } + } + /// Returns a zip file by its index. /// /// Returns an empty zip file if archive is not open. @@ -78,7 +86,7 @@ impl BufferedZipArchive { Ok(file) => Ok(BufferedZipFile::new(file)), Err(e) => Err(ZipError::out_of_range(e.to_string())), }, - None => Ok(BufferedZipFile::default()), + None => Err(ZipError::failed_precondition("Zip archive is not open")), } } @@ -95,14 +103,14 @@ impl BufferedZipArchive { Ok(file) => Ok(BufferedZipFile::new(file)), Err(e) => Err(ZipError::out_of_range(e.to_string())), }, - None => Ok(BufferedZipFile::default()), + None => Err(ZipError::failed_precondition("Zip archive is not open")), } } } #[derive(Default)] pub struct FsZipArchive { - reader: Option>, + reader: Option>>, } impl Debug for FsZipArchive { @@ -148,7 +156,7 @@ impl FsZipArchive { match File::open(path_str) { Ok(file) => match WrappedZipArchive::new(file) { Ok(reader) => { - self.reader = Some(reader); + self.reader = Some(Box::new(reader)); Ok(()) } Err(e) => Err(e.to_string()), @@ -168,6 +176,14 @@ impl FsZipArchive { get_length_impl(&self.reader) } + /// Returns the comment of the zip archive. + pub fn get_comment(&self) -> VecU8 { + match self.reader.as_ref() { + Some(reader) => VecU8::copy_from_slice(reader.comment()), + None => VecU8::default(), + } + } + /// Returns a zip file by its index. /// /// Returns an empty zip file if archive is not open. @@ -178,7 +194,7 @@ impl FsZipArchive { Ok(file) => Ok(FsZipFile::new(file)), Err(e) => Err(ZipError::out_of_range(e.to_string())), }, - None => Ok(FsZipFile::default()), + None => Err(ZipError::failed_precondition("Zip archive is not open")), } } @@ -195,14 +211,345 @@ impl FsZipArchive { Ok(file) => Ok(FsZipFile::new(file)), Err(e) => Err(ZipError::out_of_range(e.to_string())), }, - None => Ok(FsZipFile::default()), + None => Err(ZipError::failed_precondition("Zip archive is not open")), } } } -fn get_length_impl(reader: &Option>) -> usize { +fn get_length_impl(reader: &Option>>) -> usize { match reader.as_ref() { Some(r) => r.len(), None => 0, } } + +#[derive(Default)] +/// A streaming zip reader that reads from an in-memory buffer without seeking. +pub struct BufferedZipStreamReader { + reader: Option>>, + finished: bool, +} + +impl Debug for BufferedZipStreamReader { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BufferedZipStreamReader") + .field("reader", &if self.reader.is_some() { "Some(Cursor>)" } else { "None" }) + .field("finished", &self.finished) + .finish() + } +} + +impl BufferedZipStreamReader { + /// Initializes an empty `BufferedZipStreamReader`. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new `BufferedZipStreamReader` from data. + pub fn new_from_data(data: VecU8) -> Result { + Ok(Self { reader: Some(Cursor::new(data.into_vec())), finished: false }) + } + + /// Returns whether the stream reader is none (due to being default-constructed + /// or moved-from). + pub fn is_none(&self) -> bool { + self.reader.is_none() + } + + /// Returns whether the end of the archive stream has been reached. + pub fn is_finished(&self) -> bool { + self.finished + } + + /// Reads the next zip file from the stream. + /// + /// When there are no more files in the archive (start of central directory + /// is reached), returns a `BufferedZipFile` where `is_none()` returns true. + pub fn read_next_file(&mut self) -> Result, ZipError> { + if self.finished { + return Ok(BufferedZipFile::default()); + } + match self.reader.as_mut() { + Some(reader) => match zip::read::read_zipfile_from_stream(reader) { + Ok(Some(file)) => Ok(BufferedZipFile::new(file)), + Ok(None) => { + self.finished = true; + Ok(BufferedZipFile::default()) + } + Err(e) => Err(ZipError::from(e)), + }, + None => Err(ZipError::failed_precondition("Zip stream reader is not open")), + } + } + + /// Reads the next zip file from the stream with an assumed compressed size. + /// + /// When there are no more files in the archive, returns a `BufferedZipFile` + /// where `is_none()` returns true. + pub fn read_next_file_with_compressed_size( + &mut self, + compressed_size: u64, + ) -> Result, ZipError> { + if self.finished { + return Ok(BufferedZipFile::default()); + } + match self.reader.as_mut() { + Some(reader) => { + match zip::read::read_zipfile_from_stream_with_compressed_size( + reader, + compressed_size, + ) { + Ok(Some(file)) => Ok(BufferedZipFile::new(file)), + Ok(None) => { + self.finished = true; + Ok(BufferedZipFile::default()) + } + Err(e) => Err(ZipError::from(e)), + } + } + None => Err(ZipError::failed_precondition("Zip stream reader is not open")), + } + } +} + +#[derive(Default)] +/// A streaming zip reader that reads sequentially from a file without seeking. +pub struct FsZipStreamReader { + reader: Option, + finished: bool, +} + +impl Debug for FsZipStreamReader { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FsZipStreamReader") + .field("reader", &if self.reader.is_some() { "Some(File)" } else { "None" }) + .field("finished", &self.finished) + .finish() + } +} + +impl FsZipStreamReader { + /// Initializes an empty `FsZipStreamReader`. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new `FsZipStreamReader` from a path. + pub fn new_from_path(path: &[u8]) -> Result { + let mut reader = Self::default(); + if let Err(e) = reader.open(path) { + return Err(ZipError::invalid_argument(format!("Failed to open zip stream: {}", e))); + } + Ok(reader) + } + + fn open(&mut self, path: &[u8]) -> Result<(), String> { + if self.reader.is_some() { + return Err("Zip stream reader is already open".into()); + } + let path_str = match std::str::from_utf8(path) { + Ok(s) => s, + Err(e) => return Err(e.to_string()), + }; + match File::open(path_str) { + Ok(file) => { + self.reader = Some(file); + self.finished = false; + Ok(()) + } + Err(e) => Err(e.to_string()), + } + } + + /// Returns whether the stream reader is none (due to being default-constructed + /// or moved-from). + pub fn is_none(&self) -> bool { + self.reader.is_none() + } + + /// Returns whether the end of the archive stream has been reached. + pub fn is_finished(&self) -> bool { + self.finished + } + + /// Reads the next zip file from the stream. + /// + /// When there are no more files in the archive (start of central directory + /// is reached), returns a `FsZipFile` where `is_none()` returns true. + pub fn read_next_file(&mut self) -> Result, ZipError> { + if self.finished { + return Ok(FsZipFile::default()); + } + match self.reader.as_mut() { + Some(reader) => match zip::read::read_zipfile_from_stream(reader) { + Ok(Some(file)) => Ok(FsZipFile::new(file)), + Ok(None) => { + self.finished = true; + Ok(FsZipFile::default()) + } + Err(e) => Err(ZipError::from(e)), + }, + None => Err(ZipError::failed_precondition("Zip stream reader is not open")), + } + } + + /// Reads the next zip file from the stream with an assumed compressed size. + /// + /// When there are no more files in the archive, returns a `FsZipFile` + /// where `is_none()` returns true. + pub fn read_next_file_with_compressed_size( + &mut self, + compressed_size: u64, + ) -> Result, ZipError> { + if self.finished { + return Ok(FsZipFile::default()); + } + match self.reader.as_mut() { + Some(reader) => { + match zip::read::read_zipfile_from_stream_with_compressed_size( + reader, + compressed_size, + ) { + Ok(Some(file)) => Ok(FsZipFile::new(file)), + Ok(None) => { + self.finished = true; + Ok(FsZipFile::default()) + } + Err(e) => Err(ZipError::from(e)), + } + } + None => Err(ZipError::failed_precondition("Zip stream reader is not open")), + } + } +} + +/// Reads the next zip file from a buffered stream reader. +pub fn read_buffered_zipfile_from_stream( + reader: &mut BufferedZipStreamReader, +) -> Result, ZipError> { + reader.read_next_file() +} + +/// Reads the next zip file from a buffered stream reader with an assumed compressed size. +pub fn read_buffered_zipfile_from_stream_with_compressed_size( + reader: &mut BufferedZipStreamReader, + compressed_size: u64, +) -> Result, ZipError> { + reader.read_next_file_with_compressed_size(compressed_size) +} + +/// Reads the next zip file from a filesystem stream reader. +pub fn read_fs_zipfile_from_stream( + reader: &mut FsZipStreamReader, +) -> Result, ZipError> { + reader.read_next_file() +} + +/// Reads the next zip file from a filesystem stream reader with an assumed compressed size. +pub fn read_fs_zipfile_from_stream_with_compressed_size( + reader: &mut FsZipStreamReader, + compressed_size: u64, +) -> Result, ZipError> { + reader.read_next_file_with_compressed_size(compressed_size) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{BufferedZipWriter, CompressionMethod, ZipWriterFileOptions}; + use googletest::prelude::*; + + fn create_test_zip() -> VecU8 { + let mut writer = BufferedZipWriter::new_from_data(VecU8::default(), false).unwrap(); + let options = ZipWriterFileOptions::new().compression_method(CompressionMethod::Stored); + writer.start_file(b"hello.txt", options).unwrap(); + writer.write_data(VecU8::from(b"Hello, world!".to_vec())).unwrap(); + writer.start_file(b"second.txt", options).unwrap(); + writer.write_data(VecU8::from(b"Second file content".to_vec())).unwrap(); + writer.finish().unwrap() + } + + #[gtest] + fn test_buffered_stream_reader_read_all_files() { + let zip_data = create_test_zip(); + let mut reader = BufferedZipStreamReader::new_from_data(zip_data).unwrap(); + expect_false!(reader.is_none()); + expect_false!(reader.is_finished()); + + { + let mut file1 = reader.read_next_file().unwrap(); + expect_false!(file1.is_none()); + expect_eq!(file1.get_file_name().as_slice(), b"hello.txt"); + expect_true!(file1.is_file()); + expect_false!(file1.is_dir()); + expect_eq!(file1.get_file_data().unwrap().as_slice(), b"Hello, world!"); + } + + { + let mut file2 = reader.read_next_file().unwrap(); + expect_false!(file2.is_none()); + expect_eq!(file2.get_file_name().as_slice(), b"second.txt"); + expect_eq!(file2.get_file_data().unwrap().as_slice(), b"Second file content"); + } + + // Third read should return None (end of files) + { + let file3 = reader.read_next_file().unwrap(); + expect_true!(file3.is_none()); + } + expect_true!(reader.is_finished()); + + // Repeated reads after finish should be idempotent + { + let file4 = reader.read_next_file().unwrap(); + expect_true!(file4.is_none()); + } + } + + #[gtest] + fn test_buffered_stream_reader_chunked_read() { + let zip_data = create_test_zip(); + let mut reader = BufferedZipStreamReader::new_from_data(zip_data).unwrap(); + let mut file = reader.read_next_file().unwrap(); + expect_false!(file.is_none()); + + // Read in chunks of 5 bytes + let chunk1 = file.read_bytes(5).unwrap(); + expect_eq!(chunk1.as_slice(), b"Hello"); + let chunk2 = file.read_bytes(5).unwrap(); + expect_eq!(chunk2.as_slice(), b", wor"); + let chunk3 = file.read_bytes(5).unwrap(); + expect_eq!(chunk3.as_slice(), b"ld!"); + let chunk4 = file.read_bytes(5).unwrap(); + expect_true!(chunk4.is_empty()); + } + + #[gtest] + fn test_buffered_stream_reader_free_functions() { + let zip_data = create_test_zip(); + let mut reader = BufferedZipStreamReader::new_from_data(zip_data).unwrap(); + let mut file = read_buffered_zipfile_from_stream(&mut reader).unwrap(); + expect_false!(file.is_none()); + expect_eq!(file.get_file_name().as_slice(), b"hello.txt"); + expect_eq!(file.get_file_data().unwrap().as_slice(), b"Hello, world!"); + } + + #[gtest] + fn test_buffered_stream_reader_with_compressed_size() { + let zip_data = create_test_zip(); + let mut reader = BufferedZipStreamReader::new_from_data(zip_data).unwrap(); + // File 1 is uncompressed "Hello, world!" which is 13 bytes stored + let mut file = reader.read_next_file_with_compressed_size(13).unwrap(); + expect_false!(file.is_none()); + expect_eq!(file.get_file_name().as_slice(), b"hello.txt"); + expect_eq!(file.get_file_data().unwrap().as_slice(), b"Hello, world!"); + } + + #[gtest] + fn test_buffered_stream_reader_invalid_data() { + let mut reader = + BufferedZipStreamReader::new_from_data(VecU8::from(b"not a zip".to_vec())).unwrap(); + let res = reader.read_next_file(); + expect_true!(res.is_err()); + } +} diff --git a/zip/rust/write.rs b/zip/rust/write.rs index a9b77af..83815bc 100644 --- a/zip/rust/write.rs +++ b/zip/rust/write.rs @@ -6,8 +6,8 @@ use std::fmt::{Debug, Formatter}; use std::fs::{File, OpenOptions}; use std::io::{copy, Cursor, Read, Seek, Write}; use zip::{ - write::FileOptions, CompressionMethod as ZipCrateCompressionMethod, - ZipWriter as WrappedZipWriter, + write::{FileOptions, StreamWriter}, + CompressionMethod as ZipCrateCompressionMethod, ZipWriter as WrappedZipWriter, }; // Expose some of the options for writing files to the zip archive. @@ -56,10 +56,10 @@ impl From for ZipCrateCompressionMethod { match val { CompressionMethod::Deflated => ZipCrateCompressionMethod::Deflated, CompressionMethod::Stored => ZipCrateCompressionMethod::Stored, - CompressionMethod::Bzip2 => ZipCrateCompressionMethod::Bzip2, - CompressionMethod::Zstd => ZipCrateCompressionMethod::Zstd, - CompressionMethod::Lzma => ZipCrateCompressionMethod::Lzma, - CompressionMethod::Xz => ZipCrateCompressionMethod::Xz, + CompressionMethod::Bzip2 => ZipCrateCompressionMethod::BZIP2, + CompressionMethod::Zstd => ZipCrateCompressionMethod::ZSTD, + CompressionMethod::Lzma => ZipCrateCompressionMethod::LZMA, + CompressionMethod::Xz => ZipCrateCompressionMethod::XZ, CompressionMethod::Unsupported => { panic!("cannot convert CompressionMethod::Unsupported to ZipCrateCompressionMethod") } @@ -83,7 +83,7 @@ impl TryFrom<&ZipWriterFileOptions> for FileOptions<'static, ()> { if let Some(method) = val.compression_method { options = options.compression_method(method.into()); } - // third_party/rust/zip/v6/src/write.rs + // third_party/rust/zip/v8/src/write.rs // // `None` value specifies default compression level. // @@ -162,7 +162,7 @@ impl ZipWriterFileOptions { #[derive(Default)] /// A zip writer that writes to an in-memory buffer. pub struct BufferedZipWriter { - writer: Option>>>, + writer: Option>>>>, } impl Debug for BufferedZipWriter { @@ -194,13 +194,13 @@ impl BufferedZipWriter { if append { match WrappedZipWriter::new_append(cursor) { Ok(writer) => { - self.writer = Some(writer); + self.writer = Some(Box::new(writer)); Ok(()) } Err(e) => Err(e.to_string()), } } else { - self.writer = Some(WrappedZipWriter::new(cursor)); + self.writer = Some(Box::new(WrappedZipWriter::new(cursor))); Ok(()) } } @@ -266,12 +266,27 @@ impl BufferedZipWriter { pub fn write_file_content(&mut self, path: &[u8]) -> Result<(), ZipError> { write_file_content_impl(&mut self.writer, path) } + + /// Sets the archive comment. + pub fn set_comment(&mut self, comment: &[u8]) -> Result<(), ZipError> { + set_comment_impl(&mut self.writer, comment) + } + + /// Flushes any pending output. + pub fn flush(&mut self) -> Result<(), ZipError> { + flush_impl(&mut self.writer) + } + + /// Returns whether seeking is possible (always true for BufferedZipWriter). + pub fn is_seek_possible(&self) -> bool { + true + } } #[derive(Default)] /// A zip writer that writes to a file on the filesystem. pub struct FsZipWriter { - writer: Option>, + writer: Option>>, } impl Debug for FsZipWriter { @@ -311,7 +326,7 @@ impl FsZipWriter { { Ok(file) => match WrappedZipWriter::new_append(file) { Ok(writer) => { - self.writer = Some(writer); + self.writer = Some(Box::new(writer)); Ok(()) } Err(e) => Err(e.to_string()), @@ -321,7 +336,7 @@ impl FsZipWriter { } else { match OpenOptions::new().write(true).create(true).truncate(true).open(path_str) { Ok(file) => { - self.writer = Some(WrappedZipWriter::new(file)); + self.writer = Some(Box::new(WrappedZipWriter::new(file))); Ok(()) } Err(e) => Err(e.to_string()), @@ -390,14 +405,29 @@ impl FsZipWriter { pub fn write_file_content(&mut self, path: &[u8]) -> Result<(), ZipError> { write_file_content_impl(&mut self.writer, path) } + + /// Sets the archive comment. + pub fn set_comment(&mut self, comment: &[u8]) -> Result<(), ZipError> { + set_comment_impl(&mut self.writer, comment) + } + + /// Flushes any pending output. + pub fn flush(&mut self) -> Result<(), ZipError> { + flush_impl(&mut self.writer) + } + + /// Returns whether seeking is possible (always true for FsZipWriter). + pub fn is_seek_possible(&self) -> bool { + true + } } fn start_file_impl( - writer: &mut Option>, + writer: &mut Option>>, name: &[u8], options: ZipWriterFileOptions, ) -> Result<(), ZipError> { - if let Some(writer) = writer.as_mut() { + if let Some(writer) = writer.as_deref_mut() { let name_lossy = String::from_utf8_lossy(name); let name_str = name_lossy.as_ref(); match FileOptions::try_from(&options) { @@ -413,11 +443,11 @@ fn start_file_impl( } fn add_directory_impl( - writer: &mut Option>, + writer: &mut Option>>, name: &[u8], options: ZipWriterFileOptions, ) -> Result<(), ZipError> { - if let Some(writer) = writer.as_mut() { + if let Some(writer) = writer.as_deref_mut() { let name_lossy = String::from_utf8_lossy(name); let name_str = name_lossy.as_ref(); match FileOptions::try_from(&options) { @@ -433,10 +463,10 @@ fn add_directory_impl( } fn write_data_impl( - writer: &mut Option>, + writer: &mut Option>>, data: VecU8, ) -> Result<(), ZipError> { - if let Some(writer) = writer.as_mut() { + if let Some(writer) = writer.as_deref_mut() { match writer.write_all(data.as_slice()) { Ok(_) => Ok(()), Err(e) => Err(ZipError::internal(e.to_string())), @@ -447,10 +477,10 @@ fn write_data_impl( } fn do_copy_impl( - writer: &mut Option>, + writer: &mut Option>>, reader: &mut R, ) -> Result<(), ZipError> { - if let Some(writer) = writer.as_mut() { + if let Some(writer) = writer.as_deref_mut() { match copy(reader, writer) { Ok(_) => Ok(()), Err(e) => Err(ZipError::internal(e.to_string())), @@ -461,10 +491,10 @@ fn do_copy_impl( } fn write_file_content_impl( - writer: &mut Option>, + writer: &mut Option>>, path: &[u8], ) -> Result<(), ZipError> { - if let Some(writer) = writer.as_mut() { + if let Some(writer) = writer.as_deref_mut() { let path_lossy = String::from_utf8_lossy(path); let path_str = path_lossy.as_ref(); match File::open(path_str) { @@ -478,3 +508,349 @@ fn write_file_content_impl( Err(ZipError::failed_precondition("writer is not open")) } } + +fn set_comment_impl( + writer: &mut Option>>, + comment: &[u8], +) -> Result<(), ZipError> { + if let Some(writer) = writer.as_deref_mut() { + let comment_lossy = String::from_utf8_lossy(comment); + match writer.set_comment(comment_lossy.as_ref()) { + Ok(_) => Ok(()), + Err(e) => Err(ZipError::internal(e.to_string())), + } + } else { + Err(ZipError::failed_precondition("writer is not open")) + } +} + +fn flush_impl( + writer: &mut Option>>, +) -> Result<(), ZipError> { + if let Some(writer) = writer.as_deref_mut() { + match writer.flush() { + Ok(_) => Ok(()), + Err(e) => Err(ZipError::internal(e.to_string())), + } + } else { + Err(ZipError::failed_precondition("writer is not open")) + } +} + +#[derive(Default)] +/// A zip writer that streams to an in-memory buffer without seeking. +pub struct BufferedZipStreamWriter { + writer: Option>>>>>, +} + +impl Debug for BufferedZipStreamWriter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BufferedZipStreamWriter") + .field( + "writer", + &if self.writer.is_some() { + "Some(WrappedZipWriter>>>>)" + } else { + "None" + }, + ) + .finish() + } +} + +impl BufferedZipStreamWriter { + /// Creates a new empty `BufferedZipStreamWriter`. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new streaming `BufferedZipStreamWriter`. + pub fn new_stream() -> Self { + let cursor = Cursor::new(Vec::new()); + Self { writer: Some(Box::new(WrappedZipWriter::new_stream(cursor))) } + } + + /// Creates a new streaming `BufferedZipStreamWriter` initialized with data. + pub fn new_from_data(data: VecU8) -> Result { + let cursor = Cursor::new(data.into_vec()); + Ok(Self { writer: Some(Box::new(WrappedZipWriter::new_stream(cursor))) }) + } + + /// Returns whether the zip writer is none (due to being + /// default-constructed or moved-from). + pub fn is_none(&self) -> bool { + self.writer.is_none() + } + + /// Returns whether seeking is possible (always false for stream writers). + pub fn is_seek_possible(&self) -> bool { + false + } + + /// Finishes writing the zip archive and returns the buffered data. + pub fn finish(&mut self) -> Result { + if let Some(writer) = self.writer.take() { + match writer.finish() { + Ok(stream_writer) => Ok(stream_writer.into_inner().into_inner().into()), + Err(e) => Err(ZipError::internal(e.to_string())), + } + } else { + Err(ZipError::failed_precondition("writer is not open")) + } + } + + /// Creates a new file in the zip archive and starts writing to it. + pub fn start_file( + &mut self, + name: &[u8], + options: ZipWriterFileOptions, + ) -> Result<(), ZipError> { + start_file_impl(&mut self.writer, name, options) + } + + /// Adds a directory to the zip archive. + pub fn add_directory( + &mut self, + name: &[u8], + options: ZipWriterFileOptions, + ) -> Result<(), ZipError> { + add_directory_impl(&mut self.writer, name, options) + } + + /// Writes data to the current file in the zip archive. + pub fn write_data(&mut self, data: VecU8) -> Result<(), ZipError> { + write_data_impl(&mut self.writer, data) + } + + /// Writes file content from a `BufferedZipFile` to the current file in the zip archive. + pub fn write_buffered_zip_file_content( + &mut self, + file: &mut BufferedZipFile, + ) -> Result<(), ZipError> { + do_copy_impl(&mut self.writer, file) + } + + /// Writes file content from a `FsZipFile` to the current file in the zip archive. + pub fn write_fs_zip_file_content(&mut self, file: &mut FsZipFile) -> Result<(), ZipError> { + do_copy_impl(&mut self.writer, file) + } + + /// Writes file content from a path to the current file in the zip archive. + pub fn write_file_content(&mut self, path: &[u8]) -> Result<(), ZipError> { + write_file_content_impl(&mut self.writer, path) + } + + /// Sets the archive comment. + pub fn set_comment(&mut self, comment: &[u8]) -> Result<(), ZipError> { + set_comment_impl(&mut self.writer, comment) + } + + /// Flushes any pending output. + pub fn flush(&mut self) -> Result<(), ZipError> { + flush_impl(&mut self.writer) + } +} + +#[derive(Default)] +/// A zip writer that streams to a file on the filesystem without seeking. +pub struct FsZipStreamWriter { + writer: Option>>>, +} + +impl Debug for FsZipStreamWriter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FsZipStreamWriter") + .field( + "writer", + &if self.writer.is_some() { + "Some(WrappedZipWriter>)" + } else { + "None" + }, + ) + .finish() + } +} + +impl FsZipStreamWriter { + /// Creates a new `FsZipStreamWriter`. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new streaming `FsZipStreamWriter` writing to `path`. + pub fn new_from_path(path: &[u8]) -> Result { + let mut writer = Self::default(); + if let Err(e) = writer.open(path) { + return Err(ZipError::invalid_argument(format!("Failed to open zip archive: {}", e))); + } + Ok(writer) + } + + fn open(&mut self, path: &[u8]) -> Result<(), String> { + let path_lossy = String::from_utf8_lossy(path); + let path_str = path_lossy.as_ref(); + match OpenOptions::new().write(true).create(true).truncate(true).open(path_str) { + Ok(file) => { + self.writer = Some(Box::new(WrappedZipWriter::new_stream(file))); + Ok(()) + } + Err(e) => Err(e.to_string()), + } + } + + /// Returns whether the zip writer is none (due to being + /// default-constructed or moved-from). + pub fn is_none(&self) -> bool { + self.writer.is_none() + } + + /// Returns whether seeking is possible (always false for stream writers). + pub fn is_seek_possible(&self) -> bool { + false + } + + /// Finishes writing the zip archive to file. + pub fn finish(&mut self) -> Result<(), ZipError> { + if let Some(writer) = self.writer.take() { + match writer.finish() { + Ok(_) => Ok(()), + Err(e) => Err(ZipError::internal(e.to_string())), + } + } else { + Err(ZipError::failed_precondition("writer is not open")) + } + } + + /// Creates a new file in the zip archive and starts writing to it. + pub fn start_file( + &mut self, + name: &[u8], + options: ZipWriterFileOptions, + ) -> Result<(), ZipError> { + start_file_impl(&mut self.writer, name, options) + } + + /// Adds a directory to the zip archive. + pub fn add_directory( + &mut self, + name: &[u8], + options: ZipWriterFileOptions, + ) -> Result<(), ZipError> { + add_directory_impl(&mut self.writer, name, options) + } + + /// Writes data to the current file in the zip archive. + pub fn write_data(&mut self, data: VecU8) -> Result<(), ZipError> { + write_data_impl(&mut self.writer, data) + } + + /// Writes file content from a `BufferedZipFile` to the current file in the zip archive. + pub fn write_buffered_zip_file_content( + &mut self, + file: &mut BufferedZipFile, + ) -> Result<(), ZipError> { + do_copy_impl(&mut self.writer, file) + } + + /// Writes file content from a `FsZipFile` to the current file in the zip archive. + pub fn write_fs_zip_file_content(&mut self, file: &mut FsZipFile) -> Result<(), ZipError> { + do_copy_impl(&mut self.writer, file) + } + + /// Writes file content from a path to the current file in the zip archive. + pub fn write_file_content(&mut self, path: &[u8]) -> Result<(), ZipError> { + write_file_content_impl(&mut self.writer, path) + } + + /// Sets the archive comment. + pub fn set_comment(&mut self, comment: &[u8]) -> Result<(), ZipError> { + set_comment_impl(&mut self.writer, comment) + } + + /// Flushes any pending output. + pub fn flush(&mut self) -> Result<(), ZipError> { + flush_impl(&mut self.writer) + } +} + +/// Creates a new in-memory streaming zip writer. +pub fn new_buffered_zip_stream_writer() -> BufferedZipStreamWriter { + BufferedZipStreamWriter::new_stream() +} + +/// Creates a new filesystem streaming zip writer. +pub fn new_fs_zip_stream_writer(path: &[u8]) -> Result { + FsZipStreamWriter::new_from_path(path) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::BufferedZipArchive; + use googletest::prelude::*; + + #[gtest] + fn test_buffered_zip_stream_writer() { + let mut writer = BufferedZipStreamWriter::new_stream(); + expect_false!(writer.is_none()); + expect_false!(writer.is_seek_possible()); + + let options = ZipWriterFileOptions::new().compression_method(CompressionMethod::Deflated); + writer.start_file(b"test.txt", options).unwrap(); + writer.write_data(VecU8::from(b"stream writer data".to_vec())).unwrap(); + writer.add_directory(b"mydir", options).unwrap(); + writer.set_comment(b"archive comment").unwrap(); + writer.flush().unwrap(); + + let zip_bytes = writer.finish().unwrap(); + expect_false!(zip_bytes.is_empty()); + + // Verify that ZipArchive can read this streaming-written zip archive + let mut archive = BufferedZipArchive::new_from_data(zip_bytes).unwrap(); + expect_eq!(archive.get_length(), 2); + expect_eq!(archive.get_comment().as_slice(), b"archive comment"); + + { + let mut file1 = archive.get_file_by_index(0).unwrap(); + expect_eq!(file1.get_file_name().as_slice(), b"test.txt"); + expect_eq!(file1.get_file_data().unwrap().as_slice(), b"stream writer data"); + } + + { + let file2 = archive.get_file_by_index(1).unwrap(); + expect_true!(file2.is_dir()); + expect_eq!(file2.get_file_name().as_slice(), b"mydir/"); + } + } + + #[gtest] + fn test_free_functions_stream_writer() { + let mut writer = new_buffered_zip_stream_writer(); + expect_false!(writer.is_seek_possible()); + let options = ZipWriterFileOptions::new(); + writer.start_file(b"a.txt", options).unwrap(); + writer.write_data(VecU8::from(b"content".to_vec())).unwrap(); + let zip_bytes = writer.finish().unwrap(); + + let mut archive = BufferedZipArchive::new_from_data(zip_bytes).unwrap(); + expect_eq!(archive.get_length(), 1); + let mut file = archive.get_file_by_index(0).unwrap(); + expect_eq!(file.get_file_data().unwrap().as_slice(), b"content"); + } + + #[gtest] + fn test_regular_writer_is_seek_possible_and_set_comment() { + let mut writer = BufferedZipWriter::new_from_data(VecU8::default(), false).unwrap(); + expect_true!(writer.is_seek_possible()); + let options = ZipWriterFileOptions::new(); + writer.start_file(b"b.txt", options).unwrap(); + writer.write_data(VecU8::from(b"data".to_vec())).unwrap(); + writer.set_comment(b"comment on regular writer").unwrap(); + writer.flush().unwrap(); + let zip_bytes = writer.finish().unwrap(); + + let archive = BufferedZipArchive::new_from_data(zip_bytes).unwrap(); + expect_eq!(archive.get_comment().as_slice(), b"comment on regular writer"); + } +} diff --git a/zip/write.cc b/zip/write.cc index d5b4943..a986823 100644 --- a/zip/write.cc +++ b/zip/write.cc @@ -91,6 +91,21 @@ absl::Status BufferedZipWriter::WriteFileContent(absl::string_view path) { reinterpret_cast(path.data()), path.size()))); } +absl::Status BufferedZipWriter::SetComment(absl::string_view comment) { + return FromRustResultUnit(writer_.set_comment(absl::Span( + reinterpret_cast(comment.data()), comment.size()))); +} + +absl::Status BufferedZipWriter::Flush() { + return FromRustResultUnit(writer_.flush()); +} + +bool BufferedZipWriter::IsSeekPossible() const { + return writer_.is_seek_possible(); +} + +bool BufferedZipWriter::IsNone() const { return writer_.is_none(); } + absl::StatusOr FsZipWriter::NewFromPath(absl::string_view path, bool append) { ABSL_ASSIGN_OR_RETURN( @@ -148,6 +163,19 @@ absl::Status FsZipWriter::WriteFileContent(absl::string_view path) { reinterpret_cast(path.data()), path.size()))); } +absl::Status FsZipWriter::SetComment(absl::string_view comment) { + return FromRustResultUnit(writer_.set_comment(absl::Span( + reinterpret_cast(comment.data()), comment.size()))); +} + +absl::Status FsZipWriter::Flush() { + return FromRustResultUnit(writer_.flush()); +} + +bool FsZipWriter::IsSeekPossible() const { return writer_.is_seek_possible(); } + +bool FsZipWriter::IsNone() const { return writer_.is_none(); } + absl::StatusOr ZipWriter::FromFile(absl::string_view path, bool append) { ABSL_ASSIGN_OR_RETURN(FsZipWriter writer, @@ -207,4 +235,260 @@ absl::Status ZipWriter::WriteFileContent(absl::string_view path) { writer_); } +absl::Status ZipWriter::SetComment(absl::string_view comment) { + return std::visit([&](auto& writer) { return writer.SetComment(comment); }, + writer_); +} + +absl::Status ZipWriter::Flush() { + return std::visit([&](auto& writer) { return writer.Flush(); }, writer_); +} + +bool ZipWriter::IsSeekPossible() const { + return std::visit([&](const auto& writer) { return writer.IsSeekPossible(); }, + writer_); +} + +bool ZipWriter::IsNone() const { + return std::visit([&](const auto& writer) { return writer.IsNone(); }, + writer_); +} + +absl::StatusOr BufferedZipStreamWriter::New() { + return BufferedZipStreamWriter( + rust::BufferedZipStreamWriter::new_stream()); +} + +absl::StatusOr BufferedZipStreamWriter::NewFromData( + absl::string_view data) { + rust::VecU8 input_data = + rust::VecU8::copy_from_slice(absl::Span( + reinterpret_cast(data.data()), data.size())); + ABSL_ASSIGN_OR_RETURN( + rust::BufferedZipStreamWriter writer, + FromRustBufferedZipStreamWriter( + rust::BufferedZipStreamWriter::new_from_data(input_data))); + return BufferedZipStreamWriter(std::move(writer)); +} + +absl::StatusOr BufferedZipStreamWriter::Finish() { + return FromRustResultVecU8(writer_.finish()); +} + +absl::Status BufferedZipStreamWriter::StartFile( + absl::string_view file_name, const ZipWriterFileOptions& options) { + return FromRustResultUnit(writer_.start_file( + absl::Span( + reinterpret_cast(file_name.data()), file_name.size()), + options.options_)); +} + +absl::Status BufferedZipStreamWriter::AddDirectory( + absl::string_view file_name, const ZipWriterFileOptions& options) { + return FromRustResultUnit(writer_.add_directory( + absl::Span( + reinterpret_cast(file_name.data()), file_name.size()), + options.options_)); +} + +absl::Status BufferedZipStreamWriter::WriteData(absl::string_view data) { + rust::VecU8 input_data = + rust::VecU8::copy_from_slice(absl::Span( + reinterpret_cast(data.data()), data.size())); + return FromRustResultUnit(writer_.write_data(input_data)); +} + +absl::Status BufferedZipStreamWriter::WriteZipFileContent(ZipFile& file) { + return std::visit( + absl::Overload{[&](BufferedZipFile& file) { + return FromRustResultUnit( + writer_.write_buffered_zip_file_content(file.zip_)); + }, + [&](FsZipFile& file) { + return FromRustResultUnit( + writer_.write_fs_zip_file_content(file.zip_)); + }}, + file.zip_); +} + +absl::Status BufferedZipStreamWriter::WriteFileContent(absl::string_view path) { + return FromRustResultUnit( + writer_.write_file_content(absl::Span( + reinterpret_cast(path.data()), path.size()))); +} + +absl::Status BufferedZipStreamWriter::SetComment(absl::string_view comment) { + return FromRustResultUnit(writer_.set_comment(absl::Span( + reinterpret_cast(comment.data()), comment.size()))); +} + +absl::Status BufferedZipStreamWriter::Flush() { + return FromRustResultUnit(writer_.flush()); +} + +bool BufferedZipStreamWriter::IsSeekPossible() const { + return writer_.is_seek_possible(); +} + +bool BufferedZipStreamWriter::IsNone() const { return writer_.is_none(); } + +absl::StatusOr FsZipStreamWriter::NewFromPath( + absl::string_view path) { + ABSL_ASSIGN_OR_RETURN( + rust::FsZipStreamWriter writer, + FromRustFsZipStreamWriter(rust::FsZipStreamWriter::new_from_path( + absl::Span( + reinterpret_cast(path.data()), path.size())))); + return FsZipStreamWriter(std::move(writer)); +} + +absl::Status FsZipStreamWriter::Finish() { + return FromRustResultUnit(writer_.finish()); +} + +absl::Status FsZipStreamWriter::StartFile(absl::string_view file_name, + const ZipWriterFileOptions& options) { + return FromRustResultUnit(writer_.start_file( + absl::Span( + reinterpret_cast(file_name.data()), file_name.size()), + options.options_)); +} + +absl::Status FsZipStreamWriter::AddDirectory( + absl::string_view file_name, const ZipWriterFileOptions& options) { + return FromRustResultUnit(writer_.add_directory( + absl::Span( + reinterpret_cast(file_name.data()), file_name.size()), + options.options_)); +} + +absl::Status FsZipStreamWriter::WriteData(absl::string_view data) { + rust::VecU8 input_data = + rust::VecU8::copy_from_slice(absl::Span( + reinterpret_cast(data.data()), data.size())); + return FromRustResultUnit(writer_.write_data(input_data)); +} + +absl::Status FsZipStreamWriter::WriteZipFileContent(ZipFile& file) { + return std::visit( + absl::Overload{[&](BufferedZipFile& file) { + return FromRustResultUnit( + writer_.write_buffered_zip_file_content(file.zip_)); + }, + [&](FsZipFile& file) { + return FromRustResultUnit( + writer_.write_fs_zip_file_content(file.zip_)); + }}, + file.zip_); +} + +absl::Status FsZipStreamWriter::WriteFileContent(absl::string_view path) { + return FromRustResultUnit( + writer_.write_file_content(absl::Span( + reinterpret_cast(path.data()), path.size()))); +} + +absl::Status FsZipStreamWriter::SetComment(absl::string_view comment) { + return FromRustResultUnit(writer_.set_comment(absl::Span( + reinterpret_cast(comment.data()), comment.size()))); +} + +absl::Status FsZipStreamWriter::Flush() { + return FromRustResultUnit(writer_.flush()); +} + +bool FsZipStreamWriter::IsSeekPossible() const { + return writer_.is_seek_possible(); +} + +bool FsZipStreamWriter::IsNone() const { return writer_.is_none(); } + +absl::StatusOr ZipStreamWriter::FromFile( + absl::string_view path) { + ABSL_ASSIGN_OR_RETURN(FsZipStreamWriter writer, + FsZipStreamWriter::NewFromPath(path)); + return ZipStreamWriter(std::move(writer)); +} + +absl::StatusOr ZipStreamWriter::FromBuffer() { + ABSL_ASSIGN_OR_RETURN(BufferedZipStreamWriter writer, + BufferedZipStreamWriter::New()); + return ZipStreamWriter(std::move(writer)); +} + +absl::StatusOr ZipStreamWriter::FromBuffer( + absl::string_view data) { + ABSL_ASSIGN_OR_RETURN(BufferedZipStreamWriter writer, + BufferedZipStreamWriter::NewFromData(data)); + return ZipStreamWriter(std::move(writer)); +} + +absl::StatusOr ZipStreamWriter::Finish() { + return std::visit( + absl::Overload{ + [](BufferedZipStreamWriter& writer) + -> absl::StatusOr { return writer.Finish(); }, + [](FsZipStreamWriter& writer) -> absl::StatusOr { + ABSL_RETURN_IF_ERROR(writer.Finish()); + return RustVecU8Wrapper(); + }}, + writer_); +} + +absl::Status ZipStreamWriter::StartFile(absl::string_view file_name, + const ZipWriterFileOptions& options) { + return std::visit( + [&](auto& writer) { return writer.StartFile(file_name, options); }, + writer_); +} + +absl::Status ZipStreamWriter::AddDirectory( + absl::string_view file_name, const ZipWriterFileOptions& options) { + return std::visit( + [&](auto& writer) { return writer.AddDirectory(file_name, options); }, + writer_); +} + +absl::Status ZipStreamWriter::WriteData(absl::string_view data) { + return std::visit([&](auto& writer) { return writer.WriteData(data); }, + writer_); +} + +absl::Status ZipStreamWriter::WriteZipFileContent(ZipFile& file) { + return std::visit( + [&](auto& writer) { return writer.WriteZipFileContent(file); }, writer_); +} + +absl::Status ZipStreamWriter::WriteFileContent(absl::string_view path) { + return std::visit([&](auto& writer) { return writer.WriteFileContent(path); }, + writer_); +} + +absl::Status ZipStreamWriter::SetComment(absl::string_view comment) { + return std::visit([&](auto& writer) { return writer.SetComment(comment); }, + writer_); +} + +absl::Status ZipStreamWriter::Flush() { + return std::visit([&](auto& writer) { return writer.Flush(); }, writer_); +} + +bool ZipStreamWriter::IsSeekPossible() const { + return std::visit([&](const auto& writer) { return writer.IsSeekPossible(); }, + writer_); +} + +bool ZipStreamWriter::IsNone() const { + return std::visit([&](const auto& writer) { return writer.IsNone(); }, + writer_); +} + +absl::StatusOr NewBufferedZipStreamWriter() { + return BufferedZipStreamWriter::New(); +} + +absl::StatusOr NewFsZipStreamWriter(absl::string_view path) { + return FsZipStreamWriter::NewFromPath(path); +} + } // namespace security::zip diff --git a/zip/write.h b/zip/write.h index 44775b5..67b5190 100644 --- a/zip/write.h +++ b/zip/write.h @@ -16,6 +16,9 @@ namespace security::zip { class BufferedZipWriter; class FsZipWriter; +class BufferedZipStreamWriter; +class FsZipStreamWriter; +class ZipStreamWriter; class ZipWriterFileOptions { public: @@ -40,6 +43,9 @@ class ZipWriterFileOptions { rust::ZipWriterFileOptions options_; friend class BufferedZipWriter; friend class FsZipWriter; + friend class BufferedZipStreamWriter; + friend class FsZipStreamWriter; + friend class ZipStreamWriter; }; class BufferedZipWriter final { @@ -54,6 +60,10 @@ class BufferedZipWriter final { absl::Status WriteData(absl::string_view data); absl::Status WriteZipFileContent(ZipFile& file); absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; private: explicit BufferedZipWriter(rust::BufferedZipWriter writer) @@ -73,6 +83,10 @@ class FsZipWriter final { absl::Status WriteData(absl::string_view data); absl::Status WriteZipFileContent(ZipFile& file); absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; private: explicit FsZipWriter(rust::FsZipWriter writer) @@ -94,6 +108,10 @@ class ZipWriter final { absl::Status WriteData(absl::string_view data); absl::Status WriteZipFileContent(ZipFile& file); absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; private: using BackendType = std::variant; @@ -102,6 +120,81 @@ class ZipWriter final { BackendType writer_; }; +// Streaming zip writers write zip archives sequentially without seeking. +class BufferedZipStreamWriter final { + public: + static absl::StatusOr New(); + static absl::StatusOr NewFromData( + absl::string_view data); + absl::StatusOr Finish(); + absl::Status StartFile(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status AddDirectory(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status WriteData(absl::string_view data); + absl::Status WriteZipFileContent(ZipFile& file); + absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; + + private: + explicit BufferedZipStreamWriter(rust::BufferedZipStreamWriter writer) + : writer_(std::move(writer)) {} + rust::BufferedZipStreamWriter writer_; +}; + +class FsZipStreamWriter final { + public: + static absl::StatusOr NewFromPath(absl::string_view path); + absl::Status Finish(); + absl::Status StartFile(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status AddDirectory(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status WriteData(absl::string_view data); + absl::Status WriteZipFileContent(ZipFile& file); + absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; + + private: + explicit FsZipStreamWriter(rust::FsZipStreamWriter writer) + : writer_(std::move(writer)) {} + rust::FsZipStreamWriter writer_; +}; + +class ZipStreamWriter final { + public: + static absl::StatusOr FromFile(absl::string_view path); + static absl::StatusOr FromBuffer(); + static absl::StatusOr FromBuffer(absl::string_view data); + absl::StatusOr Finish(); + absl::Status StartFile(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status AddDirectory(absl::string_view file_name, + const ZipWriterFileOptions& options); + absl::Status WriteData(absl::string_view data); + absl::Status WriteZipFileContent(ZipFile& file); + absl::Status WriteFileContent(absl::string_view path); + absl::Status SetComment(absl::string_view comment); + absl::Status Flush(); + [[nodiscard]] bool IsSeekPossible() const; + [[nodiscard]] bool IsNone() const; + + private: + using BackendType = std::variant; + + explicit ZipStreamWriter(BackendType writer) : writer_(std::move(writer)) {} + BackendType writer_; +}; + +absl::StatusOr NewBufferedZipStreamWriter(); +absl::StatusOr NewFsZipStreamWriter(absl::string_view path); + } // namespace security::zip #endif // SECURITY_ZIP_WRITE_H_ diff --git a/zip/zip_benchmark.cc b/zip/zip_benchmark.cc new file mode 100644 index 0000000..a638602 --- /dev/null +++ b/zip/zip_benchmark.cc @@ -0,0 +1,232 @@ +#include +#include +#include +#include +#include + +#include "converters.h" +#include "file.h" +#include "read.h" +#include "write.h" +#include "absl/log/check.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "third_party/benchmark/include/benchmark/benchmark.h" + +namespace security::zip { +namespace { + +std::string GenerateTestData(size_t size) { + std::string data; + data.reserve(size); + for (size_t i = 0; i < size; ++i) { + data.push_back(static_cast((i * 73 + 19) % 256)); + } + return data; +} + +std::string BuildTestZip(size_t file_count, size_t file_size, + CompressionMethod method) { + absl::StatusOr writer_or = + ZipWriter::FromBuffer("", /*append=*/false); + CHECK_OK(writer_or.status()); + ZipWriter writer = *std::move(writer_or); + ZipWriterFileOptions options; + (void)options.SetCompressionMethod(method); + std::string payload = GenerateTestData(file_size); + for (size_t i = 0; i < file_count; ++i) { + std::string name = absl::StrCat("entry_", i, ".bin"); + (void)writer.StartFile(name, options); + (void)writer.WriteData(payload); + } + absl::StatusOr res_or = writer.Finish(); + CHECK_OK(res_or.status()); + RustVecU8Wrapper res = *std::move(res_or); + return std::string(res.data(), res.size()); +} + +// Benchmark streaming write throughput. +void BM_StreamWriter_Write(benchmark::State& state) { + const size_t file_count = static_cast(state.range(0)); + const size_t file_size = static_cast(state.range(1)); + const std::string payload = GenerateTestData(file_size); + + ZipWriterFileOptions options; + (void)options.SetCompressionMethod(CompressionMethod::kStored); + + for (auto _ : state) { + absl::StatusOr writer_or = ZipStreamWriter::FromBuffer(); + CHECK_OK(writer_or.status()); + ZipStreamWriter writer = *std::move(writer_or); + for (size_t i = 0; i < file_count; ++i) { + std::string name = absl::StrCat("stream_file_", i, ".bin"); + (void)writer.StartFile(name, options); + (void)writer.WriteData(payload); + } + auto result = writer.Finish(); + benchmark::DoNotOptimize(result); + } + + state.SetBytesProcessed(state.iterations() * file_count * file_size); +} +BENCHMARK(BM_StreamWriter_Write) + ->Args({1, 1024}) + ->Args({10, 1024}) + ->Args({1, 65536}) + ->Args({10, 65536}); + +// Benchmark seekable write throughput. +void BM_SeekableWriter_Write(benchmark::State& state) { + const size_t file_count = static_cast(state.range(0)); + const size_t file_size = static_cast(state.range(1)); + const std::string payload = GenerateTestData(file_size); + + ZipWriterFileOptions options; + (void)options.SetCompressionMethod(CompressionMethod::kStored); + + for (auto _ : state) { + absl::StatusOr writer_or = + ZipWriter::FromBuffer("", /*append=*/false); + CHECK_OK(writer_or.status()); + ZipWriter writer = *std::move(writer_or); + for (size_t i = 0; i < file_count; ++i) { + std::string name = absl::StrCat("seek_file_", i, ".bin"); + (void)writer.StartFile(name, options); + (void)writer.WriteData(payload); + } + auto result = writer.Finish(); + benchmark::DoNotOptimize(result); + } + + state.SetBytesProcessed(state.iterations() * file_count * file_size); +} +BENCHMARK(BM_SeekableWriter_Write) + ->Args({1, 1024}) + ->Args({10, 1024}) + ->Args({1, 65536}) + ->Args({10, 65536}); + +// Benchmark streaming write with Deflate compression. +void BM_StreamWriter_Deflated(benchmark::State& state) { + const size_t file_size = static_cast(state.range(0)); + const std::string payload = GenerateTestData(file_size); + + ZipWriterFileOptions options; + (void)options.SetCompressionMethod(CompressionMethod::kDeflated); + + for (auto _ : state) { + absl::StatusOr writer_or = ZipStreamWriter::FromBuffer(); + CHECK_OK(writer_or.status()); + ZipStreamWriter writer = *std::move(writer_or); + (void)writer.StartFile("deflated.bin", options); + (void)writer.WriteData(payload); + auto result = writer.Finish(); + benchmark::DoNotOptimize(result); + } + + state.SetBytesProcessed(state.iterations() * file_size); +} +BENCHMARK(BM_StreamWriter_Deflated)->Args({1024})->Args({65536}); + +// Benchmark streaming read throughput. +void BM_StreamReader_SequentialRead(benchmark::State& state) { + const size_t file_count = static_cast(state.range(0)); + const size_t file_size = static_cast(state.range(1)); + const std::string zip_data = + BuildTestZip(file_count, file_size, CompressionMethod::kStored); + + for (auto _ : state) { + absl::StatusOr reader_or = + ZipStreamReader::FromBuffer(zip_data); + CHECK_OK(reader_or.status()); + ZipStreamReader reader = *std::move(reader_or); + for (size_t i = 0; i < file_count; ++i) { + absl::StatusOr> file_opt_or = + reader.ReadNextFile(); + CHECK_OK(file_opt_or.status()); + std::optional file_opt = *std::move(file_opt_or); + if (!file_opt.has_value()) break; + absl::StatusOr content_or = file_opt->GetFileData(); + CHECK_OK(content_or.status()); + RustVecU8Wrapper content = *std::move(content_or); + benchmark::DoNotOptimize(content); + } + } + + state.SetBytesProcessed(state.iterations() * file_count * file_size); +} +BENCHMARK(BM_StreamReader_SequentialRead) + ->Args({1, 1024}) + ->Args({10, 1024}) + ->Args({1, 65536}) + ->Args({10, 65536}); + +// Benchmark seekable archive read throughput. +void BM_ArchiveReader_GetFileByIndex(benchmark::State& state) { + const size_t file_count = static_cast(state.range(0)); + const size_t file_size = static_cast(state.range(1)); + const std::string zip_data = + BuildTestZip(file_count, file_size, CompressionMethod::kStored); + + for (auto _ : state) { + absl::StatusOr archive_or = ZipArchive::FromBuffer(zip_data); + CHECK_OK(archive_or.status()); + ZipArchive archive = *std::move(archive_or); + for (size_t i = 0; i < file_count; ++i) { + absl::StatusOr file_or = archive.GetFileByIndex(i); + CHECK_OK(file_or.status()); + ZipFile file = *std::move(file_or); + absl::StatusOr content_or = file.GetFileData(); + CHECK_OK(content_or.status()); + RustVecU8Wrapper content = *std::move(content_or); + benchmark::DoNotOptimize(content); + } + } + + state.SetBytesProcessed(state.iterations() * file_count * file_size); +} +BENCHMARK(BM_ArchiveReader_GetFileByIndex) + ->Args({1, 1024}) + ->Args({10, 1024}) + ->Args({1, 65536}) + ->Args({10, 65536}); + +// Benchmark streaming chunked reading. +void BM_StreamReader_ChunkedRead(benchmark::State& state) { + const size_t chunk_size = static_cast(state.range(0)); + constexpr size_t kFileSize = 65536; + const std::string zip_data = + BuildTestZip(1, kFileSize, CompressionMethod::kStored); + + for (auto _ : state) { + absl::StatusOr reader_or = + ZipStreamReader::FromBuffer(zip_data); + CHECK_OK(reader_or.status()); + ZipStreamReader reader = *std::move(reader_or); + absl::StatusOr> file_opt_or = reader.ReadNextFile(); + CHECK_OK(file_opt_or.status()); + std::optional file_opt = *std::move(file_opt_or); + if (file_opt.has_value()) { + size_t total_read = 0; + while (total_read < kFileSize) { + absl::StatusOr chunk_or = + file_opt->ReadBytes(chunk_size); + CHECK_OK(chunk_or.status()); + RustVecU8Wrapper chunk = *std::move(chunk_or); + if (chunk.empty()) break; + total_read += chunk.size(); + benchmark::DoNotOptimize(chunk); + } + } + } + + state.SetBytesProcessed(state.iterations() * kFileSize); +} +BENCHMARK(BM_StreamReader_ChunkedRead) + ->Args({1024}) + ->Args({4096}) + ->Args({16384}); + +} // namespace +} // namespace security::zip