Skip to content

Implement streaming zip writer, tests, and benchmarks #124

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
copybara-service wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Implement streaming zip writer, tests, and benchmarks #124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions zip/converters.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,4 +105,42 @@ absl::StatusOr<rust::FsZipWriter> FromRustFsZipWriter(
return std::move(result_fs_zip_writer).value();
}

absl::StatusOr<rust::BufferedZipStreamReader>
FromRustBufferedZipStreamReader(
rs_std::Result<rust::BufferedZipStreamReader, rust::ZipError>
result_buffered_zip_stream_reader) {
if (!result_buffered_zip_stream_reader.has_value()) {
return ZipErrorToStatus(std::move(result_buffered_zip_stream_reader).err());
}
return std::move(result_buffered_zip_stream_reader).value();
}

absl::StatusOr<rust::FsZipStreamReader> FromRustFsZipStreamReader(
rs_std::Result<rust::FsZipStreamReader, rust::ZipError>
result_fs_zip_stream_reader) {
if (!result_fs_zip_stream_reader.has_value()) {
return ZipErrorToStatus(std::move(result_fs_zip_stream_reader).err());
}
return std::move(result_fs_zip_stream_reader).value();
}

absl::StatusOr<rust::BufferedZipStreamWriter>
FromRustBufferedZipStreamWriter(
rs_std::Result<rust::BufferedZipStreamWriter, rust::ZipError>
result_buffered_zip_stream_writer) {
if (!result_buffered_zip_stream_writer.has_value()) {
return ZipErrorToStatus(std::move(result_buffered_zip_stream_writer).err());
}
return std::move(result_buffered_zip_stream_writer).value();
}

absl::StatusOr<rust::FsZipStreamWriter> FromRustFsZipStreamWriter(
rs_std::Result<rust::FsZipStreamWriter, rust::ZipError>
result_fs_zip_stream_writer) {
if (!result_fs_zip_stream_writer.has_value()) {
return ZipErrorToStatus(std::move(result_fs_zip_stream_writer).err());
}
return std::move(result_fs_zip_stream_writer).value();
}

} // namespace security::zip
32 changes: 31 additions & 1 deletion zip/converters.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#ifndef SECURITY_ZIP_CONVERTERS_H_
#define SECURITY_ZIP_CONVERTERS_H_

#include <cstddef>
#include <utility>

#include "crubit_helpers/string_conversions.h"
Expand All @@ -22,10 +23,23 @@ class RustVecU8Wrapper {
explicit RustVecU8Wrapper(rust::VecU8 vec_u8)
: vec_u8_(std::move(vec_u8)) {}

absl::string_view AsStringView() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
[[nodiscard]] absl::string_view AsStringView() const
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return security::crubit_helpers::StringViewFromVecU8(vec_u8_);
}

[[nodiscard]] const char* data() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return AsStringView().data();
}

[[nodiscard]] size_t size() const { return AsStringView().size(); }

[[nodiscard]] bool empty() const { return AsStringView().empty(); }

explicit operator absl::string_view() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return AsStringView();
}

private:
rust::VecU8 vec_u8_;
};
Expand Down Expand Up @@ -56,6 +70,22 @@ absl::StatusOr<rust::FsZipWriter> FromRustFsZipWriter(
rs_std::Result<rust::FsZipWriter, rust::ZipError>
result_fs_zip_writer);

absl::StatusOr<rust::BufferedZipStreamReader>
FromRustBufferedZipStreamReader(
rs_std::Result<rust::BufferedZipStreamReader, rust::ZipError>
result_buffered_zip_stream_reader);
absl::StatusOr<rust::FsZipStreamReader> FromRustFsZipStreamReader(
rs_std::Result<rust::FsZipStreamReader, rust::ZipError>
result_fs_zip_stream_reader);

absl::StatusOr<rust::BufferedZipStreamWriter>
FromRustBufferedZipStreamWriter(
rs_std::Result<rust::BufferedZipStreamWriter, rust::ZipError>
result_buffered_zip_stream_writer);
absl::StatusOr<rust::FsZipStreamWriter> FromRustFsZipStreamWriter(
rs_std::Result<rust::FsZipStreamWriter, rust::ZipError>
result_fs_zip_stream_writer);

} // namespace security::zip

#endif // SECURITY_ZIP_CONVERTERS_H_
95 changes: 95 additions & 0 deletions zip/file.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#include "file.h"

#include <cstddef>
#include <cstdint>
#include <string>
#include <variant>

Expand Down Expand Up @@ -62,10 +64,44 @@ absl::StatusOr<CompressionMethod> BufferedZipFile::GetCompressionMethod()
ABSL_RETURN_IF_ERROR(CheckNone());
return ToSecurityZipCompressionMethod(zip_.get_compression_method());
}
absl::StatusOr<std::string> BufferedZipFile::GetComment() const {
ABSL_RETURN_IF_ERROR(CheckNone());
rust::VecU8 comment_vec = zip_.get_comment();
return std::string(
security::crubit_helpers::StringViewFromVecU8(comment_vec));
}
absl::StatusOr<uint64_t> BufferedZipFile::GetUncompressedSize() const {
ABSL_RETURN_IF_ERROR(CheckNone());
return zip_.get_uncompressed_size();
}
absl::StatusOr<uint64_t> BufferedZipFile::GetCompressedSize() const {
ABSL_RETURN_IF_ERROR(CheckNone());
return zip_.get_compressed_size();
}
absl::StatusOr<uint32_t> BufferedZipFile::GetCrc32() const {
ABSL_RETURN_IF_ERROR(CheckNone());
return zip_.get_crc32();
}
absl::StatusOr<uint16_t> BufferedZipFile::GetLastModifiedDate() const {
ABSL_RETURN_IF_ERROR(CheckNone());
return zip_.get_last_modified_date();
}
absl::StatusOr<uint16_t> BufferedZipFile::GetLastModifiedTime() const {
ABSL_RETURN_IF_ERROR(CheckNone());
return zip_.get_last_modified_time();
}
absl::StatusOr<uint32_t> BufferedZipFile::GetUnixMode() const {
ABSL_RETURN_IF_ERROR(CheckNone());
return zip_.get_unix_mode();
}
absl::StatusOr<RustVecU8Wrapper> BufferedZipFile::GetFileData() {
ABSL_RETURN_IF_ERROR(CheckNone());
return FromRustResultVecU8(zip_.get_file_data());
}
absl::StatusOr<RustVecU8Wrapper> BufferedZipFile::ReadBytes(size_t max_bytes) {
ABSL_RETURN_IF_ERROR(CheckNone());
return FromRustResultVecU8(zip_.read_bytes(max_bytes));
}

absl::Status FsZipFile::CheckNone() const {
if (IsNone()) {
Expand All @@ -92,10 +128,44 @@ absl::StatusOr<CompressionMethod> FsZipFile::GetCompressionMethod() const {
ABSL_RETURN_IF_ERROR(CheckNone());
return ToSecurityZipCompressionMethod(zip_.get_compression_method());
}
absl::StatusOr<std::string> FsZipFile::GetComment() const {
ABSL_RETURN_IF_ERROR(CheckNone());
rust::VecU8 comment_vec = zip_.get_comment();
return std::string(
security::crubit_helpers::StringViewFromVecU8(comment_vec));
}
absl::StatusOr<uint64_t> FsZipFile::GetUncompressedSize() const {
ABSL_RETURN_IF_ERROR(CheckNone());
return zip_.get_uncompressed_size();
}
absl::StatusOr<uint64_t> FsZipFile::GetCompressedSize() const {
ABSL_RETURN_IF_ERROR(CheckNone());
return zip_.get_compressed_size();
}
absl::StatusOr<uint32_t> FsZipFile::GetCrc32() const {
ABSL_RETURN_IF_ERROR(CheckNone());
return zip_.get_crc32();
}
absl::StatusOr<uint16_t> FsZipFile::GetLastModifiedDate() const {
ABSL_RETURN_IF_ERROR(CheckNone());
return zip_.get_last_modified_date();
}
absl::StatusOr<uint16_t> FsZipFile::GetLastModifiedTime() const {
ABSL_RETURN_IF_ERROR(CheckNone());
return zip_.get_last_modified_time();
}
absl::StatusOr<uint32_t> FsZipFile::GetUnixMode() const {
ABSL_RETURN_IF_ERROR(CheckNone());
return zip_.get_unix_mode();
}
absl::StatusOr<RustVecU8Wrapper> FsZipFile::GetFileData() {
ABSL_RETURN_IF_ERROR(CheckNone());
return FromRustResultVecU8(zip_.get_file_data());
}
absl::StatusOr<RustVecU8Wrapper> FsZipFile::ReadBytes(size_t max_bytes) {
ABSL_RETURN_IF_ERROR(CheckNone());
return FromRustResultVecU8(zip_.read_bytes(max_bytes));
}

absl::StatusOr<bool> ZipFile::IsFile() const {
return std::visit([](auto& zip) { return zip.IsFile(); }, zip_);
Expand All @@ -112,8 +182,33 @@ absl::StatusOr<std::string> ZipFile::GetFileName() const {
absl::StatusOr<CompressionMethod> ZipFile::GetCompressionMethod() const {
return std::visit([](auto& zip) { return zip.GetCompressionMethod(); }, zip_);
}
absl::StatusOr<std::string> ZipFile::GetComment() const {
return std::visit([](auto& zip) { return zip.GetComment(); }, zip_);
}
absl::StatusOr<uint64_t> ZipFile::GetUncompressedSize() const {
return std::visit([](auto& zip) { return zip.GetUncompressedSize(); }, zip_);
}
absl::StatusOr<uint64_t> ZipFile::GetCompressedSize() const {
return std::visit([](auto& zip) { return zip.GetCompressedSize(); }, zip_);
}
absl::StatusOr<uint32_t> ZipFile::GetCrc32() const {
return std::visit([](auto& zip) { return zip.GetCrc32(); }, zip_);
}
absl::StatusOr<uint16_t> ZipFile::GetLastModifiedDate() const {
return std::visit([](auto& zip) { return zip.GetLastModifiedDate(); }, zip_);
}
absl::StatusOr<uint16_t> ZipFile::GetLastModifiedTime() const {
return std::visit([](auto& zip) { return zip.GetLastModifiedTime(); }, zip_);
}
absl::StatusOr<uint32_t> ZipFile::GetUnixMode() const {
return std::visit([](auto& zip) { return zip.GetUnixMode(); }, zip_);
}
absl::StatusOr<RustVecU8Wrapper> ZipFile::GetFileData() {
return std::visit([](auto& zip) { return zip.GetFileData(); }, zip_);
}
absl::StatusOr<RustVecU8Wrapper> ZipFile::ReadBytes(size_t max_bytes) {
return std::visit([max_bytes](auto& zip) { return zip.ReadBytes(max_bytes); },
zip_);
}

} // namespace security::zip
43 changes: 40 additions & 3 deletions zip/file.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#ifndef SECURITY_ZIP_FILE_H_
#define SECURITY_ZIP_FILE_H_

#include <cstddef>
#include <cstdint>
#include <string>
#include <utility>
Expand All @@ -15,6 +16,9 @@ namespace security::zip {

class BufferedZipWriter;
class FsZipWriter;
class BufferedZipStreamWriter;
class FsZipStreamWriter;
class ZipStreamWriter;

enum class CompressionMethod : int32_t {
kStored,
Expand All @@ -31,33 +35,55 @@ class BufferedZipFile final {
: zip_(std::move(zip)) {}
absl::StatusOr<bool> IsFile() const;
absl::StatusOr<bool> IsDir() const;
bool IsNone() const;
[[nodiscard]] bool IsNone() const;
absl::StatusOr<std::string> GetFileName() const;
absl::StatusOr<CompressionMethod> GetCompressionMethod() const;
absl::StatusOr<std::string> GetComment() const;
absl::StatusOr<uint64_t> GetUncompressedSize() const;
absl::StatusOr<uint64_t> GetCompressedSize() const;
absl::StatusOr<uint32_t> GetCrc32() const;
absl::StatusOr<uint16_t> GetLastModifiedDate() const;
absl::StatusOr<uint16_t> GetLastModifiedTime() const;
absl::StatusOr<uint32_t> GetUnixMode() const;
absl::StatusOr<RustVecU8Wrapper> GetFileData();
absl::StatusOr<RustVecU8Wrapper> ReadBytes(size_t max_bytes);

private:
absl::Status CheckNone() const;
rust::BufferedZipFile zip_;
friend class BufferedZipWriter;
friend class FsZipWriter;
friend class BufferedZipStreamWriter;
friend class FsZipStreamWriter;
friend class ZipStreamWriter;
};

class FsZipFile final {
public:
explicit FsZipFile(rust::FsZipFile zip) : zip_(std::move(zip)) {}
absl::StatusOr<bool> IsFile() const;
absl::StatusOr<bool> IsDir() const;
bool IsNone() const;
[[nodiscard]] bool IsNone() const;
absl::StatusOr<std::string> GetFileName() const;
absl::StatusOr<CompressionMethod> GetCompressionMethod() const;
absl::StatusOr<std::string> GetComment() const;
absl::StatusOr<uint64_t> GetUncompressedSize() const;
absl::StatusOr<uint64_t> GetCompressedSize() const;
absl::StatusOr<uint32_t> GetCrc32() const;
absl::StatusOr<uint16_t> GetLastModifiedDate() const;
absl::StatusOr<uint16_t> GetLastModifiedTime() const;
absl::StatusOr<uint32_t> GetUnixMode() const;
absl::StatusOr<RustVecU8Wrapper> GetFileData();
absl::StatusOr<RustVecU8Wrapper> ReadBytes(size_t max_bytes);

private:
absl::Status CheckNone() const;
rust::FsZipFile zip_;
friend class BufferedZipWriter;
friend class FsZipWriter;
friend class BufferedZipStreamWriter;
friend class FsZipStreamWriter;
friend class ZipStreamWriter;
};

class ZipFile final {
Expand All @@ -72,17 +98,28 @@ class ZipFile final {

absl::StatusOr<bool> IsFile() const;
absl::StatusOr<bool> IsDir() const;
bool IsNone() const;
[[nodiscard]] bool IsNone() const;
absl::StatusOr<std::string> GetFileName() const;
absl::StatusOr<CompressionMethod> GetCompressionMethod() const;
absl::StatusOr<std::string> GetComment() const;
absl::StatusOr<uint64_t> GetUncompressedSize() const;
absl::StatusOr<uint64_t> GetCompressedSize() const;
absl::StatusOr<uint32_t> GetCrc32() const;
absl::StatusOr<uint16_t> GetLastModifiedDate() const;
absl::StatusOr<uint16_t> GetLastModifiedTime() const;
absl::StatusOr<uint32_t> GetUnixMode() const;
absl::StatusOr<RustVecU8Wrapper> GetFileData();
absl::StatusOr<RustVecU8Wrapper> ReadBytes(size_t max_bytes);

private:
using BackendType = std::variant<BufferedZipFile, FsZipFile>;
BackendType zip_;
explicit ZipFile(BackendType b) : zip_(std::move(b)) {}
friend class BufferedZipWriter;
friend class FsZipWriter;
friend class BufferedZipStreamWriter;
friend class FsZipStreamWriter;
friend class ZipStreamWriter;
};

} // namespace security::zip
Expand Down
Loading
Loading