From 966db4b452181b225752ab8164801cc21e8d3aa9 Mon Sep 17 00:00:00 2001 From: GeorgH93 Date: Tue, 14 Nov 2023 23:42:23 +0100 Subject: [PATCH] Add support for streaming archive writing and unknown filesizes in zip files --- openVulkanoCpp/IO/Archive/ArchiveBase.cpp | 2 +- openVulkanoCpp/IO/Archive/ArchiveBase.hpp | 4 +- .../IO/Archive/ArchiveConfiguration.hpp | 3 +- openVulkanoCpp/IO/Archive/ArchiveOStream.cpp | 21 ++++++++ openVulkanoCpp/IO/Archive/ArchiveOStream.hpp | 27 ++++++++++ .../IO/Archive/ArchiveStreamBufferWriter.hpp | 53 +++++++++++++++++++ openVulkanoCpp/IO/Archive/ArchiveWriter.cpp | 27 +++++++--- openVulkanoCpp/IO/Archive/ArchiveWriter.hpp | 9 +++- openVulkanoCpp/IO/Archive/IArchiveWriter.hpp | 9 ++++ .../IO/Archive/LibArchiveHelper.hpp | 2 +- .../IO/Archive/MultiPartArchiveWriter.cpp | 6 +++ .../IO/Archive/MultiPartArchiveWriter.hpp | 1 + openVulkanoCpp/IO/FileDescription.hpp | 2 + 13 files changed, 154 insertions(+), 12 deletions(-) create mode 100644 openVulkanoCpp/IO/Archive/ArchiveOStream.cpp create mode 100644 openVulkanoCpp/IO/Archive/ArchiveOStream.hpp create mode 100644 openVulkanoCpp/IO/Archive/ArchiveStreamBufferWriter.hpp diff --git a/openVulkanoCpp/IO/Archive/ArchiveBase.cpp b/openVulkanoCpp/IO/Archive/ArchiveBase.cpp index 40420fd..0f5898a 100644 --- a/openVulkanoCpp/IO/Archive/ArchiveBase.cpp +++ b/openVulkanoCpp/IO/Archive/ArchiveBase.cpp @@ -23,7 +23,7 @@ namespace OpenVulkano ChkErr(archive_free(m_archive)); } - bool ArchiveBase::ChkErr(int result) const + bool ArchiveBase::ChkErr(long result) const { return LibArchiveHelper::CheckError(result, m_archive, m_logger); } diff --git a/openVulkanoCpp/IO/Archive/ArchiveBase.hpp b/openVulkanoCpp/IO/Archive/ArchiveBase.hpp index 9bbc7d5..7d4bbc9 100644 --- a/openVulkanoCpp/IO/Archive/ArchiveBase.hpp +++ b/openVulkanoCpp/IO/Archive/ArchiveBase.hpp @@ -26,11 +26,11 @@ namespace OpenVulkano archive_entry* m_archiveEntry; std::shared_ptr m_logger; + bool ChkErr(long result) const; public: ArchiveBase(archive* arch, archive_entry* archEntry, const std::shared_ptr& logger); - virtual ~ArchiveBase(); - bool ChkErr(int result) const; + virtual ~ArchiveBase(); void SetLogger(const std::shared_ptr& logger) { m_logger = logger; } [[nodiscard]] std::shared_ptr GetLogger() const { return m_logger; } diff --git a/openVulkanoCpp/IO/Archive/ArchiveConfiguration.hpp b/openVulkanoCpp/IO/Archive/ArchiveConfiguration.hpp index 024d2e5..467a5cd 100644 --- a/openVulkanoCpp/IO/Archive/ArchiveConfiguration.hpp +++ b/openVulkanoCpp/IO/Archive/ArchiveConfiguration.hpp @@ -16,7 +16,6 @@ namespace OpenVulkano struct ArchiveConfiguration final { - ArchiveType type; CompressionType compression; int compressionLevel; @@ -34,6 +33,8 @@ namespace OpenVulkano [[nodiscard]] int GetLibArchiveArchiveType() const; [[nodiscard]] int GetLibArchiveCompressionType() const; + + [[nodiscard]] constexpr bool AllowsUnknownFileSize() const { return type == ArchiveType::ZIP; } }; namespace ArchiveConfig diff --git a/openVulkanoCpp/IO/Archive/ArchiveOStream.cpp b/openVulkanoCpp/IO/Archive/ArchiveOStream.cpp new file mode 100644 index 0000000..5af4c1f --- /dev/null +++ b/openVulkanoCpp/IO/Archive/ArchiveOStream.cpp @@ -0,0 +1,21 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#include "ArchiveOStream.hpp" +#include "ArchiveStreamBufferWriter.hpp" + +namespace OpenVulkano +{ + ArchiveOStream::ArchiveOStream(const ASBufferPtr& buffer) + : std::ostream(buffer.get()) + , m_buffer(buffer) + {} + + void ArchiveOStream::Close() + { + m_buffer->Close(); + } +} \ No newline at end of file diff --git a/openVulkanoCpp/IO/Archive/ArchiveOStream.hpp b/openVulkanoCpp/IO/Archive/ArchiveOStream.hpp new file mode 100644 index 0000000..de626f3 --- /dev/null +++ b/openVulkanoCpp/IO/Archive/ArchiveOStream.hpp @@ -0,0 +1,27 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#pragma once + +#include +#include + +namespace OpenVulkano +{ + class ArchiveStreamBufferWriter; + + class ArchiveOStream : public std::ostream + { + public: + typedef std::shared_ptr ASBufferPtr; + + explicit ArchiveOStream(const ASBufferPtr& buffer); + + void Close(); + private: + ASBufferPtr m_buffer; + }; +} \ No newline at end of file diff --git a/openVulkanoCpp/IO/Archive/ArchiveStreamBufferWriter.hpp b/openVulkanoCpp/IO/Archive/ArchiveStreamBufferWriter.hpp new file mode 100644 index 0000000..4a5e163 --- /dev/null +++ b/openVulkanoCpp/IO/Archive/ArchiveStreamBufferWriter.hpp @@ -0,0 +1,53 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#pragma once + +#include "ArchiveWriter.hpp" +#include +#include + +namespace OpenVulkano +{ + class ArchiveStreamBufferWriter final : public std::streambuf + { + ArchiveWriter* m_archive; + size_t m_maxSize, m_written; + + public: + ArchiveStreamBufferWriter(ArchiveWriter* archive, size_t size) + : m_archive(archive), m_maxSize(size), m_written(0) + {} + + std::streamsize xsputn(const char* s, std::streamsize n) override + { + if (!m_archive) return 0; + if (m_maxSize < m_written + n) n = m_maxSize - m_written; + if (n == 0) return 0; + m_archive->ChkErr(archive_write_data(m_archive->m_archive, s, n)); + m_written += n; + if (m_written == m_maxSize) Close(); + return n; + } + + void Close() + { + if (!m_archive) return; + if (m_maxSize > m_written && m_maxSize != FileDescription::UNKNOWN_SIZE) + { //Zero fill till eof + std::vector data(m_maxSize - m_written, 0); + archive_write_data(m_archive->m_archive, data.data(), data.size()); + } + m_archive->ChkErr(archive_write_finish_entry(m_archive->m_archive)); + m_archive = nullptr; + } + + ~ArchiveStreamBufferWriter() override + { + if (m_archive) Close(); + } + }; +} diff --git a/openVulkanoCpp/IO/Archive/ArchiveWriter.cpp b/openVulkanoCpp/IO/Archive/ArchiveWriter.cpp index 29902f4..af3dcf2 100644 --- a/openVulkanoCpp/IO/Archive/ArchiveWriter.cpp +++ b/openVulkanoCpp/IO/Archive/ArchiveWriter.cpp @@ -6,18 +6,16 @@ #include "ArchiveWriter.hpp" #include "LibArchiveHelper.hpp" +#include "ArchiveStreamBufferWriter.hpp" #include #include #include namespace OpenVulkano { - ArchiveWriter::ArchiveWriter(const char* fileName, const std::shared_ptr& logger) - : ArchiveWriter(fileName, ArchiveConfiguration::FromFileName(fileName), logger) - {} - ArchiveWriter::ArchiveWriter(const char* fileName, ArchiveConfiguration archiveConfiguration, const std::shared_ptr& logger) : ArchiveBase(archive_write_new(), archive_entry_new(), logger) + , m_archiveConfig(archiveConfiguration) { //TODO error handling ChkErr(archive_write_set_format(m_archive, archiveConfiguration.GetLibArchiveArchiveType())); if (archiveConfiguration.type == ArchiveType::TAR) @@ -35,6 +33,7 @@ namespace OpenVulkano ArchiveWriter::~ArchiveWriter() { + if (m_asBuffer) { m_asBuffer->Close(); m_asBuffer = nullptr; } archive_write_close(m_archive); archive_entry_free(m_archiveEntry); } @@ -48,7 +47,7 @@ namespace OpenVulkano //archive_read_disk_entry_from_file(archiveDisk, m_archiveEntry, -1, nullptr); archive_entry_set_pathname(m_archiveEntry, inArchiveName); ChkErr(archive_write_header(m_archive, m_archiveEntry)); - int result = LibArchiveHelper::CopyArchiveData(archiveDisk.get(), m_archive); + long result = LibArchiveHelper::CopyArchiveData(archiveDisk.get(), m_archive); ChkErr(result); LibArchiveHelper::CheckError(result, archiveDisk.get(), m_logger); ChkErr(archive_write_finish_entry(m_archive)); @@ -79,11 +78,27 @@ namespace OpenVulkano return true; } + ArchiveOStream ArchiveWriter::AddFileStream(const FileDescription& description) + { + WriteHeader(description); + return ArchiveOStream(m_asBuffer = std::make_shared(this, description.size)); + } + void ArchiveWriter::WriteHeader(const FileDescription& fileDescription) { + if (m_asBuffer) { m_asBuffer->Close(); m_asBuffer = nullptr; } archive_entry_clear(m_archiveEntry); archive_entry_set_pathname_utf8(m_archiveEntry, fileDescription.path.c_str()); - archive_entry_set_size(m_archiveEntry, fileDescription.size); + if (fileDescription.size != FileDescription::UNKNOWN_SIZE) + { + archive_entry_set_size(m_archiveEntry, fileDescription.size); + } + else + { + if (!m_archiveConfig.AllowsUnknownFileSize()) + throw std::invalid_argument("Only ZIP files allow files of unknown size"); + archive_entry_unset_size(m_archiveEntry); + } archive_entry_set_filetype(m_archiveEntry, LibArchiveHelper::GetFileType(fileDescription.type)); archive_entry_set_perm(m_archiveEntry, static_cast(fileDescription.permissions)); archive_entry_set_ctime(m_archiveEntry, fileDescription.createTime, 0); diff --git a/openVulkanoCpp/IO/Archive/ArchiveWriter.hpp b/openVulkanoCpp/IO/Archive/ArchiveWriter.hpp index 96e7ba6..cb18d31 100644 --- a/openVulkanoCpp/IO/Archive/ArchiveWriter.hpp +++ b/openVulkanoCpp/IO/Archive/ArchiveWriter.hpp @@ -17,11 +17,17 @@ namespace OpenVulkano class ArchiveWriter final : public ArchiveBase, public IArchiveWriter { friend MultiPartArchiveWriter; + friend ArchiveStreamBufferWriter; + ArchiveConfiguration m_archiveConfig; size_t m_bytesWritten = 0; + ArchiveOStream::ASBufferPtr m_asBuffer = nullptr; public: - ArchiveWriter(const char* fileName, const std::shared_ptr& logger = nullptr); + ArchiveWriter(const char* fileName, const std::shared_ptr& logger = nullptr) + : ArchiveWriter(fileName, ArchiveConfiguration::FromFileName(fileName), logger) + {} + ArchiveWriter(const char* fileName, ArchiveConfiguration archiveConfiguration, const std::shared_ptr& logger = nullptr); ~ArchiveWriter() override; @@ -31,6 +37,7 @@ namespace OpenVulkano bool AddFile(const FileDescription& description, const void* buffer) override; bool AddFile(const FileDescription& description, const std::vector>& buffers) override; bool AddFile(const char* fileName, const char* inArchiveName) override; + [[nodiscard]] ArchiveOStream AddFileStream(const FileDescription& description) override; private: void WriteHeader(const FileDescription& fileDescription); diff --git a/openVulkanoCpp/IO/Archive/IArchiveWriter.hpp b/openVulkanoCpp/IO/Archive/IArchiveWriter.hpp index cdac4b5..a96b6d6 100644 --- a/openVulkanoCpp/IO/Archive/IArchiveWriter.hpp +++ b/openVulkanoCpp/IO/Archive/IArchiveWriter.hpp @@ -6,6 +6,7 @@ #pragma once +#include "ArchiveOStream.hpp" #include "IO/FileDescription.hpp" #include @@ -23,6 +24,14 @@ namespace OpenVulkano virtual bool AddFile(const FileDescription& description, const std::vector>& buffers) = 0; virtual bool AddFile(const char* fileName, const char* inArchiveName) = 0; + /** + * Adds a new file to the archive as a stream. + * If used with ZIP files the size in the description might be set to UNKNOWN_SIZE. + * @param description The files description. + * @return An output stream that allows the file to be written in a streaming fashion. The stream is only valid till either the file has been fully written (reached the size given in the file description) or the next file is added to the archive. + */ + [[nodiscard]] virtual ArchiveOStream AddFileStream(const FileDescription& description) = 0; + bool AddFile(const char* filePath) { std::filesystem::path path(filePath); diff --git a/openVulkanoCpp/IO/Archive/LibArchiveHelper.hpp b/openVulkanoCpp/IO/Archive/LibArchiveHelper.hpp index 1610e14..9efe466 100644 --- a/openVulkanoCpp/IO/Archive/LibArchiveHelper.hpp +++ b/openVulkanoCpp/IO/Archive/LibArchiveHelper.hpp @@ -74,7 +74,7 @@ namespace OpenVulkano } } - inline bool CheckError(int archiveResult, struct archive* arch, const Logger::Ptr& logger) + inline bool CheckError(long archiveResult, struct archive* arch, const Logger::Ptr& logger) { if (archiveResult >= ARCHIVE_OK) return true; spdlog::level::level_enum lvl = spdlog::level::level_enum::warn; diff --git a/openVulkanoCpp/IO/Archive/MultiPartArchiveWriter.cpp b/openVulkanoCpp/IO/Archive/MultiPartArchiveWriter.cpp index 8fc898a..a08b0ac 100644 --- a/openVulkanoCpp/IO/Archive/MultiPartArchiveWriter.cpp +++ b/openVulkanoCpp/IO/Archive/MultiPartArchiveWriter.cpp @@ -74,6 +74,12 @@ namespace OpenVulkano return m_writer->AddFile(fileName, inArchiveName); } + ArchiveOStream MultiPartArchiveWriter::AddFileStream(const FileDescription& description) + { //TODO handle zip splits + CheckSize(description.size); + return m_writer->AddFileStream(description); + } + void MultiPartArchiveWriter::Move(const std::filesystem::path& newDir) { bool hasToMove = m_archives.size() > 1 || m_writer->GetTotalWrittenBytes() > 0; diff --git a/openVulkanoCpp/IO/Archive/MultiPartArchiveWriter.hpp b/openVulkanoCpp/IO/Archive/MultiPartArchiveWriter.hpp index 8d82dbf..3e08ad6 100644 --- a/openVulkanoCpp/IO/Archive/MultiPartArchiveWriter.hpp +++ b/openVulkanoCpp/IO/Archive/MultiPartArchiveWriter.hpp @@ -37,6 +37,7 @@ namespace OpenVulkano bool AddFile(const FileDescription& description, const std::vector>& buffers) override; bool AddFile(const char* fileName, const char* inArchiveName) override; using IArchiveWriter::AddFile; + [[nodiscard]] virtual ArchiveOStream AddFileStream(const FileDescription& description) override; void SetLogger(const std::shared_ptr& logger) { m_logger = logger; } [[nodiscard]] std::shared_ptr GetLogger() const { return m_logger; } diff --git a/openVulkanoCpp/IO/FileDescription.hpp b/openVulkanoCpp/IO/FileDescription.hpp index e74d6da..34c2c36 100644 --- a/openVulkanoCpp/IO/FileDescription.hpp +++ b/openVulkanoCpp/IO/FileDescription.hpp @@ -12,6 +12,8 @@ namespace OpenVulkano { struct FileDescription { + constexpr static inline size_t UNKNOWN_SIZE = SIZE_MAX; + std::filesystem::file_type type; std::string path; size_t size;