Add support for streaming archive writing and unknown filesizes in zip files

openVulkanoCpp/IO/Archive/ArchiveBase.cpp

@@ -23,7 +23,7 @@ namespace OpenVulkano
 		ChkErr(archive_free(m_archive));
 	}
 
-	bool ArchiveBase::ChkErr(int result) const
+	bool ArchiveBase::ChkErr(long result) const
 	{
 		return LibArchiveHelper::CheckError(result, m_archive, m_logger);
 	}

openVulkanoCpp/IO/Archive/ArchiveBase.hpp

@@ -26,11 +26,11 @@ namespace OpenVulkano
 		archive_entry* m_archiveEntry;
 		std::shared_ptr<spdlog::logger> m_logger;
 
+		bool ChkErr(long result) const;
 	public:
 		ArchiveBase(archive* arch, archive_entry* archEntry, const std::shared_ptr<spdlog::logger>& logger);
-		virtual ~ArchiveBase();
 
-		bool ChkErr(int result) const;
+		virtual ~ArchiveBase();
 
 		void SetLogger(const std::shared_ptr<spdlog::logger>& logger) { m_logger = logger; }
 		[[nodiscard]] std::shared_ptr<spdlog::logger> GetLogger() const { return m_logger; }

openVulkanoCpp/IO/Archive/ArchiveConfiguration.hpp

@@ -16,7 +16,6 @@ namespace OpenVulkano
 
 	struct ArchiveConfiguration final
 	{
-
 		ArchiveType type;
 		CompressionType compression;
 		int compressionLevel;
@@ -34,6 +33,8 @@ namespace OpenVulkano
 		[[nodiscard]] int GetLibArchiveArchiveType() const;
 
 		[[nodiscard]] int GetLibArchiveCompressionType() const;
+
+		[[nodiscard]] constexpr bool AllowsUnknownFileSize() const { return type == ArchiveType::ZIP; }
 	};
 
 	namespace ArchiveConfig

openVulkanoCpp/IO/Archive/ArchiveOStream.cpp

@@ -0,0 +1,21 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#include "ArchiveOStream.hpp"
+#include "ArchiveStreamBufferWriter.hpp"
+
+namespace OpenVulkano
+{
+	ArchiveOStream::ArchiveOStream(const ASBufferPtr& buffer)
+			: std::ostream(buffer.get())
+			, m_buffer(buffer)
+	{}
+
+	void ArchiveOStream::Close()
+	{
+		m_buffer->Close();
+	}
+}

openVulkanoCpp/IO/Archive/ArchiveOStream.hpp

@@ -0,0 +1,27 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#pragma once
+
+#include <ostream>
+#include <memory>
+
+namespace OpenVulkano
+{
+	class ArchiveStreamBufferWriter;
+
+	class ArchiveOStream : public std::ostream
+	{
+	public:
+		typedef std::shared_ptr<ArchiveStreamBufferWriter> ASBufferPtr;
+
+		explicit ArchiveOStream(const ASBufferPtr& buffer);
+
+		void Close();
+	private:
+		ASBufferPtr m_buffer;
+	};
+}

openVulkanoCpp/IO/Archive/ArchiveStreamBufferWriter.hpp

@@ -0,0 +1,53 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#pragma once
+
+#include "ArchiveWriter.hpp"
+#include <archive.h>
+#include <ostream>
+
+namespace OpenVulkano
+{
+	class ArchiveStreamBufferWriter final : public std::streambuf
+	{
+		ArchiveWriter* m_archive;
+		size_t m_maxSize, m_written;
+
+	public:
+		ArchiveStreamBufferWriter(ArchiveWriter* archive, size_t size)
+				: m_archive(archive), m_maxSize(size), m_written(0)
+		{}
+
+		std::streamsize xsputn(const char* s, std::streamsize n) override
+		{
+			if (!m_archive) return 0;
+			if (m_maxSize < m_written + n) n = m_maxSize - m_written;
+			if (n == 0) return 0;
+			m_archive->ChkErr(archive_write_data(m_archive->m_archive, s, n));
+			m_written += n;
+			if (m_written == m_maxSize) Close();
+			return n;
+		}
+
+		void Close()
+		{
+			if (!m_archive) return;
+			if (m_maxSize > m_written && m_maxSize != FileDescription::UNKNOWN_SIZE)
+			{ //Zero fill till eof
+				std::vector<char> data(m_maxSize - m_written, 0);
+				archive_write_data(m_archive->m_archive, data.data(), data.size());
+			}
+			m_archive->ChkErr(archive_write_finish_entry(m_archive->m_archive));
+			m_archive = nullptr;
+		}
+
+		~ArchiveStreamBufferWriter() override
+		{
+			if (m_archive) Close();
+		}
+	};
+}

openVulkanoCpp/IO/Archive/ArchiveWriter.cpp

@@ -6,18 +6,16 @@
 
 #include "ArchiveWriter.hpp"
 #include "LibArchiveHelper.hpp"
+#include "ArchiveStreamBufferWriter.hpp"
 #include <archive.h>
 #include <archive_entry.h>
 #include <iostream>
 
 namespace OpenVulkano
 {
-	ArchiveWriter::ArchiveWriter(const char* fileName, const std::shared_ptr<spdlog::logger>& logger)
-		: ArchiveWriter(fileName, ArchiveConfiguration::FromFileName(fileName), logger)
-	{}
-
 	ArchiveWriter::ArchiveWriter(const char* fileName, ArchiveConfiguration archiveConfiguration, const std::shared_ptr<spdlog::logger>& logger)
 			: ArchiveBase(archive_write_new(), archive_entry_new(), logger)
+			, m_archiveConfig(archiveConfiguration)
 	{ //TODO error handling
 		ChkErr(archive_write_set_format(m_archive, archiveConfiguration.GetLibArchiveArchiveType()));
 		if (archiveConfiguration.type == ArchiveType::TAR)
@@ -35,6 +33,7 @@ namespace OpenVulkano
 
 	ArchiveWriter::~ArchiveWriter()
 	{
+		if (m_asBuffer) { m_asBuffer->Close(); m_asBuffer = nullptr; }
 		archive_write_close(m_archive);
 		archive_entry_free(m_archiveEntry);
 	}
@@ -48,7 +47,7 @@ namespace OpenVulkano
 		//archive_read_disk_entry_from_file(archiveDisk, m_archiveEntry, -1, nullptr);
 		archive_entry_set_pathname(m_archiveEntry, inArchiveName);
 		ChkErr(archive_write_header(m_archive, m_archiveEntry));
-		int result = LibArchiveHelper::CopyArchiveData(archiveDisk.get(), m_archive);
+		long result = LibArchiveHelper::CopyArchiveData(archiveDisk.get(), m_archive);
 		ChkErr(result);
 		LibArchiveHelper::CheckError(result, archiveDisk.get(), m_logger);
 		ChkErr(archive_write_finish_entry(m_archive));
@@ -79,11 +78,27 @@ namespace OpenVulkano
 		return true;
 	}
 
+	ArchiveOStream ArchiveWriter::AddFileStream(const FileDescription& description)
+	{
+		WriteHeader(description);
+		return ArchiveOStream(m_asBuffer = std::make_shared<ArchiveStreamBufferWriter>(this, description.size));
+	}
+
 	void ArchiveWriter::WriteHeader(const FileDescription& fileDescription)
 	{
+		if (m_asBuffer) { m_asBuffer->Close(); m_asBuffer = nullptr; }
 		archive_entry_clear(m_archiveEntry);
 		archive_entry_set_pathname_utf8(m_archiveEntry, fileDescription.path.c_str());
+		if (fileDescription.size != FileDescription::UNKNOWN_SIZE)
+		{
 			archive_entry_set_size(m_archiveEntry, fileDescription.size);
+		}
+		else
+		{
+			if (!m_archiveConfig.AllowsUnknownFileSize())
+				throw std::invalid_argument("Only ZIP files allow files of unknown size");
+			archive_entry_unset_size(m_archiveEntry);
+		}
 		archive_entry_set_filetype(m_archiveEntry, LibArchiveHelper::GetFileType(fileDescription.type));
 		archive_entry_set_perm(m_archiveEntry, static_cast<mode_t>(fileDescription.permissions));
 		archive_entry_set_ctime(m_archiveEntry, fileDescription.createTime, 0);

openVulkanoCpp/IO/Archive/ArchiveWriter.hpp

@@ -17,11 +17,17 @@ namespace OpenVulkano
 	class ArchiveWriter final : public ArchiveBase, public IArchiveWriter
 	{
 		friend MultiPartArchiveWriter;
+		friend ArchiveStreamBufferWriter;
 
+		ArchiveConfiguration m_archiveConfig;
 		size_t m_bytesWritten = 0;
+		ArchiveOStream::ASBufferPtr m_asBuffer = nullptr;
 
 	public:
-		ArchiveWriter(const char* fileName, const std::shared_ptr<spdlog::logger>& logger = nullptr);
+		ArchiveWriter(const char* fileName, const std::shared_ptr<spdlog::logger>& logger = nullptr)
+				: ArchiveWriter(fileName, ArchiveConfiguration::FromFileName(fileName), logger)
+		{}
+
 		ArchiveWriter(const char* fileName, ArchiveConfiguration archiveConfiguration, const std::shared_ptr<spdlog::logger>& logger = nullptr);
 
 		~ArchiveWriter() override;
@@ -31,6 +37,7 @@ namespace OpenVulkano
 		bool AddFile(const FileDescription& description, const void* buffer) override;
 		bool AddFile(const FileDescription& description, const std::vector<std::pair<const void*, size_t>>& buffers) override;
 		bool AddFile(const char* fileName, const char* inArchiveName) override;
+		[[nodiscard]] ArchiveOStream AddFileStream(const FileDescription& description) override;
 
 	private:
 		void WriteHeader(const FileDescription& fileDescription);

openVulkanoCpp/IO/Archive/IArchiveWriter.hpp

@@ -6,6 +6,7 @@
 
 #pragma once
 
+#include "ArchiveOStream.hpp"
 #include "IO/FileDescription.hpp"
 #include <vector>
 
@@ -23,6 +24,14 @@ namespace OpenVulkano
 		virtual bool AddFile(const FileDescription& description, const std::vector<std::pair<const void*, size_t>>& buffers) = 0;
 		virtual bool AddFile(const char* fileName, const char* inArchiveName) = 0;
 
+		/**
+		 * Adds a new file to the archive as a stream.
+		 * If used with ZIP files the size in the description might be set to UNKNOWN_SIZE.
+		 * @param description The files description.
+		 * @return An output stream that allows the file to be written in a streaming fashion. The stream is only valid till either the file has been fully written (reached the size given in the file description) or the next file is added to the archive.
+		 */
+		[[nodiscard]] virtual ArchiveOStream AddFileStream(const FileDescription& description) = 0;
+
 		bool AddFile(const char* filePath)
 		{
 			std::filesystem::path path(filePath);

openVulkanoCpp/IO/Archive/LibArchiveHelper.hpp

@@ -74,7 +74,7 @@ namespace OpenVulkano
 			}
 		}
 
-		inline bool CheckError(int archiveResult, struct archive* arch, const Logger::Ptr& logger)
+		inline bool CheckError(long archiveResult, struct archive* arch, const Logger::Ptr& logger)
 		{
 			if (archiveResult >= ARCHIVE_OK) return true;
 			spdlog::level::level_enum lvl = spdlog::level::level_enum::warn;

openVulkanoCpp/IO/Archive/MultiPartArchiveWriter.cpp

@@ -74,6 +74,12 @@ namespace OpenVulkano
 		return m_writer->AddFile(fileName, inArchiveName);
 	}
 
+	ArchiveOStream MultiPartArchiveWriter::AddFileStream(const FileDescription& description)
+	{ //TODO handle zip splits
+		CheckSize(description.size);
+		return m_writer->AddFileStream(description);
+	}
+
 	void MultiPartArchiveWriter::Move(const std::filesystem::path& newDir)
 	{
 		bool hasToMove = m_archives.size() > 1 || m_writer->GetTotalWrittenBytes() > 0;

openVulkanoCpp/IO/Archive/MultiPartArchiveWriter.hpp

@@ -37,6 +37,7 @@ namespace OpenVulkano
 		bool AddFile(const FileDescription& description, const std::vector<std::pair<const void*, size_t>>& buffers) override;
 		bool AddFile(const char* fileName, const char* inArchiveName) override;
 		using IArchiveWriter::AddFile;
+		[[nodiscard]] virtual ArchiveOStream AddFileStream(const FileDescription& description) override;
 
 		void SetLogger(const std::shared_ptr<spdlog::logger>& logger) { m_logger = logger; }
 		[[nodiscard]] std::shared_ptr<spdlog::logger> GetLogger() const { return m_logger; }

openVulkanoCpp/IO/FileDescription.hpp

@@ -12,6 +12,8 @@ namespace OpenVulkano
 {
 	struct FileDescription
 	{
+		constexpr static inline size_t UNKNOWN_SIZE = SIZE_MAX;
+
 		std::filesystem::file_type type;
 		std::string path;
 		size_t size;