GeorgH93/OpenVulkano
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for streaming archive writing and unknown filesizes in zip files

Browse Source
This commit is contained in:
GeorgH93
2023-11-14 23:42:23 +01:00
parent 6356075227
commit 966db4b452
13 changed files with 154 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
openVulkanoCpp/IO/Archive/ArchiveBase.cpp
View File

@@ -23,7 +23,7 @@ namespace OpenVulkano
ChkErr(archive_free(m_archive));
}
bool ArchiveBase::ChkErr(int result) const
bool ArchiveBase::ChkErr(long result) const
{
return LibArchiveHelper::CheckError(result, m_archive, m_logger);
}

4
openVulkanoCpp/IO/Archive/ArchiveBase.hpp
View File

@@ -26,11 +26,11 @@ namespace OpenVulkano
archive_entry* m_archiveEntry;
std::shared_ptr<spdlog::logger> m_logger;
bool ChkErr(long result) const;
public:
ArchiveBase(archive* arch, archive_entry* archEntry, const std::shared_ptr<spdlog::logger>& logger);
virtual ~ArchiveBase();
bool ChkErr(int result) const;
virtual ~ArchiveBase();
void SetLogger(const std::shared_ptr<spdlog::logger>& logger) { m_logger = logger; }
[[nodiscard]] std::shared_ptr<spdlog::logger> GetLogger() const { return m_logger; }

3
openVulkanoCpp/IO/Archive/ArchiveConfiguration.hpp
View File

@@ -16,7 +16,6 @@ namespace OpenVulkano
struct ArchiveConfiguration final
{
ArchiveType type;
CompressionType compression;
int compressionLevel;
@@ -34,6 +33,8 @@ namespace OpenVulkano
[[nodiscard]] int GetLibArchiveArchiveType() const;
[[nodiscard]] int GetLibArchiveCompressionType() const;
[[nodiscard]] constexpr bool AllowsUnknownFileSize() const { return type == ArchiveType::ZIP; }
};
namespace ArchiveConfig

21
openVulkanoCpp/IO/Archive/ArchiveOStream.cpp Normal file
View File

@@ -0,0 +1,21 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
#include "ArchiveOStream.hpp"
#include "ArchiveStreamBufferWriter.hpp"
namespace OpenVulkano
{
ArchiveOStream::ArchiveOStream(const ASBufferPtr& buffer)
: std::ostream(buffer.get())
, m_buffer(buffer)
{}
void ArchiveOStream::Close()
{
m_buffer->Close();
}
}

27
openVulkanoCpp/IO/Archive/ArchiveOStream.hpp Normal file
View File

@@ -0,0 +1,27 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
#pragma once
#include <ostream>
#include <memory>
namespace OpenVulkano
{
class ArchiveStreamBufferWriter;
class ArchiveOStream : public std::ostream
{
public:
typedef std::shared_ptr<ArchiveStreamBufferWriter> ASBufferPtr;
explicit ArchiveOStream(const ASBufferPtr& buffer);
void Close();
private:
ASBufferPtr m_buffer;
};
}

53
openVulkanoCpp/IO/Archive/ArchiveStreamBufferWriter.hpp Normal file
View File

@@ -0,0 +1,53 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
#pragma once
#include "ArchiveWriter.hpp"
#include <archive.h>
#include <ostream>
namespace OpenVulkano
{
class ArchiveStreamBufferWriter final : public std::streambuf
{
ArchiveWriter* m_archive;
size_t m_maxSize, m_written;
public:
ArchiveStreamBufferWriter(ArchiveWriter* archive, size_t size)
: m_archive(archive), m_maxSize(size), m_written(0)
{}
std::streamsize xsputn(const char* s, std::streamsize n) override
{
if (!m_archive) return 0;
if (m_maxSize < m_written + n) n = m_maxSize - m_written;
if (n == 0) return 0;
m_archive->ChkErr(archive_write_data(m_archive->m_archive, s, n));
m_written += n;
if (m_written == m_maxSize) Close();
return n;
}
void Close()
{
if (!m_archive) return;
if (m_maxSize > m_written && m_maxSize != FileDescription::UNKNOWN_SIZE)
{ //Zero fill till eof
std::vector<char> data(m_maxSize - m_written, 0);
archive_write_data(m_archive->m_archive, data.data(), data.size());
}
m_archive->ChkErr(archive_write_finish_entry(m_archive->m_archive));
m_archive = nullptr;
}
~ArchiveStreamBufferWriter() override
{
if (m_archive) Close();
}
};
}

25
openVulkanoCpp/IO/Archive/ArchiveWriter.cpp
View File

@@ -6,18 +6,16 @@
#include "ArchiveWriter.hpp"
#include "LibArchiveHelper.hpp"
#include "ArchiveStreamBufferWriter.hpp"
#include <archive.h>
#include <archive_entry.h>
#include <iostream>
namespace OpenVulkano
{
ArchiveWriter::ArchiveWriter(const char* fileName, const std::shared_ptr<spdlog::logger>& logger)
: ArchiveWriter(fileName, ArchiveConfiguration::FromFileName(fileName), logger)
{}
ArchiveWriter::ArchiveWriter(const char* fileName, ArchiveConfiguration archiveConfiguration, const std::shared_ptr<spdlog::logger>& logger)
: ArchiveBase(archive_write_new(), archive_entry_new(), logger)
, m_archiveConfig(archiveConfiguration)
{ //TODO error handling
ChkErr(archive_write_set_format(m_archive, archiveConfiguration.GetLibArchiveArchiveType()));
if (archiveConfiguration.type == ArchiveType::TAR)
@@ -35,6 +33,7 @@ namespace OpenVulkano
ArchiveWriter::~ArchiveWriter()
{
if (m_asBuffer) { m_asBuffer->Close(); m_asBuffer = nullptr; }
archive_write_close(m_archive);
archive_entry_free(m_archiveEntry);
}
@@ -48,7 +47,7 @@ namespace OpenVulkano
//archive_read_disk_entry_from_file(archiveDisk, m_archiveEntry, -1, nullptr);
archive_entry_set_pathname(m_archiveEntry, inArchiveName);
ChkErr(archive_write_header(m_archive, m_archiveEntry));
int result = LibArchiveHelper::CopyArchiveData(archiveDisk.get(), m_archive);
long result = LibArchiveHelper::CopyArchiveData(archiveDisk.get(), m_archive);
ChkErr(result);
LibArchiveHelper::CheckError(result, archiveDisk.get(), m_logger);
ChkErr(archive_write_finish_entry(m_archive));
@@ -79,11 +78,27 @@ namespace OpenVulkano
return true;
}
ArchiveOStream ArchiveWriter::AddFileStream(const FileDescription& description)
{
WriteHeader(description);
return ArchiveOStream(m_asBuffer = std::make_shared<ArchiveStreamBufferWriter>(this, description.size));
}
void ArchiveWriter::WriteHeader(const FileDescription& fileDescription)
{
if (m_asBuffer) { m_asBuffer->Close(); m_asBuffer = nullptr; }
archive_entry_clear(m_archiveEntry);
archive_entry_set_pathname_utf8(m_archiveEntry, fileDescription.path.c_str());
if (fileDescription.size != FileDescription::UNKNOWN_SIZE)
{
archive_entry_set_size(m_archiveEntry, fileDescription.size);
}
else
{
if (!m_archiveConfig.AllowsUnknownFileSize())
throw std::invalid_argument("Only ZIP files allow files of unknown size");
archive_entry_unset_size(m_archiveEntry);
}
archive_entry_set_filetype(m_archiveEntry, LibArchiveHelper::GetFileType(fileDescription.type));
archive_entry_set_perm(m_archiveEntry, static_cast<mode_t>(fileDescription.permissions));
archive_entry_set_ctime(m_archiveEntry, fileDescription.createTime, 0);

9
openVulkanoCpp/IO/Archive/ArchiveWriter.hpp
View File

@@ -17,11 +17,17 @@ namespace OpenVulkano
class ArchiveWriter final : public ArchiveBase, public IArchiveWriter
{
friend MultiPartArchiveWriter;
friend ArchiveStreamBufferWriter;
ArchiveConfiguration m_archiveConfig;
size_t m_bytesWritten = 0;
ArchiveOStream::ASBufferPtr m_asBuffer = nullptr;
public:
ArchiveWriter(const char* fileName, const std::shared_ptr<spdlog::logger>& logger = nullptr);
ArchiveWriter(const char* fileName, const std::shared_ptr<spdlog::logger>& logger = nullptr)
: ArchiveWriter(fileName, ArchiveConfiguration::FromFileName(fileName), logger)
{}
ArchiveWriter(const char* fileName, ArchiveConfiguration archiveConfiguration, const std::shared_ptr<spdlog::logger>& logger = nullptr);
~ArchiveWriter() override;
@@ -31,6 +37,7 @@ namespace OpenVulkano
bool AddFile(const FileDescription& description, const void* buffer) override;
bool AddFile(const FileDescription& description, const std::vector<std::pair<const void*, size_t>>& buffers) override;
bool AddFile(const char* fileName, const char* inArchiveName) override;
[[nodiscard]] ArchiveOStream AddFileStream(const FileDescription& description) override;
private:
void WriteHeader(const FileDescription& fileDescription);

9
openVulkanoCpp/IO/Archive/IArchiveWriter.hpp
View File

@@ -6,6 +6,7 @@
#pragma once
#include "ArchiveOStream.hpp"
#include "IO/FileDescription.hpp"
#include <vector>
@@ -23,6 +24,14 @@ namespace OpenVulkano
virtual bool AddFile(const FileDescription& description, const std::vector<std::pair<const void*, size_t>>& buffers) = 0;
virtual bool AddFile(const char* fileName, const char* inArchiveName) = 0;
/**
* Adds a new file to the archive as a stream.
* If used with ZIP files the size in the description might be set to UNKNOWN_SIZE.
* @param description The files description.
* @return An output stream that allows the file to be written in a streaming fashion. The stream is only valid till either the file has been fully written (reached the size given in the file description) or the next file is added to the archive.
*/
[[nodiscard]] virtual ArchiveOStream AddFileStream(const FileDescription& description) = 0;
bool AddFile(const char* filePath)
{
std::filesystem::path path(filePath);

2
openVulkanoCpp/IO/Archive/LibArchiveHelper.hpp
View File

@@ -74,7 +74,7 @@ namespace OpenVulkano
}
}
inline bool CheckError(int archiveResult, struct archive* arch, const Logger::Ptr& logger)
inline bool CheckError(long archiveResult, struct archive* arch, const Logger::Ptr& logger)
{
if (archiveResult >= ARCHIVE_OK) return true;
spdlog::level::level_enum lvl = spdlog::level::level_enum::warn;

6
openVulkanoCpp/IO/Archive/MultiPartArchiveWriter.cpp
View File

@@ -74,6 +74,12 @@ namespace OpenVulkano
return m_writer->AddFile(fileName, inArchiveName);
}
ArchiveOStream MultiPartArchiveWriter::AddFileStream(const FileDescription& description)
{ //TODO handle zip splits
CheckSize(description.size);
return m_writer->AddFileStream(description);
}
void MultiPartArchiveWriter::Move(const std::filesystem::path& newDir)
{
bool hasToMove = m_archives.size() > 1 || m_writer->GetTotalWrittenBytes() > 0;

1
openVulkanoCpp/IO/Archive/MultiPartArchiveWriter.hpp
View File

@@ -37,6 +37,7 @@ namespace OpenVulkano
bool AddFile(const FileDescription& description, const std::vector<std::pair<const void*, size_t>>& buffers) override;
bool AddFile(const char* fileName, const char* inArchiveName) override;
using IArchiveWriter::AddFile;
[[nodiscard]] virtual ArchiveOStream AddFileStream(const FileDescription& description) override;
void SetLogger(const std::shared_ptr<spdlog::logger>& logger) { m_logger = logger; }
[[nodiscard]] std::shared_ptr<spdlog::logger> GetLogger() const { return m_logger; }

2
openVulkanoCpp/IO/FileDescription.hpp
View File

@@ -12,6 +12,8 @@ namespace OpenVulkano
{
struct FileDescription
{
constexpr static inline size_t UNKNOWN_SIZE = SIZE_MAX;
std::filesystem::file_type type;
std::string path;
size_t size;