From 58da701354979c50e9db27837c483ff67645a461 Mon Sep 17 00:00:00 2001 From: GeorgH93 Date: Wed, 9 Dec 2020 21:20:19 +0100 Subject: [PATCH] Add ArchiveWriter --- openVulkanoCpp/IO/Archive/ArchiveWriter.cpp | 218 ++++++++++++++++++ openVulkanoCpp/IO/Archive/ArchiveWriter.hpp | 60 +++++ .../IO/Archive/LibArchiveHelper.hpp | 113 +++++++++ openVulkanoCpp/IO/FileDescription.hpp | 32 +++ 4 files changed, 423 insertions(+) create mode 100644 openVulkanoCpp/IO/Archive/ArchiveWriter.cpp create mode 100644 openVulkanoCpp/IO/Archive/ArchiveWriter.hpp create mode 100644 openVulkanoCpp/IO/Archive/LibArchiveHelper.hpp create mode 100644 openVulkanoCpp/IO/FileDescription.hpp diff --git a/openVulkanoCpp/IO/Archive/ArchiveWriter.cpp b/openVulkanoCpp/IO/Archive/ArchiveWriter.cpp new file mode 100644 index 0000000..22ac2ce --- /dev/null +++ b/openVulkanoCpp/IO/Archive/ArchiveWriter.cpp @@ -0,0 +1,218 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#include "ArchiveWriter.hpp" +#include "LibArchiveHelper.hpp" +#include "Base/Utils.hpp" +#include +#include +#include + +namespace openVulkanoCpp +{ + namespace + { + constexpr int TYPE_MAP[] = { ARCHIVE_FORMAT_TAR, ARCHIVE_FORMAT_CPIO, ARCHIVE_FORMAT_ISO9660, ARCHIVE_FORMAT_ZIP, ARCHIVE_FORMAT_XAR, ARCHIVE_FORMAT_7ZIP, ARCHIVE_FORMAT_WARC, ARCHIVE_FORMAT_SHAR }; + constexpr std::string_view TYPE_NAMES[] = { ".tar", ".cpio", ".iso", ".zip", ".xar", ".7z", ".warc", ".shar" }; + + constexpr int COMP_MAP[] = { ARCHIVE_FILTER_COMPRESS, ARCHIVE_FILTER_NONE, ARCHIVE_FILTER_GZIP, ARCHIVE_FILTER_BZIP2, ARCHIVE_FILTER_XZ, ARCHIVE_FILTER_LZ4, ARCHIVE_FILTER_ZSTD }; + constexpr std::string_view COMP_NAMES[] = { ".Z", "", ".gz", ".bz2", ".xz", ".lz", ".zst" }; + + constexpr std::string_view TAR_SORTS[] = { ".tZ", ".tar", ".tgz", ".tbz2", ".txz", ".tlz", ".tzst" }; + } + + std::string ArchiveConfiguration::GetFileExtension() const + { + const std::string_view& tName = TYPE_NAMES[static_cast(type)]; + if (type == ArchiveType::TAR) + { + const std::string_view& cName = COMP_NAMES[static_cast(compression)]; + std::string extension; + extension.reserve(tName.length() + cName.length()); + extension.append(tName).append(cName); + return extension; + } + return std::string(tName); + } + + ArchiveConfiguration ArchiveConfiguration::FromFileName(const char* fileName) + { + std::string_view fName = fileName; + ArchiveConfiguration ac; + int tId = 0; + for(const std::string_view& tName : TYPE_NAMES) + { + if (tId > 0) + { + if (Utils::EndsWith(fName, tName)) + { + ac.type = static_cast(tId); + return ac; + } + } + tId++; + } + tId = 0; + for(const std::string_view& tsName : TAR_SORTS) + { + if (Utils::EndsWith(fName, tsName)) + { + ac.compression = static_cast(tId); + return ac; + } + tId++; + } + tId = 0; + for(const std::string_view& cName : COMP_NAMES) + { + std::string tName(".tar"); + tName.append(cName); + if (Utils::EndsWith(fName, tName)) + { + ac.compression = static_cast(tId); + return ac; + } + tId++; + } + return ac; + } + + ArchiveWriter::ArchiveWriter(const char* fileName) : ArchiveWriter(fileName, ArchiveConfiguration::FromFileName(fileName)) + {} + + ArchiveWriter::ArchiveWriter(const char* fileName, ArchiveConfiguration archiveConfiguration) + : m_archive(archive_write_new()), m_archiveEntry(archive_entry_new()) + { //TODO error handling + archive_write_set_format(m_archive, TYPE_MAP[static_cast(archiveConfiguration.type)]); + if (archiveConfiguration.type == ArchiveConfiguration::ArchiveType::TAR) + { + archive_write_add_filter(m_archive, COMP_MAP[static_cast(archiveConfiguration.compression)]); + } + if (archiveConfiguration.compression != ArchiveConfiguration::CompressionType::NONE && + archiveConfiguration.compressionLevel > 0) + { + std::string level = "compression-level=" + std::to_string(archiveConfiguration.compressionLevel); + archive_write_set_options(m_archive, level.c_str()); + } + archive_write_open_filename(m_archive, fileName); + } + + ArchiveWriter::~ArchiveWriter() + { + archive_entry_free(m_archiveEntry); + archive_write_free(m_archive); + } + + void ArchiveWriter::Close() + { + archive_write_close(m_archive); + } + + bool ArchiveWriter::AddFile(const char* fileName) + { + std::filesystem::path path(fileName); + if (std::filesystem::is_regular_file(path)) + { + return AddFile(fileName, path.filename().c_str()); + } + else if (std::filesystem::is_directory(path)) + { + AddFiles(fileName, path.filename().c_str()); + } + return false; + } + + bool ArchiveWriter::AddFile(const char* fileName, const char* inArchiveName) + { + archive_entry_clear(m_archiveEntry); + std::unique_ptr archiveDisk(archive_read_disk_new(), archive_free); + + archive_entry_clear(m_archiveEntry); + archive_read_disk_open(archiveDisk.get(), fileName); + archive_read_next_header2(archiveDisk.get(), m_archiveEntry); + //archive_read_disk_entry_from_file(archiveDisk, m_archiveEntry, -1, nullptr); + archive_entry_set_pathname(m_archiveEntry, inArchiveName); + archive_write_header(m_archive, m_archiveEntry); + if (LibArchiveHelper::CopyArchiveData(archiveDisk.get(), m_archive) != ARCHIVE_OK) + { + //TODO handle error + } + return true; + } + + bool ArchiveWriter::AddFile(const char* inArchiveName, const void* buffer, size_t length) + { + FileDescription description = FileDescription::MakeDescriptionForFile(inArchiveName, length); + return AddFile(description, buffer); + } + + bool ArchiveWriter::AddFile(const FileDescription& description, const void* buffer) + { + WriteHeader(description); + return archive_write_data(m_archive, buffer, description.size) == ARCHIVE_OK; + } + + bool ArchiveWriter::AddFile(const char* fileName, const std::vector>& buffers) + { + size_t size = 0; + for(const auto& buffer : buffers) + { + if (buffer.first == nullptr) continue; + size += buffer.second; + } + FileDescription description = FileDescription::MakeDescriptionForFile(fileName, size); + return AddFile(description, buffers); + } + + bool ArchiveWriter::AddFile(const FileDescription& description, const std::vector>& buffers) + { + WriteHeader(description); + for(const auto& buffer : buffers) + { + if (buffer.first == nullptr) continue; + if (archive_write_data(m_archive, buffer.first, buffer.second) != ARCHIVE_OK) + { + return false; + } + } + return true; + } + + void ArchiveWriter::WriteHeader(const FileDescription& fileDescription) + { + archive_entry_clear(m_archiveEntry); + archive_entry_set_pathname_utf8(m_archiveEntry, fileDescription.path.c_str()); + archive_entry_set_size(m_archiveEntry, fileDescription.size); + archive_entry_set_filetype(m_archiveEntry, LibArchiveHelper::GetFileType(fileDescription.type)); + archive_entry_set_perm(m_archiveEntry, static_cast(fileDescription.permissions)); + archive_entry_set_ctime(m_archiveEntry, fileDescription.createTime, 0); + archive_entry_set_mtime(m_archiveEntry, fileDescription.modTime, 0); + archive_write_header(m_archive, m_archiveEntry); + } + + bool ArchiveWriter::AddFiles(const char* dirName) + { + return AddFiles(dirName, ""); + } + + bool ArchiveWriter::AddFiles(const char* dirName, const char* inArchiveDirName) + { + return AddFiles(std::filesystem::path(dirName), std::string(inArchiveDirName)); + } + + bool ArchiveWriter::AddFiles(const std::filesystem::path& dirName, const std::string& inArchiveDirName) + { + AddFile(dirName.c_str(), inArchiveDirName.c_str()); + for(const auto& entry : std::filesystem::directory_iterator(dirName)) + { + std::string fPath = inArchiveDirName + "/" + entry.path().filename().native(); + if (entry.is_directory()) AddFiles(entry, fPath); + else AddFile(entry.path().c_str(), fPath.c_str()); + } + return true; + } + +} \ No newline at end of file diff --git a/openVulkanoCpp/IO/Archive/ArchiveWriter.hpp b/openVulkanoCpp/IO/Archive/ArchiveWriter.hpp new file mode 100644 index 0000000..bd09fa3 --- /dev/null +++ b/openVulkanoCpp/IO/Archive/ArchiveWriter.hpp @@ -0,0 +1,60 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#pragma once + +#include "IO/FileDescription.hpp" +#include +#include + +struct archive; +struct archive_entry; + +namespace openVulkanoCpp +{ + struct ArchiveConfiguration + { + enum class ArchiveType { TAR = 0, CPIO, ISO, ZIP, XAR, SEVEN_ZIP, WARC, SHAR }; + enum class CompressionType { AUTO, NONE, GZIP, BZIP2, XZ, LZ4, ZSTD }; + + ArchiveType type = ArchiveType::TAR; + CompressionType compression = CompressionType::AUTO; + int compressionLevel = -1; + + [[nodiscard]] std::string GetFileExtension() const; + static ArchiveConfiguration FromFileName(const char* fileName); + }; + + class ArchiveWriter + { + archive* m_archive; + archive_entry* m_archiveEntry; + size_t m_bytesWritten = 0; + + public: + ArchiveWriter(const char* fileName); + ArchiveWriter(const char* fileName, ArchiveConfiguration archiveConfiguration); + + ~ArchiveWriter(); + + void Close(); + + [[nodiscard]] size_t GetTotalWrittenBytes() const { return m_bytesWritten; } + + bool AddFile(const char* fileName); + bool AddFile(const char* fileName, const char* inArchiveName); + bool AddFile(const char* inArchiveName, const void* buffer, size_t length); + bool AddFile(const FileDescription& description, const void* buffer); + bool AddFile(const char* fileName, const std::vector>& buffers); + bool AddFile(const FileDescription& description, const std::vector>& buffers); + bool AddFiles(const char* dirName); + bool AddFiles(const char* dirName, const char* inArchiveDirName); + bool AddFiles(const std::filesystem::path& dirName, const std::string& inArchiveDirName); + + private: + void WriteHeader(const FileDescription& fileDescription); + }; +} \ No newline at end of file diff --git a/openVulkanoCpp/IO/Archive/LibArchiveHelper.hpp b/openVulkanoCpp/IO/Archive/LibArchiveHelper.hpp new file mode 100644 index 0000000..662df94 --- /dev/null +++ b/openVulkanoCpp/IO/Archive/LibArchiveHelper.hpp @@ -0,0 +1,113 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#pragma once + +#include +#include +#include +#include "Base/Logger.hpp" + +namespace openVulkanoCpp +{ + namespace LibArchiveHelper + { + constexpr mode_t FILE_TYPE_MAP[] = { AE_IFMT, AE_IFREG, AE_IFDIR, AE_IFLNK, AE_IFBLK, AE_IFCHR, AE_IFIFO, AE_IFSOCK, AE_IFMT }; + + constexpr std::filesystem::file_type GetFileType(mode_t type) + { + switch(type & AE_IFMT) + { + case AE_IFREG: return std::filesystem::file_type::regular; + case AE_IFDIR: return std::filesystem::file_type::directory; + case AE_IFLNK: return std::filesystem::file_type::symlink; + case AE_IFBLK: return std::filesystem::file_type::block; + case AE_IFCHR: return std::filesystem::file_type::character; + case AE_IFIFO: return std::filesystem::file_type::fifo; + case AE_IFSOCK: return std::filesystem::file_type::socket; + } + return std::filesystem::file_type::unknown; + } + + constexpr mode_t GetFileType(std::filesystem::file_type type) + { + return FILE_TYPE_MAP[static_cast(type)]; + } + + template + inline ssize_t CopyArchiveData(struct archive* archiveReader, struct archive* archiveWriter) + { + const void* buffer; + size_t size; + ssize_t offset, r; + + while (true) + { + r = archive_read_data_block(archiveReader, &buffer, &size, &offset); + if (r == ARCHIVE_EOF) return ARCHIVE_OK; + if (r < ARCHIVE_OK) return r; + if constexpr (BLOCK_WRITE) + { + r = archive_write_data_block(archiveWriter, buffer, size, offset); + } + else + { + r = archive_write_data(archiveWriter, buffer, size); + } + if (r < ARCHIVE_OK) return r; + } + } + + inline bool CheckError(int archiveResult, struct archive* arch, const Logger::Ptr& logger) + { + if (archiveResult >= ARCHIVE_OK) return true; + spdlog::level::level_enum lvl = spdlog::level::level_enum::warn; + if (archiveResult <= ARCHIVE_FATAL) lvl = spdlog::level::level_enum::critical; + else if (archiveResult <= ARCHIVE_FAILED) lvl = spdlog::level::level_enum::err; + const char* errorString = archive_error_string(arch); + logger->log(lvl, errorString); + if (archiveResult == ARCHIVE_FAILED) throw std::runtime_error(errorString); + return false; + } + } + + class LibArchiveEntryStreamBuffer final : public std::streambuf + { + private: + struct archive* m_archive; + char* m_buffer; + size_t m_currentBlockSize; + ssize_t offset, r; + + bool ReadNextBlock() + { + const void* buffer; + bool ok = archive_read_data_block(m_archive, &buffer, &m_currentBlockSize, &offset) == ARCHIVE_OK; + m_buffer = static_cast(const_cast(buffer)); + return ok; + } + + public: + LibArchiveEntryStreamBuffer(struct archive* arch) : m_archive(arch), m_currentBlockSize(0) + {} + + int underflow() override + { + if (gptr() == egptr()) + { + if (ReadNextBlock()) + { + setg(m_buffer, m_buffer, m_buffer + m_currentBlockSize); + } + else + { + return std::char_traits::eof(); + } + } + return std::char_traits::to_int_type(*this->gptr()); + } + }; +} diff --git a/openVulkanoCpp/IO/FileDescription.hpp b/openVulkanoCpp/IO/FileDescription.hpp new file mode 100644 index 0000000..3903eeb --- /dev/null +++ b/openVulkanoCpp/IO/FileDescription.hpp @@ -0,0 +1,32 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#pragma once + +#include + +namespace openVulkanoCpp +{ + struct FileDescription + { + std::filesystem::file_type type; + std::string path; + size_t size; + std::filesystem::perms permissions; + time_t createTime, modTime; + + static FileDescription MakeDescriptionForFile(const char* path, size_t size) + { + return { + std::filesystem::file_type::regular, + path, + size, + std::filesystem::perms(0644), + std::time(nullptr), std::time(nullptr) + }; + } + }; +}