From 5f85d076091b6b70c8b0b374d2d764e0cdc51f2a Mon Sep 17 00:00:00 2001 From: GeorgH93 Date: Wed, 30 Jun 2021 19:35:10 +0200 Subject: [PATCH] Add ArRecorder logic --- openVulkanoCpp/AR/ArFrame.cpp | 8 +- openVulkanoCpp/AR/ArFrame.hpp | 25 ++- openVulkanoCpp/AR/ArRecorder.cpp | 193 ++++++++++++++++++ openVulkanoCpp/AR/ArRecorder.hpp | 46 +++-- openVulkanoCpp/AR/ArSession.cpp | 2 +- openVulkanoCpp/AR/ArSession.hpp | 12 +- .../AR/Provider/Network/ArSessionStream.h | 2 - .../AR/Provider/Playback/ArFramePlayback.cpp | 4 +- .../Provider/Playback/ArSessionPlayback.cpp | 7 +- ...essionPlayback.h => ArSessionPlayback.hpp} | 2 - 10 files changed, 263 insertions(+), 38 deletions(-) create mode 100644 openVulkanoCpp/AR/ArRecorder.cpp rename openVulkanoCpp/AR/Provider/Playback/{ArSessionPlayback.h => ArSessionPlayback.hpp} (94%) diff --git a/openVulkanoCpp/AR/ArFrame.cpp b/openVulkanoCpp/AR/ArFrame.cpp index 4b72b6f..1726854 100644 --- a/openVulkanoCpp/AR/ArFrame.cpp +++ b/openVulkanoCpp/AR/ArFrame.cpp @@ -6,11 +6,17 @@ #include "ArFrame.hpp" #include "ArSession.hpp" +#include "ArRecorder.hpp" namespace openVulkanoCpp::AR { float ArFrame::GetConfidenceNormalisationFactor() const { - return session->GetSessionMetadata().GetConfidenceNormalisationFactor(); + return m_session->GetSessionMetadata().GetConfidenceNormalisationFactor(); + } + + void ArFrame::Save() + { + m_session->GetRecorder().Save(this); } } \ No newline at end of file diff --git a/openVulkanoCpp/AR/ArFrame.hpp b/openVulkanoCpp/AR/ArFrame.hpp index 09cc6ed..a893c09 100644 --- a/openVulkanoCpp/AR/ArFrame.hpp +++ b/openVulkanoCpp/AR/ArFrame.hpp @@ -14,6 +14,7 @@ #include "ArTrackingState.hpp" #include "ArFrameMetadata.hpp" #include +#include namespace openVulkanoCpp::AR { @@ -86,20 +87,24 @@ namespace openVulkanoCpp::AR class ArFrame { - std::shared_ptr session; + std::shared_ptr m_session; + size_t m_frameId; + bool m_saved = false; protected: ArFrameMetadata frameMetadata; - ArFrame(const std::shared_ptr& session) : session(session) + ArFrame(const std::shared_ptr& session, size_t frameId) : m_session(session), m_frameId(frameId) {} public: virtual ~ArFrame() = default; + [[nodiscard]] size_t GetFrameId() const { return m_frameId; } + [[nodiscard]] const ArFrameMetadata& GetFrameMetadata() const { return frameMetadata; } - [[nodiscard]] const std::shared_ptr& GetArSession() const { return session; } + [[nodiscard]] const std::shared_ptr& GetArSession() const { return m_session; } [[nodiscard]] ArTrackingState GetTrackingState() const { return frameMetadata.trackingState; }; @@ -130,5 +135,19 @@ namespace openVulkanoCpp::AR [[nodiscard]] float GetExposureTime() const { return frameMetadata.exposureTime; }; [[nodiscard]] float GetExposureOffset() const { return frameMetadata.exposureOffset; }; + + [[nodiscard]] bool IsSaved() const { return m_saved; }; + + void Save(); + + void SetSaved() { m_saved = true; } + + /** + * Uses the platforms native jpeg writer to produce a jpeg representation of the camera image. + * This might not be implemented + * @param handler Function to be called once the image has been encoded + * @return true if image was encoded, false if not or unavailable + */ + virtual bool GetCameraImageAsJpeg(const std::function& handler) { return false; }; }; } diff --git a/openVulkanoCpp/AR/ArRecorder.cpp b/openVulkanoCpp/AR/ArRecorder.cpp new file mode 100644 index 0000000..068d9d3 --- /dev/null +++ b/openVulkanoCpp/AR/ArRecorder.cpp @@ -0,0 +1,193 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#include "ArRecorder.hpp" +#include "ArSession.hpp" +#include "ArFrame.hpp" +#include "IO/Archive/MultiPartArchiveWriter.hpp" +#include "IO/Files/Pfm.hpp" +#include "IO/Files/Pnm.hpp" +#include "IO/AppFolders.hpp" +#include "Base/BlockProfiler.hpp" +#include "Image/YuvUtils.hpp" +#include +#include +//#if __has_include("turbojpeg.h") +#include +#define TURBO_JPEG +//#endif + +namespace openVulkanoCpp::AR +{ + namespace + { + std::filesystem::path GeneratePath(const std::filesystem::path& baseDir, std::string_view name) + { + std::stringstream ss; + auto t = std::time(nullptr); + auto localTime = *std::localtime(&t); + ss << std::put_time(&localTime, "%Y-%m-%d_%H_%M_%S"); + return baseDir / name / ss.str(); + } + + std::string GetFileName(size_t frameId, std::string_view fileExtension) + { + std::stringstream fnStream; + fnStream << std::setw(7) << std::setfill('0') << frameId << '.' << fileExtension; + return fnStream.str(); + } + } + + ArRecorder::ArRecorder(ArSession* session) + : m_session(session), m_path(GeneratePath(AppFolders::GetAppDataHomeDir(), "ar_recording")) + {} + + ArRecorder::~ArRecorder() = default; + + void ArRecorder::WriteColorImage(ArFrame* arFrame) + { + std::string fileName = GetFileName(arFrame->GetFrameId(), "jpg"); +#ifndef TURBO_JPEG + if (arFrame->GetCameraImageAsJpeg([&fileName, this](const char* data, size_t len){ m_colorWriter->AddFile(fileName.c_str(), data, len); })) + return; + //TODO stb??? + Logger::AR->error("Failed to create JPEG! Missing turbojpeg.h"); +#else + //TODO handle non nv12 images + auto img = arFrame->GetCameraImage(); + auto sizeLum = img.luminescenceOrColor.resolution.x * img.luminescenceOrColor.resolution.y; + auto sizeUV = img.uv.resolution.x * img.uv.resolution.y; + auto resX = img.luminescenceOrColor.resolution.x; + auto resY = img.luminescenceOrColor.resolution.y; + + tjhandle handle = tjInitCompress(); + const uint8_t* buffers[3]; + std::unique_ptr dataBuffer; + bool downsampleRecording = true; + if (downsampleRecording) + { + dataBuffer = YuvUtils::PlansFromNV12(static_cast(img.luminescenceOrColor.data), static_cast(img.uv.data), + resX, resY, img.uv.resolution.x, img.uv.resolution.y, 2, 2); + resX /= 2; + resY /= 2; + buffers[0] = dataBuffer.get(); + buffers[1] = buffers[0] + sizeLum / 4; + buffers[2] = buffers[1] + sizeUV / 4; + } + else + { + dataBuffer = std::unique_ptr(new uint8_t[sizeUV + sizeUV]); + YuvUtils::ChromaPlanesFromNV12((uint8_t*)img.uv.data, dataBuffer.get(), sizeUV); + buffers[0] = static_cast(img.luminescenceOrColor.data); + buffers[1] = dataBuffer.get(); + buffers[2] = buffers[1] + sizeUV; + } + + uint8_t* outBuffer = nullptr; + unsigned long size = 0; + if (tjCompressFromYUVPlanes(handle, buffers, resX, nullptr, resY, TJSAMP_420, &outBuffer, &size, 95, TJFLAG_FASTDCT)) + Logger::AR->error("Failed to create JPEG! {}", tjGetErrorStr()); + else + m_colorWriter->AddFile(fileName.c_str(), outBuffer, size); + tjFree(outBuffer); +#endif + } + + void ArRecorder::WriteDepthImage(ArFrame* arFrame) + { + if (!m_depthWriter || !m_confidenceWriter) return; + auto depthImg = arFrame->GetDepthImage(); + std::vector> buffers(2); + { // TODO handle alternative depth formats!!!! + BlockProfiler profile("Save AR Frame - Depth"); + PfmHeader depthHeader(static_cast(depthImg.depth.resolution.x), static_cast(depthImg.depth.resolution.y), 5.0f, false); + std::string header = depthHeader.ToString(); + buffers[0].first = header.c_str(); + buffers[0].second = header.size(); + buffers[1].first = static_cast(depthImg.depth.data); + buffers[1].second = depthImg.depth.resolution.x * depthImg.depth.resolution.y * sizeof(float); + + const std::string fileName = GetFileName(arFrame->GetFrameId(), "pfm"); + m_depthWriter->AddFile(fileName.c_str(), buffers); + } + + { + BlockProfiler profile("Save AR Frame - Confi"); + PnmHeader confidenceHeader(static_cast(depthImg.confidence.resolution.x), static_cast(depthImg.confidence.resolution.y), false, 2); + std::string header = confidenceHeader.ToString(); + buffers[0].first = header.c_str(); + buffers[0].second = header.size(); + buffers[1].first = static_cast(static_cast(depthImg.confidence.data)); + buffers[1].second = static_cast(depthImg.confidence.resolution.x * depthImg.confidence.resolution.y); + + const std::string fileName = GetFileName(arFrame->GetFrameId(), "pgm"); + m_confidenceWriter->AddFile(fileName.c_str(), buffers); + } + } + + void ArRecorder::Save(ArFrame* frame) + { + if (!m_recording || frame->IsSaved()) return; + frame->SetSaved(); + BlockProfiler profile("Save AR Frame"); + + { + BlockProfiler profile("Save AR Frame - Image"); + WriteColorImage(frame); + } + { + //BlockProfiler profile("Save AR Frame - Depth"); + WriteDepthImage(frame); + } + { + BlockProfiler profileMeta("Save AR Frame - Meta"); + std::string metaContent = frame->GetFrameMetadata().ToXML(); + std::string fileName = GetFileName(frame->GetFrameId(), "meta"); + m_metadataWriter->AddFile(fileName.c_str(), metaContent.c_str(), metaContent.size()); + } + } + + void ArRecorder::Start() + { + if (!m_colorWriter) + { + m_colorWriter = std::make_unique(m_path, "color_{:05d}.tar", ArchiveConfig::TAR); + m_depthWriter = std::make_unique(m_path, "depth_{:05d}.tar", ArchiveConfig::TAR); + m_confidenceWriter = std::make_unique(m_path, "confidence_{:05d}.tar", ArchiveConfig::TAR_GZ); + m_metadataWriter = std::make_unique(m_path, "meta_{:05d}.tar", ArchiveConfig::TAR_GZ); + + std::ofstream platformInfoStream(m_path / "ArRecording.xml"); + platformInfoStream << m_session->GetSessionMetadata().ToXML(); + platformInfoStream.close(); + } + m_recording = true; + } + + void ArRecorder::Stop() + { + if (!m_recording) return; + m_recording = false; + for(MultiPartArchiveWriter* writer : { m_colorWriter.get(), m_depthWriter.get(), m_confidenceWriter.get(), m_metadataWriter.get() }) + { + writer->Split(); + } + } + + void ArRecorder::SetRecordingPath(const std::string& path) + { + if (!m_colorWriter) + { + for (MultiPartArchiveWriter* writer: {m_colorWriter.get(), m_depthWriter.get(), m_confidenceWriter.get(), m_metadataWriter.get()}) + { + writer->Move(path); + } + std::filesystem::rename(m_path / "ArRecording.xml", path + "/ArRecording.xml"); + } + + m_persistent = true; + m_path = path; + } +} diff --git a/openVulkanoCpp/AR/ArRecorder.hpp b/openVulkanoCpp/AR/ArRecorder.hpp index e6175e6..e63d840 100644 --- a/openVulkanoCpp/AR/ArRecorder.hpp +++ b/openVulkanoCpp/AR/ArRecorder.hpp @@ -8,9 +8,18 @@ #include #include +#include + +namespace openVulkanoCpp +{ + class MultiPartArchiveWriter; +} namespace openVulkanoCpp::AR { + class ArSession; + class ArFrame; + enum class RecordingMode { /** @@ -27,23 +36,33 @@ namespace openVulkanoCpp::AR NEW_FRAME }; - class ArRecorder + class ArRecorder final { - protected: - ArRecorder() = default; + ArSession* m_session; + RecordingMode m_recordingMode = RecordingMode::MANUAL; + std::filesystem::path m_path; + std::unique_ptr m_colorWriter, m_depthWriter, m_confidenceWriter, m_metadataWriter; + bool m_recording = false, m_persistent = false; + + void WriteColorImage(ArFrame* arFrame); + void WriteDepthImage(ArFrame *arFrame); public: - virtual ~ArRecorder() = default; + ArRecorder(ArSession* session); + + ~ArRecorder(); + + void Save(ArFrame* frame); /** * Starts the recording of the owning AR session */ - virtual void Start() = 0; + void Start(); /** * Stops the recording of the owning AR session */ - virtual void Stop() = 0; + void Stop(); /** * Sets the directory into which the AR recording should be stored. @@ -51,40 +70,37 @@ namespace openVulkanoCpp::AR * If path is changed after starting the recording, the already running recording will be moved to the new path. * @param path The path to be used to store the recording */ - virtual void SetRecordingPath(const std::string& path) = 0; + void SetRecordingPath(const std::string& path); /** * Gets the current recording path * @return Current recording dir */ - [[nodiscard]] virtual const std::filesystem::path& GetRecordingPath() const = 0; + [[nodiscard]] const std::filesystem::path& GetRecordingPath() const { return m_path; } /** * Checks if a path to be used for the recording has been set. * If no path has been set the recording will be stored in a temporary location and will be deleted once the recording is stopped. * @return True if a path has been set and the recording will persist after stopping the recording. */ - [[nodiscard]] virtual bool IsPersistent() const = 0; + [[nodiscard]] bool IsPersistent() const { return m_persistent; } /** * Checks if the recording of the AR session is running * @return True if recording is started */ - [[nodiscard]] virtual bool IsRecording() const = 0; + [[nodiscard]] bool IsRecording() const { return m_recording; } /** * Sets the used recording mode * @param mode The mode that should be used to record the current session */ - void SetRecordingMode(RecordingMode mode) { recordingMode = mode; } + void SetRecordingMode(RecordingMode mode) { m_recordingMode = mode; } /** * Checks the currently used recording mode for the AR session * @return The currently used recording mode */ - [[nodiscard]] RecordingMode SetRecordingMode() { return recordingMode; } - - protected: - RecordingMode recordingMode = RecordingMode::FRAME_REQUEST; + [[nodiscard]] RecordingMode SetRecordingMode() const { return m_recordingMode; } }; } diff --git a/openVulkanoCpp/AR/ArSession.cpp b/openVulkanoCpp/AR/ArSession.cpp index 5237fa2..8003e49 100644 --- a/openVulkanoCpp/AR/ArSession.cpp +++ b/openVulkanoCpp/AR/ArSession.cpp @@ -5,7 +5,7 @@ */ #include "ArSession.hpp" -#include "Provider/Playback/ArSessionPlayback.h" +#include "Provider/Playback/ArSessionPlayback.hpp" #include "Provider/Network/ArSessionStream.h" #ifdef __APPLE__ #include "Provider/ArKit/ArSessionArKit.h" diff --git a/openVulkanoCpp/AR/ArSession.hpp b/openVulkanoCpp/AR/ArSession.hpp index e77e664..498085e 100644 --- a/openVulkanoCpp/AR/ArSession.hpp +++ b/openVulkanoCpp/AR/ArSession.hpp @@ -10,6 +10,7 @@ #include "ArConstans.hpp" #include "ArTrackingState.hpp" #include "ArSessionMetadata.hpp" +#include "ArRecorder.hpp" #include "Math/Range.hpp" #include "Base/Event.hpp" #include @@ -22,7 +23,6 @@ namespace openVulkanoCpp::AR { class ArSession; class ArFrame; - class ArRecorder; enum class ArSessionType { @@ -35,18 +35,16 @@ namespace openVulkanoCpp::AR ArSessionType sessionType = ArSessionType::NATIVE; bool uncompressed = true; bool depthSupported = false; - bool recordingSupported = false; public: ArSessionCapabilities() = default; ArSessionCapabilities(const ArType type, const ArSessionType sessionType, const bool uncompressed, const bool depthSupported, const bool recordingSupported) - : type(type), sessionType(sessionType), uncompressed(uncompressed), depthSupported(depthSupported), recordingSupported(recordingSupported) + : type(type), sessionType(sessionType), uncompressed(uncompressed), depthSupported(depthSupported) {} [[nodiscard]] bool IsUncompressed() const { return uncompressed; } [[nodiscard]] bool IsDepthSupported() const { return depthSupported; } - [[nodiscard]] bool IsRecordingSupported() const { return recordingSupported; } [[nodiscard]] bool IsNative() const { return sessionType == ArSessionType::NATIVE; } [[nodiscard]] bool IsPlayback() const { return sessionType == ArSessionType::PLAYBACK; } [[nodiscard]] bool IsStream() const { return sessionType == ArSessionType::NETWORK_STREAM; } @@ -73,7 +71,7 @@ namespace openVulkanoCpp::AR class ArSession { protected: - ArSession(const ArSessionMetadata& metadata) : metadata(metadata) {} + ArSession(const ArSessionMetadata& metadata) : metadata(metadata), recorder(this) {} public: /** @@ -173,7 +171,7 @@ namespace openVulkanoCpp::AR * The ArRecorder for this ArSession. * @return ArRecorder instance or nullptr if the ArSession does not support recording. */ - [[nodiscard]] virtual ArRecorder* GetRecorder() = 0; + [[nodiscard]] ArRecorder& GetRecorder() { return recorder; }; /** * Gets the type of the AR session. @@ -219,6 +217,8 @@ namespace openVulkanoCpp::AR ArSessionMetadata metadata; private: + ArRecorder recorder; + static std::vector> sessions; static std::weak_ptr nativeSession; }; diff --git a/openVulkanoCpp/AR/Provider/Network/ArSessionStream.h b/openVulkanoCpp/AR/Provider/Network/ArSessionStream.h index d56970b..21004cd 100644 --- a/openVulkanoCpp/AR/Provider/Network/ArSessionStream.h +++ b/openVulkanoCpp/AR/Provider/Network/ArSessionStream.h @@ -25,8 +25,6 @@ namespace openVulkanoCpp::AR::Network [[nodiscard]] std::shared_ptr GetFrame() override; - [[nodiscard]] ArRecorder* GetRecorder() override { return nullptr; } - [[nodiscard]] ArSessionType GetSessionType() override { return ArSessionType::NETWORK_STREAM; } [[nodiscard]] ArType GetArType() override; diff --git a/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp b/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp index 80c805a..585c71a 100644 --- a/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp +++ b/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp @@ -5,13 +5,13 @@ */ #include "ArFramePlayback.hpp" -#include "ArSessionPlayback.h" +#include "ArSessionPlayback.hpp" #include "Base/BlockProfiler.hpp" namespace openVulkanoCpp::AR::Playback { ArFramePlayback::ArFramePlayback(const std::shared_ptr& session, ArPlaybackReader& frameReader) - : ArFrame(session) + : ArFrame(session, frameReader.GetNextFrameId()) { BlockProfiler profile("Read_AR_Frame"); const auto data = frameReader.ReadMetadata(); diff --git a/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp b/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp index 57c4ceb..401f450 100644 --- a/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp +++ b/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp @@ -4,7 +4,7 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -#include "ArSessionPlayback.h" +#include "ArSessionPlayback.hpp" #include "ArFramePlayback.hpp" #include "Base/Logger.hpp" #include @@ -74,11 +74,6 @@ namespace openVulkanoCpp::AR::Playback return nullptr; } - ArRecorder* ArSessionPlayback::GetRecorder() - { - return nullptr; - } - ArType ArSessionPlayback::GetArType() { return capabilities.GetArType(); diff --git a/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.h b/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.hpp similarity index 94% rename from openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.h rename to openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.hpp index 01699ea..617124f 100644 --- a/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.h +++ b/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.hpp @@ -27,8 +27,6 @@ class ArSessionPlayback final : public ArSession, public std::enable_shared_from [[nodiscard]] std::shared_ptr GetFrame() override; - [[nodiscard]] ArRecorder* GetRecorder() override; - [[nodiscard]] ArSessionType GetSessionType() override { return ArSessionType::PLAYBACK; } [[nodiscard]] ArType GetArType() override;