From 7faf3a7a7d1721c7cc948f2a08866d30cce93d67 Mon Sep 17 00:00:00 2001 From: GeorgH93 Date: Mon, 26 Jul 2021 21:57:56 +0200 Subject: [PATCH] Update AR data recording to use Events for new frame recording, add more config options and prepare for async recording --- openVulkanoCpp/AR/ArFrame.cpp | 5 +- openVulkanoCpp/AR/ArFrame.hpp | 9 +- openVulkanoCpp/AR/ArRecorder.cpp | 100 ++++++++++++------ openVulkanoCpp/AR/ArRecorder.hpp | 66 ++++++++++-- openVulkanoCpp/AR/ArSession.hpp | 1 + .../Provider/ArKit/ArSessionArKitInternal.mm | 3 +- 6 files changed, 141 insertions(+), 43 deletions(-) diff --git a/openVulkanoCpp/AR/ArFrame.cpp b/openVulkanoCpp/AR/ArFrame.cpp index 1726854..8b63aa4 100644 --- a/openVulkanoCpp/AR/ArFrame.cpp +++ b/openVulkanoCpp/AR/ArFrame.cpp @@ -17,6 +17,9 @@ namespace openVulkanoCpp::AR void ArFrame::Save() { - m_session->GetRecorder().Save(this); + if (m_highRes) + m_session->GetRecorder().SaveHighResolution(shared_from_this()); + else + m_session->GetRecorder().Save(shared_from_this()); } } \ No newline at end of file diff --git a/openVulkanoCpp/AR/ArFrame.hpp b/openVulkanoCpp/AR/ArFrame.hpp index a893c09..38e32cd 100644 --- a/openVulkanoCpp/AR/ArFrame.hpp +++ b/openVulkanoCpp/AR/ArFrame.hpp @@ -46,7 +46,7 @@ namespace openVulkanoCpp::AR size_t idx = (y * luminescenceOrColor.resolution.x + x) * 3; return { lumColBuffer[idx + 2], lumColBuffer[idx + 1], lumColBuffer[idx], 255 }; } - else if (format == Format::RGB) + else if (format == Format::RGB) { size_t idx = (y * luminescenceOrColor.resolution.x + x) * 3; return { lumColBuffer[idx], lumColBuffer[idx + 1], lumColBuffer[idx + 2], 255 }; @@ -85,11 +85,12 @@ namespace openVulkanoCpp::AR Math::CameraIntrinsic intrinsic; }; - class ArFrame + class ArFrame : public std::enable_shared_from_this { std::shared_ptr m_session; size_t m_frameId; bool m_saved = false; + bool m_highRes = false; protected: ArFrameMetadata frameMetadata; @@ -142,6 +143,10 @@ namespace openVulkanoCpp::AR void SetSaved() { m_saved = true; } + void MarkHighRes() { m_highRes = true; } + + [[nodiscard]] bool IsHighResolution() const { return m_highRes; } + /** * Uses the platforms native jpeg writer to produce a jpeg representation of the camera image. * This might not be implemented diff --git a/openVulkanoCpp/AR/ArRecorder.cpp b/openVulkanoCpp/AR/ArRecorder.cpp index 068d9d3..4bfe98e 100644 --- a/openVulkanoCpp/AR/ArRecorder.cpp +++ b/openVulkanoCpp/AR/ArRecorder.cpp @@ -42,12 +42,15 @@ namespace openVulkanoCpp::AR } ArRecorder::ArRecorder(ArSession* session) - : m_session(session), m_path(GeneratePath(AppFolders::GetAppDataHomeDir(), "ar_recording")) - {} + : m_session(session) + { + m_settings.path = GeneratePath(AppFolders::GetAppDataHomeDir(), "ar_recording"); + session->OnNewFrameHighResolution += EventHandler(this, &ArRecorder::SaveHighResolution); + } ArRecorder::~ArRecorder() = default; - void ArRecorder::WriteColorImage(ArFrame* arFrame) + void ArRecorder::WriteColorImage(ArFrame* arFrame, MultiPartArchiveWriter* colorWriter, bool highRes) { std::string fileName = GetFileName(arFrame->GetFrameId(), "jpg"); #ifndef TURBO_JPEG @@ -66,8 +69,7 @@ namespace openVulkanoCpp::AR tjhandle handle = tjInitCompress(); const uint8_t* buffers[3]; std::unique_ptr dataBuffer; - bool downsampleRecording = true; - if (downsampleRecording) + if (m_settings.downsampleColor && !highRes) { dataBuffer = YuvUtils::PlansFromNV12(static_cast(img.luminescenceOrColor.data), static_cast(img.uv.data), resX, resY, img.uv.resolution.x, img.uv.resolution.y, 2, 2); @@ -91,12 +93,12 @@ namespace openVulkanoCpp::AR if (tjCompressFromYUVPlanes(handle, buffers, resX, nullptr, resY, TJSAMP_420, &outBuffer, &size, 95, TJFLAG_FASTDCT)) Logger::AR->error("Failed to create JPEG! {}", tjGetErrorStr()); else - m_colorWriter->AddFile(fileName.c_str(), outBuffer, size); + colorWriter->AddFile(fileName.c_str(), outBuffer, size); tjFree(outBuffer); #endif } - void ArRecorder::WriteDepthImage(ArFrame* arFrame) + void ArRecorder::WriteDepthImage(ArFrame* arFrame, MultiPartArchiveWriter* depthWriter, MultiPartArchiveWriter* confWriter) { if (!m_depthWriter || !m_confidenceWriter) return; auto depthImg = arFrame->GetDepthImage(); @@ -111,7 +113,7 @@ namespace openVulkanoCpp::AR buffers[1].second = depthImg.depth.resolution.x * depthImg.depth.resolution.y * sizeof(float); const std::string fileName = GetFileName(arFrame->GetFrameId(), "pfm"); - m_depthWriter->AddFile(fileName.c_str(), buffers); + depthWriter->AddFile(fileName.c_str(), buffers); } { @@ -124,29 +126,31 @@ namespace openVulkanoCpp::AR buffers[1].second = static_cast(depthImg.confidence.resolution.x * depthImg.confidence.resolution.y); const std::string fileName = GetFileName(arFrame->GetFrameId(), "pgm"); - m_confidenceWriter->AddFile(fileName.c_str(), buffers); + confWriter->AddFile(fileName.c_str(), buffers); } } - void ArRecorder::Save(ArFrame* frame) + void ArRecorder::Write(ArFrame* frame, bool highRes) { - if (!m_recording || frame->IsSaved()) return; + if (frame->IsSaved()) return; frame->SetSaved(); + bool useHighResWriter = highRes && m_settings.highResFramesInSeparateArchive; BlockProfiler profile("Save AR Frame"); - - { - BlockProfiler profile("Save AR Frame - Image"); - WriteColorImage(frame); - } - { - //BlockProfiler profile("Save AR Frame - Depth"); - WriteDepthImage(frame); - } { BlockProfiler profileMeta("Save AR Frame - Meta"); std::string metaContent = frame->GetFrameMetadata().ToXML(); std::string fileName = GetFileName(frame->GetFrameId(), "meta"); - m_metadataWriter->AddFile(fileName.c_str(), metaContent.c_str(), metaContent.size()); + (useHighResWriter ? m_highResWriter : m_metadataWriter)->AddFile(fileName.c_str(), metaContent.c_str(), metaContent.size()); + } + { + BlockProfiler profile("Save AR Frame - Image"); + WriteColorImage(frame, useHighResWriter ? m_highResWriter.get() : m_colorWriter.get(), highRes); + } + { + //BlockProfiler profile("Save AR Frame - Depth"); + WriteDepthImage(frame, + useHighResWriter ? m_highResWriter.get() : m_depthWriter.get(), + useHighResWriter ? m_highResWriter.get() : m_confidenceWriter.get()); } } @@ -154,12 +158,13 @@ namespace openVulkanoCpp::AR { if (!m_colorWriter) { - m_colorWriter = std::make_unique(m_path, "color_{:05d}.tar", ArchiveConfig::TAR); - m_depthWriter = std::make_unique(m_path, "depth_{:05d}.tar", ArchiveConfig::TAR); - m_confidenceWriter = std::make_unique(m_path, "confidence_{:05d}.tar", ArchiveConfig::TAR_GZ); - m_metadataWriter = std::make_unique(m_path, "meta_{:05d}.tar", ArchiveConfig::TAR_GZ); + m_colorWriter = std::make_unique(m_settings.path, "color_{:05d}.tar", ArchiveConfig::TAR, m_settings.archiveSize, true); + m_depthWriter = std::make_unique(m_settings.path, "depth_{:05d}.tar", ArchiveConfig::TAR, m_settings.archiveSize, true); + m_confidenceWriter = std::make_unique(m_settings.path, "confidence_{:05d}.tar.gz", ArchiveConfig::TAR_GZ, m_settings.archiveSize, true); + m_metadataWriter = std::make_unique(m_settings.path, "meta_{:05d}.tar.gz", ArchiveConfig::TAR_GZ, m_settings.archiveSize, true); + m_highResWriter = std::make_unique(m_settings.path, "highres_{:05d}.tar", ArchiveConfig::TAR, m_settings.archiveSize, true); - std::ofstream platformInfoStream(m_path / "ArRecording.xml"); + std::ofstream platformInfoStream(m_settings.path / "ArRecording.xml"); platformInfoStream << m_session->GetSessionMetadata().ToXML(); platformInfoStream.close(); } @@ -170,7 +175,7 @@ namespace openVulkanoCpp::AR { if (!m_recording) return; m_recording = false; - for(MultiPartArchiveWriter* writer : { m_colorWriter.get(), m_depthWriter.get(), m_confidenceWriter.get(), m_metadataWriter.get() }) + for(MultiPartArchiveWriter* writer : { m_colorWriter.get(), m_depthWriter.get(), m_confidenceWriter.get(), m_metadataWriter.get(), m_highResWriter.get() }) { writer->Split(); } @@ -180,14 +185,49 @@ namespace openVulkanoCpp::AR { if (!m_colorWriter) { - for (MultiPartArchiveWriter* writer: {m_colorWriter.get(), m_depthWriter.get(), m_confidenceWriter.get(), m_metadataWriter.get()}) + for (MultiPartArchiveWriter* writer: { m_colorWriter.get(), m_depthWriter.get(), m_confidenceWriter.get(), m_metadataWriter.get(), m_highResWriter.get() }) { writer->Move(path); } - std::filesystem::rename(m_path / "ArRecording.xml", path + "/ArRecording.xml"); + std::filesystem::rename(m_settings.path / "ArRecording.xml", path + "/ArRecording.xml"); } m_persistent = true; - m_path = path; + m_settings.path = path; + } + + void ArRecorder::SetRecordingMode(RecordingMode mode) + { + if (m_settings.recordingMode == mode) return; + if (m_settings.recordingMode == RecordingMode::NEW_FRAME && m_newFrameHandler) + { + m_session->OnNewFrame -= m_newFrameHandler; + m_newFrameHandler = nullptr; + } + m_settings.recordingMode = mode; + if (m_settings.recordingMode == RecordingMode::NEW_FRAME) + { + m_newFrameHandler = m_session->OnNewFrame += EventHandler(this, &ArRecorder::Save); + } + } + + void ArRecorder::Save(const std::shared_ptr& frame) + { + if (!m_recording) return; + if (m_settings.asyncRecording) + { + + } + else Write(frame.get()); + } + + void ArRecorder::SaveHighResolution(const std::shared_ptr& frame) + { + if (!m_recording) return; + if (m_settings.asyncRecording) + { + + } + else Write(frame.get(), true); } } diff --git a/openVulkanoCpp/AR/ArRecorder.hpp b/openVulkanoCpp/AR/ArRecorder.hpp index e63d840..81054f1 100644 --- a/openVulkanoCpp/AR/ArRecorder.hpp +++ b/openVulkanoCpp/AR/ArRecorder.hpp @@ -6,12 +6,14 @@ #pragma once +#include "Math/ByteSize.hpp" #include #include #include namespace openVulkanoCpp { + class IEventHandler; class MultiPartArchiveWriter; } @@ -36,23 +38,37 @@ namespace openVulkanoCpp::AR NEW_FRAME }; + struct RecordingSettings + { + RecordingMode recordingMode = RecordingMode::MANUAL; + std::filesystem::path path; + size_t archiveSize = 2_GiB; + bool downsampleColor = false; + bool highResFramesInSeparateArchive = true; + bool asyncRecording = false; + }; + class ArRecorder final { ArSession* m_session; - RecordingMode m_recordingMode = RecordingMode::MANUAL; - std::filesystem::path m_path; - std::unique_ptr m_colorWriter, m_depthWriter, m_confidenceWriter, m_metadataWriter; + std::unique_ptr m_colorWriter, m_depthWriter, m_confidenceWriter, m_metadataWriter, m_highResWriter; + RecordingSettings m_settings; bool m_recording = false, m_persistent = false; - void WriteColorImage(ArFrame* arFrame); - void WriteDepthImage(ArFrame *arFrame); + IEventHandler* m_newFrameHandler = nullptr; + + void Write(ArFrame* frame, bool highRes = false); + void WriteColorImage(ArFrame* arFrame, MultiPartArchiveWriter* colorWriter, bool highRes); + void WriteDepthImage(ArFrame *arFrame, MultiPartArchiveWriter* depthWriter, MultiPartArchiveWriter* confWriter); public: ArRecorder(ArSession* session); ~ArRecorder(); - void Save(ArFrame* frame); + void Save(const std::shared_ptr& frame); + + void SaveHighResolution(const std::shared_ptr& frame); /** * Starts the recording of the owning AR session @@ -76,7 +92,7 @@ namespace openVulkanoCpp::AR * Gets the current recording path * @return Current recording dir */ - [[nodiscard]] const std::filesystem::path& GetRecordingPath() const { return m_path; } + [[nodiscard]] const std::filesystem::path& GetRecordingPath() const { return m_settings.path; } /** * Checks if a path to be used for the recording has been set. @@ -95,12 +111,44 @@ namespace openVulkanoCpp::AR * Sets the used recording mode * @param mode The mode that should be used to record the current session */ - void SetRecordingMode(RecordingMode mode) { m_recordingMode = mode; } + void SetRecordingMode(RecordingMode mode); /** * Checks the currently used recording mode for the AR session * @return The currently used recording mode */ - [[nodiscard]] RecordingMode SetRecordingMode() const { return m_recordingMode; } + [[nodiscard]] RecordingMode GetRecordingMode() const { return m_settings.recordingMode; } + + /** + * If enabled color images will be reduced to 1/4 of the original resolution (both axes will have half the resolution). + * This option can be used to reduce the used storage space and storage bandwidth during recording. + * + * @param downsample true = downsample images; false = no downsampling + */ + void SetDownsampleColorImages(bool downsample = true) { m_settings.downsampleColor = downsample; } + + /** + * Checks if color image downsampling is enabled. + * @return true = downsample images; false = no downsampling + */ + bool GetDownsampleColorImages() const { return m_settings.downsampleColor; } + + /** + * If enabled the requested high resolution images will be stored in a separate archive ("highres.tar") instead of the normal ar recording archives. + * @param separateArchive true = using the separate archive file; false = using the normal archive files + */ + void SetHighResFramesInSeparateArchive(bool separateArchive = true) { m_settings.highResFramesInSeparateArchive = separateArchive; } + + /** + * Checks if a separate archive is used for the high res frames. + * @return true = using the separate archive file; false = using the normal archive files + */ + bool GetHighResFramesInSeparateArchive() const { return m_settings.highResFramesInSeparateArchive; } + + void SetArchivePartMaxFileSize(size_t maxPartSize = 2_GiB) { m_settings.archiveSize = maxPartSize; } + + size_t GetArchivePartMaxFileSize() const { m_settings.archiveSize; } + + const RecordingSettings& GetRecordingSettings() const { return m_settings; } }; } diff --git a/openVulkanoCpp/AR/ArSession.hpp b/openVulkanoCpp/AR/ArSession.hpp index 97ba562..19b3f18 100644 --- a/openVulkanoCpp/AR/ArSession.hpp +++ b/openVulkanoCpp/AR/ArSession.hpp @@ -203,6 +203,7 @@ namespace openVulkanoCpp::AR Event<> OnNewFrameAvailable; Event&> OnNewFrame; + Event&> OnNewFrameHighResolution; Event OnSessionInterruptionChange; Event OnTrackingStateChanged; Event OnNewCameraTransformation; diff --git a/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm b/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm index fbf9f7b..165b9fa 100644 --- a/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm +++ b/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm @@ -74,6 +74,8 @@ namespace openVulkanoCpp::AR::ArKit std::shared_ptr ArSessionArKitInternal::GetFrame() { + if (GetRecorder().GetRecordingMode() == RecordingMode::FRAME_REQUEST) + GetRecorder().Save(m_frame); return m_frame; } @@ -99,7 +101,6 @@ namespace openVulkanoCpp::AR::ArKit } OnNewFrameAvailable(); m_frame = arFrame; - GetRecorder().Save((ArFrameArKit*)arFrame.get()); } void ArSessionArKitInternal::OnArSessionInterruptedChanged(ARSession* session, bool interrupted)