From 4572cfbf53d4f8e4cce026fb0bac30c25de57b08 Mon Sep 17 00:00:00 2001 From: GeorgH93 Date: Sun, 10 Sep 2023 13:59:11 +0200 Subject: [PATCH] Make AR playback frame loading async --- openVulkanoCpp/AR/ArRecorder.hpp | 2 +- .../Provider/ArKit/ArSessionArKitInternal.mm | 8 +-- .../AR/Provider/Playback/ArFramePlayback.cpp | 2 +- .../Provider/Playback/ArSessionPlayback.cpp | 55 ++++++++++--------- .../Provider/Playback/ArSessionPlayback.hpp | 6 ++ 5 files changed, 40 insertions(+), 33 deletions(-) diff --git a/openVulkanoCpp/AR/ArRecorder.hpp b/openVulkanoCpp/AR/ArRecorder.hpp index 8ce62e6..f107af8 100644 --- a/openVulkanoCpp/AR/ArRecorder.hpp +++ b/openVulkanoCpp/AR/ArRecorder.hpp @@ -50,7 +50,7 @@ namespace openVulkanoCpp::AR size_t archiveSize = 2_GiB; bool downsampleColor = true; bool highResFramesInSeparateArchive = true; - bool asyncRecording = false; + bool asyncRecording = true; }; class ArRecorder final diff --git a/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm b/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm index ee1b0ed..4c4beda 100644 --- a/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm +++ b/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm @@ -34,9 +34,7 @@ namespace openVulkanoCpp::AR::ArKit if (ss.gcount() > 1) ss << ", "; ss << format.imageResolution.width << 'x' << format.imageResolution.height; ss << '@' << format.framesPerSecond; - if (@available - (iOS - 16.0, *)) + if (@available(iOS 16.0, *)) { if (format.videoHDRSupported) ss << "_HDR"; if (format.isRecommendedForHighResolutionFrameCapturing) ss << "_HdCaptureRecommended"; @@ -49,9 +47,7 @@ namespace openVulkanoCpp::AR::ArKit void LogFormat(std::string_view formatUsage, ARVideoFormat* format) { bool hdr = false, hdCapture = false; - if (@available - (iOS - 16.0, *)) + if (@available(iOS 16.0, *)) { hdr = format.videoHDRSupported; hdCapture = format.isRecommendedForHighResolutionFrameCapturing; diff --git a/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp b/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp index 585c71a..89bc168 100644 --- a/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp +++ b/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp @@ -13,7 +13,7 @@ namespace openVulkanoCpp::AR::Playback ArFramePlayback::ArFramePlayback(const std::shared_ptr& session, ArPlaybackReader& frameReader) : ArFrame(session, frameReader.GetNextFrameId()) { - BlockProfiler profile("Read_AR_Frame"); + //BlockProfiler profile("Read_AR_Frame"); const auto data = frameReader.ReadMetadata(); frameMetadata = ArFrameMetadata::FromXML(data.Data(), data.Size()); colorImgData = frameReader.ReadColorImage(); diff --git a/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp b/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp index dd93563..54f5880 100644 --- a/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp +++ b/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp @@ -16,9 +16,15 @@ namespace openVulkanoCpp::AR::Playback { capabilities = ArSessionCapabilities(metadata.type, ArSessionType::PLAYBACK, false, metadata.depthFormat != ArDepthFormat::UNAVAILABLE); constants = { Math::Matrix4f(1), metadata.confidenceRange }; + + m_playbackReaderThread = std::thread([this](){ReadWorker();}); } - ArSessionPlayback::~ArSessionPlayback() = default; + ArSessionPlayback::~ArSessionPlayback() + { + Stop(); + if (m_playbackReaderThread.joinable()) m_playbackReaderThread.join(); + } void ArSessionPlayback::Start() { @@ -37,15 +43,29 @@ namespace openVulkanoCpp::AR::Playback std::shared_ptr ArSessionPlayback::GetFrame() { - try + while(IsRunning() && !m_nextFrame) { std::this_thread::yield(); } + auto frame = m_nextFrame; + m_nextFrame = nullptr; + return frame; + } + + ArType ArSessionPlayback::GetArType() + { + return capabilities.GetArType(); + } + + void ArSessionPlayback::ReadWorker() + { + while (playbackReader.HasNext() && IsRunning()) { - if (playbackReader.HasNext()) + while (m_nextFrame) { std::this_thread::yield(); } + try { std::shared_ptr frame = std::make_shared(shared_from_this(), playbackReader); - //if (lastTimestamp == frame->GetTimestamp()) return nullptr; lastTimestamp = frame->GetTimestamp(); - + //TODO try to keep original frame timing // Trigger events + OnNewFrameAvailable(); OnNewFrame(frame); OnNewCameraTransformation(frame->GetCameraTransformation()); if (OnNewCameraViewMatrix.HasHandlers()) @@ -53,29 +73,14 @@ namespace openVulkanoCpp::AR::Playback auto view = frame->GetCameraViewForCurrentDeviceOrientation(); OnNewCameraViewMatrix(view); } - if (playbackReader.HasNext()) - { - OnNewFrameAvailable(); - } - else - { - OnSessionInterruptionChange(true); - } - - return frame; + m_nextFrame = frame; + } + catch (const std::exception& e) + { + Logger::AR->error("Failed to read AR frame: {}", e.what()); } - } - catch (const std::exception& e) - { - Logger::AR->error("Failed to read AR frame: {}", e.what()); } Stop(); OnSessionInterruptionChange(true); - return nullptr; - } - - ArType ArSessionPlayback::GetArType() - { - return capabilities.GetArType(); } } diff --git a/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.hpp b/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.hpp index 617124f..9459277 100644 --- a/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.hpp +++ b/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.hpp @@ -32,9 +32,15 @@ class ArSessionPlayback final : public ArSession, public std::enable_shared_from [[nodiscard]] ArType GetArType() override; private: + void ReadWorker(); + + Math::Timestamp lastTimestamp; const std::string recordingPath; const bool autoAdvance; ArPlaybackReader playbackReader; + + std::shared_ptr m_nextFrame; + std::thread m_playbackReaderThread; }; }