Make AR playback frame loading async

openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm

@@ -34,9 +34,7 @@ namespace openVulkanoCpp::AR::ArKit
 				if (ss.gcount() > 1) ss << ", ";
 				ss << format.imageResolution.width << 'x' << format.imageResolution.height;
 				ss << '@' << format.framesPerSecond;
-				if (@available
-				(iOS
-				16.0, *))
+				if (@available(iOS 16.0, *))
 				{
 					if (format.videoHDRSupported) ss << "_HDR";
 					if (format.isRecommendedForHighResolutionFrameCapturing) ss << "_HdCaptureRecommended";
@@ -49,9 +47,7 @@ namespace openVulkanoCpp::AR::ArKit
 		void LogFormat(std::string_view formatUsage, ARVideoFormat* format)
 		{
 			bool hdr = false, hdCapture = false;
-			if (@available
-			(iOS
-			16.0, *))
+			if (@available(iOS 16.0, *))
 			{
 				hdr = format.videoHDRSupported;
 				hdCapture = format.isRecommendedForHighResolutionFrameCapturing;

openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp

@@ -13,7 +13,7 @@ namespace openVulkanoCpp::AR::Playback
 	ArFramePlayback::ArFramePlayback(const std::shared_ptr<ArSessionPlayback>& session, ArPlaybackReader& frameReader)
 			: ArFrame(session, frameReader.GetNextFrameId())
 	{
-		BlockProfiler profile("Read_AR_Frame");
+		//BlockProfiler profile("Read_AR_Frame");
 		const auto data = frameReader.ReadMetadata();
 		frameMetadata = ArFrameMetadata::FromXML(data.Data(), data.Size());
 		colorImgData = frameReader.ReadColorImage();

openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp

@@ -16,9 +16,15 @@ namespace openVulkanoCpp::AR::Playback
 	{
 		capabilities = ArSessionCapabilities(metadata.type, ArSessionType::PLAYBACK, false, metadata.depthFormat != ArDepthFormat::UNAVAILABLE);
 		constants = { Math::Matrix4f(1), metadata.confidenceRange };
+
+		m_playbackReaderThread = std::thread([this](){ReadWorker();});
 	}
 
-	ArSessionPlayback::~ArSessionPlayback() = default;
+	ArSessionPlayback::~ArSessionPlayback()
+	{
+		Stop();
+		if (m_playbackReaderThread.joinable()) m_playbackReaderThread.join();
+	}
 
 	void ArSessionPlayback::Start()
 	{
@@ -37,15 +43,29 @@ namespace openVulkanoCpp::AR::Playback
 
 	std::shared_ptr<ArFrame> ArSessionPlayback::GetFrame()
 	{
-		try
+		while(IsRunning() && !m_nextFrame) { std::this_thread::yield(); }
+		auto frame = m_nextFrame;
+		m_nextFrame = nullptr;
+		return frame;
+	}
+
+	ArType ArSessionPlayback::GetArType()
+	{
+		return capabilities.GetArType();
+	}
+
+	void ArSessionPlayback::ReadWorker()
+	{
+		while (playbackReader.HasNext() && IsRunning())
 		{
-			if (playbackReader.HasNext())
+			while (m_nextFrame) { std::this_thread::yield(); }
+			try
 			{
 				std::shared_ptr<ArFrame> frame = std::make_shared<ArFramePlayback>(shared_from_this(), playbackReader);
-				//if (lastTimestamp == frame->GetTimestamp()) return nullptr;
 				lastTimestamp = frame->GetTimestamp();
-
+				//TODO try to keep original frame timing
 				// Trigger events
+				OnNewFrameAvailable();
 				OnNewFrame(frame);
 				OnNewCameraTransformation(frame->GetCameraTransformation());
 				if (OnNewCameraViewMatrix.HasHandlers())
@@ -53,29 +73,14 @@ namespace openVulkanoCpp::AR::Playback
 					auto view = frame->GetCameraViewForCurrentDeviceOrientation();
 					OnNewCameraViewMatrix(view);
 				}
-				if (playbackReader.HasNext())
-				{
-					OnNewFrameAvailable();
-				}
-				else
-				{
-					OnSessionInterruptionChange(true);
-				}
-
-				return frame;
+				m_nextFrame = frame;
+			}
+			catch (const std::exception& e)
+			{
+				Logger::AR->error("Failed to read AR frame: {}", e.what());
 			}
-		}
-		catch (const std::exception& e)
-		{
-			Logger::AR->error("Failed to read AR frame: {}", e.what());
 		}
 		Stop();
 		OnSessionInterruptionChange(true);
-		return nullptr;
-	}
-
-	ArType ArSessionPlayback::GetArType()
-	{
-		return capabilities.GetArType();
 	}
 }

openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.hpp

@@ -32,9 +32,15 @@ class ArSessionPlayback final : public ArSession, public std::enable_shared_from
 		[[nodiscard]] ArType GetArType() override;
 
 	private:
+		void ReadWorker();
+
+
 		Math::Timestamp lastTimestamp;
 		const std::string recordingPath;
 		const bool autoAdvance;
 		ArPlaybackReader playbackReader;
+
+		std::shared_ptr<ArFrame> m_nextFrame;
+		std::thread m_playbackReaderThread;
 	};
 }