Add ArRecorder logic

openVulkanoCpp/AR/ArFrame.cpp

@@ -6,11 +6,17 @@
 
 #include "ArFrame.hpp"
 #include "ArSession.hpp"
+#include "ArRecorder.hpp"
 
 namespace openVulkanoCpp::AR
 {
 	float ArFrame::GetConfidenceNormalisationFactor() const
 	{
-		return session->GetSessionMetadata().GetConfidenceNormalisationFactor();
+		return m_session->GetSessionMetadata().GetConfidenceNormalisationFactor();
+	}
+
+	void ArFrame::Save()
+	{
+		m_session->GetRecorder().Save(this);
 	}
 }

openVulkanoCpp/AR/ArFrame.hpp

@@ -14,6 +14,7 @@
 #include "ArTrackingState.hpp"
 #include "ArFrameMetadata.hpp"
 #include <memory>
+#include <functional>
 
 namespace openVulkanoCpp::AR
 {
@@ -86,20 +87,24 @@ namespace openVulkanoCpp::AR
 
 	class ArFrame
 	{
-		std::shared_ptr<ArSession> session;
+		std::shared_ptr<ArSession> m_session;
+		size_t m_frameId;
+		bool m_saved = false;
 
 	protected:
 		ArFrameMetadata frameMetadata;
 
-		ArFrame(const std::shared_ptr<ArSession>& session) : session(session)
+		ArFrame(const std::shared_ptr<ArSession>& session, size_t frameId) : m_session(session), m_frameId(frameId)
 		{}
 
 	public:
 		virtual ~ArFrame() = default;
 
+		[[nodiscard]] size_t GetFrameId() const { return m_frameId; }
+
 		[[nodiscard]] const ArFrameMetadata& GetFrameMetadata() const { return frameMetadata; }
 
-		[[nodiscard]] const std::shared_ptr<ArSession>& GetArSession() const { return session; }
+		[[nodiscard]] const std::shared_ptr<ArSession>& GetArSession() const { return m_session; }
 
 		[[nodiscard]] ArTrackingState GetTrackingState() const { return frameMetadata.trackingState; };
 
@@ -130,5 +135,19 @@ namespace openVulkanoCpp::AR
 		[[nodiscard]] float GetExposureTime() const { return frameMetadata.exposureTime; };
 
 		[[nodiscard]] float GetExposureOffset() const { return frameMetadata.exposureOffset; };
+
+		[[nodiscard]] bool IsSaved() const { return m_saved; };
+
+		void Save();
+
+		void SetSaved() { m_saved = true; }
+
+		/**
+		 * Uses the platforms native jpeg writer to produce a jpeg representation of the camera image.
+		 * This might not be implemented
+		 * @param handler Function to be called once the image has been encoded
+		 * @return true if image was encoded, false if not or unavailable
+		 */
+		virtual bool GetCameraImageAsJpeg(const std::function<void(const char*, size_t)>& handler) { return false; };
 	};
 }

openVulkanoCpp/AR/ArRecorder.cpp

@@ -0,0 +1,193 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#include "ArRecorder.hpp"
+#include "ArSession.hpp"
+#include "ArFrame.hpp"
+#include "IO/Archive/MultiPartArchiveWriter.hpp"
+#include "IO/Files/Pfm.hpp"
+#include "IO/Files/Pnm.hpp"
+#include "IO/AppFolders.hpp"
+#include "Base/BlockProfiler.hpp"
+#include "Image/YuvUtils.hpp"
+#include <fstream>
+#include <iostream>
+//#if __has_include("turbojpeg.h")
+#include <turbojpeg.h>
+#define TURBO_JPEG
+//#endif
+
+namespace openVulkanoCpp::AR
+{
+	namespace
+	{
+		std::filesystem::path GeneratePath(const std::filesystem::path& baseDir, std::string_view name)
+		{
+			std::stringstream ss;
+			auto t = std::time(nullptr);
+			auto localTime = *std::localtime(&t);
+			ss << std::put_time(&localTime, "%Y-%m-%d_%H_%M_%S");
+			return baseDir / name / ss.str();
+		}
+
+		std::string GetFileName(size_t frameId, std::string_view fileExtension)
+		{
+			std::stringstream fnStream;
+			fnStream << std::setw(7) << std::setfill('0') << frameId << '.' << fileExtension;
+			return fnStream.str();
+		}
+	}
+
+	ArRecorder::ArRecorder(ArSession* session)
+			: m_session(session), m_path(GeneratePath(AppFolders::GetAppDataHomeDir(), "ar_recording"))
+	{}
+
+	ArRecorder::~ArRecorder() = default;
+
+	void ArRecorder::WriteColorImage(ArFrame* arFrame)
+	{
+		std::string fileName = GetFileName(arFrame->GetFrameId(), "jpg");
+#ifndef TURBO_JPEG
+		if (arFrame->GetCameraImageAsJpeg([&fileName, this](const char* data, size_t len){ m_colorWriter->AddFile(fileName.c_str(), data, len); }))
+			return;
+		//TODO stb???
+		Logger::AR->error("Failed to create JPEG! Missing turbojpeg.h");
+#else
+		//TODO handle non nv12 images
+		auto img = arFrame->GetCameraImage();
+		auto sizeLum = img.luminescenceOrColor.resolution.x * img.luminescenceOrColor.resolution.y;
+		auto sizeUV = img.uv.resolution.x * img.uv.resolution.y;
+		auto resX = img.luminescenceOrColor.resolution.x;
+		auto resY = img.luminescenceOrColor.resolution.y;
+
+		tjhandle handle = tjInitCompress();
+		const uint8_t* buffers[3];
+		std::unique_ptr<uint8_t[]> dataBuffer;
+		bool downsampleRecording = true;
+		if (downsampleRecording)
+		{
+			dataBuffer = YuvUtils::PlansFromNV12(static_cast<uint8_t*>(img.luminescenceOrColor.data), static_cast<uint8_t*>(img.uv.data),
+												 resX, resY, img.uv.resolution.x, img.uv.resolution.y, 2, 2);
+			resX /= 2;
+			resY /= 2;
+			buffers[0] = dataBuffer.get();
+			buffers[1] = buffers[0] + sizeLum / 4;
+			buffers[2] = buffers[1] + sizeUV / 4;
+		}
+		else
+		{
+			dataBuffer = std::unique_ptr<uint8_t[]>(new uint8_t[sizeUV + sizeUV]);
+			YuvUtils::ChromaPlanesFromNV12((uint8_t*)img.uv.data, dataBuffer.get(), sizeUV);
+			buffers[0] = static_cast<uint8_t*>(img.luminescenceOrColor.data);
+			buffers[1] = dataBuffer.get();
+			buffers[2] = buffers[1] + sizeUV;
+		}
+
+		uint8_t* outBuffer = nullptr;
+		unsigned long size = 0;
+		if (tjCompressFromYUVPlanes(handle, buffers, resX, nullptr, resY, TJSAMP_420, &outBuffer, &size, 95, TJFLAG_FASTDCT))
+			Logger::AR->error("Failed to create JPEG! {}", tjGetErrorStr());
+		else
+			m_colorWriter->AddFile(fileName.c_str(), outBuffer, size);
+		tjFree(outBuffer);
+#endif
+	}
+
+	void ArRecorder::WriteDepthImage(ArFrame* arFrame)
+	{
+		if (!m_depthWriter || !m_confidenceWriter) return;
+		auto depthImg = arFrame->GetDepthImage();
+		std::vector<std::pair<const void*, size_t>> buffers(2);
+		{ // TODO handle alternative depth formats!!!!
+			BlockProfiler profile("Save AR Frame - Depth");
+			PfmHeader depthHeader(static_cast<uint32_t>(depthImg.depth.resolution.x), static_cast<uint32_t>(depthImg.depth.resolution.y), 5.0f, false);
+			std::string header = depthHeader.ToString();
+			buffers[0].first = header.c_str();
+			buffers[0].second = header.size();
+			buffers[1].first = static_cast<const char *>(depthImg.depth.data);
+			buffers[1].second = depthImg.depth.resolution.x * depthImg.depth.resolution.y * sizeof(float);
+
+			const std::string fileName = GetFileName(arFrame->GetFrameId(), "pfm");
+			m_depthWriter->AddFile(fileName.c_str(), buffers);
+		}
+
+		{
+			BlockProfiler profile("Save AR Frame - Confi");
+			PnmHeader confidenceHeader(static_cast<uint32_t>(depthImg.confidence.resolution.x), static_cast<uint32_t>(depthImg.confidence.resolution.y), false, 2);
+			std::string header = confidenceHeader.ToString();
+			buffers[0].first = header.c_str();
+			buffers[0].second = header.size();
+			buffers[1].first = static_cast<const char *>(static_cast<void*>(depthImg.confidence.data));
+			buffers[1].second = static_cast<size_t>(depthImg.confidence.resolution.x * depthImg.confidence.resolution.y);
+
+			const std::string fileName = GetFileName(arFrame->GetFrameId(), "pgm");
+			m_confidenceWriter->AddFile(fileName.c_str(), buffers);
+		}
+	}
+
+	void ArRecorder::Save(ArFrame* frame)
+	{
+		if (!m_recording || frame->IsSaved()) return;
+		frame->SetSaved();
+		BlockProfiler profile("Save AR Frame");
+
+		{
+			BlockProfiler profile("Save AR Frame - Image");
+			WriteColorImage(frame);
+		}
+		{
+			//BlockProfiler profile("Save AR Frame - Depth");
+			WriteDepthImage(frame);
+		}
+		{
+			BlockProfiler profileMeta("Save AR Frame -  Meta");
+			std::string metaContent = frame->GetFrameMetadata().ToXML();
+			std::string fileName = GetFileName(frame->GetFrameId(), "meta");
+			m_metadataWriter->AddFile(fileName.c_str(), metaContent.c_str(), metaContent.size());
+		}
+	}
+
+	void ArRecorder::Start()
+	{
+		if (!m_colorWriter)
+		{
+			m_colorWriter = std::make_unique<MultiPartArchiveWriter>(m_path, "color_{:05d}.tar", ArchiveConfig::TAR);
+			m_depthWriter = std::make_unique<MultiPartArchiveWriter>(m_path, "depth_{:05d}.tar", ArchiveConfig::TAR);
+			m_confidenceWriter = std::make_unique<MultiPartArchiveWriter>(m_path, "confidence_{:05d}.tar", ArchiveConfig::TAR_GZ);
+			m_metadataWriter = std::make_unique<MultiPartArchiveWriter>(m_path, "meta_{:05d}.tar", ArchiveConfig::TAR_GZ);
+
+			std::ofstream platformInfoStream(m_path / "ArRecording.xml");
+			platformInfoStream << m_session->GetSessionMetadata().ToXML();
+			platformInfoStream.close();
+		}
+		m_recording = true;
+	}
+
+	void ArRecorder::Stop()
+	{
+		if (!m_recording) return;
+		m_recording = false;
+		for(MultiPartArchiveWriter* writer : { m_colorWriter.get(), m_depthWriter.get(), m_confidenceWriter.get(), m_metadataWriter.get() })
+		{
+			writer->Split();
+		}
+	}
+
+	void ArRecorder::SetRecordingPath(const std::string& path)
+	{
+		if (!m_colorWriter)
+		{
+			for (MultiPartArchiveWriter* writer: {m_colorWriter.get(), m_depthWriter.get(), m_confidenceWriter.get(), m_metadataWriter.get()})
+			{
+				writer->Move(path);
+			}
+			std::filesystem::rename(m_path / "ArRecording.xml", path + "/ArRecording.xml");
+		}
+
+		m_persistent = true;
+		m_path = path;
+	}
+}

openVulkanoCpp/AR/ArRecorder.hpp

@@ -8,9 +8,18 @@
 
 #include <string>
 #include <filesystem>
+#include <memory>
+
+namespace openVulkanoCpp
+{
+	class MultiPartArchiveWriter;
+}
 
 namespace openVulkanoCpp::AR
 {
+	class ArSession;
+	class ArFrame;
+
 	enum class RecordingMode
 	{
 		/**
@@ -27,23 +36,33 @@ namespace openVulkanoCpp::AR
 		NEW_FRAME
 	};
 
-	class ArRecorder
+	class ArRecorder final
 	{
-	protected:
+		ArSession* m_session;
-		ArRecorder() = default;
+		RecordingMode m_recordingMode = RecordingMode::MANUAL;
+		std::filesystem::path m_path;
+		std::unique_ptr<MultiPartArchiveWriter> m_colorWriter, m_depthWriter, m_confidenceWriter, m_metadataWriter;
+		bool m_recording = false, m_persistent = false;
+
+		void WriteColorImage(ArFrame* arFrame);
+		void WriteDepthImage(ArFrame *arFrame);
 
 	public:
-		virtual ~ArRecorder() = default;
+		ArRecorder(ArSession* session);
+
+		~ArRecorder();
+
+		void Save(ArFrame* frame);
 
 		/**
 		 * Starts the recording of the owning AR session
 		 */
-		virtual void Start() = 0;
+		void Start();
 
 		/**
 		 * Stops the recording of the owning AR session
 		 */
-		virtual void Stop() = 0;
+		void Stop();
 
 		/**
 		 * Sets the directory into which the AR recording should be stored.
@@ -51,40 +70,37 @@ namespace openVulkanoCpp::AR
 		 * If path is changed after starting the recording, the already running recording will be moved to the new path.
 		 * @param path The path to be used to store the recording
 		 */
-		virtual void SetRecordingPath(const std::string& path) = 0;
+		void SetRecordingPath(const std::string& path);
 
 		/**
 		 * Gets the current recording path
 		 * @return Current recording dir
 		 */
-		[[nodiscard]] virtual const std::filesystem::path& GetRecordingPath() const = 0;
+		[[nodiscard]] const std::filesystem::path& GetRecordingPath() const { return m_path; }
 
 		/**
 		 * Checks if a path to be used for the recording has been set.
 		 * If no path has been set the recording will be stored in a temporary location and will be deleted once the recording is stopped.
 		 * @return True if a path has been set and the recording will persist after stopping the recording.
 		 */
-		[[nodiscard]] virtual bool IsPersistent() const = 0;
+		[[nodiscard]] bool IsPersistent() const { return m_persistent; }
 
 		/**
 		 * Checks if the recording of the AR session is running
 		 * @return True if recording is started
 		 */
-		[[nodiscard]] virtual bool IsRecording() const = 0;
+		[[nodiscard]] bool IsRecording() const { return m_recording; }
 
 		/**
 		 * Sets the used recording mode
 		 * @param mode The mode that should be used to record the current session
 		 */
-		void SetRecordingMode(RecordingMode mode) { recordingMode = mode; }
+		void SetRecordingMode(RecordingMode mode) { m_recordingMode = mode; }
 
 		/**
 		 * Checks the currently used recording mode for the AR session
 		 * @return The currently used recording mode
 		 */
-		[[nodiscard]] RecordingMode SetRecordingMode() { return recordingMode; }
+		[[nodiscard]] RecordingMode SetRecordingMode() const { return m_recordingMode; }
-
-	protected:
-		RecordingMode recordingMode = RecordingMode::FRAME_REQUEST;
 	};
 }

openVulkanoCpp/AR/ArSession.cpp

@@ -5,7 +5,7 @@
  */
 
 #include "ArSession.hpp"
-#include "Provider/Playback/ArSessionPlayback.h"
+#include "Provider/Playback/ArSessionPlayback.hpp"
 #include "Provider/Network/ArSessionStream.h"
 #ifdef __APPLE__
 #include "Provider/ArKit/ArSessionArKit.h"

openVulkanoCpp/AR/ArSession.hpp

@@ -10,6 +10,7 @@
 #include "ArConstans.hpp"
 #include "ArTrackingState.hpp"
 #include "ArSessionMetadata.hpp"
+#include "ArRecorder.hpp"
 #include "Math/Range.hpp"
 #include "Base/Event.hpp"
 #include <memory>
@@ -22,7 +23,6 @@ namespace openVulkanoCpp::AR
 {
 	class ArSession;
 	class ArFrame;
-	class ArRecorder;
 
 	enum class ArSessionType
 	{
@@ -35,18 +35,16 @@ namespace openVulkanoCpp::AR
 		ArSessionType sessionType = ArSessionType::NATIVE;
 		bool uncompressed = true;
 		bool depthSupported = false;
-		bool recordingSupported = false;
 
 	public:
 		ArSessionCapabilities() = default;
 
 		ArSessionCapabilities(const ArType type, const ArSessionType sessionType, const bool uncompressed, const bool depthSupported, const bool recordingSupported)
-			: type(type), sessionType(sessionType), uncompressed(uncompressed), depthSupported(depthSupported), recordingSupported(recordingSupported)
+			: type(type), sessionType(sessionType), uncompressed(uncompressed), depthSupported(depthSupported)
 		{}
 
 		[[nodiscard]] bool IsUncompressed() const { return uncompressed; }
 		[[nodiscard]] bool IsDepthSupported() const { return depthSupported; }
-		[[nodiscard]] bool IsRecordingSupported() const { return recordingSupported; }
 		[[nodiscard]] bool IsNative() const { return sessionType == ArSessionType::NATIVE; }
 		[[nodiscard]] bool IsPlayback() const { return sessionType == ArSessionType::PLAYBACK; }
 		[[nodiscard]] bool IsStream() const { return sessionType == ArSessionType::NETWORK_STREAM; }
@@ -73,7 +71,7 @@ namespace openVulkanoCpp::AR
 	class ArSession
 	{
 	protected:
-		ArSession(const ArSessionMetadata& metadata) : metadata(metadata) {}
+		ArSession(const ArSessionMetadata& metadata) : metadata(metadata), recorder(this) {}
 
 	public:
 		/**
@@ -173,7 +171,7 @@ namespace openVulkanoCpp::AR
 		 * The ArRecorder for this ArSession.
 		 * @return ArRecorder instance or nullptr if the ArSession does not support recording.
 		 */
-		[[nodiscard]] virtual ArRecorder* GetRecorder() = 0;
+		[[nodiscard]] ArRecorder& GetRecorder() { return recorder; };
 
 		/**
 		 * Gets the type of the AR session.
@@ -219,6 +217,8 @@ namespace openVulkanoCpp::AR
 		ArSessionMetadata metadata;
 
 	private:
+		ArRecorder recorder;
+
 		static std::vector<std::weak_ptr<ArSession>> sessions;
 		static std::weak_ptr<ArSession> nativeSession;
 	};

openVulkanoCpp/AR/Provider/Network/ArSessionStream.h

@@ -25,8 +25,6 @@ namespace openVulkanoCpp::AR::Network
 
 		[[nodiscard]] std::shared_ptr<ArFrame> GetFrame() override;
 
-		[[nodiscard]] ArRecorder* GetRecorder() override { return nullptr; }
-
 		[[nodiscard]] ArSessionType GetSessionType() override { return ArSessionType::NETWORK_STREAM; }
 
 		[[nodiscard]] ArType GetArType() override;

openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp

@@ -5,13 +5,13 @@
  */
 
 #include "ArFramePlayback.hpp"
-#include "ArSessionPlayback.h"
+#include "ArSessionPlayback.hpp"
 #include "Base/BlockProfiler.hpp"
 
 namespace openVulkanoCpp::AR::Playback
 {
 	ArFramePlayback::ArFramePlayback(const std::shared_ptr<ArSessionPlayback>& session, ArPlaybackReader& frameReader)
-			: ArFrame(session)
+			: ArFrame(session, frameReader.GetNextFrameId())
 	{
 		BlockProfiler profile("Read_AR_Frame");
 		const auto data = frameReader.ReadMetadata();

openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp

@@ -4,7 +4,7 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
-#include "ArSessionPlayback.h"
+#include "ArSessionPlayback.hpp"
 #include "ArFramePlayback.hpp"
 #include "Base/Logger.hpp"
 #include <filesystem>
@@ -74,11 +74,6 @@ namespace openVulkanoCpp::AR::Playback
 		return nullptr;
 	}
 
-	ArRecorder* ArSessionPlayback::GetRecorder()
-	{
-		return nullptr;
-	}
-
 	ArType ArSessionPlayback::GetArType()
 	{
 		return capabilities.GetArType();

openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.hpp

@@ -27,8 +27,6 @@ class ArSessionPlayback final : public ArSession, public std::enable_shared_from
 
 		[[nodiscard]] std::shared_ptr<ArFrame> GetFrame() override;
 
-		[[nodiscard]] ArRecorder* GetRecorder() override;
-
 		[[nodiscard]] ArSessionType GetSessionType() override { return ArSessionType::PLAYBACK; }
 
 		[[nodiscard]] ArType GetArType() override;