diff --git a/openVulkanoCpp/AR/ArFrame.cpp b/openVulkanoCpp/AR/ArFrame.cpp new file mode 100644 index 0000000..0de779c --- /dev/null +++ b/openVulkanoCpp/AR/ArFrame.cpp @@ -0,0 +1,16 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#include "ArFrame.h" +#include "ArSession.h" + +namespace openVulkanoCpp::AR +{ + float ArFrame::GetConfidenceNormalisationFactor() const + { + return session->GetSessionMetadata().GetConfidenceNormalisationFactor(); + } +} \ No newline at end of file diff --git a/openVulkanoCpp/AR/ArFrame.h b/openVulkanoCpp/AR/ArFrame.h index 2897a97..09cc6ed 100644 --- a/openVulkanoCpp/AR/ArFrame.h +++ b/openVulkanoCpp/AR/ArFrame.h @@ -12,9 +12,13 @@ #include "Math/CameraIntrinsic.hpp" #include "ArDepthFormat.hpp" #include "ArTrackingState.hpp" +#include "ArFrameMetadata.hpp" +#include namespace openVulkanoCpp::AR { + class ArSession; + class ArImage { public: @@ -82,32 +86,49 @@ namespace openVulkanoCpp::AR class ArFrame { + std::shared_ptr session; + protected: - ArFrame() = default; + ArFrameMetadata frameMetadata; + + ArFrame(const std::shared_ptr& session) : session(session) + {} public: virtual ~ArFrame() = default; - [[nodiscard]] virtual ArTrackingState GetTrackingState() = 0; + [[nodiscard]] const ArFrameMetadata& GetFrameMetadata() const { return frameMetadata; } - [[nodiscard]] virtual Math::PoseF GetPose() = 0; + [[nodiscard]] const std::shared_ptr& GetArSession() const { return session; } - [[nodiscard]] virtual Math::Timestamp GetTimestamp() = 0; + [[nodiscard]] ArTrackingState GetTrackingState() const { return frameMetadata.trackingState; }; - //[[nodiscard]] virtual Math::Timestamp GetTimestampDepth() = 0; + [[nodiscard]] virtual Math::PoseF GetPose() const {}; //TODO + + [[nodiscard]] Math::Timestamp GetTimestamp() const { return frameMetadata.timestamp; }; + + [[nodiscard]] Math::Timestamp GetTimestampDepth() const { return frameMetadata.timestampDepth; }; [[nodiscard]] virtual ArImagePlanar GetCameraImage() = 0; [[nodiscard]] virtual ArDepthImage GetDepthImage() = 0; - [[nodiscard]] virtual const Math::Matrix4f& GetCameraTransformation() = 0; + [[nodiscard]] const Math::Matrix4f& GetCameraTransformation() const { return frameMetadata.transformation; }; [[nodiscard]] virtual Math::Matrix4f GetCameraViewForCurrentDeviceOrientation() = 0; + [[nodiscard]] const Math::Matrix4f& GetCameraProjection() const { return frameMetadata.projection; } + [[nodiscard]] virtual Math::Matrix4f GetCameraProjection(Math::Vector2f viewportSize, float near = 0.25f, float far = 250.0f) = 0; - [[nodiscard]] virtual float GetConfidenceNormalisationFactor() = 0; + [[nodiscard]] float GetConfidenceNormalisationFactor() const; - [[nodiscard]] virtual float GetColorTemperature() const = 0; + [[nodiscard]] float GetColorTemperature() const { return frameMetadata.lightColorTemp; }; + + [[nodiscard]] float GetLightIntensity() const { return frameMetadata.lightIntensity; }; + + [[nodiscard]] float GetExposureTime() const { return frameMetadata.exposureTime; }; + + [[nodiscard]] float GetExposureOffset() const { return frameMetadata.exposureOffset; }; }; } diff --git a/openVulkanoCpp/AR/ArFrameMetadata.cpp b/openVulkanoCpp/AR/ArFrameMetadata.cpp new file mode 100644 index 0000000..fbea473 --- /dev/null +++ b/openVulkanoCpp/AR/ArFrameMetadata.cpp @@ -0,0 +1,143 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#include "ArFrameMetadata.hpp" +#include +#include +#include + +namespace openVulkanoCpp::AR +{ + namespace + { + template + T ReadMat(const pugi::xml_node& matrixNode) + { + T mat(0); + auto rowNode = matrixNode.first_child(); + for (int row = 0; row < S && rowNode; row++, rowNode = rowNode.next_sibling()) + { + auto columnNode = rowNode.first_child(); + for (int column = 0; column < S && columnNode; column++, columnNode = columnNode.next_sibling()) + { + mat[column][row] = columnNode.text().as_float(); + } + } + return mat; + } + + template + void MatToXML(const glm::mat& mat, std::ostream& stream, const std::string& nl = "\n") + { + auto nlRow = nl + "\t"; + stream << nl; + for(int r = 0 ; r < SIZE; r++) + { + stream << "" << nlRow; + for (int c = 0; c < SIZE; c++) + { + stream << "" << mat[c][r] << ""; + } + stream << nl << ""; + } + } + + template + void MatToYaml(const glm::mat& mat, std::ostream& stream, const std::string& nl = "\n") + { + for(int r = 0 ; r < SIZE; r++) + { + stream << nl << "- ["; + for (int c = 0; c < SIZE; c++) + { + if (c) stream << ','; + stream << ' ' << mat[c][r]; + } + stream << " ]"; + } + } + } + + ArFrameMetadata ArFrameMetadata::FromXML(const char* xml, size_t length) + { + ArFrameMetadata frameData; + + pugi::xml_document doc; + pugi::xml_parse_result result = doc.load_buffer(xml, length); + if (!result) throw std::runtime_error("Failed to parse frame metadata"); + pugi::xml_node nodeFrame = doc.child("arframe"); + + pugi::xml_node nodeCamera = nodeFrame.child("camera"); + frameData.transformation = ReadMat(nodeCamera.child("transform")); + frameData.projection = ReadMat(nodeCamera.child("projection")); + pugi::xml_node nodeRes = nodeCamera.child("resolution"); + Math::Vector2i resolution = { nodeRes.child("width").text().as_int(), nodeRes.child("height").text().as_int() }; + frameData.intrinsic = { ReadMat(nodeCamera.child("intrinsics")), resolution }; + frameData.exposureTime = nodeCamera.child("exposureDuration").text().as_float(); + frameData.exposureOffset = nodeCamera.child("exposureOffset").text().as_float(); + + pugi::xml_node nodeLight = nodeFrame.child("light"); + frameData.lightIntensity = nodeLight.child("ambientIntensity").text().as_float(); + frameData.lightColorTemp = nodeLight.child("ambientColorTemp").text().as_float(); + + frameData.timestamp = nodeFrame.child("timestamp").text().as_double(); + frameData.trackingState = ArTrackingState::GetFromName(nodeFrame.child("trackingState").child("camera").text().as_string()); + + return frameData; + } + + ArFrameMetadata ArFrameMetadata::FromYaml(const char* xml, size_t length) + { + ArFrameMetadata frameData; + //TODO + return frameData; + } + + std::string ArFrameMetadata::ToYaml() const + { + std::stringstream meta; + meta << std::setprecision(std::numeric_limits::digits10); + + meta << "camera:\n transform: "; + MatToYaml(transformation, meta, "\n "); + meta << "\n projection"; + MatToYaml(projection, meta, "\n "); + meta << "\n resolution:\n width: " << intrinsic.GetResolution().x << "\n height: " << intrinsic.GetResolution().y; + meta << "\n intrinsics: "; + MatToYaml(intrinsic.cameraMatrix, meta, "\n "); + meta << "\n exposureDuration: " << exposureTime << "\n exposureOffset: " << exposureOffset; + meta << "\ntimestamp: " << timestamp.GetSeconds(); + meta << "\ntimestampDepth: " << timestampDepth.GetNanos(); + meta << "\ntrackingState: \n camera: " << trackingState.GetName(); + meta << "\nlight:\n intensity: " << lightIntensity << "\n colorTemp: " << lightColorTemp; + + return meta.str(); + } + + std::string ArFrameMetadata::ToXML() const + { + std::stringstream meta; + meta << std::setprecision(std::numeric_limits::digits10); + + meta << "\n\t\n\t\t"; + MatToXML(transformation, meta, "\n\t\t\t"); + meta << "\n\t\t\n\t\t"; + MatToXML(projection, meta, "\n\t\t\t"); + meta << "\n\t\t\n\t\t\n\t\t\t" << intrinsic.GetResolution().x << "\n\t\t\t"; + meta << "" << intrinsic.GetResolution().y << "\n\t\t\n\t\t"; + MatToXML(intrinsic.cameraMatrix, meta, "\n\t\t\t"); + meta << "\n\t\t" << exposureTime << "\n\t\t" << exposureOffset; + meta << "\n\t\n\t"; + meta << "" << timestamp.GetSeconds() << "\n\t"; + meta << "" << timestampDepth.GetSeconds() << "\n\t"; + meta << "" << "\n\t\t" << trackingState.GetName() << "" << "\n\t"; + meta << "\n\t\t" << lightIntensity << "\n\t\t"; + meta << "" << lightColorTemp << "\n\t"; + meta << "\n"; + + return meta.str(); + } +} \ No newline at end of file diff --git a/openVulkanoCpp/AR/ArFrameMetadata.hpp b/openVulkanoCpp/AR/ArFrameMetadata.hpp new file mode 100644 index 0000000..d0dd39e --- /dev/null +++ b/openVulkanoCpp/AR/ArFrameMetadata.hpp @@ -0,0 +1,31 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#pragma once + +#include "ArTrackingState.hpp" +#include "Math/CameraIntrinsic.hpp" +#include "Math/Timestamp.hpp" + +namespace openVulkanoCpp::AR +{ + struct ArFrameMetadata + { + Math::CameraIntrinsicWithResolution intrinsic; + Math::Matrix4f transformation, projection; + float exposureTime, exposureOffset, lightIntensity, lightColorTemp; + Math::Timestamp timestamp, timestampDepth; + ArTrackingState trackingState; + + [[nodiscard]] std::string ToYaml() const; + + [[nodiscard]] std::string ToXML() const; + + static ArFrameMetadata FromXML(const char* xml, size_t length); + + static ArFrameMetadata FromYaml(const char* yaml, size_t length); + }; +} \ No newline at end of file diff --git a/openVulkanoCpp/AR/ArSession.cpp b/openVulkanoCpp/AR/ArSession.cpp index 89be2c1..456c1c8 100644 --- a/openVulkanoCpp/AR/ArSession.cpp +++ b/openVulkanoCpp/AR/ArSession.cpp @@ -56,12 +56,11 @@ namespace openVulkanoCpp::AR ArCreateResult ArSession::CreatePlayback(const std::string& recordingPath, bool autoAdvance) { - return { nullptr, ArCreateResult::FAILED_NOT_SUPPORTED, "not yet implemented" }; try { - /*const auto session = std::make_shared(recordingPath, autoAdvance); + const auto session = std::make_shared(recordingPath, autoAdvance); sessions.push_back(session); - return { session, ArCreateResult::SUCCESS, "" };*/ + return { session, ArCreateResult::SUCCESS, "" }; } catch (const std::exception& e) { // TODO error handling diff --git a/openVulkanoCpp/AR/ArSession.h b/openVulkanoCpp/AR/ArSession.h index 08e805a..e77e664 100644 --- a/openVulkanoCpp/AR/ArSession.h +++ b/openVulkanoCpp/AR/ArSession.h @@ -9,6 +9,7 @@ #include "ArType.hpp" #include "ArConstans.hpp" #include "ArTrackingState.hpp" +#include "ArSessionMetadata.hpp" #include "Math/Range.hpp" #include "Base/Event.hpp" #include @@ -71,6 +72,9 @@ namespace openVulkanoCpp::AR class ArSession { + protected: + ArSession(const ArSessionMetadata& metadata) : metadata(metadata) {} + public: /** * Creates a platform native AR session. nullptr if failed to create session. @@ -195,6 +199,8 @@ namespace openVulkanoCpp::AR [[nodiscard]] bool GetShouldAttemptRelocalization() { return shouldAttemptRelocalization; } + [[nodiscard]] const ArSessionMetadata& GetSessionMetadata() const { return metadata; } + void SetShouldAttemptRelocalization(bool attemptReloc) { shouldAttemptRelocalization = attemptReloc; } Event<> OnNewFrameAvailable; @@ -210,6 +216,7 @@ namespace openVulkanoCpp::AR bool running = false, paused = false, shouldAttemptRelocalization = false; ArSessionCapabilities capabilities; ArConstants constants; + ArSessionMetadata metadata; private: static std::vector> sessions; diff --git a/openVulkanoCpp/AR/ArSessionMetadata.cpp b/openVulkanoCpp/AR/ArSessionMetadata.cpp new file mode 100644 index 0000000..de67b75 --- /dev/null +++ b/openVulkanoCpp/AR/ArSessionMetadata.cpp @@ -0,0 +1,87 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#include "ArSessionMetadata.hpp" +#include +#include +#include +#include + +namespace openVulkanoCpp::AR +{ + ArSessionMetadata ArSessionMetadata::FromXML(const std::string& filePath) + { + pugi::xml_document doc; + pugi::xml_parse_result result = doc.load_file(filePath.c_str()); + if (!result) throw std::runtime_error("Failed to read metadata xml file."); + pugi::xml_node metaNode = doc.child("arPlatformInfo"); + const char* type = metaNode.child("type").text().as_string(); + int minRange = metaNode.child("minConfidence").text().as_int(); + int maxRange = metaNode.child("maxConfidence").text().as_int(); + const char* depthType = metaNode.child("depthType").text().as_string(); + return { ArType::GetFromHumanReadableName(type), + ArDepthFormat::GetFromAltName(depthType).value_or(ArDepthFormat::UNAVAILABLE), + Math::Range(minRange, maxRange) + }; + } + + ArSessionMetadata ArSessionMetadata::FromYaml(const std::string& filePath) + { + //TODO + /*YAML::Node meta = YAML::LoadFile(filePath); + + return { + ArType::GetFromName(meta["Type"].Scalar()), + ArDepthFormat::GetFromName(meta["DepthType"].Scalar()).value_or(ArDepthFormat::UNAVAILABLE), + Math::Range(meta["MinConfidence"].as(), meta["MaxConfidence"].as()) + };*/ + } + + ArSessionMetadata::ArSessionMetadata(const std::string& dirPath) + { + std::optional metaFromFile; + std::filesystem::path rPath(dirPath); + if (!is_directory(rPath)) throw std::runtime_error("Ar recording path must be a directory!"); + std::filesystem::path xmlInfoPath(dirPath + "/arType.info"); + if (std::filesystem::exists(xmlInfoPath)) + { + metaFromFile = FromXML(xmlInfoPath.string()); + } + xmlInfoPath = { dirPath + "/ArRecording.xml" }; + if (std::filesystem::exists(xmlInfoPath)) + { + metaFromFile = FromXML(xmlInfoPath.string()); + } + std::filesystem::path ymlInfoPath(dirPath + "/ArRecording.yml"); + if (std::filesystem::exists(ymlInfoPath)) + { + metaFromFile = FromYaml(ymlInfoPath.string()); + } + if (!metaFromFile) throw std::runtime_error("Ar recording dir is missing platform metadata file!"); + type = metaFromFile->type; + depthFormat = metaFromFile->depthFormat; + confidenceRange = metaFromFile->confidenceRange; + } + + std::string ArSessionMetadata::ToXML() const + { + std::stringstream ss; + ss << "\n\t" << type.GetHumanReadableName() << "\n\t"; + ss << static_cast(confidenceRange.min) << "\n\t"; + ss << static_cast(confidenceRange.max) << "\n\t"; + ss << depthFormat.GetAltName() << "\n"; + return ss.str(); + } + + std::string ArSessionMetadata::ToYaml() const + { + std::stringstream ss; + ss << "Type: " << type.GetName() << "\nDepthType: " << depthFormat.GetName(); + ss << "\nMinConfidence: " << static_cast(confidenceRange.min); + ss << "\nMaxConfidence: " << static_cast(confidenceRange.max); + return ss.str(); + } +} \ No newline at end of file diff --git a/openVulkanoCpp/AR/ArSessionMetadata.hpp b/openVulkanoCpp/AR/ArSessionMetadata.hpp new file mode 100644 index 0000000..9f52b32 --- /dev/null +++ b/openVulkanoCpp/AR/ArSessionMetadata.hpp @@ -0,0 +1,44 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#pragma once + +#include "AR/ArType.hpp" +#include "AR/ArDepthFormat.hpp" +#include "Math/Range.hpp" + +namespace openVulkanoCpp::AR +{ + struct ArSessionMetadata + { + ArType type; + ArDepthFormat depthFormat; + Math::Range confidenceRange; + + ArSessionMetadata() noexcept + : ArSessionMetadata(ArType::UNKNOWN, ArDepthFormat::UNAVAILABLE, {0,0}) + {} + + ArSessionMetadata(ArType type, ArDepthFormat format, Math::Range confRange) noexcept + : type(type), depthFormat(format), confidenceRange(confRange) + {} + + ArSessionMetadata(const std::string& dirPath); + + [[nodiscard]] std::string ToXML() const; + + [[nodiscard]] std::string ToYaml() const; + + float GetConfidenceNormalisationFactor() const + { + return 1.0f / static_cast(confidenceRange.max); + } + + static ArSessionMetadata FromXML(const std::string& filePath); + + static ArSessionMetadata FromYaml(const std::string& filePath); + }; +} \ No newline at end of file diff --git a/openVulkanoCpp/AR/Provider/Network/ArSessionStream.cpp b/openVulkanoCpp/AR/Provider/Network/ArSessionStream.cpp index 70b42e5..1d60d4a 100644 --- a/openVulkanoCpp/AR/Provider/Network/ArSessionStream.cpp +++ b/openVulkanoCpp/AR/Provider/Network/ArSessionStream.cpp @@ -9,6 +9,7 @@ namespace openVulkanoCpp::AR::Network { ArSessionStream::ArSessionStream(const std::string& serverAddress, std::optional requestConfig) + : ArSession({}) { // TODO } diff --git a/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.hpp b/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.hpp index c3987e5..62f6faf 100644 --- a/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.hpp +++ b/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.hpp @@ -8,7 +8,6 @@ #include "AR/ArFrame.h" #include "ArPlaybackReader.hpp" -#include "ArPlaybackData.hpp" namespace openVulkanoCpp::AR::Playback { @@ -19,34 +18,20 @@ namespace openVulkanoCpp::AR::Playback public: ArFramePlayback(const std::shared_ptr& session, ArPlaybackReader& frameReader); - ArTrackingState GetTrackingState() override; - - Math::PoseF GetPose() override; - - Math::Timestamp GetTimestamp() override; - ArImagePlanar GetCameraImage() override; ArDepthImage GetDepthImage() override; - const Math::Matrix4f& GetCameraTransformation() override; - Math::Matrix4f GetCameraViewForCurrentDeviceOrientation() override; Math::Matrix4f GetCameraProjection(Math::Vector2f viewportSize, float near, float far) override; - float GetConfidenceNormalisationFactor() override; - - [[nodiscard]] float GetColorTemperature() const override; - private: - ArPlaybackFrameData frameData; ArImagePlanar colorImage; ArDepthImage depthImage; std::unique_ptr confImgData; std::unique_ptr depthImgData; ColorImg colorImgData; - //cv::Mat colorImgData; }; }