From 5d7ef4914232d7d67f923e404f00bfa116801ef1 Mon Sep 17 00:00:00 2001 From: Georg Hagen Date: Wed, 8 Jan 2025 23:56:10 +0100 Subject: [PATCH] Add support for front facing ar experience --- openVulkanoCpp/AR/ArDepthFormat.hpp | 4 +-- openVulkanoCpp/AR/ArSession.hpp | 3 +- .../AR/Provider/ArKit/ArFrameArKit.h | 1 + .../AR/Provider/ArKit/ArFrameArKit.mm | 35 +++++++++++++++++-- .../Provider/ArKit/ArSessionArKitInternal.h | 2 +- .../Provider/ArKit/ArSessionArKitInternal.mm | 34 ++++++++++++------ 6 files changed, 61 insertions(+), 18 deletions(-) diff --git a/openVulkanoCpp/AR/ArDepthFormat.hpp b/openVulkanoCpp/AR/ArDepthFormat.hpp index 11600f1..b332c66 100644 --- a/openVulkanoCpp/AR/ArDepthFormat.hpp +++ b/openVulkanoCpp/AR/ArDepthFormat.hpp @@ -17,7 +17,7 @@ namespace OpenVulkano::AR }; public: - enum Format : uint8_t { UNAVAILABLE = 0, METER_FP32, METER_FP64, MILLIMETER_U16, MILLIMETER_U32 }; + enum Format : uint8_t { UNAVAILABLE = 0, METER_FP16, METER_FP32, METER_FP64, MILLIMETER_U16, MILLIMETER_U32 }; ArDepthFormat() : ArDepthFormat(UNAVAILABLE) {} @@ -56,4 +56,4 @@ namespace OpenVulkano::AR private: Format m_format; }; -} \ No newline at end of file +} diff --git a/openVulkanoCpp/AR/ArSession.hpp b/openVulkanoCpp/AR/ArSession.hpp index cb215ff..d3c61a8 100644 --- a/openVulkanoCpp/AR/ArSession.hpp +++ b/openVulkanoCpp/AR/ArSession.hpp @@ -75,6 +75,7 @@ namespace OpenVulkano::AR bool preferHDR = true; bool sceneReconstruction = false; bool planeDetection = false; + bool useFrontCamera = false; int preferredResolution = 0; }; @@ -265,4 +266,4 @@ namespace OpenVulkano::AR static std::vector> sessions; static std::weak_ptr nativeSession; }; -} \ No newline at end of file +} diff --git a/openVulkanoCpp/AR/Provider/ArKit/ArFrameArKit.h b/openVulkanoCpp/AR/Provider/ArKit/ArFrameArKit.h index ba8fb3c..d34967f 100644 --- a/openVulkanoCpp/AR/Provider/ArKit/ArFrameArKit.h +++ b/openVulkanoCpp/AR/Provider/ArKit/ArFrameArKit.h @@ -36,6 +36,7 @@ namespace OpenVulkano::AR::ArKit private: ARFrame* m_arKitFrame; bool m_lockedColor, m_lockedDepth; + uint8_t m_frameDepthQuality = 0; ArImagePlanar m_colorImage; ArDepthImage m_depthImage; }; diff --git a/openVulkanoCpp/AR/Provider/ArKit/ArFrameArKit.mm b/openVulkanoCpp/AR/Provider/ArKit/ArFrameArKit.mm index f93c12a..3adfa61 100644 --- a/openVulkanoCpp/AR/Provider/ArKit/ArFrameArKit.mm +++ b/openVulkanoCpp/AR/Provider/ArKit/ArFrameArKit.mm @@ -42,6 +42,10 @@ namespace OpenVulkano::AR::ArKit { return GetSize(arKitFrame.smoothedSceneDepth.depthMap); } + if(arKitFrame.capturedDepthData) + { + return GetSize(arKitFrame.capturedDepthData.depthDataMap); + } } return { -1, -1 }; } @@ -106,9 +110,22 @@ namespace OpenVulkano::AR::ArKit assert(format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); #endif - m_depthImage.format = ArDepthFormat::METER_FP32; + m_depthImage.format = ArDepthFormat::UNAVAILABLE; m_depthImage.depth.resolution = GetDepthSize(arKitFrame); - m_depthImage.confidence.resolution = m_depthImage.depth.resolution; + if (m_arKitFrame.sceneDepth || m_arKitFrame.smoothedSceneDepth) + { + m_depthImage.format = ArDepthFormat::METER_FP32; + m_depthImage.confidence.resolution = m_depthImage.depth.resolution; + } + else if (m_arKitFrame.capturedDepthData) + { + auto depthFormat = arKitFrame.capturedDepthData.depthDataType; + if (arKitFrame.capturedDepthData.depthDataType == kCVPixelFormatType_DepthFloat32) m_depthImage.format = ArDepthFormat::METER_FP32; + else if (arKitFrame.capturedDepthData.depthDataType == kCVPixelFormatType_DepthFloat16) m_depthImage.format = ArDepthFormat::METER_FP16; + else Logger::AR->info("Received depth frame with unsuported format: {} ({})", + depthFormat == kCVPixelFormatType_DisparityFloat16 ? "DisparityFloat16" : (depthFormat == kCVPixelFormatType_DisparityFloat32 ? "DisparityFloat32" : "Unknown"), depthFormat); + m_depthImage.confidence.resolution = { 1, 1 }; + } m_depthImage.intrinsic = frameMetadata.intrinsic.GetForResolution(m_depthImage.depth.resolution); } @@ -125,11 +142,15 @@ namespace OpenVulkano::AR::ArKit CVPixelBufferUnlockBaseAddress(m_arKitFrame.sceneDepth.depthMap, kCVPixelBufferLock_ReadOnly); CVPixelBufferUnlockBaseAddress(m_arKitFrame.sceneDepth.confidenceMap, kCVPixelBufferLock_ReadOnly); } - else + else if(m_arKitFrame.smoothedSceneDepth) { CVPixelBufferUnlockBaseAddress(m_arKitFrame.smoothedSceneDepth.depthMap, kCVPixelBufferLock_ReadOnly); CVPixelBufferUnlockBaseAddress(m_arKitFrame.smoothedSceneDepth.confidenceMap, kCVPixelBufferLock_ReadOnly); } + else + { + CVPixelBufferUnlockBaseAddress(m_arKitFrame.capturedDepthData.depthDataMap, kCVPixelBufferLock_ReadOnly); + } } [m_arKitFrame release]; } @@ -180,6 +201,14 @@ namespace OpenVulkano::AR::ArKit m_depthImage.confidence.data = CVPixelBufferGetBaseAddress(m_arKitFrame.smoothedSceneDepth.confidenceMap); m_lockedDepth = true; } + else if(m_arKitFrame.capturedDepthData) + { + CVPixelBufferLockBaseAddress(m_arKitFrame.capturedDepthData.depthDataMap, kCVPixelBufferLock_ReadOnly); + m_depthImage.depth.data = CVPixelBufferGetBaseAddress(m_arKitFrame.capturedDepthData.depthDataMap); + m_frameDepthQuality = static_cast(m_arKitFrame.capturedDepthData.depthDataQuality) + static_cast(m_arKitFrame.capturedDepthData.depthDataAccuracy); + m_depthImage.confidence.data = &m_frameDepthQuality; + m_lockedDepth = true; + } } } return m_depthImage; diff --git a/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.h b/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.h index 75c7e10..872baae 100644 --- a/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.h +++ b/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.h @@ -62,7 +62,7 @@ namespace OpenVulkano::AR::ArKit private: ArKitDelegate* m_arKitDelegate; - ARWorldTrackingConfiguration* m_arConfig; + ARConfiguration* m_arConfig; ARSession* m_arSession; Vulkan::MetalTextureCache m_textureCache; /*#if (__cplusplus >= 202002L) diff --git a/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm b/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm index c255852..3f485a9 100644 --- a/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm +++ b/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm @@ -32,10 +32,10 @@ namespace OpenVulkano::AR::ArKit { namespace { - std::string SupportedVideoFormatsToString() + std::string SupportedVideoFormatsToString(bool frontCam) { std::stringstream ss("[ "); - auto vidFormats = [ARWorldTrackingConfiguration supportedVideoFormats]; + auto vidFormats = frontCam ? [ARFaceTrackingConfiguration supportedVideoFormats] : [ARWorldTrackingConfiguration supportedVideoFormats]; for (ARVideoFormat* format: vidFormats) { if (ss.tellp() > std::streampos(5)) ss << ", "; @@ -69,25 +69,37 @@ namespace OpenVulkano::AR::ArKit : m_frameId(0) { m_arKitDelegate = [[ArKitDelegate alloc] initWithFrameHandler:this]; - m_arConfig = [ARWorldTrackingConfiguration new]; - if (config.enableDepth) + if (config.useFrontCamera) { - m_arConfig.frameSemantics = ARFrameSemanticSceneDepth; + ARFaceTrackingConfiguration* arConfig = [ARFaceTrackingConfiguration new]; + arConfig.worldTrackingEnabled = true; + m_arConfig = arConfig; } - if (config.sceneReconstruction) + else { - m_arConfig.sceneReconstruction = ARSceneReconstructionMesh; + ARWorldTrackingConfiguration* arConfig = [ARWorldTrackingConfiguration new]; + if (config.enableDepth) + { + arConfig.frameSemantics = ARFrameSemanticSceneDepth; + } + if (config.sceneReconstruction) + { + arConfig.sceneReconstruction = ARSceneReconstructionMesh; + } + arConfig.userFaceTrackingEnabled = true; + arConfig.planeDetection = config.planeDetection ? (ARPlaneDetectionVertical | ARPlaneDetectionHorizontal) : ARPlaneDetectionNone; + arConfig.autoFocusEnabled = config.autoFocus; + m_arConfig = arConfig; } - m_arConfig.planeDetection = config.planeDetection ? (ARPlaneDetectionVertical | ARPlaneDetectionHorizontal) : ARPlaneDetectionNone; - m_arConfig.autoFocusEnabled = config.autoFocus; // Set video format { - Logger::AR->debug("Supported Video Formats: {}", SupportedVideoFormatsToString()); + Logger::AR->debug("Supported Video Formats: {}", SupportedVideoFormatsToString(config.useFrontCamera)); //TODO handle ar video format in settings if (@available(iOS 16.0, *)) { - if (config.preferHDR) m_arConfig.videoFormat = [ARWorldTrackingConfiguration recommendedVideoFormatForHighResolutionFrameCapturing]; + if (config.preferHDR && !config.useFrontCamera) + m_arConfig.videoFormat = [ARWorldTrackingConfiguration recommendedVideoFormatForHighResolutionFrameCapturing]; //TODO resolution handling } LogFormat("Using video format", m_arConfig.videoFormat);