GeorgH93/OpenVulkano
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for front facing ar experience

Browse Source
This commit is contained in:
Georg Hagen
2025-01-08 23:56:10 +01:00
parent f0cab05512
commit 5d7ef49142
6 changed files with 61 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
openVulkanoCpp/AR/ArDepthFormat.hpp
View File

@@ -17,7 +17,7 @@ namespace OpenVulkano::AR
};
public:
enum Format : uint8_t { UNAVAILABLE = 0, METER_FP32, METER_FP64, MILLIMETER_U16, MILLIMETER_U32 };
enum Format : uint8_t { UNAVAILABLE = 0, METER_FP16, METER_FP32, METER_FP64, MILLIMETER_U16, MILLIMETER_U32 };
ArDepthFormat() : ArDepthFormat(UNAVAILABLE) {}
@@ -56,4 +56,4 @@ namespace OpenVulkano::AR
private:
Format m_format;
};
}
}

3
openVulkanoCpp/AR/ArSession.hpp
View File

@@ -75,6 +75,7 @@ namespace OpenVulkano::AR
bool preferHDR = true;
bool sceneReconstruction = false;
bool planeDetection = false;
bool useFrontCamera = false;
int preferredResolution = 0;
};
@@ -265,4 +266,4 @@ namespace OpenVulkano::AR
static std::vector<std::weak_ptr<ArSession>> sessions;
static std::weak_ptr<ArSession> nativeSession;
};
}
}

1
openVulkanoCpp/AR/Provider/ArKit/ArFrameArKit.h
View File

@@ -36,6 +36,7 @@ namespace OpenVulkano::AR::ArKit
private:
ARFrame* m_arKitFrame;
bool m_lockedColor, m_lockedDepth;
uint8_t m_frameDepthQuality = 0;
ArImagePlanar m_colorImage;
ArDepthImage m_depthImage;
};

35
openVulkanoCpp/AR/Provider/ArKit/ArFrameArKit.mm
View File

@@ -42,6 +42,10 @@ namespace OpenVulkano::AR::ArKit
{
return GetSize(arKitFrame.smoothedSceneDepth.depthMap);
}
if(arKitFrame.capturedDepthData)
{
return GetSize(arKitFrame.capturedDepthData.depthDataMap);
}
}
return { -1, -1 };
}
@@ -106,9 +110,22 @@ namespace OpenVulkano::AR::ArKit
assert(format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
#endif
m_depthImage.format = ArDepthFormat::METER_FP32;
m_depthImage.format = ArDepthFormat::UNAVAILABLE;
m_depthImage.depth.resolution = GetDepthSize(arKitFrame);
m_depthImage.confidence.resolution = m_depthImage.depth.resolution;
if (m_arKitFrame.sceneDepth || m_arKitFrame.smoothedSceneDepth)
{
m_depthImage.format = ArDepthFormat::METER_FP32;
m_depthImage.confidence.resolution = m_depthImage.depth.resolution;
}
else if (m_arKitFrame.capturedDepthData)
{
auto depthFormat = arKitFrame.capturedDepthData.depthDataType;
if (arKitFrame.capturedDepthData.depthDataType == kCVPixelFormatType_DepthFloat32) m_depthImage.format = ArDepthFormat::METER_FP32;
else if (arKitFrame.capturedDepthData.depthDataType == kCVPixelFormatType_DepthFloat16) m_depthImage.format = ArDepthFormat::METER_FP16;
else Logger::AR->info("Received depth frame with unsuported format: {} ({})",
depthFormat == kCVPixelFormatType_DisparityFloat16 ? "DisparityFloat16" : (depthFormat == kCVPixelFormatType_DisparityFloat32 ? "DisparityFloat32" : "Unknown"), depthFormat);
m_depthImage.confidence.resolution = { 1, 1 };
}
m_depthImage.intrinsic = frameMetadata.intrinsic.GetForResolution(m_depthImage.depth.resolution);
}
@@ -125,11 +142,15 @@ namespace OpenVulkano::AR::ArKit
CVPixelBufferUnlockBaseAddress(m_arKitFrame.sceneDepth.depthMap, kCVPixelBufferLock_ReadOnly);
CVPixelBufferUnlockBaseAddress(m_arKitFrame.sceneDepth.confidenceMap, kCVPixelBufferLock_ReadOnly);
}
else
else if(m_arKitFrame.smoothedSceneDepth)
{
CVPixelBufferUnlockBaseAddress(m_arKitFrame.smoothedSceneDepth.depthMap, kCVPixelBufferLock_ReadOnly);
CVPixelBufferUnlockBaseAddress(m_arKitFrame.smoothedSceneDepth.confidenceMap, kCVPixelBufferLock_ReadOnly);
}
else
{
CVPixelBufferUnlockBaseAddress(m_arKitFrame.capturedDepthData.depthDataMap, kCVPixelBufferLock_ReadOnly);
}
}
[m_arKitFrame release];
}
@@ -180,6 +201,14 @@ namespace OpenVulkano::AR::ArKit
m_depthImage.confidence.data = CVPixelBufferGetBaseAddress(m_arKitFrame.smoothedSceneDepth.confidenceMap);
m_lockedDepth = true;
}
else if(m_arKitFrame.capturedDepthData)
{
CVPixelBufferLockBaseAddress(m_arKitFrame.capturedDepthData.depthDataMap, kCVPixelBufferLock_ReadOnly);
m_depthImage.depth.data = CVPixelBufferGetBaseAddress(m_arKitFrame.capturedDepthData.depthDataMap);
m_frameDepthQuality = static_cast<uint8_t>(m_arKitFrame.capturedDepthData.depthDataQuality) + static_cast<uint8_t>(m_arKitFrame.capturedDepthData.depthDataAccuracy);
m_depthImage.confidence.data = &m_frameDepthQuality;
m_lockedDepth = true;
}
}
}
return m_depthImage;

2
openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.h
View File

@@ -62,7 +62,7 @@ namespace OpenVulkano::AR::ArKit
private:
ArKitDelegate* m_arKitDelegate;
ARWorldTrackingConfiguration* m_arConfig;
ARConfiguration* m_arConfig;
ARSession* m_arSession;
Vulkan::MetalTextureCache m_textureCache;
/*#if (__cplusplus >= 202002L)

34
openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm
View File

@@ -32,10 +32,10 @@ namespace OpenVulkano::AR::ArKit
{
namespace
{
std::string SupportedVideoFormatsToString()
std::string SupportedVideoFormatsToString(bool frontCam)
{
std::stringstream ss("[ ");
auto vidFormats = [ARWorldTrackingConfiguration supportedVideoFormats];
auto vidFormats = frontCam ? [ARFaceTrackingConfiguration supportedVideoFormats] : [ARWorldTrackingConfiguration supportedVideoFormats];
for (ARVideoFormat* format: vidFormats)
{
if (ss.tellp() > std::streampos(5)) ss << ", ";
@@ -69,25 +69,37 @@ namespace OpenVulkano::AR::ArKit
: m_frameId(0)
{
m_arKitDelegate = [[ArKitDelegate alloc] initWithFrameHandler:this];
m_arConfig = [ARWorldTrackingConfiguration new];
if (config.enableDepth)
if (config.useFrontCamera)
{
m_arConfig.frameSemantics = ARFrameSemanticSceneDepth;
ARFaceTrackingConfiguration* arConfig = [ARFaceTrackingConfiguration new];
arConfig.worldTrackingEnabled = true;
m_arConfig = arConfig;
}
if (config.sceneReconstruction)
else
{
m_arConfig.sceneReconstruction = ARSceneReconstructionMesh;
ARWorldTrackingConfiguration* arConfig = [ARWorldTrackingConfiguration new];
if (config.enableDepth)
{
arConfig.frameSemantics = ARFrameSemanticSceneDepth;
}
if (config.sceneReconstruction)
{
arConfig.sceneReconstruction = ARSceneReconstructionMesh;
}
arConfig.userFaceTrackingEnabled = true;
arConfig.planeDetection = config.planeDetection ? (ARPlaneDetectionVertical | ARPlaneDetectionHorizontal) : ARPlaneDetectionNone;
arConfig.autoFocusEnabled = config.autoFocus;
m_arConfig = arConfig;
}
m_arConfig.planeDetection = config.planeDetection ? (ARPlaneDetectionVertical | ARPlaneDetectionHorizontal) : ARPlaneDetectionNone;
m_arConfig.autoFocusEnabled = config.autoFocus;
// Set video format
{
Logger::AR->debug("Supported Video Formats: {}", SupportedVideoFormatsToString());
Logger::AR->debug("Supported Video Formats: {}", SupportedVideoFormatsToString(config.useFrontCamera));
//TODO handle ar video format in settings
if (@available(iOS 16.0, *))
{
if (config.preferHDR) m_arConfig.videoFormat = [ARWorldTrackingConfiguration recommendedVideoFormatForHighResolutionFrameCapturing];
if (config.preferHDR && !config.useFrontCamera)
m_arConfig.videoFormat = [ARWorldTrackingConfiguration recommendedVideoFormatForHighResolutionFrameCapturing];
//TODO resolution handling
}
LogFormat("Using video format", m_arConfig.videoFormat);