GeorgH93/OpenVulkano
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request 'Add more arkit stuff' (#95) from offscreen into master

Browse Source 
Reviewed-on: https://git.madvoxel.net/OpenVulkano/OpenVulkano/pulls/95
This commit is contained in:
Georg Hagen
2024-08-01 00:20:47 +02:00
parent 4e7c4327fe 91ac458e18
commit 342bd2f285
14 changed files with 82 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
openVulkanoCpp/AR/ArSession.cpp
View File

@@ -9,10 +9,10 @@
#include "Provider/Network/ArSessionStream.h"
#ifdef __APPLE__
#include <TargetConditionals.h>
#if TARGET_OS_MAC
#include "Provider/ArSessionNull.hpp"
#else
#if TARGET_OS_IOS == 1
#include "Provider/ArKit/ArSessionArKit.h"
#else
#include "Provider/ArSessionNull.hpp"
#endif
#else
#ifdef ANDROID

16
openVulkanoCpp/AR/ArSession.hpp
View File

@@ -45,16 +45,22 @@ namespace OpenVulkano::AR
ArSessionType sessionType = ArSessionType::NATIVE;
bool uncompressed = true;
bool depthSupported = false;
bool supportsExposureLocking = false;
bool supportsWhitebalanceLocking = false;
bool hasFlashlight = false;
public:
ArSessionCapabilities() = default;
ArSessionCapabilities(const ArType type, const ArSessionType sessionType, const bool uncompressed, const bool depthSupported)
: type(type), sessionType(sessionType), uncompressed(uncompressed), depthSupported(depthSupported)
ArSessionCapabilities(const ArType type, const ArSessionType sessionType, const bool uncompressed, const bool depthSupported, const bool supportsExposureLocking, const bool suppoertWhitebalanceLocking, const bool hasFlash)
: type(type), sessionType(sessionType), uncompressed(uncompressed), depthSupported(depthSupported), supportsExposureLocking(supportsExposureLocking), supportsWhitebalanceLocking(supportsWhitebalanceLocking), hasFlashlight(hasFlash)
{}
[[nodiscard]] bool IsUncompressed() const { return uncompressed; }
[[nodiscard]] bool IsDepthSupported() const { return depthSupported; }
[[nodiscard]] bool IsExposureLockSupported() const { return supportsExposureLocking; }
[[nodiscard]] bool IsWhitebalanceLockSupported() const { return supportsWhitebalanceLocking; }
[[nodiscard]] bool HasFlashlight() const { return hasFlashlight; }
[[nodiscard]] bool IsNative() const { return sessionType == ArSessionType::NATIVE; }
[[nodiscard]] bool IsPlayback() const { return sessionType == ArSessionType::PLAYBACK; }
[[nodiscard]] bool IsStream() const { return sessionType == ArSessionType::NETWORK_STREAM; }
@@ -213,6 +219,12 @@ namespace OpenVulkano::AR
* @param renderer The renderer to be used to create textures.
*/
virtual void SetRenderer(IRenderer* renderer) = 0;
virtual void LockExposureTime(bool locked) {};
virtual void LockWhitebalance(bool locked) {};
virtual void SetFlashlightOn(bool on) {};
/**
* Gets the capabilities for this ArSession.

2
openVulkanoCpp/AR/Provider/ArKit/ArSessionArKit.mm
View File

@@ -14,7 +14,7 @@ namespace OpenVulkano::AR::ArKit
ArSessionCapabilities QueryNativeCapabilities()
{
bool supportsDepth = [ARWorldTrackingConfiguration supportsFrameSemantics:ARFrameSemanticSceneDepth];
ArSessionCapabilities capabilities(ArType::AR_KIT, ArSessionType::NATIVE, true, supportsDepth);
ArSessionCapabilities capabilities(ArType::AR_KIT, ArSessionType::NATIVE, true, supportsDepth, true, true, true);
return capabilities;
}
}

9
openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.h
View File

@@ -50,6 +50,10 @@ namespace OpenVulkano::AR::ArKit
bool ArShouldAttemptRelocalization();
void SetRenderer(IRenderer* renderer) override;
void LockExposureTime(bool locked) override;
void LockWhitebalance(bool locked) override;
void SetFlashlightOn(bool on) override;
protected:
Scene::Texture * MakeTexture(ArFrame *frame) override;
@@ -66,6 +70,9 @@ namespace OpenVulkano::AR::ArKit
#else*/
SpintexProtectedObject<std::shared_ptr<ArFrame>> m_frame;
//#endif
std::atomic_size_t m_frameId;
std::atomic_size_t m_frameId = 0;
bool m_lockedConfiguration = false;
bool m_lockedExposure = false;
bool m_lockedWhitebalance = false;
};
}

28
openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm
View File

@@ -22,6 +22,7 @@
#import <ARKit/ARConfiguration.h>
#import <ARKit/ARCamera.h>
#import <ARKit/ARKit.h>
#import <AVFoundation/AVCaptureDevice.h>
#import <CoreVideo/CoreVideo.h>
#import "ArKitDelegate.h"
@@ -138,6 +139,11 @@ namespace OpenVulkano::AR::ArKit
void ArSessionArKitInternal::Stop()
{
if (m_lockedConfiguration)
{
m_lockedConfiguration = false;
[[ARWorldTrackingConfiguration configurableCaptureDeviceForPrimaryCamera] unlockForConfiguration];
}
OnStopped();
[m_arSession pause];
/*#if (__cplusplus >= 202002L)
@@ -183,6 +189,27 @@ namespace OpenVulkano::AR::ArKit
Logger::AR->warn("Failed to perform high resolution still frame capture: requires iOS 16 or later!");
}
}
void ArSessionArKitInternal::LockExposureTime(bool locked)
{
AVCaptureDevice* dev = [ARWorldTrackingConfiguration configurableCaptureDeviceForPrimaryCamera];
if (!m_lockedConfiguration) m_lockedConfiguration |=[dev lockForConfiguration:nil];
[dev setExposureMode: locked ? AVCaptureExposureModeLocked : AVCaptureExposureModeContinuousAutoExposure];
}
void ArSessionArKitInternal::LockWhitebalance(bool locked)
{
AVCaptureDevice* dev = [ARWorldTrackingConfiguration configurableCaptureDeviceForPrimaryCamera];
if (!m_lockedConfiguration) m_lockedConfiguration |=[dev lockForConfiguration:nil];
[dev setWhiteBalanceMode: locked ? AVCaptureWhiteBalanceModeLocked : AVCaptureWhiteBalanceModeContinuousAutoWhiteBalance];
}
void ArSessionArKitInternal::SetFlashlightOn(bool on)
{
AVCaptureDevice* dev = [ARWorldTrackingConfiguration configurableCaptureDeviceForPrimaryCamera];
if (!m_lockedConfiguration) m_lockedConfiguration |=[dev lockForConfiguration:nil];
[dev setTorchMode: on ? AVCaptureTorchModeOn : AVCaptureTorchModeOff];
}
// AR Kit delegate events
@@ -201,6 +228,7 @@ namespace OpenVulkano::AR::ArKit
}
OnNewFrameAvailable();
m_frame = arFrame;
Logger::AR->info("Exposure: {}; ColTemp: {}", 1.0/frame.camera.exposureDuration, frame.lightEstimate.ambientColorTemperature);
}
void ArSessionArKitInternal::OnArSessionInterruptedChanged(ARSession* session, bool interrupted)

2
openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp
View File

@@ -17,7 +17,7 @@ namespace OpenVulkano::AR::Playback
ArSessionPlayback::ArSessionPlayback(const std::string& recordingPath, bool autoAdvance)
: ArSession(ArSessionMetadata(recordingPath)), recordingPath(recordingPath), autoAdvance(autoAdvance), playbackReader(recordingPath)
{
capabilities = ArSessionCapabilities(metadata.type, ArSessionType::PLAYBACK, false, metadata.depthFormat != ArDepthFormat::UNAVAILABLE);
capabilities = ArSessionCapabilities(metadata.type, ArSessionType::PLAYBACK, false, metadata.depthFormat != ArDepthFormat::UNAVAILABLE, false, false, false);
constants = { Math::Matrix4f(1), metadata.confidenceRange };
m_playbackReaderThread = std::thread([this](){ReadWorker();});

2
openVulkanoCpp/Host/iOS/OpenVulkanoView.mm
View File

@@ -9,7 +9,7 @@
#include "Base/Logger.hpp"
#include "Input/Touch/InputDeviceTouch.hpp"
#include "Input/InputManager.hpp"
#include "MetalViewWindow.h"
#include "Host/Apple/MetalViewWindow.h"
#include <map>
#import <MetalKit/MTKView.h>

9
openVulkanoCpp/Scene/Camera.hpp
View File

@@ -138,11 +138,10 @@ namespace OpenVulkano::Scene
[[nodiscard]] auto GetForward() const { return GetViewDirection(); }
[[nodiscard]] const Math::Matrix4f& GetViewMatrix() const { return m_view; }
[[nodiscard]] Math::Frustum GetFrustum() const
{
return {m_viewProjection};
}
[[nodiscard]] const Math::Matrix4f& GetProjectionMatrix() const { return m_projection; }
[[nodiscard]] Math::Frustum GetFrustum() const { return {m_viewProjection}; }
[[nodiscard]] float GetScaleFactor() const { return m_scaleFactor; }
[[nodiscard]] float GetPixelScaleFactor() const { return m_perPixelScaleFactor; }

1
openVulkanoCpp/Scene/Prefabs/ArBackgroundDrawable.cpp
View File

@@ -25,6 +25,7 @@ namespace OpenVulkano::Scene
m_shader.AddShaderProgram(ShaderProgramType::FRAGMENT, "Shader/background");
m_shader.AddDescriptorSetLayoutBinding(UniformBuffer::DESCRIPTOR_SET_LAYOUT_BINDING);
m_shader.AddDescriptorSetLayoutBinding(Texture::DESCRIPTOR_SET_LAYOUT_BINDING);
m_shader.depthCompareOp = CompareOp::LESS_OR_EQUAL;
SetShader(&m_shader);
m_intrinsicsBuffer.Init(sizeof(Math::CameraIntrinsicWithResolution), &FALLBACK_INTRINSICS);
m_intrinsicsBuffer.updateFrequency = UpdateFrequency::Always;

4
openVulkanoCpp/Scene/Prefabs/GridDrawable.cpp
View File

@@ -16,7 +16,9 @@ namespace OpenVulkano::Scene
m_shader.AddShaderProgram(ShaderProgramType::VERTEX, "Shader/grid");
m_shader.AddShaderProgram(ShaderProgramType::FRAGMENT, "Shader/grid");
m_shader.alphaBlend = true;
m_shader.depthTest = true;
m_shader.depthWrite = false;
SetShader(&m_shader);
}
}
}

6
openVulkanoCpp/Scene/SamplerConfig.hpp
View File

@@ -64,8 +64,8 @@ namespace OpenVulkano::Scene
: magFilter(magFilter), minFilter(minFilter), mipmapMode(mipmapMode), edgeModeU(edgeMode), edgeModeV(edgeMode), edgeModeW(edgeMode)
{}
SamplerConfig(uint32_t flags = 0, TextureSamplerFilter magFilter = TextureSamplerFilter::NEAREST,
TextureSamplerFilter minFilter = TextureSamplerFilter::NEAREST,
SamplerConfig(uint32_t flags = 0, TextureSamplerFilter magFilter = TextureSamplerFilter::LINEAR,
TextureSamplerFilter minFilter = TextureSamplerFilter::LINEAR,
TextureSamplerMipmapMode mipmapMode = TextureSamplerMipmapMode::NEAREST,
TextureSamplerEdgeMode edgeModeU = TextureSamplerEdgeMode::REPEAT,
TextureSamplerEdgeMode edgeModeV = TextureSamplerEdgeMode::REPEAT,
@@ -86,4 +86,4 @@ namespace OpenVulkano::Scene
// Default configs
static const SamplerConfig DEFAULT;
};
}
}

13
openVulkanoCpp/Scene/Shader/Shader.hpp
View File

@@ -40,6 +40,18 @@ namespace OpenVulkano::Scene
PATCH_LIST
};
enum class CompareOp : uint32_t
{
NEVER = 0,
LESS = 1,
EQUAL = 2,
LESS_OR_EQUAL = 3,
GREATER = 4,
NOT_EQUAL = 5,
GREATER_OR_EQUAL = 6,
ALWAYS = 7
};
struct ShaderProgram
{
ShaderProgramType type;
@@ -77,6 +89,7 @@ namespace OpenVulkano::Scene
Topology topology = Topology::TRIANGLE_LIST;
CullMode cullMode = CullMode::BACK;
ICloseable* renderShader = nullptr;
CompareOp depthCompareOp = CompareOp::LESS;
bool alphaBlend = false; // TODO allow fine control over blending
bool depthTest = true;
bool depthWrite = true;

3
openVulkanoCpp/Shader/background.vert
View File

@@ -27,10 +27,9 @@ layout(set = 2, binding = 0) uniform RealCameraData
layout(location = 0) out vec2 textureCoordinates;
layout(location = 1) out float scaleOut;
const float FLOAT_MAX_LESS_THAN_1 = 0.999999940395355224609;
// Background plane positions are in clipped space
const vec4 PLANE[4] = vec4[](
vec4(1, -1, FLOAT_MAX_LESS_THAN_1, 1), vec4(-1, -1, FLOAT_MAX_LESS_THAN_1, 1), vec4(1, 1, FLOAT_MAX_LESS_THAN_1, 1), vec4(-1, 1, FLOAT_MAX_LESS_THAN_1, 1)
vec4(1, -1, 1, 1), vec4(-1, -1, 1, 1), vec4(1, 1, 1, 1), vec4(-1, 1, 1, 1)
);
const vec2 TEX_COORDS[4] = vec2[](
vec2(1, 0), vec2(0, 0), vec2(1, 1), vec2(0, 1)

2
openVulkanoCpp/Vulkan/Scene/VulkanShader.cpp
View File

@@ -80,7 +80,7 @@ namespace OpenVulkano::Vulkan
vk::PipelineRasterizationStateCreateInfo rasterizer = {};
rasterizer.cullMode = static_cast<vk::CullModeFlagBits>(shader->cullMode);
vk::PipelineMultisampleStateCreateInfo msaa = {};
vk::PipelineDepthStencilStateCreateInfo depth = { {}, shader->depthTest, shader->depthWrite, vk::CompareOp::eLess };
vk::PipelineDepthStencilStateCreateInfo depth = { {}, shader->depthTest, shader->depthWrite, static_cast<vk::CompareOp>(shader->depthCompareOp) };
depth.maxDepthBounds = 1;
vk::PipelineColorBlendAttachmentState colorBlendAttachment = {};
colorBlendAttachment.colorWriteMask = vk::ColorComponentFlagBits::eA | vk::ColorComponentFlagBits::eB | vk::ColorComponentFlagBits::eG | vk::ColorComponentFlagBits::eR;