Merge pull request 'Add more arkit stuff' (#95) from offscreen into master

Reviewed-on: https://git.madvoxel.net/OpenVulkano/OpenVulkano/pulls/95

openVulkanoCpp/AR/ArSession.cpp

@@ -9,10 +9,10 @@
 #include "Provider/Network/ArSessionStream.h"
 #ifdef __APPLE__
 #include <TargetConditionals.h>
-#if TARGET_OS_MAC
-#include "Provider/ArSessionNull.hpp"
-#else
+#if TARGET_OS_IOS == 1
 #include "Provider/ArKit/ArSessionArKit.h"
+#else
+#include "Provider/ArSessionNull.hpp"
 #endif
 #else
 #ifdef ANDROID

openVulkanoCpp/AR/ArSession.hpp

@@ -45,16 +45,22 @@ namespace OpenVulkano::AR
 		ArSessionType sessionType = ArSessionType::NATIVE;
 		bool uncompressed = true;
 		bool depthSupported = false;
+		bool supportsExposureLocking = false;
+		bool supportsWhitebalanceLocking = false;
+		bool hasFlashlight = false;
 
 	public:
 		ArSessionCapabilities() = default;
 
-		ArSessionCapabilities(const ArType type, const ArSessionType sessionType, const bool uncompressed, const bool depthSupported)
-			: type(type), sessionType(sessionType), uncompressed(uncompressed), depthSupported(depthSupported)
+		ArSessionCapabilities(const ArType type, const ArSessionType sessionType, const bool uncompressed, const bool depthSupported, const bool supportsExposureLocking, const bool suppoertWhitebalanceLocking, const bool hasFlash)
+			: type(type), sessionType(sessionType), uncompressed(uncompressed), depthSupported(depthSupported), supportsExposureLocking(supportsExposureLocking), supportsWhitebalanceLocking(supportsWhitebalanceLocking), hasFlashlight(hasFlash)
 		{}
 
 		[[nodiscard]] bool IsUncompressed() const { return uncompressed; }
 		[[nodiscard]] bool IsDepthSupported() const { return depthSupported; }
+		[[nodiscard]] bool IsExposureLockSupported() const { return supportsExposureLocking; }
+		[[nodiscard]] bool IsWhitebalanceLockSupported() const { return supportsWhitebalanceLocking; }
+		[[nodiscard]] bool HasFlashlight() const { return hasFlashlight; }
 		[[nodiscard]] bool IsNative() const { return sessionType == ArSessionType::NATIVE; }
 		[[nodiscard]] bool IsPlayback() const { return sessionType == ArSessionType::PLAYBACK; }
 		[[nodiscard]] bool IsStream() const { return sessionType == ArSessionType::NETWORK_STREAM; }
@@ -214,6 +220,12 @@ namespace OpenVulkano::AR
 		 */
 		virtual void SetRenderer(IRenderer* renderer) = 0;
 		
+		virtual void LockExposureTime(bool locked) {};
+		
+		virtual void LockWhitebalance(bool locked) {};
+		
+		virtual void SetFlashlightOn(bool on) {};
+
 		/**
 		 * Gets the capabilities for this ArSession.
 		 * @return The capabilities for the current AR session.

openVulkanoCpp/AR/Provider/ArKit/ArSessionArKit.mm

@@ -14,7 +14,7 @@ namespace OpenVulkano::AR::ArKit
 		ArSessionCapabilities QueryNativeCapabilities()
 		{
 			bool supportsDepth = [ARWorldTrackingConfiguration supportsFrameSemantics:ARFrameSemanticSceneDepth];
-			ArSessionCapabilities capabilities(ArType::AR_KIT, ArSessionType::NATIVE, true, supportsDepth);
+			ArSessionCapabilities capabilities(ArType::AR_KIT, ArSessionType::NATIVE, true, supportsDepth, true, true, true);
 			return capabilities;
 		}
 	}

openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.h

@@ -51,6 +51,10 @@ namespace OpenVulkano::AR::ArKit
 		
 		void SetRenderer(IRenderer* renderer) override;
 		
+		void LockExposureTime(bool locked) override;
+		void LockWhitebalance(bool locked) override;
+		void SetFlashlightOn(bool on) override;
+
 	protected:
 		Scene::Texture * MakeTexture(ArFrame *frame) override;
 
@@ -66,6 +70,9 @@ namespace OpenVulkano::AR::ArKit
 #else*/
 		SpintexProtectedObject<std::shared_ptr<ArFrame>> m_frame;
 //#endif
-		std::atomic_size_t m_frameId;
+		std::atomic_size_t m_frameId = 0;
+		bool m_lockedConfiguration = false;
+		bool m_lockedExposure = false;
+		bool m_lockedWhitebalance = false;
 	};
 }

openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm

@@ -22,6 +22,7 @@
 #import <ARKit/ARConfiguration.h>
 #import <ARKit/ARCamera.h>
 #import <ARKit/ARKit.h>
+#import <AVFoundation/AVCaptureDevice.h>
 #import <CoreVideo/CoreVideo.h>
 #import "ArKitDelegate.h"
 
@@ -138,6 +139,11 @@ namespace OpenVulkano::AR::ArKit
 
 	void ArSessionArKitInternal::Stop()
 	{
+		if (m_lockedConfiguration)
+		{
+			m_lockedConfiguration = false;
+			[[ARWorldTrackingConfiguration configurableCaptureDeviceForPrimaryCamera] unlockForConfiguration];
+		}
 		OnStopped();
 		[m_arSession pause];
 /*#if (__cplusplus >= 202002L)
@@ -184,6 +190,27 @@ namespace OpenVulkano::AR::ArKit
 		}
 	}
 	
+	void ArSessionArKitInternal::LockExposureTime(bool locked)
+	{
+		AVCaptureDevice* dev = [ARWorldTrackingConfiguration configurableCaptureDeviceForPrimaryCamera];
+		if (!m_lockedConfiguration) m_lockedConfiguration |=[dev lockForConfiguration:nil];
+		[dev setExposureMode: locked ? AVCaptureExposureModeLocked : AVCaptureExposureModeContinuousAutoExposure];
+	}
+	
+	void ArSessionArKitInternal::LockWhitebalance(bool locked)
+	{
+		AVCaptureDevice* dev = [ARWorldTrackingConfiguration configurableCaptureDeviceForPrimaryCamera];
+		if (!m_lockedConfiguration) m_lockedConfiguration |=[dev lockForConfiguration:nil];
+		[dev setWhiteBalanceMode: locked ? AVCaptureWhiteBalanceModeLocked : AVCaptureWhiteBalanceModeContinuousAutoWhiteBalance];
+	}
+	
+	void ArSessionArKitInternal::SetFlashlightOn(bool on)
+	{
+		AVCaptureDevice* dev = [ARWorldTrackingConfiguration configurableCaptureDeviceForPrimaryCamera];
+		if (!m_lockedConfiguration) m_lockedConfiguration |=[dev lockForConfiguration:nil];
+		[dev setTorchMode: on ? AVCaptureTorchModeOn : AVCaptureTorchModeOff];
+	}
+
 
 	// AR Kit delegate events
 	void ArSessionArKitInternal::OnArNewFrame(ARSession* session, ARFrame* frame)
@@ -201,6 +228,7 @@ namespace OpenVulkano::AR::ArKit
 		}
 		OnNewFrameAvailable();
 		m_frame = arFrame;
+		Logger::AR->info("Exposure: {}; ColTemp: {}", 1.0/frame.camera.exposureDuration, frame.lightEstimate.ambientColorTemperature);
 	}
 
 	void ArSessionArKitInternal::OnArSessionInterruptedChanged(ARSession* session, bool interrupted)

openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp

@@ -17,7 +17,7 @@ namespace OpenVulkano::AR::Playback
 	ArSessionPlayback::ArSessionPlayback(const std::string& recordingPath, bool autoAdvance)
 		: ArSession(ArSessionMetadata(recordingPath)), recordingPath(recordingPath), autoAdvance(autoAdvance), playbackReader(recordingPath)
 	{
-		capabilities = ArSessionCapabilities(metadata.type, ArSessionType::PLAYBACK, false, metadata.depthFormat != ArDepthFormat::UNAVAILABLE);
+		capabilities = ArSessionCapabilities(metadata.type, ArSessionType::PLAYBACK, false, metadata.depthFormat != ArDepthFormat::UNAVAILABLE, false, false, false);
 		constants = { Math::Matrix4f(1), metadata.confidenceRange };
 
 		m_playbackReaderThread = std::thread([this](){ReadWorker();});

openVulkanoCpp/Host/iOS/OpenVulkanoView.mm

@@ -9,7 +9,7 @@
 #include "Base/Logger.hpp"
 #include "Input/Touch/InputDeviceTouch.hpp"
 #include "Input/InputManager.hpp"
-#include "MetalViewWindow.h"
+#include "Host/Apple/MetalViewWindow.h"
 #include <map>
 
 #import <MetalKit/MTKView.h>

openVulkanoCpp/Scene/Camera.hpp

@@ -139,10 +139,9 @@ namespace OpenVulkano::Scene
 
 		[[nodiscard]] const Math::Matrix4f& GetViewMatrix() const { return m_view; }
 
-		[[nodiscard]] Math::Frustum GetFrustum() const
-		{
-			return {m_viewProjection};
-		}
+		[[nodiscard]] const Math::Matrix4f& GetProjectionMatrix() const { return m_projection; }
+
+		[[nodiscard]] Math::Frustum GetFrustum() const { return {m_viewProjection}; }
 
 		[[nodiscard]] float GetScaleFactor() const { return m_scaleFactor; }
 		[[nodiscard]] float GetPixelScaleFactor() const { return m_perPixelScaleFactor; }

openVulkanoCpp/Scene/Prefabs/ArBackgroundDrawable.cpp

@@ -25,6 +25,7 @@ namespace OpenVulkano::Scene
 		m_shader.AddShaderProgram(ShaderProgramType::FRAGMENT, "Shader/background");
 		m_shader.AddDescriptorSetLayoutBinding(UniformBuffer::DESCRIPTOR_SET_LAYOUT_BINDING);
 		m_shader.AddDescriptorSetLayoutBinding(Texture::DESCRIPTOR_SET_LAYOUT_BINDING);
+		m_shader.depthCompareOp = CompareOp::LESS_OR_EQUAL;
 		SetShader(&m_shader);
 		m_intrinsicsBuffer.Init(sizeof(Math::CameraIntrinsicWithResolution), &FALLBACK_INTRINSICS);
 		m_intrinsicsBuffer.updateFrequency = UpdateFrequency::Always;

openVulkanoCpp/Scene/Prefabs/GridDrawable.cpp

@@ -16,6 +16,8 @@ namespace OpenVulkano::Scene
 		m_shader.AddShaderProgram(ShaderProgramType::VERTEX, "Shader/grid");
 		m_shader.AddShaderProgram(ShaderProgramType::FRAGMENT, "Shader/grid");
 		m_shader.alphaBlend = true;
+		m_shader.depthTest = true;
+		m_shader.depthWrite = false;
 
 		SetShader(&m_shader);
 	}

openVulkanoCpp/Scene/SamplerConfig.hpp

@@ -64,8 +64,8 @@ namespace OpenVulkano::Scene
 				: magFilter(magFilter), minFilter(minFilter), mipmapMode(mipmapMode), edgeModeU(edgeMode), edgeModeV(edgeMode), edgeModeW(edgeMode)
 		{}
 
-		SamplerConfig(uint32_t flags = 0, TextureSamplerFilter magFilter = TextureSamplerFilter::NEAREST,
-				TextureSamplerFilter minFilter = TextureSamplerFilter::NEAREST,
+		SamplerConfig(uint32_t flags = 0, TextureSamplerFilter magFilter = TextureSamplerFilter::LINEAR,
+				TextureSamplerFilter minFilter = TextureSamplerFilter::LINEAR,
 				TextureSamplerMipmapMode mipmapMode = TextureSamplerMipmapMode::NEAREST,
 				TextureSamplerEdgeMode edgeModeU = TextureSamplerEdgeMode::REPEAT,
 				TextureSamplerEdgeMode edgeModeV = TextureSamplerEdgeMode::REPEAT,

openVulkanoCpp/Scene/Shader/Shader.hpp

@@ -40,6 +40,18 @@ namespace OpenVulkano::Scene
 		PATCH_LIST
 	};
 
+	enum class CompareOp : uint32_t
+	{
+		NEVER = 0,
+		LESS = 1,
+		EQUAL = 2,
+		LESS_OR_EQUAL = 3,
+		GREATER = 4,
+		NOT_EQUAL = 5,
+		GREATER_OR_EQUAL = 6,
+		ALWAYS = 7
+	};
+
 	struct ShaderProgram
 	{
 		ShaderProgramType type;
@@ -77,6 +89,7 @@ namespace OpenVulkano::Scene
 		Topology topology = Topology::TRIANGLE_LIST;
 		CullMode cullMode = CullMode::BACK;
 		ICloseable* renderShader = nullptr;
+		CompareOp depthCompareOp = CompareOp::LESS;
 		bool alphaBlend = false; // TODO allow fine control over blending
 		bool depthTest = true;
 		bool depthWrite = true;

openVulkanoCpp/Shader/background.vert

@@ -27,10 +27,9 @@ layout(set = 2, binding = 0) uniform RealCameraData
 layout(location = 0) out vec2 textureCoordinates;
 layout(location = 1) out float scaleOut;
 
-const float FLOAT_MAX_LESS_THAN_1 = 0.999999940395355224609;
 // Background plane positions are in clipped space
 const vec4 PLANE[4] = vec4[](
-        vec4(1, -1, FLOAT_MAX_LESS_THAN_1, 1), vec4(-1, -1, FLOAT_MAX_LESS_THAN_1, 1), vec4(1, 1, FLOAT_MAX_LESS_THAN_1, 1), vec4(-1, 1, FLOAT_MAX_LESS_THAN_1, 1)
+        vec4(1, -1, 1, 1), vec4(-1, -1, 1, 1), vec4(1, 1, 1, 1), vec4(-1, 1, 1, 1)
     );
 const vec2 TEX_COORDS[4] = vec2[](
         vec2(1, 0), vec2(0, 0), vec2(1, 1), vec2(0, 1)

openVulkanoCpp/Vulkan/Scene/VulkanShader.cpp

@@ -80,7 +80,7 @@ namespace OpenVulkano::Vulkan
 		vk::PipelineRasterizationStateCreateInfo rasterizer = {};
 		rasterizer.cullMode = static_cast<vk::CullModeFlagBits>(shader->cullMode);
 		vk::PipelineMultisampleStateCreateInfo msaa = {};
-		vk::PipelineDepthStencilStateCreateInfo depth = { {}, shader->depthTest, shader->depthWrite, vk::CompareOp::eLess };
+		vk::PipelineDepthStencilStateCreateInfo depth = { {}, shader->depthTest, shader->depthWrite, static_cast<vk::CompareOp>(shader->depthCompareOp) };
 		depth.maxDepthBounds = 1;
 		vk::PipelineColorBlendAttachmentState colorBlendAttachment = {};
 		colorBlendAttachment.colorWriteMask = vk::ColorComponentFlagBits::eA | vk::ColorComponentFlagBits::eB | vk::ColorComponentFlagBits::eG | vk::ColorComponentFlagBits::eR;