Add texture cache for ar frame textures

openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.h

@@ -8,6 +8,8 @@
 
 #include "ArSessionArKit.h"
 #include "Data/Concurent/MutexProtectedObject.hpp"
+#include "Scene/Texture.hpp"
+#include "Vulkan/Metal/MetalTextureCache.h"
 #include <atomic>
 
 #import <ARKit/ARSession.h>
@@ -46,9 +48,11 @@ namespace OpenVulkano::AR::ArKit
 		void OnArCameraTrackingChange(ARSession* session, ARCamera* camera);
 		void OnArAnchorsUpdate(NSArray<__kindof ARAnchor*>* anchors);
 		bool ArShouldAttemptRelocalization();
+		
+		void SetRenderer(IRenderer* renderer) override;
 
 	protected:
-		Scene::Texture * MakeTexture(OpenVulkano::AR::ArFrame *frame) override;
+		Scene::Texture * MakeTexture(ArFrame *frame) override;
 
 		void ReturnTexture(Scene::Texture *texture) override;
 
@@ -56,6 +60,7 @@ namespace OpenVulkano::AR::ArKit
 		ArKitDelegate* m_arKitDelegate;
 		ARWorldTrackingConfiguration* m_arConfig;
 		ARSession* m_arSession;
+		Vulkan::MetalTextureCache m_textureCache;
 /*#if (__cplusplus >= 202002L)
 		std::atomic<std::shared_ptr<ArFrame>> m_frame;
 #else*/

openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm

@@ -4,11 +4,17 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
+// Makre usre the Molten include is first!
+#include <MoltenVK/mvk_vulkan.h>
+#include <vulkan/vulkan_metal.h>
+
 #include "ArSessionArKitInternal.h"
 #include "ArFrameArKit.h"
 #include "ArTrackingStateConverter.h"
 #include "Base/Logger.hpp"
 #include "IO/AppFolders.hpp"
+#include "Vulkan/Renderer.hpp"
+#include "Vulkan/Scene/VulkanTexture.hpp"
 #include <sstream>
 
 #import <ARKit/ARSession.h>
@@ -102,6 +108,25 @@ namespace OpenVulkano::AR::ArKit
 		[m_arKitDelegate release];
 	}
 
+	void ArSessionArKitInternal::SetRenderer(IRenderer* renderer)
+	{
+		m_textureCache.Init(renderer);
+	}
+
+	Scene::Texture* ArSessionArKitInternal::MakeTexture(ArFrame* frame)
+	{
+		if (!m_textureCache) [[unlikely]]
+			throw std::runtime_error("No renderer set for which to produce textures");
+		ArFrameArKit* arFrame = static_cast<ArFrameArKit*>(frame);
+		ARFrame* arKitFrame = arFrame->GetArKitFrame();
+		return m_textureCache.Get(arKitFrame.capturedImage, MTLPixelFormatR8Unorm);
+	}
+
+	void ArSessionArKitInternal::ReturnTexture(Scene::Texture* texture)
+	{
+		m_textureCache.ReturnTexture(texture);
+	}
+
 	void ArSessionArKitInternal::Start()
 	{
 		[m_arSession runWithConfiguration:m_arConfig];

openVulkanoCpp/Vulkan/Metal/MetalTextureCache.h

@@ -0,0 +1,35 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#pragma once
+
+#include "MetalBackedTexture.h"
+
+#import <CoreVideo/CVPixelBuffer.h>
+#import <CoreVideo/CVMetalTextureCache.h>
+
+namespace OpenVulkano::Vulkan
+{
+	class MetalTextureCache
+	{
+		CVMetalTextureCacheRef m_textureCache = nullptr;
+		Vulkan::ResourceManager* m_resourceManager = nullptr;
+		std::map<void*, MetalBackedTexture> m_mtlToVkTextureMap;
+
+	public:
+		~MetalTextureCache() { if (m_resourceManager) Close(); }
+
+		void Init(IRenderer* renderer);
+
+		void Close();
+
+		Scene::Texture* Get(CVPixelBufferRef pixelBuffer, MTLPixelFormat pixelFormat);
+
+		void ReturnTexture(Scene::Texture* texture);
+
+		operator bool() const { return m_resourceManager; }
+	};
+}

openVulkanoCpp/Vulkan/Metal/MetalTextureCache.mm

@@ -0,0 +1,63 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#include "MetalTextureCache.h"
+#include "Vulkan/Renderer.hpp"
+
+namespace OpenVulkano::Vulkan
+{
+	void MetalTextureCache::Init(IRenderer* renderer)
+	{
+		if (m_resourceManager) Close();
+		if (!renderer) return;
+		auto vkRenderer = dynamic_cast<Vulkan::Renderer*>(renderer);
+		if (!vkRenderer) throw std::invalid_argument("The Metal Texture Cache only supports Vulkan renderer!");
+		// Setup texture cache
+		VkExportMetalDeviceInfoEXT metalDeviceInfo {};
+		metalDeviceInfo.sType = VK_STRUCTURE_TYPE_EXPORT_METAL_DEVICE_INFO_EXT;
+		metalDeviceInfo.pNext = nullptr;
+		VkExportMetalObjectsInfoEXT exportInfo { VK_STRUCTURE_TYPE_EXPORT_METAL_OBJECTS_INFO_EXT, &metalDeviceInfo };
+		VkDevice vkDevice = vkRenderer->GetContext().device->device;
+		vkExportMetalObjectsEXT(vkDevice, &exportInfo);
+		CVReturn result = CVMetalTextureCacheCreate(nil, nil, metalDeviceInfo.mtlDevice, nil, &m_textureCache);
+		if (result != kCVReturnSuccess)
+		{
+			Logger::AR->error("Failed to create metal texture cache! Status code: {}", result);
+		}
+		m_resourceManager = &vkRenderer->GetResourceManager();
+	}
+
+	Scene::Texture* MetalTextureCache::Get(CVPixelBufferRef pixelBuffer, MTLPixelFormat pixelFormat)
+	{
+		CVMetalTextureRef mtlTextureRef;
+		auto width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);
+		auto height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);
+		//TODO check pixel format from buffer?
+		CVMetalTextureCacheCreateTextureFromImage(nil, m_textureCache, pixelBuffer, nil, pixelFormat, width, height, 0, &mtlTextureRef);
+		id<MTLTexture> mtlTexture = CVMetalTextureGetTexture(mtlTextureRef);
+		CVBufferRelease(mtlTextureRef);
+		MetalBackedTexture& texture = m_mtlToVkTextureMap[static_cast<void*>(mtlTexture)];
+		if (!texture)
+		{ // Init texture
+			texture.Init(m_resourceManager, mtlTexture, false);
+			Logger::RENDER->info("Metal Texture Cache grew to: {}", m_mtlToVkTextureMap.size());
+		}
+		return &texture;
+	}
+
+	void MetalTextureCache::ReturnTexture(Scene::Texture* texture)
+	{
+
+	}
+
+	void MetalTextureCache::Close()
+	{
+		m_mtlToVkTextureMap.clear();
+		m_resourceManager = nullptr;
+		//TODO delete the texture cache object?
+		m_textureCache = nullptr;
+	}
+}