From bbaf8933ae9c1d4e79b62d7ac3dee8e1eba83738 Mon Sep 17 00:00:00 2001 From: Georg Hagen Date: Tue, 9 Jul 2024 13:43:09 +0200 Subject: [PATCH] Add texture cache for ar frame textures --- .../Provider/ArKit/ArSessionArKitInternal.h | 7 ++- .../Provider/ArKit/ArSessionArKitInternal.mm | 25 ++++++++ .../Vulkan/Metal/MetalTextureCache.h | 35 +++++++++++ .../Vulkan/Metal/MetalTextureCache.mm | 63 +++++++++++++++++++ 4 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 openVulkanoCpp/Vulkan/Metal/MetalTextureCache.h create mode 100644 openVulkanoCpp/Vulkan/Metal/MetalTextureCache.mm diff --git a/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.h b/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.h index bf60aaa..5fc8f8d 100644 --- a/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.h +++ b/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.h @@ -8,6 +8,8 @@ #include "ArSessionArKit.h" #include "Data/Concurent/MutexProtectedObject.hpp" +#include "Scene/Texture.hpp" +#include "Vulkan/Metal/MetalTextureCache.h" #include #import @@ -46,9 +48,11 @@ namespace OpenVulkano::AR::ArKit void OnArCameraTrackingChange(ARSession* session, ARCamera* camera); void OnArAnchorsUpdate(NSArray<__kindof ARAnchor*>* anchors); bool ArShouldAttemptRelocalization(); + + void SetRenderer(IRenderer* renderer) override; protected: - Scene::Texture * MakeTexture(OpenVulkano::AR::ArFrame *frame) override; + Scene::Texture * MakeTexture(ArFrame *frame) override; void ReturnTexture(Scene::Texture *texture) override; @@ -56,6 +60,7 @@ namespace OpenVulkano::AR::ArKit ArKitDelegate* m_arKitDelegate; ARWorldTrackingConfiguration* m_arConfig; ARSession* m_arSession; + Vulkan::MetalTextureCache m_textureCache; /*#if (__cplusplus >= 202002L) std::atomic> m_frame; #else*/ diff --git a/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm b/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm index 6874e6c..9ab0568 100644 --- a/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm +++ b/openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm @@ -4,11 +4,17 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +// Makre usre the Molten include is first! +#include +#include + #include "ArSessionArKitInternal.h" #include "ArFrameArKit.h" #include "ArTrackingStateConverter.h" #include "Base/Logger.hpp" #include "IO/AppFolders.hpp" +#include "Vulkan/Renderer.hpp" +#include "Vulkan/Scene/VulkanTexture.hpp" #include #import @@ -102,6 +108,25 @@ namespace OpenVulkano::AR::ArKit [m_arKitDelegate release]; } + void ArSessionArKitInternal::SetRenderer(IRenderer* renderer) + { + m_textureCache.Init(renderer); + } + + Scene::Texture* ArSessionArKitInternal::MakeTexture(ArFrame* frame) + { + if (!m_textureCache) [[unlikely]] + throw std::runtime_error("No renderer set for which to produce textures"); + ArFrameArKit* arFrame = static_cast(frame); + ARFrame* arKitFrame = arFrame->GetArKitFrame(); + return m_textureCache.Get(arKitFrame.capturedImage, MTLPixelFormatR8Unorm); + } + + void ArSessionArKitInternal::ReturnTexture(Scene::Texture* texture) + { + m_textureCache.ReturnTexture(texture); + } + void ArSessionArKitInternal::Start() { [m_arSession runWithConfiguration:m_arConfig]; diff --git a/openVulkanoCpp/Vulkan/Metal/MetalTextureCache.h b/openVulkanoCpp/Vulkan/Metal/MetalTextureCache.h new file mode 100644 index 0000000..d6817c3 --- /dev/null +++ b/openVulkanoCpp/Vulkan/Metal/MetalTextureCache.h @@ -0,0 +1,35 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#pragma once + +#include "MetalBackedTexture.h" + +#import +#import + +namespace OpenVulkano::Vulkan +{ + class MetalTextureCache + { + CVMetalTextureCacheRef m_textureCache = nullptr; + Vulkan::ResourceManager* m_resourceManager = nullptr; + std::map m_mtlToVkTextureMap; + + public: + ~MetalTextureCache() { if (m_resourceManager) Close(); } + + void Init(IRenderer* renderer); + + void Close(); + + Scene::Texture* Get(CVPixelBufferRef pixelBuffer, MTLPixelFormat pixelFormat); + + void ReturnTexture(Scene::Texture* texture); + + operator bool() const { return m_resourceManager; } + }; +} diff --git a/openVulkanoCpp/Vulkan/Metal/MetalTextureCache.mm b/openVulkanoCpp/Vulkan/Metal/MetalTextureCache.mm new file mode 100644 index 0000000..5b0e7aa --- /dev/null +++ b/openVulkanoCpp/Vulkan/Metal/MetalTextureCache.mm @@ -0,0 +1,63 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#include "MetalTextureCache.h" +#include "Vulkan/Renderer.hpp" + +namespace OpenVulkano::Vulkan +{ + void MetalTextureCache::Init(IRenderer* renderer) + { + if (m_resourceManager) Close(); + if (!renderer) return; + auto vkRenderer = dynamic_cast(renderer); + if (!vkRenderer) throw std::invalid_argument("The Metal Texture Cache only supports Vulkan renderer!"); + // Setup texture cache + VkExportMetalDeviceInfoEXT metalDeviceInfo {}; + metalDeviceInfo.sType = VK_STRUCTURE_TYPE_EXPORT_METAL_DEVICE_INFO_EXT; + metalDeviceInfo.pNext = nullptr; + VkExportMetalObjectsInfoEXT exportInfo { VK_STRUCTURE_TYPE_EXPORT_METAL_OBJECTS_INFO_EXT, &metalDeviceInfo }; + VkDevice vkDevice = vkRenderer->GetContext().device->device; + vkExportMetalObjectsEXT(vkDevice, &exportInfo); + CVReturn result = CVMetalTextureCacheCreate(nil, nil, metalDeviceInfo.mtlDevice, nil, &m_textureCache); + if (result != kCVReturnSuccess) + { + Logger::AR->error("Failed to create metal texture cache! Status code: {}", result); + } + m_resourceManager = &vkRenderer->GetResourceManager(); + } + + Scene::Texture* MetalTextureCache::Get(CVPixelBufferRef pixelBuffer, MTLPixelFormat pixelFormat) + { + CVMetalTextureRef mtlTextureRef; + auto width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0); + auto height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0); + //TODO check pixel format from buffer? + CVMetalTextureCacheCreateTextureFromImage(nil, m_textureCache, pixelBuffer, nil, pixelFormat, width, height, 0, &mtlTextureRef); + id mtlTexture = CVMetalTextureGetTexture(mtlTextureRef); + CVBufferRelease(mtlTextureRef); + MetalBackedTexture& texture = m_mtlToVkTextureMap[static_cast(mtlTexture)]; + if (!texture) + { // Init texture + texture.Init(m_resourceManager, mtlTexture, false); + Logger::RENDER->info("Metal Texture Cache grew to: {}", m_mtlToVkTextureMap.size()); + } + return &texture; + } + + void MetalTextureCache::ReturnTexture(Scene::Texture* texture) + { + + } + + void MetalTextureCache::Close() + { + m_mtlToVkTextureMap.clear(); + m_resourceManager = nullptr; + //TODO delete the texture cache object? + m_textureCache = nullptr; + } +}