diff --git a/openVulkanoCpp/AR/ArFrame.hpp b/openVulkanoCpp/AR/ArFrame.hpp index ae769a3..db0ce51 100644 --- a/openVulkanoCpp/AR/ArFrame.hpp +++ b/openVulkanoCpp/AR/ArFrame.hpp @@ -31,6 +31,7 @@ namespace OpenVulkano::AR public: void* data; Math::Vector2ui resolution; + uint32_t numChannels = 1; }; class ArImagePlanar @@ -49,12 +50,12 @@ namespace OpenVulkano::AR const uint8_t* lumColBuffer = static_cast(luminescenceOrColor.data); if (format == Format::BGR) { - size_t idx = (y * luminescenceOrColor.resolution.x + x) * 3; + size_t idx = (y * luminescenceOrColor.resolution.x + x) * luminescenceOrColor.numChannels; return { lumColBuffer[idx + 2], lumColBuffer[idx + 1], lumColBuffer[idx], 255 }; } else if (format == Format::RGB) { - size_t idx = (y * luminescenceOrColor.resolution.x + x) * 3; + size_t idx = (y * luminescenceOrColor.resolution.x + x) * luminescenceOrColor.numChannels; return { lumColBuffer[idx], lumColBuffer[idx + 1], lumColBuffer[idx + 2], 255 }; } diff --git a/openVulkanoCpp/AR/ArSession.hpp b/openVulkanoCpp/AR/ArSession.hpp index 058853f..5a47143 100644 --- a/openVulkanoCpp/AR/ArSession.hpp +++ b/openVulkanoCpp/AR/ArSession.hpp @@ -19,9 +19,14 @@ #include #include -namespace OpenVulkano::Scene +namespace OpenVulkano { - class Texture; + class IRenderer; + + namespace Scene + { + class Texture; + } } namespace OpenVulkano::AR @@ -203,6 +208,12 @@ namespace OpenVulkano::AR virtual void RequestHighResolutionFrame() {} + /** + * Sets the renderer that should be used for texture creation. + * @param renderer The renderer to be used to create textures. + */ + virtual void SetRenderer(IRenderer* renderer) = 0; + /** * Gets the capabilities for this ArSession. * @return The capabilities for the current AR session. diff --git a/openVulkanoCpp/AR/Provider/Network/ArSessionStream.h b/openVulkanoCpp/AR/Provider/Network/ArSessionStream.h index d75d19c..799943b 100644 --- a/openVulkanoCpp/AR/Provider/Network/ArSessionStream.h +++ b/openVulkanoCpp/AR/Provider/Network/ArSessionStream.h @@ -33,6 +33,8 @@ namespace OpenVulkano::AR::Network [[nodiscard]] ArSessionType GetSessionType() override { return ArSessionType::NETWORK_STREAM; } [[nodiscard]] ArType GetArType() override; + + void SetRenderer(IRenderer* renderer) {} // TODO }; } diff --git a/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp b/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp index ec77265..c7f4807 100644 --- a/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp +++ b/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp @@ -30,6 +30,7 @@ namespace OpenVulkano::AR::Playback colorImage.intrinsic = frameMetadata.intrinsic.GetForResolution({ colorImgData.cols, colorImgData.rows }); colorImage.format = ArImagePlanar::Format::RGB; colorImage.luminescenceOrColor = { colorImgData.data, { colorImgData.cols, colorImgData.rows }}; + colorImage.luminescenceOrColor.numChannels = colorImgData.channels; SetSaved(); } diff --git a/openVulkanoCpp/AR/Provider/Playback/ArPlaybackReader.cpp b/openVulkanoCpp/AR/Provider/Playback/ArPlaybackReader.cpp index 94ffce3..1ae47bd 100644 --- a/openVulkanoCpp/AR/Provider/Playback/ArPlaybackReader.cpp +++ b/openVulkanoCpp/AR/Provider/Playback/ArPlaybackReader.cpp @@ -23,13 +23,12 @@ namespace OpenVulkano::AR::Playback tjhandle jpegDecompressor = tjInitDecompress(); tjDecompressHeader2(jpegDecompressor, compressedImage, jpegSize, &img.cols, &img.rows, &jpegSubsamp); - img.channels = 3; - img.dataPtr = std::shared_ptr(new uint8_t[img.cols * img.rows * 3]); + img.channels = 4; + img.dataPtr = std::shared_ptr(new uint8_t[img.cols * img.rows * img.channels]); img.data = img.dataPtr.get(); //TODO is it better to not map to rgb? to keep the same pipeline as on device - tjDecompress2(jpegDecompressor, compressedImage, jpegSize, img.data, img.cols, 0/*pitch*/, img.rows, TJPF_RGB, TJFLAG_FASTDCT); - //tjDecompressToYUV2(jpegDecompressor, compressedImage, jpegSize, img.data, img.cols, img.rows, 1, TJFLAG_FASTDCT); + tjDecompress2(jpegDecompressor, compressedImage, jpegSize, img.data, img.cols, 0/*pitch*/, img.rows, TJPF_RGBA, TJFLAG_FASTDCT); tjDestroy(jpegDecompressor); //auto buff = new uint8_t[img.cols * img.rows * 3]; diff --git a/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp b/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp index 7ce3844..5d1792b 100644 --- a/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp +++ b/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp @@ -7,6 +7,7 @@ #include "ArSessionPlayback.hpp" #include "ArFramePlayback.hpp" #include "Base/Logger.hpp" +#include "Scene/Texture.hpp" #include namespace OpenVulkano::AR::Playback @@ -91,10 +92,32 @@ namespace OpenVulkano::AR::Playback Scene::Texture* ArSessionPlayback::MakeTexture(OpenVulkano::AR::ArFrame* frame) { - return nullptr; //TODO + Scene::Texture* texture; + if (!m_textureCache.empty()) + { + texture = m_textureCache.back(); + m_textureCache.pop_back(); + } + else + { + texture = new Scene::Texture(); + texture->format = DataFormat::R8G8B8A8_UNORM; + texture->updateFrequency = Scene::UpdateFrequency::Always; + } + auto img = frame->GetCameraImage(); + texture->resolution = { img.luminescenceOrColor.resolution , 1 }; + texture->textureBuffer = img.luminescenceOrColor.data; + texture->size = img.luminescenceOrColor.resolution.x * img.luminescenceOrColor.resolution.y * img.luminescenceOrColor.numChannels; + texture->updated = true; + return texture; } void ArSessionPlayback::ReturnTexture(Scene::Texture* texture) + { + m_textureCache.push_back(texture); + } + + void ArSessionPlayback::SetRenderer(IRenderer* renderer) { //TODO } diff --git a/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.hpp b/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.hpp index e3f77eb..fa481ee 100644 --- a/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.hpp +++ b/openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.hpp @@ -31,6 +31,8 @@ class ArSessionPlayback final : public ArSession, public std::enable_shared_from [[nodiscard]] ArType GetArType() override; + void SetRenderer(IRenderer* renderer); + protected: Scene::Texture * MakeTexture(OpenVulkano::AR::ArFrame *frame) override; @@ -48,5 +50,7 @@ class ArSessionPlayback final : public ArSession, public std::enable_shared_from std::atomic_bool m_frameConsumed = true; std::shared_ptr m_nextFrame; std::thread m_playbackReaderThread; + + std::vector m_textureCache; }; } diff --git a/openVulkanoCpp/Scene/Prefabs/ArBackgroundDrawable.cpp b/openVulkanoCpp/Scene/Prefabs/ArBackgroundDrawable.cpp new file mode 100644 index 0000000..d44e3f3 --- /dev/null +++ b/openVulkanoCpp/Scene/Prefabs/ArBackgroundDrawable.cpp @@ -0,0 +1,64 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#include "ArBackgroundDrawable.hpp" +#include "AR/ArSession.hpp" +#include "AR/ArFrame.hpp" +#include "Base/Logger.hpp" + +namespace OpenVulkano::Scene +{ + ArBackgroundDrawable::ArBackgroundDrawable(const Ptr& arSession) + : Drawable(DrawEncoder::GetDrawEncoder(), DrawPhase::BACKGROUND) + , m_arSession(arSession) + { + m_shader.topology = Topology::TRIANGLE_STRIP; + m_shader.AddShaderProgram(ShaderProgramType::VERTEX, "Shader/background"); + m_shader.AddShaderProgram(ShaderProgramType::FRAGMENT, "Shader/background"); + //m_shader.AddDescriptorSetLayoutBinding(DESCRIPTOR_SET_LAYOUT_BINDING); + m_shader.AddDescriptorSetLayoutBinding(Texture::DESCRIPTOR_SET_LAYOUT_BINDING); + SetShader(&m_shader); + if (m_arSession) + m_arSession->OnNewFrame += EventHandler(this, &ArBackgroundDrawable::OnNewArFrame); + } + + void ArBackgroundDrawable::Init(const Ptr& arSession) + { + if (m_arSession) + { + if (m_arSession == arSession) return; + Close(); + } + if (!arSession) return; + m_arSession = arSession; + m_arSession->OnNewFrame += EventHandler(this, &ArBackgroundDrawable::OnNewArFrame); + } + + void ArBackgroundDrawable::Close() + { + m_arSession->OnNewFrame -= EventHandler(this, &ArBackgroundDrawable::OnNewArFrame); + m_nextFrame = nullptr; + Drawable::Close(); + } + + void ArBackgroundDrawable::OnNewArFrame(const Ptr& frame) + { + m_nextFrame = frame; + } + + void ArBackgroundDrawable::Tick() + { + m_lastFrame = nullptr; + if (m_nextFrame) + { + m_lastFrame = std::move(m_currentFrame); + m_currentFrame = std::move(m_nextFrame); + m_nextFrame = nullptr; + } + if (m_currentFrame) m_texture = m_currentFrame->GetImageTexture(); + if (!m_texture) m_texture = &Texture::PLACEHOLDER; + } +} \ No newline at end of file diff --git a/openVulkanoCpp/Scene/Prefabs/ArBackgroundDrawable.hpp b/openVulkanoCpp/Scene/Prefabs/ArBackgroundDrawable.hpp new file mode 100644 index 0000000..206600e --- /dev/null +++ b/openVulkanoCpp/Scene/Prefabs/ArBackgroundDrawable.hpp @@ -0,0 +1,49 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#pragma once + +#include "Base/Wrapper.hpp" +#include "Scene/Drawable.hpp" +#include "Scene/Shader/Shader.hpp" +#include "Scene/Texture.hpp" + +namespace OpenVulkano +{ + namespace AR + { + class ArSession; + class ArFrame; + } + + namespace Scene + { + class ArBackgroundDrawable final : public Drawable + { + static constexpr inline DescriptorSetLayoutBinding DESCRIPTOR_SET_LAYOUT_BINDING = { 0, DescriptorSetLayoutBinding::Type::TYPE_UNIFORM_BUFFER, 1, ShaderProgramType::ALL_GRAPHICS }; + + Shader m_shader; + Ptr m_arSession; + Ptr m_nextFrame, m_currentFrame, m_lastFrame; + const Texture* m_texture; + + void OnNewArFrame(const Ptr& frame); + + public: + ArBackgroundDrawable(const Ptr& arSession = nullptr); + + ~ArBackgroundDrawable() override { if (m_arSession) ArBackgroundDrawable::Close(); } + + void Init(const Ptr& arSession); + + void Tick(); + + void Close() override; + + const Texture* GetTexture() const { return m_texture; } + }; + } +} \ No newline at end of file diff --git a/openVulkanoCpp/Vulkan/Scene/ArBackgroundDrawableVulkanEncoder.cpp b/openVulkanoCpp/Vulkan/Scene/ArBackgroundDrawableVulkanEncoder.cpp new file mode 100644 index 0000000..5566105 --- /dev/null +++ b/openVulkanoCpp/Vulkan/Scene/ArBackgroundDrawableVulkanEncoder.cpp @@ -0,0 +1,34 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#include "Scene/Prefabs/ArBackgroundDrawable.hpp" +#include "VulkanGeometry.hpp" +#include "Vulkan/VulkanDrawContext.hpp" +#include "Vulkan/Scene/VulkanTexture.hpp" + +using namespace OpenVulkano::Scene; + +namespace OpenVulkano::Vulkan +{ + void EncodeArBackgroundDrawable(Drawable* instance, Vulkan::VulkanDrawContext* drawContext) + { + ArBackgroundDrawable* bgDrawable = static_cast(instance); + bgDrawable->Tick(); + const Texture* texture = bgDrawable->GetTexture(); + VulkanTexture* vkTexture = static_cast(texture->renderTexture); + if (!vkTexture) + { + vkTexture = drawContext->renderer->GetResourceManager().PrepareTexture(const_cast(texture)); + } + vkTexture->Record(drawContext, 2); + drawContext->commandBuffer.draw(4, 1, 0, 0); + } +} + +namespace +{ + void* arBgDrawableVulkanEncoderReg = DrawEncoder::RegisterVulkanEncodeFunction(&OpenVulkano::Vulkan::EncodeArBackgroundDrawable); +} \ No newline at end of file