Add ArBackgroundDrawable

openVulkanoCpp/AR/ArFrame.hpp

@@ -31,6 +31,7 @@ namespace OpenVulkano::AR
 	public:
 		void* data;
 		Math::Vector2ui resolution;
+		uint32_t numChannels = 1;
 	};
 
 	class ArImagePlanar
@@ -49,12 +50,12 @@ namespace OpenVulkano::AR
 			const uint8_t* lumColBuffer = static_cast<const uint8_t*>(luminescenceOrColor.data);
 			if (format == Format::BGR)
 			{
-				size_t idx = (y * luminescenceOrColor.resolution.x + x) * 3;
+				size_t idx = (y * luminescenceOrColor.resolution.x + x) * luminescenceOrColor.numChannels;
 				return { lumColBuffer[idx + 2], lumColBuffer[idx + 1], lumColBuffer[idx], 255 };
 			}
 			else if (format == Format::RGB)
 			{
-				size_t idx = (y * luminescenceOrColor.resolution.x + x) * 3;
+				size_t idx = (y * luminescenceOrColor.resolution.x + x) * luminescenceOrColor.numChannels;
 				return { lumColBuffer[idx], lumColBuffer[idx + 1], lumColBuffer[idx + 2], 255 };
 			}
 

openVulkanoCpp/AR/ArSession.hpp

@@ -19,9 +19,14 @@
 #include <optional>
 #include <future>
 
-namespace OpenVulkano::Scene
+namespace OpenVulkano
 {
+	class IRenderer;
+
+	namespace Scene
+	{
 		class Texture;
+	}
 }
 
 namespace OpenVulkano::AR
@@ -203,6 +208,12 @@ namespace OpenVulkano::AR
 
 		virtual void RequestHighResolutionFrame() {}
 
+		/**
+		 * Sets the renderer that should be used for texture creation.
+		 * @param renderer The renderer to be used to create textures.
+		 */
+		virtual void SetRenderer(IRenderer* renderer) = 0;
+
 		/**
 		 * Gets the capabilities for this ArSession.
 		 * @return The capabilities for the current AR session.

openVulkanoCpp/AR/Provider/Network/ArSessionStream.h

@@ -33,6 +33,8 @@ namespace OpenVulkano::AR::Network
 		[[nodiscard]] ArSessionType GetSessionType() override { return ArSessionType::NETWORK_STREAM; }
 
 		[[nodiscard]] ArType GetArType() override;
+
+		void SetRenderer(IRenderer* renderer) {} // TODO
 	};
 }
 

openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp

@@ -30,6 +30,7 @@ namespace OpenVulkano::AR::Playback
 		colorImage.intrinsic = frameMetadata.intrinsic.GetForResolution({ colorImgData.cols, colorImgData.rows });
 		colorImage.format = ArImagePlanar::Format::RGB;
 		colorImage.luminescenceOrColor = { colorImgData.data, { colorImgData.cols, colorImgData.rows }};
+		colorImage.luminescenceOrColor.numChannels = colorImgData.channels;
 		SetSaved();
 	}
 

openVulkanoCpp/AR/Provider/Playback/ArPlaybackReader.cpp

@@ -23,13 +23,12 @@ namespace OpenVulkano::AR::Playback
 		tjhandle jpegDecompressor = tjInitDecompress();
 		tjDecompressHeader2(jpegDecompressor, compressedImage, jpegSize, &img.cols, &img.rows, &jpegSubsamp);
 
-		img.channels = 3;
+		img.channels = 4;
-		img.dataPtr = std::shared_ptr<uint8_t>(new uint8_t[img.cols * img.rows * 3]);
+		img.dataPtr = std::shared_ptr<uint8_t>(new uint8_t[img.cols * img.rows * img.channels]);
 		img.data = img.dataPtr.get();
 
 		//TODO is it better to not map to rgb? to keep the same pipeline as on device
-		tjDecompress2(jpegDecompressor, compressedImage, jpegSize, img.data, img.cols, 0/*pitch*/, img.rows, TJPF_RGB, TJFLAG_FASTDCT);
+		tjDecompress2(jpegDecompressor, compressedImage, jpegSize, img.data, img.cols, 0/*pitch*/, img.rows, TJPF_RGBA, TJFLAG_FASTDCT);
-		//tjDecompressToYUV2(jpegDecompressor, compressedImage, jpegSize, img.data, img.cols, img.rows, 1, TJFLAG_FASTDCT);
 		tjDestroy(jpegDecompressor);
 
 		//auto buff = new uint8_t[img.cols * img.rows * 3];

openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.cpp

@@ -7,6 +7,7 @@
 #include "ArSessionPlayback.hpp"
 #include "ArFramePlayback.hpp"
 #include "Base/Logger.hpp"
+#include "Scene/Texture.hpp"
 #include <filesystem>
 
 namespace OpenVulkano::AR::Playback
@@ -91,10 +92,32 @@ namespace OpenVulkano::AR::Playback
 
 	Scene::Texture* ArSessionPlayback::MakeTexture(OpenVulkano::AR::ArFrame* frame)
 	{
-		return nullptr; //TODO
+		Scene::Texture* texture;
+		if (!m_textureCache.empty())
+		{
+			texture = m_textureCache.back();
+			m_textureCache.pop_back();
+		}
+		else
+		{
+			texture = new Scene::Texture();
+			texture->format = DataFormat::R8G8B8A8_UNORM;
+			texture->updateFrequency = Scene::UpdateFrequency::Always;
+		}
+		auto img = frame->GetCameraImage();
+		texture->resolution = { img.luminescenceOrColor.resolution , 1 };
+		texture->textureBuffer = img.luminescenceOrColor.data;
+		texture->size = img.luminescenceOrColor.resolution.x * img.luminescenceOrColor.resolution.y * img.luminescenceOrColor.numChannels;
+		texture->updated = true;
+		return texture;
 	}
 
 	void ArSessionPlayback::ReturnTexture(Scene::Texture* texture)
+	{
+		m_textureCache.push_back(texture);
+	}
+
+	void ArSessionPlayback::SetRenderer(IRenderer* renderer)
 	{
 		//TODO
 	}

openVulkanoCpp/AR/Provider/Playback/ArSessionPlayback.hpp

@@ -31,6 +31,8 @@ class ArSessionPlayback final : public ArSession, public std::enable_shared_from
 
 		[[nodiscard]] ArType GetArType() override;
 
+		void SetRenderer(IRenderer* renderer);
+
 	protected:
 		Scene::Texture * MakeTexture(OpenVulkano::AR::ArFrame *frame) override;
 
@@ -48,5 +50,7 @@ class ArSessionPlayback final : public ArSession, public std::enable_shared_from
 		std::atomic_bool m_frameConsumed = true;
 		std::shared_ptr<ArFrame> m_nextFrame;
 		std::thread m_playbackReaderThread;
+
+		std::vector<Scene::Texture*> m_textureCache;
 	};
 }

openVulkanoCpp/Scene/Prefabs/ArBackgroundDrawable.cpp

@@ -0,0 +1,64 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#include "ArBackgroundDrawable.hpp"
+#include "AR/ArSession.hpp"
+#include "AR/ArFrame.hpp"
+#include "Base/Logger.hpp"
+
+namespace OpenVulkano::Scene
+{
+	ArBackgroundDrawable::ArBackgroundDrawable(const Ptr<AR::ArSession>& arSession)
+			: Drawable(DrawEncoder::GetDrawEncoder<ArBackgroundDrawable>(), DrawPhase::BACKGROUND)
+			, m_arSession(arSession)
+	{
+		m_shader.topology = Topology::TRIANGLE_STRIP;
+		m_shader.AddShaderProgram(ShaderProgramType::VERTEX, "Shader/background");
+		m_shader.AddShaderProgram(ShaderProgramType::FRAGMENT, "Shader/background");
+		//m_shader.AddDescriptorSetLayoutBinding(DESCRIPTOR_SET_LAYOUT_BINDING);
+		m_shader.AddDescriptorSetLayoutBinding(Texture::DESCRIPTOR_SET_LAYOUT_BINDING);
+		SetShader(&m_shader);
+		if (m_arSession)
+			m_arSession->OnNewFrame += EventHandler(this, &ArBackgroundDrawable::OnNewArFrame);
+	}
+
+	void ArBackgroundDrawable::Init(const Ptr<AR::ArSession>& arSession)
+	{
+		if (m_arSession)
+		{
+			if (m_arSession == arSession) return;
+			Close();
+		}
+		if (!arSession) return;
+		m_arSession = arSession;
+		m_arSession->OnNewFrame += EventHandler(this, &ArBackgroundDrawable::OnNewArFrame);
+	}
+
+	void ArBackgroundDrawable::Close()
+	{
+		m_arSession->OnNewFrame -= EventHandler(this, &ArBackgroundDrawable::OnNewArFrame);
+		m_nextFrame = nullptr;
+		Drawable::Close();
+	}
+
+	void ArBackgroundDrawable::OnNewArFrame(const Ptr<AR::ArFrame>& frame)
+	{
+		m_nextFrame = frame;
+	}
+
+	void ArBackgroundDrawable::Tick()
+	{
+		m_lastFrame = nullptr;
+		if (m_nextFrame)
+		{
+			m_lastFrame = std::move(m_currentFrame);
+			m_currentFrame = std::move(m_nextFrame);
+			m_nextFrame = nullptr;
+		}
+		if (m_currentFrame) m_texture = m_currentFrame->GetImageTexture();
+		if (!m_texture) m_texture = &Texture::PLACEHOLDER;
+	}
+}

openVulkanoCpp/Scene/Prefabs/ArBackgroundDrawable.hpp

@@ -0,0 +1,49 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#pragma once
+
+#include "Base/Wrapper.hpp"
+#include "Scene/Drawable.hpp"
+#include "Scene/Shader/Shader.hpp"
+#include "Scene/Texture.hpp"
+
+namespace OpenVulkano
+{
+	namespace AR
+	{
+		class ArSession;
+		class ArFrame;
+	}
+
+	namespace Scene
+	{
+		class ArBackgroundDrawable final : public Drawable
+		{
+			static constexpr inline DescriptorSetLayoutBinding DESCRIPTOR_SET_LAYOUT_BINDING = { 0, DescriptorSetLayoutBinding::Type::TYPE_UNIFORM_BUFFER, 1, ShaderProgramType::ALL_GRAPHICS };
+
+			Shader m_shader;
+			Ptr<AR::ArSession> m_arSession;
+			Ptr<AR::ArFrame> m_nextFrame, m_currentFrame, m_lastFrame;
+			const Texture* m_texture;
+
+			void OnNewArFrame(const Ptr<AR::ArFrame>& frame);
+
+		public:
+			ArBackgroundDrawable(const Ptr<AR::ArSession>& arSession = nullptr);
+
+			~ArBackgroundDrawable() override { if (m_arSession) ArBackgroundDrawable::Close(); }
+
+			void Init(const Ptr<AR::ArSession>& arSession);
+
+			void Tick();
+
+			void Close() override;
+
+			const Texture* GetTexture() const { return m_texture; }
+		};
+	}
+}

openVulkanoCpp/Vulkan/Scene/ArBackgroundDrawableVulkanEncoder.cpp

@@ -0,0 +1,34 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#include "Scene/Prefabs/ArBackgroundDrawable.hpp"
+#include "VulkanGeometry.hpp"
+#include "Vulkan/VulkanDrawContext.hpp"
+#include "Vulkan/Scene/VulkanTexture.hpp"
+
+using namespace OpenVulkano::Scene;
+
+namespace OpenVulkano::Vulkan
+{
+	void EncodeArBackgroundDrawable(Drawable* instance, Vulkan::VulkanDrawContext* drawContext)
+	{
+		ArBackgroundDrawable* bgDrawable = static_cast<ArBackgroundDrawable*>(instance);
+		bgDrawable->Tick();
+		const Texture* texture = bgDrawable->GetTexture();
+		VulkanTexture* vkTexture = static_cast<VulkanTexture*>(texture->renderTexture);
+		if (!vkTexture)
+		{
+			vkTexture = drawContext->renderer->GetResourceManager().PrepareTexture(const_cast<Texture*>(texture));
+		}
+		vkTexture->Record(drawContext, 2);
+		drawContext->commandBuffer.draw(4, 1, 0, 0);
+	}
+}
+
+namespace
+{
+	void* arBgDrawableVulkanEncoderReg = DrawEncoder::RegisterVulkanEncodeFunction<ArBackgroundDrawable>(&OpenVulkano::Vulkan::EncodeArBackgroundDrawable);
+}