From f042c5f37ae1b57487835882839d99f288530cf0 Mon Sep 17 00:00:00 2001 From: Georg Hagen Date: Tue, 27 May 2025 00:01:41 +0200 Subject: [PATCH] Expand ArFramePlayback to allow to be loaded from single file ArFrames saved with "SaveToFile" --- .../AR/Provider/Playback/ArFramePlayback.cpp | 66 +++++++++++++++---- .../AR/Provider/Playback/ArFramePlayback.hpp | 17 +++-- .../AR/Provider/Playback/ArPlaybackReader.cpp | 53 +++++++-------- .../AR/Provider/Playback/ArPlaybackReader.hpp | 8 ++- 4 files changed, 95 insertions(+), 49 deletions(-) diff --git a/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp b/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp index e583124..49ae123 100644 --- a/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp +++ b/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.cpp @@ -19,26 +19,68 @@ namespace OpenVulkano::AR::Playback if (session->GetCapabilities().IsDepthSupported()) { auto depth = frameReader.ReadDepthImage(); - confImgData = std::move(depth.confidence.image); - depthImgData = std::move(depth.depth.image); - depthImage.format = session->GetSessionMetadata().depthFormat; - depthImage.intrinsic = frameMetadata.intrinsic.GetForResolution({depth.depth.header.width, depth.depth.header.height}); - depthImage.depth.data = depthImgData.get(); - depthImage.depth.resolution = {depth.depth.header.width, depth.depth.header.height}; - depthImage.confidence.data = confImgData.get(); - depthImage.confidence.resolution = {depth.confidence.header.width, depth.confidence.header.height}; + SetDepthImgInfo(depth); } if (session->IsLoadColorEnabled()) { colorImgData = frameReader.ReadColorImage(); - colorImage.intrinsic = frameMetadata.intrinsic.GetForResolution({colorImgData.cols, colorImgData.rows}); - colorImage.format = ArImagePlanar::Format::RGB; - colorImage.luminescenceOrColor = {colorImgData.data, {colorImgData.cols, colorImgData.rows}}; - colorImage.luminescenceOrColor.numChannels = colorImgData.channels; + SetColorImgInfo(); } SetSaved(); } + ArFramePlayback::ArFramePlayback(const std::shared_ptr& session, const std::filesystem::path& imagePath) + : ArFrame(session, 99999) // TODO find better way to generate id + { + ArchiveReader reader(imagePath); + DepthImage dImage; + while(reader.HasNext()) + { + reader.GetNextFileAsStream([this, &dImage](const FileDescription& desc, std::istream& data) + { + if (desc.path.find("depth") != std::string::npos) + { + dImage.depth.TryRead(data); + } + else if (desc.path.find("confidence") != std::string::npos) + { + dImage.confidence.TryRead(data); + } + else if (desc.path.find("metadata") != std::string::npos) + { + std::ostringstream buffer; + buffer << data.rdbuf(); + std::string str = buffer.str(); + frameMetadata = ArFrameMetadata::FromContent(str.data(), str.size()); + } + }); + } + colorImgData.channels = ColorImg::RGBA; // TODO test if defaulting to NV12 + colorImgData.Decode(Utils::ReadFile(imagePath)); + SetDepthImgInfo(dImage); + SetSaved(); + } + + void ArFramePlayback::SetDepthImgInfo(DepthImage& depth) + { + confImgData = std::move(depth.confidence.image); + depthImgData = std::move(depth.depth.image); + depthImage.format = ArDepthFormat::METER_FP32; //TODO session->GetSessionMetadata().depthFormat; + depthImage.intrinsic = frameMetadata.intrinsic.GetForResolution({depth.depth.header.width, depth.depth.header.height}); + depthImage.depth.data = depthImgData.get(); + depthImage.depth.resolution = {depth.depth.header.width, depth.depth.header.height}; + depthImage.confidence.data = confImgData.get(); + depthImage.confidence.resolution = {depth.confidence.header.width, depth.confidence.header.height}; + } + + void ArFramePlayback::SetColorImgInfo() + { + colorImage.intrinsic = frameMetadata.intrinsic.GetForResolution({colorImgData.cols, colorImgData.rows}); + colorImage.format = ArImagePlanar::Format::RGB; + colorImage.luminescenceOrColor = {colorImgData.data, {colorImgData.cols, colorImgData.rows}}; + colorImage.luminescenceOrColor.numChannels = colorImgData.channels; + } + ArImagePlanar ArFramePlayback::GetCameraImage() { return colorImage; diff --git a/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.hpp b/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.hpp index 0fceaa8..883366f 100644 --- a/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.hpp +++ b/openVulkanoCpp/AR/Provider/Playback/ArFramePlayback.hpp @@ -15,9 +15,18 @@ namespace OpenVulkano::AR::Playback class ArFramePlayback final : public ArFrame { + ArImagePlanar colorImage; + ArDepthImage depthImage; + + std::unique_ptr confImgData; + std::unique_ptr depthImgData; + ColorImg colorImgData; + public: ArFramePlayback(const std::shared_ptr& session, ArPlaybackReader& frameReader); + ArFramePlayback(const std::shared_ptr& session, const std::filesystem::path& imagePath); + ArImagePlanar GetCameraImage() override; ArDepthImage GetDepthImage() override; @@ -27,11 +36,7 @@ namespace OpenVulkano::AR::Playback Math::Matrix4f GetCameraProjection(Math::Vector2f viewportSize, float near, float far) override; private: - ArImagePlanar colorImage; - ArDepthImage depthImage; - - std::unique_ptr confImgData; - std::unique_ptr depthImgData; - ColorImg colorImgData; + void SetDepthImgInfo(DepthImage& depth); + void SetColorImgInfo(); }; } diff --git a/openVulkanoCpp/AR/Provider/Playback/ArPlaybackReader.cpp b/openVulkanoCpp/AR/Provider/Playback/ArPlaybackReader.cpp index d8705f6..4d2dffc 100644 --- a/openVulkanoCpp/AR/Provider/Playback/ArPlaybackReader.cpp +++ b/openVulkanoCpp/AR/Provider/Playback/ArPlaybackReader.cpp @@ -15,64 +15,61 @@ namespace OpenVulkano::AR::Playback { - void ArPlaybackReader::ReadColorImage(ColorImg& img) + void ColorImg::Decode(const Array& dataIn) { //BlockProfiler profiler("Load jpeg"); - std::optional>> file = ReadColorImageRaw(); - long unsigned int jpegSize = file->second.Size(); - unsigned char* compressedImage = reinterpret_cast(file->second.Data()); + long unsigned int jpegSize = dataIn.Size(); + unsigned char* compressedImage = const_cast(reinterpret_cast(dataIn.Data())); int jpegSubsamp; tjhandle jpegDecompressor = tjInitDecompress(); - tjDecompressHeader2(jpegDecompressor, compressedImage, jpegSize, &img.cols, &img.rows, &jpegSubsamp); + tjDecompressHeader2(jpegDecompressor, compressedImage, jpegSize, &cols, &rows, &jpegSubsamp); - if (img.UseRGB()) + if (UseRGB()) { - img.channels = 4; - if (!img.data) img.Allocate(); + channels = 4; + if (!data) Allocate(); - tjDecompress2(jpegDecompressor, compressedImage, jpegSize, img.data, img.cols, 0/*pitch*/, img.rows, TJPF_RGBA, TJFLAG_FASTDCT); + tjDecompress2(jpegDecompressor, compressedImage, jpegSize, data, cols, 0/*pitch*/, rows, TJPF_RGBA, TJFLAG_FASTDCT); } - else if (img.channels == 1) + else if (channels == 1) { - if (!img.data) img.Allocate(); + if (!data) Allocate(); - tjDecompress2(jpegDecompressor, compressedImage, jpegSize, img.data, img.cols, 0/*pitch*/, img.rows, TJPF_GRAY, TJFLAG_FASTDCT); + tjDecompress2(jpegDecompressor, compressedImage, jpegSize, data, cols, 0/*pitch*/, rows, TJPF_GRAY, TJFLAG_FASTDCT); } - else if (img.channels != 0) + else if (channels != 0) { - if (!img.data) img.Allocate(); + if (!data) Allocate(); std::array planes; - size_t chromaSize = img.cols * img.rows / 4; - planes[0] = img.data; - if (!img.dataUV) img.dataUV = img.data + (img.cols * img.rows); + size_t chromaSize = cols * rows / 4; + planes[0] = data; + if (!dataUV) dataUV = data + (cols * rows); Unique chromaTmp; - if (img.channels == ColorImg::NV12 && !img.dataPtr) + if (channels == ColorImg::NV12 && !dataPtr) { - chromaTmp = std::make_unique(img.cols * img.rows / 2); + chromaTmp = std::make_unique(cols * rows / 2); planes[1] = chromaTmp.get(); } else { - planes[1] = img.dataUV; - if (img.channels == ColorImg::NV12) planes[1] += chromaSize; + planes[1] = dataUV; + if (channels == ColorImg::NV12) planes[1] += chromaSize; } planes[2] = planes[1] + chromaSize; - std::array rowSize = { img.cols, img.cols / 2, img.cols / 2 }; + std::array rowSize = { cols, cols / 2, cols / 2 }; tjDecompressToYUVPlanes(jpegDecompressor, compressedImage, jpegSize, planes.data(), - img.cols, rowSize.data(), img.rows, TJFLAG_FASTDCT); + cols, rowSize.data(), rows, TJFLAG_FASTDCT); - if (img.channels == ColorImg::NV12) + if (channels == ColorImg::NV12) { - YuvUtils::NV12FromChromaPlanes(planes[1], img.data + (img.cols * img.rows), chromaSize); + YuvUtils::NV12FromChromaPlanes(planes[1], data + (cols * rows), chromaSize); } } tjDestroy(jpegDecompressor); - - m_imgReadSize += 1000 + file.value().first.size; } } @@ -87,7 +84,7 @@ namespace OpenVulkano::AR::Playback ColorImg img; auto file = m_archiveColor.GetNextFile(); img.dataPtr = std::shared_ptr( - stbi_load_from_memory(reinterpret_cast(file->second.Data()), file->second.Size(), &img.cols, &img.rows, &img.channels, 3), + stbi_load_from_memory(reinterpret_cast(data.Data()), data.Size(), &img.cols, &img.rows, &img.channels, 3), &stbi_image_free); img.data = img.dataPtr.get(); m_imgReadSize += 1000 + file.value().first.size; diff --git a/openVulkanoCpp/AR/Provider/Playback/ArPlaybackReader.hpp b/openVulkanoCpp/AR/Provider/Playback/ArPlaybackReader.hpp index 254b784..5da70b4 100644 --- a/openVulkanoCpp/AR/Provider/Playback/ArPlaybackReader.hpp +++ b/openVulkanoCpp/AR/Provider/Playback/ArPlaybackReader.hpp @@ -37,6 +37,8 @@ namespace OpenVulkano::AR::Playback } bool UseRGB() const { return channels == RGBA; } + + void Decode(const Array& data); }; class ArPlaybackReader final @@ -72,14 +74,14 @@ namespace OpenVulkano::AR::Playback { ColorImg img; img.channels = ColorImg::RGBA; // TODO test if defaulting to NV12 - ReadColorImage(img); + std::optional>> file = ReadColorImageRaw(); + img.Decode(file->second); + m_imgReadSize += 1000 + file.value().first.size; return img; } std::optional>> ReadColorImageRaw() { return m_archiveColor.GetNextFile(); } - void ReadColorImage(ColorImg& img); - DepthImage ReadDepthImage() { DepthImage img;