From 2a1157ef71d16127f5c483966ac26b44ae319780 Mon Sep 17 00:00:00 2001 From: Georg Hagen Date: Wed, 8 Jan 2025 15:49:19 +0100 Subject: [PATCH] Add handling for padded buffers --- openVulkanoCpp/AR/ArFrame.hpp | 7 ++-- openVulkanoCpp/AR/ArRecorder.cpp | 9 +++-- .../AR/Provider/ArKit/ArFrameArKit.mm | 4 ++- openVulkanoCpp/Image/YuvUtils.hpp | 34 ++++++++++++++----- 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/openVulkanoCpp/AR/ArFrame.hpp b/openVulkanoCpp/AR/ArFrame.hpp index c06fcc4..3088974 100644 --- a/openVulkanoCpp/AR/ArFrame.hpp +++ b/openVulkanoCpp/AR/ArFrame.hpp @@ -32,6 +32,7 @@ namespace OpenVulkano::AR void* data; Math::Vector2ui resolution; uint32_t numChannels = 1; + uint32_t rowPadding = 0; }; class ArImagePlanar @@ -64,10 +65,10 @@ namespace OpenVulkano::AR return { lumColBuffer[idx], lumColBuffer[idx + 1], lumColBuffer[idx + 2], 255 }; } - int iUV = ((y / 2) * uv.resolution.x + x / 2) * 2; + int iUV = ((y / 2) * (uv.resolution.x + uv.rowPadding / 2) + x / 2) * 2; Math::Vector4uc sample( - lumColBuffer[y * luminescenceOrColor.resolution.x + x], + lumColBuffer[y * (luminescenceOrColor.resolution.x + luminescenceOrColor.rowPadding) + x], static_cast(uv.data)[iUV], static_cast(uv.data)[iUV+1], 255); @@ -95,6 +96,8 @@ namespace OpenVulkano::AR ArImage confidence; ArDepthFormat format; Math::CameraIntrinsic intrinsic; + + operator bool() const { return depth.data; } }; class ArFrame : public std::enable_shared_from_this diff --git a/openVulkanoCpp/AR/ArRecorder.cpp b/openVulkanoCpp/AR/ArRecorder.cpp index 1b84b1d..f4f414d 100644 --- a/openVulkanoCpp/AR/ArRecorder.cpp +++ b/openVulkanoCpp/AR/ArRecorder.cpp @@ -78,7 +78,7 @@ namespace OpenVulkano::AR if (m_settings.downsampleColor && !highRes) { dataBuffer = YuvUtils::PlansFromNV12(static_cast(img.luminescenceOrColor.data), static_cast(img.uv.data), - resX, resY, img.uv.resolution.x, img.uv.resolution.y, 2, 2); + resX, resY, img.uv.resolution.x, img.uv.resolution.y, 2, 2, img.luminescenceOrColor.rowPadding, img.uv.rowPadding); resX /= 2; resY /= 2; buffers[0] = dataBuffer.get(); @@ -88,7 +88,7 @@ namespace OpenVulkano::AR else { dataBuffer = std::unique_ptr(new uint8_t[sizeUV + sizeUV]); - YuvUtils::ChromaPlanesFromNV12((uint8_t*)img.uv.data, dataBuffer.get(), sizeUV); + YuvUtils::ChromaPlanesFromNV12((uint8_t*)img.uv.data, dataBuffer.get(), img.uv.resolution.x, img.uv.resolution.y, img.uv.rowPadding); buffers[0] = static_cast(img.luminescenceOrColor.data); buffers[1] = dataBuffer.get(); buffers[2] = buffers[1] + sizeUV; @@ -284,7 +284,10 @@ namespace OpenVulkano::AR { requestExit = true; newDataAvailable.notify_one(); - if (processingThread.joinable()) processingThread.join(); + if (std::this_thread::get_id() != processingThread.get_id()) + { + if (processingThread.joinable()) processingThread.join(); + } else processingThread.detach(); } void ArRecorder::AsyncProcessor::Queue(const std::shared_ptr& frame, bool highRes) diff --git a/openVulkanoCpp/AR/Provider/ArKit/ArFrameArKit.mm b/openVulkanoCpp/AR/Provider/ArKit/ArFrameArKit.mm index ebbd00a..f93c12a 100644 --- a/openVulkanoCpp/AR/Provider/ArKit/ArFrameArKit.mm +++ b/openVulkanoCpp/AR/Provider/ArKit/ArFrameArKit.mm @@ -97,9 +97,11 @@ namespace OpenVulkano::AR::ArKit m_colorImage.intrinsic = Math::CameraIntrinsic(frameMetadata.intrinsic); m_colorImage.format = ArImagePlanar::Format::NV12; m_colorImage.luminescenceOrColor.resolution = GetSize(arKitFrame.capturedImage); + m_colorImage.luminescenceOrColor.rowPadding = CVPixelBufferGetBytesPerRowOfPlane(arKitFrame.capturedImage, 0) - m_colorImage.luminescenceOrColor.resolution.x; m_colorImage.uv.resolution = m_colorImage.luminescenceOrColor.resolution / 2u; + m_colorImage.uv.rowPadding = CVPixelBufferGetBytesPerRowOfPlane(arKitFrame.capturedImage, 1) - m_colorImage.uv.resolution.x * 2; #ifdef DEBUG - assert(m_colorImage.uv.resolution.x == GetSize(arKitFrame.capturedImage, 1).x); + assert(m_colorImage.uv.resolution == Math::Vector2ui(GetSize(arKitFrame.capturedImage, 1))); auto format = CVPixelBufferGetPixelFormatType(arKitFrame.capturedImage); assert(format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); #endif diff --git a/openVulkanoCpp/Image/YuvUtils.hpp b/openVulkanoCpp/Image/YuvUtils.hpp index 7e5ee1c..7f91e67 100644 --- a/openVulkanoCpp/Image/YuvUtils.hpp +++ b/openVulkanoCpp/Image/YuvUtils.hpp @@ -25,29 +25,45 @@ namespace OpenVulkano } } - void ChromaPlanesFromNV12(unsigned char* __restrict src, unsigned char* __restrict dest, int chromaChannelPixelCount) + void ChromaPlanesFromNV12(unsigned char* __restrict src, unsigned char* __restrict dest, uint32_t chromaChannelWidth, uint32_t chromaChannelHeight, uint32_t chromaRowPadding) { - for (int i = 0; i < chromaChannelPixelCount; i++) + uint8_t* dest2 = dest + chromaChannelWidth * chromaChannelHeight; + for (uint32_t row = 0; row < chromaChannelHeight; row++) { - dest[i] = src[i * 2]; - dest[i + chromaChannelPixelCount] = src[i * 2 + 1]; + for (int col = 0; col < chromaChannelWidth; col++, dest++, dest2++, src += 2) + { + *dest = src[0]; + *dest2 = src[1]; + } + src += chromaRowPadding; } } std::unique_ptr PlansFromNV12( unsigned char* __restrict srcLum, unsigned char* __restrict srcChroma, uint32_t sizeLumX, uint32_t sizeLumY, - uint32_t sizeChromaComponentX, uint32_t sizeChromaComponentY, uint32_t strideX = 1, uint32_t strideY = 1) + uint32_t sizeChromaComponentX, uint32_t sizeChromaComponentY, uint32_t strideX = 1, uint32_t strideY = 1, uint32_t srcLumRowPadding = 0, uint32_t srcChromaRowPadding = 0) { std::unique_ptr buffer(new uint8_t[(sizeLumX / strideX) * (sizeLumY / strideY) + 2 * (sizeChromaComponentX / strideX) * (sizeChromaComponentY / strideY)]); if (strideX == 1 && strideY == 1) { - memcpy(buffer.get(), srcLum, sizeLumX * sizeLumY); - ChromaPlanesFromNV12(srcChroma, buffer.get() + sizeLumX * sizeLumY, sizeChromaComponentX * sizeChromaComponentY); + if (srcLumRowPadding) + { + uint32_t srcOffset = 0, destOffset = 0; + for(uint32_t y = 0, srcOffset = 0, destOffset = 0; y < sizeLumY; y++, srcOffset += sizeLumX + srcLumRowPadding, destOffset += sizeLumX) + { + memcpy(buffer.get() + destOffset, srcLum + srcOffset, sizeLumX); + } + } + else + { + memcpy(buffer.get(), srcLum, sizeLumX * sizeLumY); + } + ChromaPlanesFromNV12(srcChroma, buffer.get() + sizeLumX * sizeLumY, sizeChromaComponentX, sizeChromaComponentY, srcChromaRowPadding); } else { uint8_t* dest = buffer.get(); - size_t rowStride = strideY * sizeLumX; + size_t rowStride = strideY * (sizeLumX + srcLumRowPadding); for(uint32_t y = 0; y < sizeLumY; y += strideY, srcLum += rowStride) { for(uint32_t x = 0; x < sizeLumX; x += strideX, dest++) @@ -56,7 +72,7 @@ namespace OpenVulkano } } uint32_t chromaCount = (sizeChromaComponentX / strideX) * (sizeChromaComponentY / strideY); - rowStride = strideY * sizeChromaComponentX * 2; + rowStride = strideY * (sizeChromaComponentX * 2 + srcChromaRowPadding); for(uint32_t y = 0; y < sizeChromaComponentY; y += strideY, srcChroma += rowStride) { for (uint32_t x = 0; x < sizeChromaComponentX * 2; x += 2 * strideX, dest++)