GeorgH93/OpenVulkano
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update AR data recording to use Events for new frame recording, add more config options and prepare for async recording

Browse Source
This commit is contained in:
GeorgH93
2021-07-26 21:57:56 +02:00
parent 440b6ca0e5
commit 7faf3a7a7d
6 changed files with 141 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
openVulkanoCpp/AR/ArFrame.cpp
View File

@@ -17,6 +17,9 @@ namespace openVulkanoCpp::AR
void ArFrame::Save()
{
m_session->GetRecorder().Save(this);
if (m_highRes)
m_session->GetRecorder().SaveHighResolution(shared_from_this());
else
m_session->GetRecorder().Save(shared_from_this());
}
}

9
openVulkanoCpp/AR/ArFrame.hpp
View File

@@ -46,7 +46,7 @@ namespace openVulkanoCpp::AR
size_t idx = (y * luminescenceOrColor.resolution.x + x) * 3;
return { lumColBuffer[idx + 2], lumColBuffer[idx + 1], lumColBuffer[idx], 255 };
}
else if (format == Format::RGB)
else if (format == Format::RGB)
{
size_t idx = (y * luminescenceOrColor.resolution.x + x) * 3;
return { lumColBuffer[idx], lumColBuffer[idx + 1], lumColBuffer[idx + 2], 255 };
@@ -85,11 +85,12 @@ namespace openVulkanoCpp::AR
Math::CameraIntrinsic intrinsic;
};
class ArFrame
class ArFrame : public std::enable_shared_from_this<ArFrame>
{
std::shared_ptr<ArSession> m_session;
size_t m_frameId;
bool m_saved = false;
bool m_highRes = false;
protected:
ArFrameMetadata frameMetadata;
@@ -142,6 +143,10 @@ namespace openVulkanoCpp::AR
void SetSaved() { m_saved = true; }
void MarkHighRes() { m_highRes = true; }
[[nodiscard]] bool IsHighResolution() const { return m_highRes; }
/**
* Uses the platforms native jpeg writer to produce a jpeg representation of the camera image.
* This might not be implemented

100
openVulkanoCpp/AR/ArRecorder.cpp
View File

@@ -42,12 +42,15 @@ namespace openVulkanoCpp::AR
}
ArRecorder::ArRecorder(ArSession* session)
: m_session(session), m_path(GeneratePath(AppFolders::GetAppDataHomeDir(), "ar_recording"))
{}
: m_session(session)
{
m_settings.path = GeneratePath(AppFolders::GetAppDataHomeDir(), "ar_recording");
session->OnNewFrameHighResolution += EventHandler(this, &ArRecorder::SaveHighResolution);
}
ArRecorder::~ArRecorder() = default;
void ArRecorder::WriteColorImage(ArFrame* arFrame)
void ArRecorder::WriteColorImage(ArFrame* arFrame, MultiPartArchiveWriter* colorWriter, bool highRes)
{
std::string fileName = GetFileName(arFrame->GetFrameId(), "jpg");
#ifndef TURBO_JPEG
@@ -66,8 +69,7 @@ namespace openVulkanoCpp::AR
tjhandle handle = tjInitCompress();
const uint8_t* buffers[3];
std::unique_ptr<uint8_t[]> dataBuffer;
bool downsampleRecording = true;
if (downsampleRecording)
if (m_settings.downsampleColor && !highRes)
{
dataBuffer = YuvUtils::PlansFromNV12(static_cast<uint8_t*>(img.luminescenceOrColor.data), static_cast<uint8_t*>(img.uv.data),
resX, resY, img.uv.resolution.x, img.uv.resolution.y, 2, 2);
@@ -91,12 +93,12 @@ namespace openVulkanoCpp::AR
if (tjCompressFromYUVPlanes(handle, buffers, resX, nullptr, resY, TJSAMP_420, &outBuffer, &size, 95, TJFLAG_FASTDCT))
Logger::AR->error("Failed to create JPEG! {}", tjGetErrorStr());
else
m_colorWriter->AddFile(fileName.c_str(), outBuffer, size);
colorWriter->AddFile(fileName.c_str(), outBuffer, size);
tjFree(outBuffer);
#endif
}
void ArRecorder::WriteDepthImage(ArFrame* arFrame)
void ArRecorder::WriteDepthImage(ArFrame* arFrame, MultiPartArchiveWriter* depthWriter, MultiPartArchiveWriter* confWriter)
{
if (!m_depthWriter || !m_confidenceWriter) return;
auto depthImg = arFrame->GetDepthImage();
@@ -111,7 +113,7 @@ namespace openVulkanoCpp::AR
buffers[1].second = depthImg.depth.resolution.x * depthImg.depth.resolution.y * sizeof(float);
const std::string fileName = GetFileName(arFrame->GetFrameId(), "pfm");
m_depthWriter->AddFile(fileName.c_str(), buffers);
depthWriter->AddFile(fileName.c_str(), buffers);
}
{
@@ -124,29 +126,31 @@ namespace openVulkanoCpp::AR
buffers[1].second = static_cast<size_t>(depthImg.confidence.resolution.x * depthImg.confidence.resolution.y);
const std::string fileName = GetFileName(arFrame->GetFrameId(), "pgm");
m_confidenceWriter->AddFile(fileName.c_str(), buffers);
confWriter->AddFile(fileName.c_str(), buffers);
}
}
void ArRecorder::Save(ArFrame* frame)
void ArRecorder::Write(ArFrame* frame, bool highRes)
{
if (!m_recording || frame->IsSaved()) return;
if (frame->IsSaved()) return;
frame->SetSaved();
bool useHighResWriter = highRes && m_settings.highResFramesInSeparateArchive;
BlockProfiler profile("Save AR Frame");
{
BlockProfiler profile("Save AR Frame - Image");
WriteColorImage(frame);
}
{
//BlockProfiler profile("Save AR Frame - Depth");
WriteDepthImage(frame);
}
{
BlockProfiler profileMeta("Save AR Frame - Meta");
std::string metaContent = frame->GetFrameMetadata().ToXML();
std::string fileName = GetFileName(frame->GetFrameId(), "meta");
m_metadataWriter->AddFile(fileName.c_str(), metaContent.c_str(), metaContent.size());
(useHighResWriter ? m_highResWriter : m_metadataWriter)->AddFile(fileName.c_str(), metaContent.c_str(), metaContent.size());
}
{
BlockProfiler profile("Save AR Frame - Image");
WriteColorImage(frame, useHighResWriter ? m_highResWriter.get() : m_colorWriter.get(), highRes);
}
{
//BlockProfiler profile("Save AR Frame - Depth");
WriteDepthImage(frame,
useHighResWriter ? m_highResWriter.get() : m_depthWriter.get(),
useHighResWriter ? m_highResWriter.get() : m_confidenceWriter.get());
}
}
@@ -154,12 +158,13 @@ namespace openVulkanoCpp::AR
{
if (!m_colorWriter)
{
m_colorWriter = std::make_unique<MultiPartArchiveWriter>(m_path, "color_{:05d}.tar", ArchiveConfig::TAR);
m_depthWriter = std::make_unique<MultiPartArchiveWriter>(m_path, "depth_{:05d}.tar", ArchiveConfig::TAR);
m_confidenceWriter = std::make_unique<MultiPartArchiveWriter>(m_path, "confidence_{:05d}.tar", ArchiveConfig::TAR_GZ);
m_metadataWriter = std::make_unique<MultiPartArchiveWriter>(m_path, "meta_{:05d}.tar", ArchiveConfig::TAR_GZ);
m_colorWriter = std::make_unique<MultiPartArchiveWriter>(m_settings.path, "color_{:05d}.tar", ArchiveConfig::TAR, m_settings.archiveSize, true);
m_depthWriter = std::make_unique<MultiPartArchiveWriter>(m_settings.path, "depth_{:05d}.tar", ArchiveConfig::TAR, m_settings.archiveSize, true);
m_confidenceWriter = std::make_unique<MultiPartArchiveWriter>(m_settings.path, "confidence_{:05d}.tar.gz", ArchiveConfig::TAR_GZ, m_settings.archiveSize, true);
m_metadataWriter = std::make_unique<MultiPartArchiveWriter>(m_settings.path, "meta_{:05d}.tar.gz", ArchiveConfig::TAR_GZ, m_settings.archiveSize, true);
m_highResWriter = std::make_unique<MultiPartArchiveWriter>(m_settings.path, "highres_{:05d}.tar", ArchiveConfig::TAR, m_settings.archiveSize, true);
std::ofstream platformInfoStream(m_path / "ArRecording.xml");
std::ofstream platformInfoStream(m_settings.path / "ArRecording.xml");
platformInfoStream << m_session->GetSessionMetadata().ToXML();
platformInfoStream.close();
}
@@ -170,7 +175,7 @@ namespace openVulkanoCpp::AR
{
if (!m_recording) return;
m_recording = false;
for(MultiPartArchiveWriter* writer : { m_colorWriter.get(), m_depthWriter.get(), m_confidenceWriter.get(), m_metadataWriter.get() })
for(MultiPartArchiveWriter* writer : { m_colorWriter.get(), m_depthWriter.get(), m_confidenceWriter.get(), m_metadataWriter.get(), m_highResWriter.get() })
{
writer->Split();
}
@@ -180,14 +185,49 @@ namespace openVulkanoCpp::AR
{
if (!m_colorWriter)
{
for (MultiPartArchiveWriter* writer: {m_colorWriter.get(), m_depthWriter.get(), m_confidenceWriter.get(), m_metadataWriter.get()})
for (MultiPartArchiveWriter* writer: { m_colorWriter.get(), m_depthWriter.get(), m_confidenceWriter.get(), m_metadataWriter.get(), m_highResWriter.get() })
{
writer->Move(path);
}
std::filesystem::rename(m_path / "ArRecording.xml", path + "/ArRecording.xml");
std::filesystem::rename(m_settings.path / "ArRecording.xml", path + "/ArRecording.xml");
}
m_persistent = true;
m_path = path;
m_settings.path = path;
}
void ArRecorder::SetRecordingMode(RecordingMode mode)
{
if (m_settings.recordingMode == mode) return;
if (m_settings.recordingMode == RecordingMode::NEW_FRAME && m_newFrameHandler)
{
m_session->OnNewFrame -= m_newFrameHandler;
m_newFrameHandler = nullptr;
}
m_settings.recordingMode = mode;
if (m_settings.recordingMode == RecordingMode::NEW_FRAME)
{
m_newFrameHandler = m_session->OnNewFrame += EventHandler(this, &ArRecorder::Save);
}
}
void ArRecorder::Save(const std::shared_ptr<ArFrame>& frame)
{
if (!m_recording) return;
if (m_settings.asyncRecording)
{
}
else Write(frame.get());
}
void ArRecorder::SaveHighResolution(const std::shared_ptr<ArFrame>& frame)
{
if (!m_recording) return;
if (m_settings.asyncRecording)
{
}
else Write(frame.get(), true);
}
}

66
openVulkanoCpp/AR/ArRecorder.hpp
View File

@@ -6,12 +6,14 @@
#pragma once
#include "Math/ByteSize.hpp"
#include <string>
#include <filesystem>
#include <memory>
namespace openVulkanoCpp
{
class IEventHandler;
class MultiPartArchiveWriter;
}
@@ -36,23 +38,37 @@ namespace openVulkanoCpp::AR
NEW_FRAME
};
struct RecordingSettings
{
RecordingMode recordingMode = RecordingMode::MANUAL;
std::filesystem::path path;
size_t archiveSize = 2_GiB;
bool downsampleColor = false;
bool highResFramesInSeparateArchive = true;
bool asyncRecording = false;
};
class ArRecorder final
{
ArSession* m_session;
RecordingMode m_recordingMode = RecordingMode::MANUAL;
std::filesystem::path m_path;
std::unique_ptr<MultiPartArchiveWriter> m_colorWriter, m_depthWriter, m_confidenceWriter, m_metadataWriter;
std::unique_ptr<MultiPartArchiveWriter> m_colorWriter, m_depthWriter, m_confidenceWriter, m_metadataWriter, m_highResWriter;
RecordingSettings m_settings;
bool m_recording = false, m_persistent = false;
void WriteColorImage(ArFrame* arFrame);
void WriteDepthImage(ArFrame *arFrame);
IEventHandler* m_newFrameHandler = nullptr;
void Write(ArFrame* frame, bool highRes = false);
void WriteColorImage(ArFrame* arFrame, MultiPartArchiveWriter* colorWriter, bool highRes);
void WriteDepthImage(ArFrame *arFrame, MultiPartArchiveWriter* depthWriter, MultiPartArchiveWriter* confWriter);
public:
ArRecorder(ArSession* session);
~ArRecorder();
void Save(ArFrame* frame);
void Save(const std::shared_ptr<ArFrame>& frame);
void SaveHighResolution(const std::shared_ptr<ArFrame>& frame);
/**
* Starts the recording of the owning AR session
@@ -76,7 +92,7 @@ namespace openVulkanoCpp::AR
* Gets the current recording path
* @return Current recording dir
*/
[[nodiscard]] const std::filesystem::path& GetRecordingPath() const { return m_path; }
[[nodiscard]] const std::filesystem::path& GetRecordingPath() const { return m_settings.path; }
/**
* Checks if a path to be used for the recording has been set.
@@ -95,12 +111,44 @@ namespace openVulkanoCpp::AR
* Sets the used recording mode
* @param mode The mode that should be used to record the current session
*/
void SetRecordingMode(RecordingMode mode) { m_recordingMode = mode; }
void SetRecordingMode(RecordingMode mode);
/**
* Checks the currently used recording mode for the AR session
* @return The currently used recording mode
*/
[[nodiscard]] RecordingMode SetRecordingMode() const { return m_recordingMode; }
[[nodiscard]] RecordingMode GetRecordingMode() const { return m_settings.recordingMode; }
/**
* If enabled color images will be reduced to 1/4 of the original resolution (both axes will have half the resolution).
* This option can be used to reduce the used storage space and storage bandwidth during recording.
*
* @param downsample true = downsample images; false = no downsampling
*/
void SetDownsampleColorImages(bool downsample = true) { m_settings.downsampleColor = downsample; }
/**
* Checks if color image downsampling is enabled.
* @return true = downsample images; false = no downsampling
*/
bool GetDownsampleColorImages() const { return m_settings.downsampleColor; }
/**
* If enabled the requested high resolution images will be stored in a separate archive ("highres.tar") instead of the normal ar recording archives.
* @param separateArchive true = using the separate archive file; false = using the normal archive files
*/
void SetHighResFramesInSeparateArchive(bool separateArchive = true) { m_settings.highResFramesInSeparateArchive = separateArchive; }
/**
* Checks if a separate archive is used for the high res frames.
* @return true = using the separate archive file; false = using the normal archive files
*/
bool GetHighResFramesInSeparateArchive() const { return m_settings.highResFramesInSeparateArchive; }
void SetArchivePartMaxFileSize(size_t maxPartSize = 2_GiB) { m_settings.archiveSize = maxPartSize; }
size_t GetArchivePartMaxFileSize() const { m_settings.archiveSize; }
const RecordingSettings& GetRecordingSettings() const { return m_settings; }
};
}

1
openVulkanoCpp/AR/ArSession.hpp
View File

@@ -203,6 +203,7 @@ namespace openVulkanoCpp::AR
Event<> OnNewFrameAvailable;
Event<const std::shared_ptr<ArFrame>&> OnNewFrame;
Event<const std::shared_ptr<ArFrame>&> OnNewFrameHighResolution;
Event<bool> OnSessionInterruptionChange;
Event<ArTrackingState> OnTrackingStateChanged;
Event<const Math::Matrix4f&> OnNewCameraTransformation;

3
openVulkanoCpp/AR/Provider/ArKit/ArSessionArKitInternal.mm
View File

@@ -74,6 +74,8 @@ namespace openVulkanoCpp::AR::ArKit
std::shared_ptr<ArFrame> ArSessionArKitInternal::GetFrame()
{
if (GetRecorder().GetRecordingMode() == RecordingMode::FRAME_REQUEST)
GetRecorder().Save(m_frame);
return m_frame;
}
@@ -99,7 +101,6 @@ namespace openVulkanoCpp::AR::ArKit
}
OnNewFrameAvailable();
m_frame = arFrame;
GetRecorder().Save((ArFrameArKit*)arFrame.get());
}
void ArSessionArKitInternal::OnArSessionInterruptedChanged(ARSession* session, bool interrupted)