Separate camera code

src/CMakeLists.txt

@@ -46,6 +46,7 @@ add_subdirectory(Cemu)
 add_subdirectory(config)
 add_subdirectory(input)
 add_subdirectory(audio)
+add_subdirectory(camera)
 add_subdirectory(util)
 add_subdirectory(imgui)
 add_subdirectory(resource)
@@ -119,6 +120,7 @@ set_target_properties(CemuBin PROPERTIES
 target_link_libraries(CemuBin PRIVATE
 	CemuAudio
 	CemuCafe
+	CemuCamera
 	CemuCommon
 	CemuComponents
 	CemuConfig

src/Cafe/CMakeLists.txt

@@ -257,8 +257,6 @@ add_library(CemuCafe
   OS/libs/avm/avm.h
   OS/libs/camera/camera.cpp
   OS/libs/camera/camera.h
-  OS/libs/camera/Rgb2Nv12.h
-  OS/libs/camera/Rgb2Nv12.cpp
   OS/libs/coreinit/coreinit_Alarm.cpp
   OS/libs/coreinit/coreinit_Alarm.h
   OS/libs/coreinit/coreinit_Atomic.cpp
@@ -535,6 +533,7 @@ target_include_directories(CemuCafe PUBLIC "../")
 target_link_libraries(CemuCafe PRIVATE
 	CemuAsm
 	CemuAudio
+    CemuCamera
 	CemuCommon
 	CemuComponents
 	CemuConfig
@@ -556,7 +555,6 @@ target_link_libraries(CemuCafe PRIVATE
 	ZArchive::zarchive
 	ZLIB::ZLIB
 	zstd::zstd
-    openpnp-capture
 )
 
 if (ENABLE_WAYLAND)

src/Cafe/OS/libs/camera/camera.cpp

@@ -2,23 +2,20 @@
 #include "Cafe/OS/common/OSCommon.h"
 #include "camera.h"
 
-#include "Rgb2Nv12.h"
 #include "Cafe/OS/RPL/rpl.h"
 #include "Cafe/OS/libs/coreinit/coreinit_Alarm.h"
 #include "Cafe/OS/libs/coreinit/coreinit_Time.h"
 #include "Cafe/HW/Espresso/PPCCallback.h"
 #include "util/helpers/helpers.h"
 
-#include <util/helpers/ringbuffer.h>
-#include <openpnp-capture.h>
-
+#include "util/helpers/ringbuffer.h"
+#include "camera/CameraManager.h"
 namespace camera
 {
-	constexpr static size_t g_width = 640;
-	constexpr static size_t g_height = 480;
-	constexpr static size_t g_pitch = 768;
+	constexpr unsigned CAMERA_WIDTH = 640;
+	constexpr unsigned CAMERA_HEIGHT = 480;
 
-	enum class CAMError : sint32
+	enum class CAMStatus : sint32
 	{
 		Success = 0,
 		InvalidArg = -1,
@@ -27,7 +24,7 @@ namespace camera
 		InsufficientMemory = -5,
 		NotReady = -6,
 		Uninitialized = -8,
-		DeviceInitFailed = -9,
+		UVCError = -9,
 		DecoderInitFailed = -10,
 		DeviceInUse = -12,
 		DecoderSessionFailed = -13,
@@ -68,7 +65,7 @@ namespace camera
 	{
 		CAMImageInfo imageInfo;
 		uint32be workMemorySize;
-		MEMPTR<void> workMemory;
+		MEMPTR<void> workMemoryData;
 		MEMPTR<void> callback;
 		betype<CAMForceDisplay> forceDisplay;
 		betype<CAMFps> fps;
@@ -79,8 +76,8 @@ namespace camera
 
 	struct CAMTargetSurface
 	{
-		/* +0x00 */ sint32be surfaceSize;
-		/* +0x04 */ MEMPTR<void> surfacePtr;
+		/* +0x00 */ sint32be size;
+		/* +0x04 */ MEMPTR<uint8_t> data;
 		/* +0x08 */ uint32be height;
 		/* +0x0C */ uint32be width;
 		/* +0x10 */ uint32be ukn10;
@@ -92,149 +89,74 @@ namespace camera
 	struct CAMDecodeEventParam
 	{
 		betype<CAMEventType> type;
-		MEMPTR<void> buffer;
+		MEMPTR<void> data;
 		uint32be channel;
 		uint32be errored;
 	};
 	static_assert(sizeof(CAMDecodeEventParam) == 0x10);
 
-	std::recursive_mutex g_cameraMutex;
+	constexpr static int32_t CAM_HANDLE = 0;
 
-	SysAllocator<CAMDecodeEventParam> g_cameraEventData;
-	SysAllocator<coreinit::OSAlarm_t> g_cameraAlarm;
+	struct
+	{
+		std::recursive_mutex mutex{};
+		bool initialized = false;
+		bool shouldTriggerCallback = false;
+		std::atomic_bool isOpen = false;
+		MEMPTR<void> eventCallback = nullptr;
+		RingBuffer<MEMPTR<uint8_t>, 20> inTargetBuffers{};
+		RingBuffer<MEMPTR<uint8_t>, 20> outTargetBuffers{};
+		std::thread updateThread;
+	} s_instance;
 
-	void DecodeAlarmCallback(PPCInterpreter_t*);
+	SysAllocator<CAMDecodeEventParam> s_cameraEventData;
+	SysAllocator<coreinit::OSAlarm_t> s_cameraAlarm;
 
-	class CAMInstance
+	void UpdateLoop()
 	{
-		constexpr static auto nv12BufferSize = g_pitch * g_height * 3 / 2;
-
-	  public:
-		CAMInstance(CAMFps frameRate, MEMPTR<void> callbackPtr)
-			: m_capCtx(Cap_createContext()),
-			  m_capNv12Buffer(new uint8_t[nv12BufferSize]),
-			  m_frameRate(30),
-			  m_callbackPtr(callbackPtr)
+		while (s_instance.isOpen)
 		{
-			if (callbackPtr.IsNull() || frameRate != CAMFps::_15 && frameRate != CAMFps::_30)
-				throw CAMError::InvalidArg;
-			coreinit::OSCreateAlarm(g_cameraAlarm.GetPtr());
-			coreinit::OSSetPeriodicAlarm(g_cameraAlarm.GetPtr(),
-										 coreinit::OSGetTime(),
-										 coreinit::EspressoTime::GetTimerClock() / m_frameRate,
-										 RPLLoader_MakePPCCallable(DecodeAlarmCallback));
-		}
-		~CAMInstance()
 			{
-			m_capWorker.request_stop();
-			m_capWorker.join();
-			coreinit::OSCancelAlarm(g_cameraAlarm.GetPtr());
-			Cap_releaseContext(m_capCtx);
-		}
-
-		void OnAlarm()
+				std::scoped_lock lock(s_instance.mutex);
+				auto surfaceBuffer = s_instance.inTargetBuffers.Pop();
+				if (surfaceBuffer.IsNull())
 				{
-			const auto surface = m_targetSurfaceQueue.Pop();
-			if (surface.IsNull())
-				return;
-			{
-				std::scoped_lock lock(m_callbackMutex);
-				std::memcpy(surface->surfacePtr.GetPtr(), m_capNv12Buffer.get(), nv12BufferSize);
-			}
-			g_cameraEventData->type = CAMEventType::Decode;
-			g_cameraEventData->buffer = surface->surfacePtr;
-			g_cameraEventData->channel = 0;
-			g_cameraEventData->errored = false;
-			PPCCoreCallback(m_callbackPtr, g_cameraEventData.GetPtr());
-		}
-
-		void Worker(std::stop_token stopToken, CapFormatID formatId)
-		{
-			SetThreadName("CAMWorker");
-			const auto stream = Cap_openStream(m_capCtx, m_capDeviceId, formatId);
-			constexpr static auto rgbBufferSize = g_width * g_height * 3;
-			auto rgbBuffer = std::make_unique<uint8[]>(rgbBufferSize);
-			while (!stopToken.stop_requested())
-			{
-				if (!m_opened)
-				{
-					std::this_thread::sleep_for(std::chrono::milliseconds(20));
-					std::this_thread::yield();
 					continue;
 				}
-				auto res = Cap_captureFrame(m_capCtx, stream, rgbBuffer.get(), rgbBufferSize);
-				if (res == CAPRESULT_OK)
-				{
-					std::scoped_lock lock(m_callbackMutex);
-					Rgb2Nv12(rgbBuffer.get(), g_width, g_height, m_capNv12Buffer.get(), g_pitch);
-				}
-			}
-			Cap_closeStream(m_capCtx, stream);
+				CameraManager::instance().GetNV12Data(surfaceBuffer.GetPtr());
+				s_instance.outTargetBuffers.Push(surfaceBuffer);
 			}
 
-		CAMError Open()
-		{
-			if (m_opened)
-				return CAMError::DeviceInUse;
-			const auto formatCount = Cap_getNumFormats(m_capCtx, m_capDeviceId);
-			int formatId = -1;
-			for (auto i = 0; i < formatCount; ++i)
-			{
-				CapFormatInfo formatInfo;
-				Cap_getFormatInfo(m_capCtx, m_capDeviceId, i, &formatInfo);
-				if (formatInfo.width == g_width && formatInfo.height == g_height && formatInfo.fps == m_frameRate)
-				{
-					formatId = i;
-					break;
 		}
 	}
-			if (formatId == -1)
-			{
-				cemuLog_log(LogType::Force, "camera: Open failed, {}x{} @ {} fps not supported by host camera", g_width, g_height, m_frameRate);
-				return CAMError::DeviceInitFailed;
-			}
-			m_targetSurfaceQueue.Clear();
-			m_opened = true;
-			m_capWorker = std::jthread(&CAMInstance::Worker, this, formatId);
-			return CAMError::Success;
-		}
 
-		CAMError Close()
+	void AlarmCallback(PPCInterpreter_t*)
 	{
-			m_opened = false;
-			return CAMError::Success;
+		s_cameraEventData->type = CAMEventType::Decode;
+		s_cameraEventData->channel = 0;
+
+		if (s_instance.shouldTriggerCallback)
+		{
+			s_instance.shouldTriggerCallback = false;
+			s_cameraEventData->data = nullptr;
+			s_cameraEventData->errored = false;
 		}
-
-		CAMError SubmitTargetSurface(MEMPTR<CAMTargetSurface> surface)
+		else if (s_instance.isOpen)
 		{
-			cemu_assert(surface->surfaceSize >= nv12BufferSize);
-			if (!m_opened)
-				return CAMError::NotReady;
-			if (!m_targetSurfaceQueue.Push(surface))
-				return CAMError::SurfaceQueueFull;
-			cemuLog_logDebug(LogType::Force, "camera: surface {} submitted", surface->surfacePtr);
-			return CAMError::Success;
+			if (auto buffer = s_instance.outTargetBuffers.Pop())
+			{
+				s_cameraEventData->data = buffer;
+				s_cameraEventData->errored = false;
+				PPCCoreCallback(s_instance.eventCallback, s_cameraEventData.GetMPTR());
 			}
-
-	  private:
-		CapContext m_capCtx;
-		CapDeviceID m_capDeviceId = 0;
-		std::unique_ptr<uint8[]> m_capNv12Buffer;
-		std::jthread m_capWorker;
-		std::mutex m_callbackMutex{};
-		uint32 m_frameRate;
-		MEMPTR<void> m_callbackPtr;
-		RingBuffer<MEMPTR<CAMTargetSurface>, 20> m_targetSurfaceQueue{};
-		std::atomic_bool m_opened = false;
-	};
-	std::optional<CAMInstance> g_camInstance;
-
-	void DecodeAlarmCallback(PPCInterpreter_t*)
+			else
 			{
-		std::scoped_lock camLock(g_cameraMutex);
-		if (!g_camInstance)
+				s_cameraEventData->data = nullptr;
+				s_cameraEventData->errored = true;
+			}
+		}
+		else
 			return;
-		g_camInstance->OnAlarm();
 	}
 
 	sint32 CAMGetMemReq(CAMImageInfo*)
@@ -242,76 +164,110 @@ namespace camera
 		return 1 * 1024; // always return 1KB
 	}
 
-	CAMError CAMCheckMemSegmentation(void* base, uint32 size)
+	CAMStatus CAMCheckMemSegmentation(void* base, uint32 size)
 	{
-		return CAMError::Success; // always return success
+		if (!base || size == 0)
+			return CAMStatus::InvalidArg;
+		return CAMStatus::Success;
 	}
 
-	sint32 CAMInit(uint32 cameraId, CAMInitInfo_t* camInitInfo, betype<CAMError>* error)
+	sint32 CAMInit(uint32 cameraId, CAMInitInfo_t* initInfo, betype<CAMStatus>* error)
 	{
-		if (g_camInstance)
+		*error = CAMStatus::Success;
+		std::scoped_lock lock(s_instance.mutex);
+		if (s_instance.initialized)
 		{
-			*error = CAMError::DeviceInUse;
+			*error = CAMStatus::DeviceInUse;
 			return -1;
 		}
-		std::unique_lock _lock(g_cameraMutex);
-		const auto& [t, height, width] = camInitInfo->imageInfo;
-		cemu_assert(width == g_width && height == g_height);
-		try
+
+		if (!initInfo || !initInfo->workMemoryData ||
+			!match_any_of(initInfo->forceDisplay, CAMForceDisplay::None, CAMForceDisplay::DRC) ||
+			!match_any_of(initInfo->fps, CAMFps::_15, CAMFps::_30) ||
+			initInfo->imageInfo.type != CAMImageType::Default)
 		{
-			g_camInstance.emplace(camInitInfo->fps, camInitInfo->callback);
-		}
-		catch (CAMError e)
-		{
-			*error = e;
+			*error = CAMStatus::InvalidArg;
 			return -1;
 		}
-		*error = CAMError::Success;
+
+		cemu_assert_debug(initInfo->forceDisplay != CAMForceDisplay::DRC);
+		cemu_assert_debug(initInfo->workMemorySize != 0);
+		cemu_assert_debug(initInfo->imageInfo.type == CAMImageType::Default);
+
+		auto frameRate = initInfo->fps == CAMFps::_15 ? 15 : 30;
+		s_instance.initialized = true;
+		s_instance.shouldTriggerCallback = true;
+		s_instance.eventCallback = initInfo->callback;
+		coreinit::OSCreateAlarm(s_cameraAlarm.GetPtr());
+		coreinit::OSSetPeriodicAlarm(s_cameraAlarm.GetPtr(),
+									 coreinit::OSGetTime(),
+									 coreinit::EspressoTime::GetTimerClock() / frameRate,
+									 RPLLoader_MakePPCCallable(AlarmCallback));
 		return 0;
 	}
 
+	CAMStatus CAMClose(sint32 camHandle)
+	{
+		if (camHandle != CAM_HANDLE)
+			return CAMStatus::InvalidHandle;
+		std::scoped_lock lock(s_instance.mutex);
+		if (!s_instance.initialized || !s_instance.isOpen)
+			return CAMStatus::Uninitialized;
+		s_instance.isOpen = false;
+		s_instance.updateThread.join();
+		CameraManager::instance().Close();
+		return CAMStatus::Success;
+	}
+
+	CAMStatus CAMOpen(sint32 camHandle)
+	{
+		if (camHandle != CAM_HANDLE)
+			return CAMStatus::InvalidHandle;
+		auto lock = std::scoped_lock(s_instance.mutex);
+		if (!s_instance.initialized)
+			return CAMStatus::Uninitialized;
+		if (s_instance.isOpen)
+			return CAMStatus::DeviceInUse;
+		if (!CameraManager::instance().Open(false))
+			return CAMStatus::UVCError;
+		s_instance.inTargetBuffers.Clear();
+		s_instance.outTargetBuffers.Clear();
+		s_instance.isOpen = true;
+		s_instance.updateThread = std::thread(UpdateLoop);
+		return CAMStatus::Success;
+	}
+
+	CAMStatus CAMSubmitTargetSurface(sint32 camHandle, CAMTargetSurface* targetSurface)
+	{
+		if (camHandle != CAM_HANDLE)
+			return CAMStatus::InvalidHandle;
+		if (!targetSurface || targetSurface->data.IsNull() || targetSurface->size < 1)
+			return CAMStatus::InvalidArg;
+		auto lock = std::scoped_lock(s_instance.mutex);
+		if (!s_instance.initialized)
+			return CAMStatus::Uninitialized;
+		if (!s_instance.inTargetBuffers.Push(targetSurface->data))
+			return CAMStatus::SurfaceQueueFull;
+		return CAMStatus::Success;
+	}
+
 	void CAMExit(sint32 camHandle)
 	{
-		if (camHandle != 0 || !g_camInstance)
+		if (camHandle != CAM_HANDLE)
 			return;
-		g_camInstance.reset();
-	}
-
-	CAMError CAMClose(sint32 camHandle)
-	{
-		if (camHandle != 0)
-			return CAMError::InvalidHandle;
-		std::scoped_lock lock(g_cameraMutex);
-		if (!g_camInstance)
-			return CAMError::Uninitialized;
-		return g_camInstance->Close();
-	}
-
-	CAMError CAMOpen(sint32 camHandle)
-	{
-		if (camHandle != 0)
-			return CAMError::InvalidHandle;
-		auto lock = std::scoped_lock(g_cameraMutex);
-		if (!g_camInstance)
-			return CAMError::Uninitialized;
-		return g_camInstance->Open();
-	}
-
-	CAMError CAMSubmitTargetSurface(sint32 camHandle, CAMTargetSurface* targetSurface)
-	{
-		if (camHandle != 0)
-			return CAMError::InvalidHandle;
-		if (!targetSurface || targetSurface->surfacePtr.IsNull() || targetSurface->surfaceSize < 1)
-			return CAMError::InvalidArg;
-		auto lock = std::scoped_lock(g_cameraMutex);
-		if (!g_camInstance)
-			return CAMError::Uninitialized;
-		return g_camInstance->SubmitTargetSurface(targetSurface);
+		std::scoped_lock lock(s_instance.mutex);
+		if (!s_instance.initialized)
+			return;
+		if (s_instance.isOpen)
+			CAMClose(camHandle);
+		s_instance.initialized = false;
+		s_instance.shouldTriggerCallback = false;
+		coreinit::OSCancelAlarm(s_cameraAlarm);
 	}
 
 	void reset()
 	{
-		g_camInstance.reset();
+		CAMExit(0);
 	}
 
 	void load()

src/camera/CMakeLists.txt

@@ -0,0 +1,8 @@
+add_library(CemuCamera
+    CameraManager.cpp
+    CameraManager.h
+    Rgb2Nv12.cpp
+    Rgb2Nv12.h
+)
+target_include_directories(CemuCamera PUBLIC "../")
+target_link_libraries(CemuCamera PRIVATE CemuCommon CemuUtil PUBLIC openpnp-capture)

src/camera/CameraManager.cpp

@@ -0,0 +1,99 @@
+#include "CameraManager.h"
+#include "Rgb2Nv12.h"
+#include "util/helpers/helpers.h"
+
+constexpr unsigned CAMERA_WIDTH = 640;
+constexpr unsigned CAMERA_HEIGHT = 480;
+constexpr unsigned CAMERA_PITCH = 768;
+
+CameraManager::CameraManager()
+	: m_ctx(Cap_createContext()),
+	  m_rgbBuffer(CAMERA_WIDTH * CAMERA_HEIGHT * 3),
+	  m_nv12Buffer(CAMERA_PITCH * CAMERA_HEIGHT * 3 / 2),
+	  m_refCount(0)
+{
+	if (Cap_getDeviceCount(m_ctx) > 0)
+		m_device = 0;
+}
+CameraManager::~CameraManager()
+{
+	Close();
+	Cap_releaseContext(m_ctx);
+}
+
+void CameraManager::SetDevice(uint deviceNo)
+{
+	std::scoped_lock lock(m_mutex);
+	if (m_device == deviceNo)
+		return;
+	if (m_stream)
+		Cap_closeStream(m_ctx, *m_stream);
+	m_device = deviceNo;
+}
+bool CameraManager::Open(bool weak)
+{
+	std::scoped_lock lock(m_mutex);
+	if (!m_device)
+		return false;
+	if (m_refCount == 0)
+	{
+		const auto formatCount = Cap_getNumFormats(m_ctx, *m_device);
+		CapFormatID formatId = 0;
+		for (; formatId < formatCount; ++formatId)
+		{
+			CapFormatInfo formatInfo;
+			if (Cap_getFormatInfo(m_ctx, *m_device, formatId, &formatInfo) != CAPRESULT_OK)
+				continue;
+			if (formatInfo.width == CAMERA_WIDTH && formatInfo.height == CAMERA_HEIGHT)
+				break;
+		}
+		if (formatId == formatCount)
+			return false;
+		auto stream = Cap_openStream(m_ctx, *m_device, formatId);
+		if (stream == -1)
+			return false;
+		m_capturing = true;
+		m_stream = stream;
+		m_captureThread = std::thread(&CameraManager::CaptureWorker, this);
+	}
+	else if (!weak)
+		m_refCount += 1;
+	return true;
+}
+void CameraManager::Close()
+{
+	std::scoped_lock lock(m_mutex);
+	if (m_refCount == 0)
+		return;
+	m_refCount -= 1;
+	if (m_refCount != 0)
+		return;
+	Cap_closeStream(m_ctx, *m_stream);
+	m_stream = std::nullopt;
+	m_capturing = false;
+	m_captureThread.join();
+}
+
+void CameraManager::GetNV12Data(uint8_t* nv12Buffer) const
+{
+	std::shared_lock lock(m_mutex);
+	std::ranges::copy(m_nv12Buffer, nv12Buffer);
+}
+void CameraManager::CaptureWorker()
+{
+	SetThreadName("CameraManager");
+	while (m_capturing)
+	{
+		{
+			std::scoped_lock lock(m_mutex);
+			bool frameAvailable = Cap_hasNewFrame(m_ctx, *m_stream);
+			if (frameAvailable &&
+				Cap_captureFrame(m_ctx, *m_stream, m_rgbBuffer.data(), m_rgbBuffer.size()) == CAPRESULT_OK)
+			{
+				Rgb2Nv12(m_rgbBuffer.data(), CAMERA_WIDTH, CAMERA_HEIGHT, m_nv12Buffer.data(), CAMERA_PITCH);
+			}
+		}
+		std::this_thread::sleep_for(std::chrono::milliseconds(16));
+		std::this_thread::yield();
+	}
+}

src/camera/CameraManager.h

@@ -0,0 +1,29 @@
+#pragma once
+#include <shared_mutex>
+
+#include <openpnp-capture.h>
+#include "util/helpers/Singleton.h"
+class CameraManager : public Singleton<CameraManager>
+{
+	CapContext m_ctx;
+	std::optional<CapDeviceID> m_device;
+	std::optional<CapStream> m_stream;
+	std::vector<uint8_t> m_rgbBuffer;
+	std::vector<uint8_t> m_nv12Buffer;
+	int m_refCount;
+	std::thread m_captureThread;
+	std::atomic_bool m_capturing;
+	mutable std::shared_mutex m_mutex;
+public:
+	CameraManager();
+	~CameraManager();
+
+	void SetDevice(uint deviceNo);
+
+	bool Open(bool weak);
+	void Close();
+
+	void GetNV12Data(uint8_t* nv12Buffer) const;
+private:
+	void CaptureWorker();
+};