#include "UIVideoFrameGrabber.h" #include "SurveyUi.h" #include "UiApp.h" #include #include #include #include #include #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) #include #include #include #else #include #include #include #include #include #endif // Uncomment this line to select any camera - otherwise we will use backface only. //#define SELECT_ANY_CAMERA // Uncomment this line to log details of all camera formats. #define LOG_ALL_CAMERA_FORMATS UIVideoFrameGrabber* UIVideoFrameGrabber::_This = nullptr; struct VideoContext { VideoContext(QImage& Buffer, std::mutex& mutex) : videoFrame(Buffer), writeLock(mutex) {} QImage& videoFrame; std::mutex& writeLock; QCamera* camera = nullptr; #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) class MyVideoSurface* surface = nullptr; #else QMediaCaptureSession* session = nullptr; QVideoSink* sink = nullptr; #endif }; #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) class MyVideoSurface : public QAbstractVideoSurface { VideoContext* context; public: MyVideoSurface(VideoContext* C, QObject* parent = nullptr) : QAbstractVideoSurface(parent) { context = C; } virtual bool start(const QVideoSurfaceFormat& format) { App.ScLog("#### start MyVideoSurface"); return QAbstractVideoSurface::start(format); } QList supportedPixelFormats( QAbstractVideoBuffer::HandleType handleType = QAbstractVideoBuffer::NoHandle) const { if (handleType == QAbstractVideoBuffer::NoHandle) { App.ScLog("supportedPixelFormats"); return QList() << QVideoFrame::Format_NV21; } else { App.ScLog(QString("supportedPixelFormats : unknown ") + QString::number((int)handleType) ); return QList(); } } bool present(const QVideoFrame& frame) { if (frame.isValid()) { std::lock_guard lock(context->writeLock); context->videoFrame = frame.image().convertToFormat(QImage::Format_RGB888); return true; } return false; } }; #else // Qt 6.0.0 or later namespace { const QSize TargetResolution(640, 480); const double TargetAspectRatio = (double)TargetResolution.width() / (double)TargetResolution.height(); const int TargetFrameRate = 15; // Returns a "score" for the camera format. A value of 0.0 means this is exactly // the format we want. The higher the value, the worse a match this is for the // format we want. // double CameraFormatScore(const QCameraFormat& format) { const QVideoFrameFormat::PixelFormat pixelFormat = format.pixelFormat(); const QSize& resolution = format.resolution(); const float frameRate = format.minFrameRate(); // There are only three formats that we know are good - the rest are a bit of // an unknown. double pixelFormatScore = (pixelFormat == QVideoFrameFormat::Format_NV12 || pixelFormat == QVideoFrameFormat::Format_NV21 || pixelFormat == QVideoFrameFormat::Format_RGBA8888) ? 0.0 : 1.0; double aspectRatio = (double)resolution.width() / (double)resolution.height(); double aspectRatioScore = fabs(aspectRatio - TargetAspectRatio); int targetWidth = TargetResolution.width(); int thisWidth = resolution.width(); double widthScore = fabs((targetWidth - thisWidth) / (double)targetWidth); // If the width is less than what we want, make the score worse (larger). if (thisWidth < targetWidth) { widthScore *= 4.0 * (double)targetWidth / (double)thisWidth; } int targetHeight = TargetResolution.height(); int thisHeight = resolution.height(); double heightScore = fabs((targetHeight - thisHeight) / (double)targetHeight); // If the height is less than what we want, make the score worse (larger). if (thisHeight < targetHeight) { heightScore *= 4.0 * (double)targetHeight / (double)thisHeight; } double sizeScore = widthScore + heightScore; double frameRateScore = fabs(frameRate - TargetFrameRate); // If the frame rate is less than what we want, make the score worse (larger). // We really don't want low frame rates, so we use a multiplier of 10.0 here. if (frameRate < TargetFrameRate) { frameRateScore *= 10.0 * (double)TargetFrameRate / (double)frameRate; } // The final score is a weighed combination of the separate values. These are // weighted according to how much we care about them. For example, we really // want a good pixel format, so that is weighted the highest. double score = pixelFormatScore * 10000.0 + aspectRatioScore * 100.0 + sizeScore * 10.0 + frameRateScore; return score; } bool CameraFormatSorter(const QCameraFormat& a, const QCameraFormat& b) { return CameraFormatScore(a) < CameraFormatScore(b); } } #endif UIVideoFrameGrabber* UIVideoFrameGrabber::Instance() { if (_This == nullptr) { _This = new UIVideoFrameGrabber(); } return _This; } UIVideoFrameGrabber::UIVideoFrameGrabber() { _VideoContext = nullptr; } UIVideoFrameGrabber::~UIVideoFrameGrabber() { CloseVideo(); if (_This == this) { _This = nullptr; } } void UIVideoFrameGrabber::OpenVideo(class QImage& Buffer, std::mutex& mutex) { if (!IsOpen()) { _VideoContext = new VideoContext(Buffer, mutex); if (!StartVideo()) { CloseVideo(); } } } void UIVideoFrameGrabber::CloseVideo() { if (IsOpen()) { StopVideo(); delete _VideoContext; _VideoContext = nullptr; } } bool UIVideoFrameGrabber::IsOpen() const { return _VideoContext != nullptr; } bool UIVideoFrameGrabber::StartVideo() { bool ok = false; if (_VideoContext->camera == nullptr) { App.ScLog("UIVideoFrameGrabber StartVideo"); QCamera* selectedCamera = nullptr; #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) const QCameraInfo* selectedCameraInfo = nullptr; // See the comment below in the Qt 6 code section. #ifdef SELECT_ANY_CAMERA QList cam_list = QCameraInfo::availableCameras(QCamera::UnspecifiedPosition); #else QList cam_list = QCameraInfo::availableCameras(QCamera::BackFace); #endif for (int I = 0; I < cam_list.count(); I++) { QCamera* camera = new QCamera(cam_list[I]); QCamera::CaptureModes mode = QCamera::CaptureViewfinder; if (!camera->isCaptureModeSupported(mode)) { delete camera; } else { selectedCamera = camera; selectedCameraInfo = &cam_list[I]; break; } } #else // Qt 6.0.0 or later // We normally select only the "backface" camera, but if SELECT_ANY_CAMERA is // defined, then any camera on the device can be used. You can then specify a // camera by hardcoding its index below - for example, specifying a camera // index of 1 will select a USB webcam on some laptops, which can be useful // for testing purposes. #ifdef SELECT_ANY_CAMERA bool backfaceOnly = false; // int preferredCameraIndex = -1; int preferredCameraIndex = 0; // Will select a USB webcam on some laptops #else bool backfaceOnly = true; int preferredCameraIndex = -1; #endif const QCameraDevice* selectedCameraDevice = nullptr; QList cameraDevices = QMediaDevices::videoInputs(); App.ScLog("Camera devices available: " + QString::number(cameraDevices.count())); if (preferredCameraIndex != -1 && preferredCameraIndex < cameraDevices.count()) { selectedCameraDevice = &cameraDevices[preferredCameraIndex]; selectedCamera = new QCamera(*selectedCameraDevice); } else { for (const QCameraDevice& cameraDevice : cameraDevices) { if (!backfaceOnly || cameraDevice.position() == QCameraDevice::BackFace) { selectedCamera = new QCamera(cameraDevice); selectedCameraDevice = &cameraDevice; break; } } } #endif if (selectedCamera != nullptr) { #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) _VideoContext->camera = selectedCamera; _VideoContext->surface = new MyVideoSurface(_VideoContext); QCameraViewfinderSettings settings; settings = _VideoContext->camera->viewfinderSettings(); settings.setResolution(QSize(640, 480)); settings.setMinimumFrameRate(15.0); settings.setMaximumFrameRate(15.0); settings.setPixelAspectRatio(QSize(1, 1)); settings.setPixelFormat(QVideoFrame::PixelFormat::Format_NV21); //settings.setPixelFormat(QVideoFrame::PixelFormat::Format_RGB32); //settings.setPixelFormat(QVideoFrame::PixelFormat::Format_YV12); _VideoContext->camera->setViewfinderSettings(settings); _VideoContext->camera->setViewfinder(_VideoContext->surface); _VideoContext->camera->setCaptureMode(QCamera::CaptureViewfinder); App.ScLog("AR background camera start"); _VideoContext->camera->load(); _VideoContext->camera->start(); LogStatus(); LogCameraDetails(selectedCameraInfo, _VideoContext->camera); #else // Qt 6.0.0 or later LogCameraDetails(selectedCameraDevice); #ifdef LOG_ALL_CAMERA_FORMATS LogAllCameraFormats(selectedCameraDevice); #endif SelectVideoFormat(selectedCamera, selectedCameraDevice); QMediaCaptureSession* session = new QMediaCaptureSession(); QVideoSink* sink = new QVideoSink(); session->setCamera(selectedCamera); session->setVideoSink(sink); connect(sink, &QVideoSink::videoFrameChanged, this, &UIVideoFrameGrabber::onVideoFrameChanged); App.ScLog("AR background camera start"); selectedCamera->start(); _VideoContext->camera = selectedCamera; _VideoContext->session = session; _VideoContext->sink = sink; LogStatus(); #endif ok = true; } } return ok; } void UIVideoFrameGrabber::StopVideo() { #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) if (_VideoContext->camera != nullptr) { App.ScLog("UIVideoFrameGrabber StopVideo"); _VideoContext->camera->stop(); _VideoContext->camera->unload(); _VideoContext->surface->stop(); delete _VideoContext->camera; _VideoContext->camera = nullptr; delete _VideoContext->surface; _VideoContext->surface = nullptr; } #else if (_VideoContext->camera != nullptr) { App.ScLog("UIVideoFrameGrabber StopVideo"); _VideoContext->camera->stop(); delete _VideoContext->camera; _VideoContext->camera = nullptr; delete _VideoContext->session; _VideoContext->session = nullptr; delete _VideoContext->sink; _VideoContext->sink = nullptr; } #endif } #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) void UIVideoFrameGrabber::LogCameraDetails(const QCameraInfo* info, QCamera* camera) { App.ScLog(QString("Camera : ") + info->deviceName().toUtf8().data()); App.ScLog(QString("Camera Supports QCamera::CaptureVideo :") + (camera->isCaptureModeSupported(QCamera::CaptureVideo) ? "true" : "false")); App.ScLog(QString("Camera Supports QCamera::CaptureViewfinder :") + (camera->isCaptureModeSupported(QCamera::CaptureViewfinder) ? "true" : "false")); QString Msg = "Camera Supports pixel formats :"; QList list = camera->supportedViewfinderPixelFormats(); for (int I = 0; I < list.count(); I++) { Msg += QString::number((int)list[I]) + ", "; } App.ScLog(Msg); Msg = "Camera Supports resolutions :"; QList list2 = camera->supportedViewfinderResolutions(); for (int I = 0; I < list2.count(); I++) { Msg += QString("[") + QString::number(list2[I].width()) + "," + QString::number(list2[I].height()) + "],"; } App.ScLog(Msg); Msg = "Camera Supports frame rates :"; QList list3 = camera->supportedViewfinderFrameRateRanges(); for (int I = 0; I < list3.count(); I++) { Msg += QString("[") + QString::number((int)list3[I].minimumFrameRate) + "," + QString::number((int)list3[I].maximumFrameRate) + "],"; } App.ScLog(Msg); /* This to useful but too much spam for normal logging. App.ScLog("Current viewfinder settings"); LogSettings(camera->viewfinderSettings()); QList list4 = camera->supportedViewfinderSettings(); for (int I = 0; I < list4.count(); I++) { App.ScLog("------------ supported settings"); LogSettings(list4[I]); } */ } void UIVideoFrameGrabber::LogSettings(const QCameraViewfinderSettings& settings) { QString Msg = QString("settings : resolution [") + QString::number(settings.resolution().width()) + "," + QString::number(settings.resolution().height()) + "],"; App.ScLog(Msg); Msg = QString("settings : format ") + QString::number(settings.pixelFormat()); App.ScLog(Msg); Msg = QString("settings : resolution [") + QString::number((int)settings.minimumFrameRate()) + "," + QString::number((int)settings.maximumFrameRate()) + "],"; App.ScLog(Msg); QSize AspectRatio = settings.pixelAspectRatio(); Msg = QString("settings : aspect ratio [") + QString::number(AspectRatio.width()) + "," + QString::number(AspectRatio.height()) + "],"; App.ScLog(Msg); } void UIVideoFrameGrabber::LogStatus() { App.ScLog(QString("camera state : ") + QString::number(_VideoContext->camera->state())); App.ScLog(QString("camera status : ") + QString::number(_VideoContext->camera->status())); App.ScLog(QString("camera availability : ") + QString::number(_VideoContext->camera->availability())); App.ScLog(QString("camera error : ") + _VideoContext->camera->errorString().toUtf8().data()); App.ScLog(QString("camera surface error : ") + QString::number(_VideoContext->surface->error())); App.ScLog(QString("camera surface isActive : ") + (_VideoContext->surface->isActive() ? "true" : "false")); App.ScLog(QString("camera surface pixelformat : ") + QString::number(_VideoContext->surface->surfaceFormat().pixelFormat())); } #else // Qt 6.0.0 or later void UIVideoFrameGrabber::onVideoFrameChanged(const QVideoFrame& frame) { static bool firstFrame = true; //------------------------------------------------------------------------- // NOTE: As a temporary debug measure while we get this working for Qt 6, we will // log details about the first video frame, and once every 10 seconds thereafter. // Once released, we should not log this often, if at all! //------------------------------------------------------------------------- const int LogInterval = 10000; // ms static QElapsedTimer logTimer; bool log = false; // If the frame rate is too high, we'll need to throttle it back. A timer interval // of 60 ms should produce a frame rate of about 15 FPS, allowing for a bit of // overhead. const int FrameInterval = 60; // ms static QElapsedTimer frameRateTimer; if (firstFrame) { log = true; logTimer.start(); frameRateTimer.start(); } if (logTimer.elapsed() > LogInterval) { log = true; logTimer.restart(); } if (!frame.isValid()) { if (log) { App.ScLog("UIVideoFrameGrabber: Frame invalid"); } } else if (firstFrame || frameRateTimer.elapsed() > FrameInterval) { frameRateTimer.restart(); QImage image = frame.toImage(); if (log) { QVideoFrameFormat::PixelFormat format = frame.pixelFormat(); QSize frameSize = frame.size(); QSize imageSize = image.size(); App.ScLog("UIVideoFrameGrabber: Frame format = " + QString::number((int)format) + ", frame size = (" + QString::number(frameSize.width()) + ", " + QString::number(frameSize.height()) + ")" + ", image size = (" + QString::number(imageSize.width()) + ", " + QString::number(imageSize.height()) + ")"); } if (image.size() != TargetResolution) { // TODO_Qt6: Are we using the right aspect ratio option when scaling here...? image = image.scaled(TargetResolution, Qt::KeepAspectRatioByExpanding); if (log) { QSize size = image.size(); App.ScLog("UIVideoFrameGrabber: Scaled size = (" + QString::number(size.width()) + ", " + QString::number(size.height()) + ")"); } } std::lock_guard lock(_VideoContext->writeLock); _VideoContext->videoFrame = image.convertToFormat(QImage::Format_RGB888); } firstFrame = false; } void UIVideoFrameGrabber::SelectVideoFormat(QCamera* selectedCamera, const QCameraDevice* selectedCameraDevice) { QList cameraFormats = selectedCameraDevice->videoFormats(); double bestScore = DBL_MAX; qsizetype bestIndex = cameraFormats.size(); // Default to off-the-end/invalid // Find the camera format with the best (i.e. lowest) "score" - that will be the // closest match for the desired pixel format, aspect ratio, size, and frame rate. for (qsizetype i = 0; i < cameraFormats.size(); i++) { double score = CameraFormatScore(cameraFormats[i]); if (score == 0.0) { // This format is just what we want - use it. bestIndex = i; break; } if (score < bestScore) { bestScore = score; bestIndex = i; } } if (bestIndex < cameraFormats.size()) { selectedCamera->setCameraFormat(cameraFormats[bestIndex]); App.ScLog("Camera selected format:"); LogCameraFormat(selectedCamera->cameraFormat()); } } void UIVideoFrameGrabber::LogCameraDetails(const QCameraDevice* device) { App.ScLog("Camera: " + device->description()); App.ScLog("Camera video format count: " + QString::number(device->videoFormats().count())); } void UIVideoFrameGrabber::LogAllCameraFormats(const QCameraDevice* device) { App.ScLog("Camera supported video formats (Pixel format | Resolution | Frame rates):"); QList cameraFormats = device->videoFormats(); for (QCameraFormat& format : cameraFormats) { LogCameraFormat(format); } std::sort(cameraFormats.begin(), cameraFormats.end(), CameraFormatSorter); App.ScLog("Camera supported video formats - SORTED (Pixel format | Resolution | Frame rates):"); for (QCameraFormat& format : cameraFormats) { LogCameraFormat(format); } } void UIVideoFrameGrabber::LogCameraFormat(const QCameraFormat& format) { QVideoFrameFormat::PixelFormat pixelFormat = format.pixelFormat(); QSize resolution = format.resolution(); int minFrameRate = (int)format.minFrameRate(); int maxFrameRate = (int)format.maxFrameRate(); QString Msg = QString(" ") + QString::number((int)pixelFormat); Msg += QString(" | [") + QString::number(resolution.width()) + "," + QString::number(resolution.height()) + "]"; Msg += QString(" | [") + QString::number(minFrameRate) + " - " + QString::number(maxFrameRate) + "]"; App.ScLog(Msg); } void UIVideoFrameGrabber::LogStatus() { App.ScLog(QString("Camera isActive: ") + (_VideoContext->camera->isActive() ? "true" : "false")); App.ScLog(QString("Camera error: ") + _VideoContext->camera->errorString().toUtf8().data()); } #endif