Skip to content

Commit

Permalink
New LUT calibration based on mp4 test videos (part I) (#896)
Browse files Browse the repository at this point in the history
* to revert

* LUT rework

* Test images creator

* Test video

* Use vectors

* Extract classes & namespaces

* Add CapturedColor

* Optimize board creator

* Make board verifier work

* Read CRC from the test board

* Parse and verify colors of all test boards

* Print error stats

* Correction

* Calibration for new boards. 17*17*17 YUV samples.

* Parse text HDR LUT to binary format

* Add save raw image

* Const correction

* Add detection of bt.2020 gamma in the input data

* Add more colorspace math

* More math

* Update LutCalibrator.cpp

* Verify source: only NV12/YUYV/MJPEG source are supported for the calibration process

* Add precise scanner

* Testing more conditions and deviation of YUV coefs

* Optimize matrix multiplication

* Extract methods

* Optimize YUV coef matrix creation

* Implemented LUT saving

* Add support for Flatbuffers NV12 calibration

* Save captured YUV colors in HyperHDR home folder

* HDR10 calibration is working again also using video test files

* Fix errors (1)

* Test C++17

* Fix errors (2)

* HDR10 calibrator: fix errors

* HDR10 calibration is working again also using video test files

* HDR10 calibration is working again also using video test files

* Added a new nice web wizard for LUT calibration

* Added BT.2100 HLG support

* Added BT.2100 HLG support (2)

* Added BT.2100 HLG support (3)

* Added BT.2100 HLG support (4)

* Verify if BT.2100 HLG OOTF is present

* More detailed scan

* Faster HLG OOTF when disabled

* More precise calculation

* Fix calibration for 1280x720 capturing setting

* Support for SDR calibration

* More precise color capture saving/loading

New calibration_captured_yuv.txt format

* Add support for SDR (BT2020 in SRGB) calibration

* Grabber benchmark support for flatbuffers

* Add quarter of frame mode for flatbuffers

* Improvements

* Fix

* Important calibration fix and stronger noise resistance

* Wider search

* Two stage calibration

* Refactoring

* Improvements

* Focus on yellow,scan more vertexes,error may go up

* Improvements

* Improvements

* Multi-threading support for LUT calibration

* Multi-threading support for LUT creator

* Add 'detailed calibration' option

* Add LCH color correction

* Run calibrator on separate thread (nonblocking UI)

* Add PQ in sRGB linear signal detection

* Pause grabber/flatbuffers while calibrating LUT

* Change rounding for backlight

* Fix serial port discovery in UI

* Added 3'th LCH layer, increased number of vertices, more CPU needed

* Add more test vertexes (959), change to square errors

* Add more brown color vertexes
  • Loading branch information
awawa-dev authored Nov 8, 2024
1 parent 6294960 commit 81a93c4
Show file tree
Hide file tree
Showing 61 changed files with 4,617 additions and 1,567 deletions.
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,6 @@
[submodule "external/stb"]
path = external/stb
url = https://github.com/nothings/stb.git
[submodule "external/linalg"]
path = external/linalg
url = https://github.com/sgorsten/linalg.git
13 changes: 5 additions & 8 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -550,14 +550,11 @@ if(NOT CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-psabi")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-psabi")
endif()
if(COMPILER_SUPPORTS_CXX17 AND Qt_VERSION EQUAL 6)
message(STATUS "Enabling support for C++17 for QT6")
if(COMPILER_SUPPORTS_CXX17)
message(STATUS "Enabling support for C++17")
set(CMAKE_CXX_STANDARD 17)
elseif(COMPILER_SUPPORTS_CXX11)
message(STATUS "Enabling support for C++11")
set(CMAKE_CXX_STANDARD 11)
else()
message(STATUS "No support for C++11 detected. Compilation will most likely fail on your compiler")
message(STATUS "No support for C++17 detected. Compilation will most likely fail on your compiler")
endif()
else()
include(CheckCXXCompilerFlag)
Expand Down Expand Up @@ -691,8 +688,8 @@ add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_D
include (${CMAKE_CURRENT_SOURCE_DIR}/cmake/packages.cmake)

# external targets
if (WIN32 AND TARGET stb AND TARGET flatbuffers AND TARGET protobuf-nanopb AND TARGET lunasvg AND TARGET flatc AND TARGET qmqtt AND TARGET liblzma AND TARGET sqlite3)
set_target_properties(stb qmqtt flatbuffers protobuf-nanopb lunasvg flatc resources uninstall liblzma sqlite3 PROPERTIES FOLDER ExternalLibsTargets)
if (WIN32 AND TARGET stb AND TARGET flatbuffers AND TARGET protobuf-nanopb AND TARGET lunasvg AND TARGET flatc AND TARGET qmqtt AND TARGET liblzma AND TARGET sqlite3 AND TARGET precompiled_hyperhdr_headers)
set_target_properties(stb qmqtt flatbuffers protobuf-nanopb lunasvg flatc resources uninstall liblzma sqlite3 precompiled_hyperhdr_headers PROPERTIES FOLDER ExternalLibsTargets)
else()
set_target_properties(resources uninstall PROPERTIES FOLDER ExternalLibsTargets)
endif()
Expand Down
8 changes: 8 additions & 0 deletions external/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,14 @@ if(ENABLE_WS281XPWM)
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/rpi_ws281x)
endif()

#=============================================================================
# LINALG
#=============================================================================

add_library(linalg INTERFACE)
target_compile_definitions(linalg INTERFACE LINALG_FORWARD_COMPATIBLE )
target_include_directories(linalg INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/linalg")

#=============================================================================
# LUNASVG
#=============================================================================
Expand Down
1 change: 1 addition & 0 deletions external/linalg
Submodule linalg added at 4460f1
16 changes: 11 additions & 5 deletions include/api/BaseAPI.h
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
#pragma once

#ifndef PCH_ENABLED
#include <QThread>
#include <memory>
#endif

#include <base/HyperHdrInstance.h>
#include <base/HyperHdrManager.h>
#include <base/AccessManager.h>
Expand All @@ -18,12 +23,12 @@ class BaseAPI : public QObject

struct ImageCmdData
{
int priority;
int priority = 0;
QString origin;
int64_t duration;
int width;
int height;
int scale;
int64_t duration = 0;
int width = 0;
int height = 0;
int scale = 0;
QString format;
QString imgName;
QString imagedata;
Expand Down Expand Up @@ -122,6 +127,7 @@ class BaseAPI : public QObject
std::shared_ptr<GrabberHelper> _systemGrabber;
std::shared_ptr<PerformanceCounters> _performanceCounters;
std::shared_ptr<DiscoveryWrapper> _discoveryWrapper;
std::unique_ptr<QThread, std::function<void(QThread*)>> _lutCalibratorThread;

struct {
bool init = false;
Expand Down
4 changes: 2 additions & 2 deletions include/api/CallbackAPI.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ class CallbackAPI : public BaseAPI
void subscribe(QJsonArray subsArr);

protected:
std::unique_ptr<LutCalibrator> _lutCalibrator;
Image<ColorRgb> _liveImage;

void stopDataConnections() override = 0;
Expand All @@ -59,11 +58,12 @@ private slots:
void instancesListChangedHandler();
void tokenChangeHandler(const QVector<AccessManager::AuthDefinition>& def);
void signalBenchmarkUpdateHandler(int status, QString message);
void lutCalibrationUpdateHandler(const QJsonObject& data);
void performanceUpdateHandler(const QJsonObject& data);
#ifdef ENABLE_BONJOUR
void signalDiscoveryFoundServiceHandler(DiscoveryRecord::Service type, QList<DiscoveryRecord> records);
#endif
public slots:
void lutCalibrationUpdateHandler(const QJsonObject& data);

private:
QStringList _availableCommands;
Expand Down
9 changes: 3 additions & 6 deletions include/base/Grabber.h
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,7 @@ class Grabber : public DetectionAutomatic, public DetectionManual, protected Lut

void setSignalDetectionEnable(bool enable);

void setAutoSignalDetectionEnable(bool enable);

void benchmarkCapture(int status, QString message);
void setAutoSignalDetectionEnable(bool enable);

QList<Grabber::DevicePropertiesItem> getVideoDeviceModesFullInfo(const QString& devicePath);

Expand Down Expand Up @@ -163,6 +161,8 @@ public slots:

QStringList getVideoDevices() const;

void signalSetLutHandler(MemoryBuffer<uint8_t>* lut);

signals:
void SignalNewCapturedFrame(const Image<ColorRgb>& image);

Expand Down Expand Up @@ -268,9 +268,6 @@ public slots:
bool _signalDetectionEnabled;
bool _signalAutoDetectionEnabled;
QSemaphore _synchro;

int _benchmarkStatus;
QString _benchmarkMessage;
};

bool sortDevicePropertiesItem(const Grabber::DevicePropertiesItem& v1, const Grabber::DevicePropertiesItem& v2);
3 changes: 0 additions & 3 deletions include/base/GrabberWrapper.h
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,6 @@ public slots:
void stop();
void revive();

void benchmarkCapture(int status, QString message);

QJsonObject getJsonInfo();

QJsonDocument startCalibration();
Expand All @@ -57,7 +55,6 @@ private slots:
void SignalNewVideoImage(const QString& name, const Image<ColorRgb>& image);
void SignalVideoStreamChanged(QString device, QString videoMode);
void SignalCecKeyPressed(int key);
void SignalBenchmarkUpdate(int status, QString message);
void SignalInstancePauseChanged(int instance, bool isEnabled);
void SignalSetNewComponentStateToAllInstances(hyperhdr::Components component, bool enable);
void SignalSaveCalibration(QString saveData);
Expand Down
7 changes: 7 additions & 0 deletions include/base/HyperHdrManager.h
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#endif

#include <image/ColorRgb.h>
#include <utils/VideoBenchmark.h>
#include <utils/Logger.h>
#include <utils/settings.h>
#include <utils/Components.h>
Expand Down Expand Up @@ -43,6 +44,8 @@ class HyperHdrManager : public QObject
bool areInstancesReady();

public slots:
void handleRequestComponent(hyperhdr::Components component, int hyperHdrInd, bool listen);

void setSmoothing(int time);

void setSignalStateByCEC(bool enable);
Expand Down Expand Up @@ -101,6 +104,9 @@ public slots:

void SignalInstancePauseChanged(int instance, bool isEnabled);

void SignalBenchmarkUpdate(int status, QString message);
void SignalBenchmarkCapture(int status, QString message);

private slots:
void handleInstanceJustStarted();

Expand All @@ -126,4 +132,5 @@ private slots:
int _fireStarter;

QMap<quint8, PendingRequests> _pendingRequests;
VideoBenchmark _videoBenchmark;
};
4 changes: 4 additions & 0 deletions include/flatbuffers/server/FlatBuffersServer.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ class FlatBuffersServer : public QObject, protected LutLoader
void SignalImportFromProto(int priority, int duration, const Image<ColorRgb>& image, QString clientDescription);

public slots:
void handleRequestComponent(hyperhdr::Components component, int hyperHdrInd, bool listen);
void signalSetLutHandler(MemoryBuffer<uint8_t>* lut);
void handleSettingsUpdate(settings::type type, const QJsonDocument& config);
void initServer();
int getHdrToneMappingEnabled();
Expand Down Expand Up @@ -70,4 +72,6 @@ private slots:
QString _userLutFile;
PixelFormat _currentLutPixelFormat;
int _flatbufferToneMappingMode;
bool _quarterOfFrameMode;
bool _active;
};
8 changes: 8 additions & 0 deletions include/image/ColorRgb.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@
#include <sstream>
#include <cstdint>
#include <iostream>
#include <cmath>
#endif



struct ColorRgb
{
uint8_t red = 0;
Expand Down Expand Up @@ -100,6 +103,11 @@ struct ColorRgb
return (x < 0) ? 0 : ((x > 255) ? 255 : uint8_t(x));
}

inline static uint8_t round(double x)
{
return (x < 0) ? 0 : ((x > 255) ? 255 : static_cast<uint8_t>(std::lround(x)));
}

static void rgb2hsv(uint8_t red, uint8_t green, uint8_t blue, uint16_t& _hue, uint8_t& _saturation, uint8_t& _value);
static void hsv2rgb(uint16_t hue, uint8_t saturation, uint8_t value, uint8_t& red, uint8_t& green, uint8_t& blue);
static void rgb2hsl(uint8_t red, uint8_t green, uint8_t blue, uint16_t& hue, float& saturation, float& luminance);
Expand Down
9 changes: 9 additions & 0 deletions include/image/Image.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

#include <image/ImageData.h>

enum class PixelFormat;

template <typename ColorSpace>
class Image
{
Expand Down Expand Up @@ -36,6 +38,10 @@ class Image

ColorSpace& operator()(unsigned x, unsigned y);

void setOriginFormat(PixelFormat pf);

PixelFormat getOriginFormat() const;

void resize(unsigned width, unsigned height);

uint8_t* rawMem();
Expand All @@ -46,6 +52,9 @@ class Image

void clear();

bool save(const char* filename) const;

private:
std::shared_ptr<ImageData<ColorSpace>> _sharedData;
PixelFormat _pixelFormat;
};
127 changes: 127 additions & 0 deletions include/lut-calibrator/BestResult.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
#pragma once

/* BestResult.h
*
* MIT License
*
* Copyright (c) 2020-2024 awawa-dev
*
* Project homesite: https://github.com/awawa-dev/HyperHDR
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

#ifndef PCH_ENABLED
#include <QJsonArray>
#include <QJsonObject>
#include <QFile>
#include <QDateTime>
#include <QThread>

#include <cmath>
#include <cfloat>
#include <climits>
#endif

#include <linalg.h>
#include <lut-calibrator/ColorSpace.h>
#include <lut-calibrator/BoardUtils.h>
#include <lut-calibrator/YuvConverter.h>

using namespace linalg;
using namespace aliases;
using namespace ColorSpaceMath;
using namespace BoardUtils;

struct LchLists
{
std::list<double4> low;
std::list<double4> mid;
std::list<double4> high;
};

struct BestResult
{
YuvConverter::YUV_COEFS coef = YuvConverter::YUV_COEFS::BT601;
double4x4 coefMatrix;
double2 coefDelta;
int coloredAspectMode = 0;
std::pair<double3, double3> colorAspect;
double3 aspect;
int bt2020Range = 0;
int altConvert = 0;
double3x3 altPrimariesToSrgb;
ColorSpaceMath::HDR_GAMMA gamma = ColorSpaceMath::HDR_GAMMA::PQ;
double gammaHLG = 0;
double nits = 0;
bool lchEnabled = false;
LchLists lchPrimaries;

struct Signal
{
YuvConverter::COLOR_RANGE range = YuvConverter::COLOR_RANGE::FULL;
double yRange = 0;
double upYLimit = 0;
double downYLimit = 0;
double yShift = 0;
} signal;

long long int minError = MAX_CALIBRATION_ERROR;

void serializePrimaries(std::stringstream& out) const
{
for (const auto& p : { lchPrimaries.low, lchPrimaries.mid, lchPrimaries.high })
{
out << std::endl << "\t\t\tstd::list<double4>{" << std::endl << "\t\t\t\t";
for (const auto& v : p)
{
out << "double4"; ColorSpaceMath::serialize(out, v); out << ", ";
}
out << std::endl << "\t\t\t}," << std::endl;
}
}

void serialize(std::stringstream& out) const
{
out.precision(12);
out << "/*" << std::endl;
out << "BestResult bestResult;" << std::endl;
out << "bestResult.coef = YuvConverter::YUV_COEFS(" << std::to_string(coef) << ");" << std::endl;
out << "bestResult.coefMatrix = double4x4"; ColorSpaceMath::serialize(out, coefMatrix); out << ";" << std::endl;
out << "bestResult.coefDelta = double2"; ColorSpaceMath::serialize(out, coefDelta); out << ";" << std::endl;
out << "bestResult.coloredAspectMode = " << std::to_string(coloredAspectMode) << ";" << std::endl;
out << "bestResult.colorAspect = std::pair<double3, double3>(double3"; ColorSpaceMath::serialize(out, colorAspect.first); out << ", double3"; ColorSpaceMath::serialize(out, colorAspect.second); out << ");" << std::endl;
out << "bestResult.aspect = double3"; ColorSpaceMath::serialize(out, aspect); out << ";" << std::endl;
out << "bestResult.bt2020Range = " << std::to_string(bt2020Range) << ";" << std::endl;
out << "bestResult.altConvert = " << std::to_string(altConvert) << ";" << std::endl;
out << "bestResult.altPrimariesToSrgb = double3x3"; ColorSpaceMath::serialize(out, altPrimariesToSrgb); out << ";" << std::endl;
out << "bestResult.gamma = ColorSpaceMath::HDR_GAMMA(" << std::to_string(gamma) << ");" << std::endl;
out << "bestResult.gammaHLG = " << std::to_string(gammaHLG) << ";" << std::endl;
out << "bestResult.lchEnabled = " << std::to_string(lchEnabled) << ";" << std::endl;
out << "bestResult.lchPrimaries = LchLists{"; serializePrimaries(out); out << "\t\t};" << std::endl;
out << "bestResult.nits = " << std::to_string(nits) << ";" << std::endl;
out << "bestResult.signal.range = YuvConverter::COLOR_RANGE(" << std::to_string(signal.range) << ");" << std::endl;
out << "bestResult.signal.yRange = " << std::to_string(signal.yRange) << ";" << std::endl;
out << "bestResult.signal.upYLimit = " << std::to_string(signal.upYLimit) << ";" << std::endl;
out << "bestResult.signal.downYLimit = " << std::to_string(signal.downYLimit) << ";" << std::endl;
out << "bestResult.signal.yShift = " << std::to_string(signal.yShift) << ";" << std::endl;
out << "bestResult.minError = " << std::to_string(std::round(minError * 100.0) / 30000.0) << ";" << std::endl;
out << "*/" << std::endl;
}
};
Loading

0 comments on commit 81a93c4

Please sign in to comment.