diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 15df1c7..f5bcce0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -75,7 +75,7 @@ jobs: run: | pytest --disable-warnings tests/test_module_loading.py - gtests: + gtests-onnx: runs-on: ubuntu-latest strategy: matrix: @@ -98,17 +98,14 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt pip install -r requirements-engagement.txt - pip install -r requirements-torch.txt pip install -r tests/requirements.txt - name: Set PYTHONPATH environment variable run: echo "PYTHONPATH=$(pwd)" >> $GITHUB_ENV - - name: Download and unpack LibTorch and ONNXRuntime + - name: Download and unpack ONNXRuntime run: | - wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.6.0%2Bcpu.zip -O libtorch.zip wget https://github.com/microsoft/onnxruntime/releases/download/v1.20.1/onnxruntime-linux-x64-1.20.1.tgz -O onnxruntime.tgz - unzip libtorch.zip mkdir onnxruntime tar -xzf onnxruntime.tgz -C onnxruntime --strip-components=1 # Workaround for Linux build in CI @@ -124,7 +121,61 @@ jobs: - name: Build EmotiEffCppLib run: | mkdir build - cmake -S emotieffcpplib -B build -DWITH_TORCH="$PWD/libtorch" -DWITH_ONNX="$PWD/onnxruntime" -DBUILD_TESTS=ON + cmake -S emotieffcpplib -B build -DWITH_ONNX="$PWD/onnxruntime" -DBUILD_TESTS=ON + cmake --build build --config Release -- -j$(nproc) + + - name: Prepare torch models + run: python3 models/prepare_models_for_emotieffcpplib.py + + - name: Run GTests + run: | + EMOTIEFFLIB_ROOT=$PWD ./build/bin/unit_tests + + gtests-torch: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10"] + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 1 # shallow clone + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + sudo apt-get update && sudo apt-get install -y cmake g++ make wget libopencv-dev python3 + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-engagement.txt + pip install -r requirements-torch.txt + pip install -r tests/requirements.txt + + - name: Set PYTHONPATH environment variable + run: echo "PYTHONPATH=$(pwd)" >> $GITHUB_ENV + + - name: Download and unpack LibTorch + run: | + wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.6.0%2Bcpu.zip -O libtorch.zip + unzip libtorch.zip + + - name: Download and unpack test data + run: | + cd tests/ + ./download_test_data.sh + tar -xzf data.tar.gz + cd .. + + - name: Build EmotiEffCppLib + run: | + mkdir build + cmake -S emotieffcpplib -B build -DWITH_TORCH="$PWD/libtorch" -DBUILD_TESTS=ON cmake --build build --config Release -- -j$(nproc) - name: Prepare torch models diff --git a/docs/tutorials/cpp/One image emotion recognition.ipynb b/docs/tutorials/cpp/One image emotion recognition.ipynb index 1c058b6..248fe33 100644 --- a/docs/tutorials/cpp/One image emotion recognition.ipynb +++ b/docs/tutorials/cpp/One image emotion recognition.ipynb @@ -156,10 +156,10 @@ "#include \n", "#include \n", "\n", - "#include \n", - "#include \n", - "#include \n", - "#include \n", + "#include \n", + "#include \n", + "#include \n", + "#include \n", "\n", "#include \"xtl/xbase64.hpp\"\n", "#include \"nlohmann/json.hpp\"" diff --git a/docs/tutorials/cpp/Predict emotions on video.ipynb b/docs/tutorials/cpp/Predict emotions on video.ipynb index 74d2c6e..d2df7d7 100644 --- a/docs/tutorials/cpp/Predict emotions on video.ipynb +++ b/docs/tutorials/cpp/Predict emotions on video.ipynb @@ -156,10 +156,10 @@ "#include \n", "#include \n", "\n", - "#include \n", - "#include \n", - "#include \n", - "#include \n", + "#include \n", + "#include \n", + "#include \n", + "#include \n", "\n", "#include \"xtl/xbase64.hpp\"\n", "#include \"nlohmann/json.hpp\"" diff --git a/docs/tutorials/cpp/Predict engagement and emotions on video.ipynb b/docs/tutorials/cpp/Predict engagement and emotions on video.ipynb index 2058e9b..0e6d107 100644 --- a/docs/tutorials/cpp/Predict engagement and emotions on video.ipynb +++ b/docs/tutorials/cpp/Predict engagement and emotions on video.ipynb @@ -156,10 +156,10 @@ "#include \n", "#include \n", "\n", - "#include \n", - "#include \n", - "#include \n", - "#include \n", + "#include \n", + "#include \n", + "#include \n", + "#include \n", "\n", "#include \"xtl/xbase64.hpp\"\n", "#include \"nlohmann/json.hpp\"" diff --git a/emotieffcpplib/3rdparty/xtensor b/emotieffcpplib/3rdparty/xtensor index 47dd7a4..1089342 160000 --- a/emotieffcpplib/3rdparty/xtensor +++ b/emotieffcpplib/3rdparty/xtensor @@ -1 +1 @@ -Subproject commit 47dd7a4057cf2aa8f2e191f4b1a0663ed83b8980 +Subproject commit 108934209249b2ff25e5a08284ace2b6e0cd6f30 diff --git a/emotieffcpplib/3rdparty/xtl b/emotieffcpplib/3rdparty/xtl index d11fb6b..174e99d 160000 --- a/emotieffcpplib/3rdparty/xtl +++ b/emotieffcpplib/3rdparty/xtl @@ -1 +1 @@ -Subproject commit d11fb6b5f4c417025124ed2c62175284846a1914 +Subproject commit 174e99da6e27c3ee5837fb74e7e65330af4cd2a6 diff --git a/emotieffcpplib/include/emotiefflib/facial_analysis.h b/emotieffcpplib/include/emotiefflib/facial_analysis.h index 4a65831..32d7d94 100644 --- a/emotieffcpplib/include/emotiefflib/facial_analysis.h +++ b/emotieffcpplib/include/emotiefflib/facial_analysis.h @@ -10,7 +10,7 @@ #include #include #include -#include +#include namespace EmotiEffLib { diff --git a/emotieffcpplib/src/backends/onnx/facial_analysis.cpp b/emotieffcpplib/src/backends/onnx/facial_analysis.cpp index 99ef402..50049f2 100644 --- a/emotieffcpplib/src/backends/onnx/facial_analysis.cpp +++ b/emotieffcpplib/src/backends/onnx/facial_analysis.cpp @@ -5,9 +5,9 @@ #include "emotiefflib/backends/onnx/facial_analysis.h" -#include -#include -#include +#include +#include +#include namespace { /** diff --git a/emotieffcpplib/src/backends/torch/facial_analysis.cpp b/emotieffcpplib/src/backends/torch/facial_analysis.cpp index d631d2b..d7a2fb9 100644 --- a/emotieffcpplib/src/backends/torch/facial_analysis.cpp +++ b/emotieffcpplib/src/backends/torch/facial_analysis.cpp @@ -5,7 +5,7 @@ #include "emotiefflib/backends/torch/facial_analysis.h" -#include +#include namespace { /** diff --git a/emotieffcpplib/src/facial_analysis.cpp b/emotieffcpplib/src/facial_analysis.cpp index f8c739a..8b446fa 100644 --- a/emotieffcpplib/src/facial_analysis.cpp +++ b/emotieffcpplib/src/facial_analysis.cpp @@ -4,14 +4,18 @@ */ #include "emotiefflib/facial_analysis.h" +#ifdef WITH_ONNX #include "emotiefflib/backends/onnx/facial_analysis.h" +#endif +#ifdef WITH_TORCH #include "emotiefflib/backends/torch/facial_analysis.h" +#endif #include -#include -#include -#include +#include +#include +#include namespace fs = std::filesystem; @@ -41,17 +45,27 @@ std::unique_ptr EmotiEffLibRecognizer::createInstance(const std::string& backend, const std::string& fullPipelineModelPath) { checkBackend(backend); +#ifdef WITH_TORCH if (backend == "torch") return std::make_unique(fullPipelineModelPath); +#elif defined(WITH_ONNX) return std::make_unique(fullPipelineModelPath); +#else + throw std::runtime_error("EmotiEffCppLib wasn't compiled with any backend support."); +#endif } std::unique_ptr EmotiEffLibRecognizer::createInstance(const EmotiEffLibConfig& config) { checkBackend(config.backend); +#ifdef WITH_TORCH if (config.backend == "torch") return std::make_unique(config); +#elif defined(WITH_ONNX) return std::make_unique(config); +#else + throw std::runtime_error("EmotiEffCppLib wasn't compiled with any backend support."); +#endif } xt::xarray EmotiEffLibRecognizer::extractFeatures(const std::vector& faceImgs) { diff --git a/emotieffcpplib/tests/tests_facial_analisys.cpp b/emotieffcpplib/tests/tests_facial_analisys.cpp index 2d50838..b2927f3 100644 --- a/emotieffcpplib/tests/tests_facial_analisys.cpp +++ b/emotieffcpplib/tests/tests_facial_analisys.cpp @@ -5,10 +5,10 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include namespace fs = std::filesystem; @@ -473,6 +473,12 @@ INSTANTIATE_TEST_SUITE_P( TEST_P(EmotiEffLibOnlyModelTests, OneImageFeatures) { std::string modelName = GetParam(); auto facialImages = getOneImageFaces(); + auto supportedBackends = EmotiEffLib::getAvailableBackends(); + auto findOnnx = std::find(supportedBackends.begin(), supportedBackends.end(), "onnx"); + auto findTorch = std::find(supportedBackends.begin(), supportedBackends.end(), "torch"); + if (findOnnx == supportedBackends.end() or findTorch == supportedBackends.end()) { + GTEST_SKIP() << "Skipping test because of unsupported backend."; + } fs::path modelPath(getEmotiEffLibRootDir()); modelPath = modelPath / "models" / "emotieffcpplib_prepared_models"; @@ -498,6 +504,12 @@ TEST_P(EmotiEffLibOnlyModelTests, OneImageFeatures) { TEST_P(EmotiEffLibOnlyModelTests, OneImageMultiFeatures) { std::string modelName = GetParam(); auto facialImages = getOneImageFaces(); + auto supportedBackends = EmotiEffLib::getAvailableBackends(); + auto findOnnx = std::find(supportedBackends.begin(), supportedBackends.end(), "onnx"); + auto findTorch = std::find(supportedBackends.begin(), supportedBackends.end(), "torch"); + if (findOnnx == supportedBackends.end() or findTorch == supportedBackends.end()) { + GTEST_SKIP() << "Skipping test because of unsupported backend."; + } fs::path modelPath(getEmotiEffLibRootDir()); modelPath = modelPath / "models" / "emotieffcpplib_prepared_models"; @@ -560,26 +572,35 @@ TEST(EmotiEffLibTests, CheckIncorrectConfig) { } catch (...) { FAIL(); } + auto supportedBackends = EmotiEffLib::getAvailableBackends(); config = {.backend = "torch", .classifierPath = "bla-bla", .modelName = "bla-bla"}; - try { - EmotiEffLib::EmotiEffLibRecognizer::createInstance(config); - FAIL(); - } catch (const std::runtime_error& e) { - EXPECT_EQ("fullPipelineEmotionModelPath or featureExtractorPath MUST be specified in the " - "EmotiEffLibConfig.", - std::string(e.what())); - } catch (...) { - FAIL(); + if (std::find(supportedBackends.begin(), supportedBackends.end(), config.backend) != + supportedBackends.end()) { + try { + EmotiEffLib::EmotiEffLibRecognizer::createInstance(config); + FAIL(); + } catch (const std::runtime_error& e) { + EXPECT_EQ( + "fullPipelineEmotionModelPath or featureExtractorPath MUST be specified in the " + "EmotiEffLibConfig.", + std::string(e.what())); + } catch (...) { + FAIL(); + } } config.backend = "onnx"; - try { - EmotiEffLib::EmotiEffLibRecognizer::createInstance(config); - FAIL(); - } catch (const std::runtime_error& e) { - EXPECT_EQ("fullPipelineEmotionModelPath or featureExtractorPath MUST be specified in the " - "EmotiEffLibConfig.", - std::string(e.what())); - } catch (...) { - FAIL(); + if (std::find(supportedBackends.begin(), supportedBackends.end(), config.backend) != + supportedBackends.end()) { + try { + EmotiEffLib::EmotiEffLibRecognizer::createInstance(config); + FAIL(); + } catch (const std::runtime_error& e) { + EXPECT_EQ( + "fullPipelineEmotionModelPath or featureExtractorPath MUST be specified in the " + "EmotiEffLibConfig.", + std::string(e.what())); + } catch (...) { + FAIL(); + } } } diff --git a/models/prepare_models_for_emotieffcpplib.py b/models/prepare_models_for_emotieffcpplib.py index b77a63a..e90bffd 100644 --- a/models/prepare_models_for_emotieffcpplib.py +++ b/models/prepare_models_for_emotieffcpplib.py @@ -60,7 +60,7 @@ def trace_model( img_size = 224 input_shape = (1, 3, img_size, img_size) model_example = torch.rand(*input_shape) - model = torch.load(torch_model, map_location=torch.device("cpu")) + model = torch.load(torch_model, map_location=torch.device("cpu"), weights_only=False) # pylint: disable=no-else-return if model_name in ("mbf_va_mtl.pt", "mobilevit_va_mtl.pt"): # This is a workaround but it still is not working because of shapes issues in runtime