Move benchmarks outside of tests directory (#2670)

ocelotl · web-flow · commit 5ff46ac9d159 · 2024-07-09T17:17:00.000-06:00
Fixes #2669
diff --git a/.github/workflows/instrumentations_0.yml b/.github/workflows/instrumentations_0.yml
@@ -126,5 +126,5 @@ jobs:
             .tox
             ~/.cache/pip
           key: v7-build-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', 'gen-requirements.txt', 'dev-requirements.txt') }}
-      - name: run pytest without --benchmark-skip
+      - name: run tox
         run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra
diff --git a/.github/workflows/instrumentations_1.yml b/.github/workflows/instrumentations_1.yml
@@ -25,14 +25,15 @@ jobs:
       matrix:
         python-version: [py38, py39, py310, py311, py312, pypy3]
         package:
-          # Only add here packages that do not have benchmark tests
           - "urllib"
           - "urllib3"
           - "wsgi"
           - "distro"
           - "richconsole"
           - "psycopg"
           - "prometheus-remote-write"
+          - "sdk-extension-aws"
+          - "propagator-aws-xray"
           - "propagator-ot-trace"
           - "resource-detector-azure"
           - "resource-detector-container"
@@ -58,5 +59,5 @@ jobs:
             .tox
             ~/.cache/pip
           key: v7-build-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', 'gen-requirements.txt', 'dev-requirements.txt') }}
-      - name: run pytest without --benchmark-skip
+      - name: run tox
         run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra
diff --git a/.github/workflows/instrumentations_2.yml b/.github/workflows/instrumentations_2.yml
diff --git a/.gitignore b/.gitignore
@@ -58,3 +58,6 @@ _build/
 # mypy
 .mypy_cache/
 target
+
+# Benchmark result files
+*-benchmark.json
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -96,9 +96,7 @@ for more detail on available tox commands.
 
 ### Benchmarks
 
-Performance progression of benchmarks for packages distributed by OpenTelemetry Python can be viewed as a [graph of throughput vs commit history](https://opentelemetry-python-contrib.readthedocs.io/en/latest/performance/benchmarks.html). From the linked page, you can download a JSON file with the performance results.
-
-Running the `tox` tests also runs the performance tests if any are available. Benchmarking tests are done with `pytest-benchmark` and they output a table with results to the console.
+Some packages have benchmark tests. To run them, run `tox -f benchmark`. Benchmark tests use `pytest-benchmark` and they output a table with results to the console.
 
 To write benchmarks, simply use the [pytest benchmark fixture](https://pytest-benchmark.readthedocs.io/en/latest/usage.html#usage) like the following:
 
@@ -114,10 +112,10 @@ def test_simple_start_span(benchmark):
     benchmark(benchmark_start_as_current_span, "benchmarkedSpan", 42)
 ```
 
-Make sure the test file is under the `tests/performance/benchmarks/` folder of
+Make sure the test file is under the `benchmarks/` folder of
 the package it is benchmarking and further has a path that corresponds to the
 file in the package it is testing. Make sure that the file name begins with
-`test_benchmark_`. (e.g. `propagator/opentelemetry-propagator-aws-xray/tests/performance/benchmarks/trace/propagation/test_benchmark_aws_xray_propagator.py`)
+`test_benchmark_`. (e.g. `propagator/opentelemetry-propagator-aws-xray/benchmarks/trace/propagation/test_benchmark_aws_xray_propagator.py`)
 
 ## Pull Requests
 
diff --git a/propagator/opentelemetry-propagator-aws-xray/benchmark-requirements.txt b/propagator/opentelemetry-propagator-aws-xray/benchmark-requirements.txt
@@ -0,0 +1 @@
+pytest-benchmark==4.0.0
diff --git a/propagator/opentelemetry-propagator-aws-xray/benchmarks/test_benchmark_aws_xray_propagator.py b/propagator/opentelemetry-propagator-aws-xray/benchmarks/test_benchmark_aws_xray_propagator.py
diff --git a/sdk-extension/opentelemetry-sdk-extension-aws/benchmark-requirements.txt b/sdk-extension/opentelemetry-sdk-extension-aws/benchmark-requirements.txt
@@ -0,0 +1 @@
+pytest-benchmark==4.0.0
diff --git a/sdk-extension/opentelemetry-sdk-extension-aws/benchmarks/trace/test_benchmark_aws_xray_ids_generator.py b/sdk-extension/opentelemetry-sdk-extension-aws/benchmarks/trace/test_benchmark_aws_xray_ids_generator.py
diff --git a/tox.ini b/tox.ini
@@ -20,6 +20,7 @@ envlist =
     py3{8,9,10,11,12}-test-sdk-extension-aws
     pypy3-test-sdk-extension-aws
     lint-sdk-extension-aws
+    benchmark-sdk-extension-aws
 
     ; opentelemetry-distro
     py3{8,9,10,11,12}-test-distro
@@ -314,6 +315,7 @@ envlist =
     py3{8,9,10,11,12}-test-propagator-aws-xray
     pypy3-test-propagator-aws-xray
     lint-propagator-aws-xray
+    benchmark-propagator-aws-xray
 
     ; opentelemetry-propagator-ot-trace
     py3{8,9,10,11,12}-test-propagator-ot-trace
@@ -741,6 +743,7 @@ commands_pre =
   sdk-extension-aws: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk
   sdk-extension-aws: pip install opentelemetry-test-utils@{env:CORE_REPO}\#egg=opentelemetry-test-utils&subdirectory=tests/opentelemetry-test-utils
   sdk-extension-aws: pip install -r {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws/test-requirements.txt
+  benchmark-sdk-extension-aws: pip install -r {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws/benchmark-requirements.txt
 
   resource-detector-container: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api
   resource-detector-container: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions
@@ -765,6 +768,7 @@ commands_pre =
   propagator-aws-xray: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk
   propagator-aws-xray: pip install opentelemetry-test-utils@{env:CORE_REPO}\#egg=opentelemetry-test-utils&subdirectory=tests/opentelemetry-test-utils
   propagator-aws-xray: pip install -r {toxinidir}/propagator/opentelemetry-propagator-aws-xray/test-requirements.txt
+  benchmark-propagator-aws-xray: pip install -r {toxinidir}/propagator/opentelemetry-propagator-aws-xray/benchmark-requirements.txt
 
   processor-baggage: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api
   processor-baggage: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions
@@ -1094,6 +1098,7 @@ commands =
   lint-sdk-extension-aws: isort --diff --check-only --settings-path {toxinidir}/.isort.cfg {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws
   lint-sdk-extension-aws: flake8 --config {toxinidir}/.flake8 {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws
   lint-sdk-extension-aws: sh -c "cd sdk-extension && pylint --rcfile ../.pylintrc opentelemetry-sdk-extension-aws"
+  benchmark-sdk-extension-aws: pytest {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws/benchmarks {posargs} --benchmark-json=sdk-extension-aws-benchmark.json
 
   test-resource-detector-container: pytest {toxinidir}/resource/opentelemetry-resource-detector-container/tests {posargs}
   lint-resource-detector-container: black --diff --check --config {toxinidir}/pyproject.toml {toxinidir}/resource/opentelemetry-resource-detector-container
@@ -1118,6 +1123,7 @@ commands =
   lint-propagator-aws-xray: isort --diff --check-only --settings-path {toxinidir}/.isort.cfg {toxinidir}/propagator/opentelemetry-propagator-aws-xray
   lint-propagator-aws-xray: flake8 --config {toxinidir}/.flake8 {toxinidir}/propagator/opentelemetry-propagator-aws-xray
   lint-propagator-aws-xray: sh -c "cd propagator && pylint --rcfile ../.pylintrc opentelemetry-propagator-aws-xray"
+  benchmark-propagator-aws-xray: pytest {toxinidir}/propagator/opentelemetry-propagator-aws-xray/benchmarks {posargs} --benchmark-json=propagator-aws-xray-benchmark.json
 
   test-propagator-ot-trace: pytest {toxinidir}/propagator/opentelemetry-propagator-ot-trace/tests {posargs}
   lint-propagator-ot-trace: black --diff --check --config {toxinidir}/pyproject.toml {toxinidir}/propagator/opentelemetry-propagator-ot-trace
