feat: modernize Python tooling (pyproject.toml + uv)

.github/workflows/mysql-migrations-check.yml

@@ -30,30 +30,22 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y libxmlsec1-dev
-    - name: Get pip cache dir
-      id: pip-cache-dir
-      run: |
-        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
-    - name: Cache pip dependencies
-      id: cache-dependencies
-      uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+    - name: Install uv
+      uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
       with:
-        path: ${{ steps.pip-cache-dir.outputs.dir }}
-        key: ${{ runner.os }}-pip-${{ hashFiles('requirements/pip_tools.txt') }}
-        restore-keys: ${{ runner.os }}-pip-
+        enable-cache: true
 
     - name: Ubunto and sql versions
       run: |
         lsb_release -a
         mysql -V
     - name: Install Python Dependencies
       run: |
-        pip install -r requirements/pip-tools.txt
-        pip install -r requirements/production.txt
-        pip uninstall -y mysqlclient
-        pip install --no-binary mysqlclient mysqlclient
-        pip uninstall -y xmlsec
-        pip install --no-binary xmlsec xmlsec==1.3.13
+        uv sync --group production
+        uv pip uninstall mysqlclient
+        uv pip install --no-binary mysqlclient mysqlclient
+        uv pip uninstall xmlsec
+        uv pip install --no-binary xmlsec xmlsec==1.3.13
     - name: Initiate services
       run: |
         sudo /etc/init.d/mysql start
@@ -74,4 +66,4 @@ jobs:
       run: |
         echo "CREATE DATABASE IF NOT EXISTS enterprise_catalog;" | sudo mysql -u root
         echo "Running the migrations"
-        python manage.py migrate --settings=enterprise_catalog.settings.test
+        uv run python manage.py migrate --settings=enterprise_catalog.settings.test

.gitignore

@@ -65,6 +65,7 @@ diff_*.html
 *.report
 report
 venv
+.venv
 acceptance_tests.*.log
 acceptance_tests.*.png
 

Makefile

@@ -1,6 +1,6 @@
 .DEFAULT_GOAL := help
 TOX = ''
-.PHONY: help clean piptools requirements dev_requirements \
+.PHONY: help clean requirements dev_requirements \
         doc_requirements prod_requirements static shell test coverage \
         isort_check isort style lint quality pii_check validate \
         migrate html_coverage upgrade extract_translation dummy_translations \
@@ -11,11 +11,6 @@ TOX = ''
         dev.up.build dev.down dev.destroy dev.stop docker_build \
         shellcheck check_keywords install_transifex_client
 
-COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt
-.PHONY: $(COMMON_CONSTRAINTS_TXT)
-$(COMMON_CONSTRAINTS_TXT):
-	wget -O "$(@)" https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt || touch "$(@)"
-
 define BROWSER_PYSCRIPT
 import os, webbrowser, sys
 try:
@@ -29,7 +24,7 @@ export BROWSER_PYSCRIPT
 BROWSER := python3 -c "$$BROWSER_PYSCRIPT"
 
 ifdef TOXENV
-TOX := tox -- #to isolate each tox environment if TOXENV is defined
+TOX := uv run tox -- #to isolate each tox environment if TOXENV is defined
 endif
 
 help: ## display this help message
@@ -42,19 +37,16 @@ clean: ## delete generated byte code and coverage reports
 	rm -rf assets
 	rm -rf pii_report
 
-piptools: ## install pinned version of pip-compile and pip-sync
-	pip install -r requirements/pip-tools.txt
-
-requirements: piptools dev_requirements ## sync to default requirements
+requirements: dev_requirements ## sync to default requirements
 
 dev_requirements: ## sync to requirements for local development
-	pip-sync -q requirements/dev.txt
+	uv sync --group dev
 
 doc_requirements:
-	pip-sync -q requirements/doc.txt
+	uv sync --group doc
 
-production-requirements: piptools ## install requirements for production
-	pip-sync -q requirements/production.txt
+production-requirements: ## install requirements for production
+	uv sync --group production
 
 static: ## generate static files
 	python3 manage.py collectstatic --noinput
@@ -96,28 +88,9 @@ migrate: ## apply database migrations
 html_coverage: ## generate and view HTML coverage report
 	coverage html && open htmlcov/index.html
 
-upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
-upgrade: $(COMMON_CONSTRAINTS_TXT) piptools ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
-	# Make sure to compile files after any other files they include!
-	sed 's/Django<5.0//g' requirements/common_constraints.txt > requirements/common_constraints.tmp
-	mv requirements/common_constraints.tmp requirements/common_constraints.txt
-	sed 's/django-simple-history==3.0.0//g' requirements/common_constraints.txt > requirements/common_constraints.tmp
-	mv requirements/common_constraints.tmp requirements/common_constraints.txt
-	pip-compile --allow-unsafe --rebuild --upgrade -o requirements/pip.txt requirements/pip.in
-	pip-compile --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in
-	pip install -qr requirements/pip.txt
-	pip install -qr requirements/pip-tools.txt
-	pip-compile --upgrade -o requirements/base.txt requirements/base.in
-	pip-compile --upgrade -o requirements/test.txt requirements/test.in
-	pip-compile --upgrade -o requirements/doc.txt requirements/doc.in
-	pip-compile --upgrade -o requirements/quality.txt requirements/quality.in
-	pip-compile --upgrade -o requirements/validation.txt requirements/validation.in
-	pip-compile --upgrade -o requirements/dev.txt requirements/dev.in
-	pip-compile --upgrade -o requirements/production.txt requirements/production.in
-	# Let tox control the Django version for tests
-	grep -e "^django==" requirements/base.txt > requirements/django.txt
-	sed '/^[dD]jango==/d' requirements/test.txt > requirements/test.tmp
-	mv requirements/test.tmp requirements/test.txt
+upgrade: ## upgrade all packages in uv.lock and sync constraints from edx-lint
+	uv run --with edx-lint edx_lint write_uv_constraints pyproject.toml
+	uv lock --upgrade
 
 extract_translations: ## extract strings to be translated, outputting .mo files
 	python3 manage.py makemessages -l en -v1 -d django

pyproject.toml

@@ -0,0 +1,154 @@
+[project]
+name = "enterprise-catalog"
+version = "0.1.0"
+requires-python = ">=3.12"
+
+[dependency-groups]
+base = [
+    "analytics-python",
+    "Django",
+    "django-celery-results",
+    "django-config-models",
+    "django-cors-headers",
+    "django-crum",
+    "django-extensions",
+    "django-import-export",
+    "django-model-utils",
+    "django-simple-history",
+    "djangorestframework",
+    "djangorestframework-xml",
+    "djangoql",
+    "drf-spectacular",
+    "edx-auth-backends",
+    "edx-celeryutils",
+    "edx-django-release-util",
+    "edx-django-utils",
+    "edx-drf-extensions",
+    "edx_rbac",
+    "edx-rest-api-client",
+    "edx-toggles",
+    "mysqlclient",
+    "pymemcache",
+    "pytz",
+    "jsonfield2",
+    "celery",
+    "redis",
+    "rules",
+    "simplejson",
+    "zipp",
+    "algoliasearch",
+    "social-auth-app-django",
+    "xlsxwriter",
+    "django-clearcache",
+    "django-log-request-id",
+    "openai",
+    "scikit-learn",
+]
+test = [
+    {include-group = "base"},
+    "code-annotations",
+    "coverage",
+    "ddt",
+    "django-dynamic-fixture",
+    "edx-lint",
+    "factory-boy",
+    "pytest-cov",
+    "pytest-django",
+    "responses",
+]
+quality = [
+    {include-group = "test"},
+    "isort",
+    "pycodestyle",
+    "pydocstyle",
+]
+doc = [
+    {include-group = "test"},
+    "doc8",
+    "sphinx-book-theme",
+    "readme_renderer",
+    "Sphinx",
+]
+production = [
+    {include-group = "base"},
+    "gevent",
+    "gunicorn",
+    "python-memcached",
+    "PyYAML>=5.1",
+]
+ci = [
+    "tox",
+    "tox-uv",
+]
+dev = [
+    {include-group = "test"},
+    {include-group = "quality"},
+    {include-group = "ci"},
+    "diff-cover==4.0.0",
+    "edx-i18n-tools",
+    "django-debug-toolbar",
+    "gunicorn",
+    "inflect",
+    "python-memcached",
+]
+
+[tool.uv]
+package = false
+# Every `uv sync`/`uv run` invocation in this repo names an explicit --group.
+# Without this, uv's implicit default group (named "dev") would be synced *in
+# addition* to whatever --group is passed, silently pulling the entire
+# dev/test/quality/ci superset (and mysqlclient) into every target, defeating
+# the point of having separate groups.
+default-groups = []
+#  DO NOT EDIT constraint-dependencies DIRECTLY.
+#  This list is managed by `edx_lint write_uv_constraints`
+#  and will be overwritten the next time `make upgrade` is run.
+#  - GLOBAL constraints: edit edx_lint/files/common_constraints.txt
+#  - REPO-SPECIFIC constraints: edit [tool.edx_lint].uv_constraints in this file
+constraint-dependencies = [
+    "Django>=5.2,<5.3",
+    "elasticsearch<7.14.0",
+    "django-simple-history>=3.4.0",
+    "celery>=5.2.2,<6.0.0",
+    "diff-cover==4.0.0",
+    "openai<=1.13.3",
+    "algoliasearch<4",
+]
+
+[tool.edx_lint]
+uv_constraints = [
+    # using LTS django version
+    "Django>=5.2,<5.3",
+    # At least version 3.4.0 of django simple history
+    "django-simple-history>=3.4.0",
+    # As it is not clarified what exact breaking changes will be introduced as per
+    # the next major release, ensure the installed version is within boundaries.
+    "celery>=5.2.2,<6.0.0",
+    # diff-cover latest requires (pluggy>=0.13.1,<0.14.0)
+    # which conflicts with pytest(pluggy>=0.12,<2.0.0) and tox(pluggy>0.12) both of these fetch pluggy==1.0.0
+    # but diff-cover latest has a pin (pluggy<1.0.0a1)
+    # Using the same version of diff-cover which is being used currently in edx-platform to avoid this conflict.
+    "diff-cover==4.0.0",
+    # To avoid any breaking changes
+    "openai<=1.13.3",
+    # algoliasearch 4.0 is not backwards compatible with 3.0
+    "algoliasearch<4",
+]
+
+[tool.isort]
+indent = '    '
+line_length = 80
+multi_line_output = 3
+lines_after_imports = 2
+include_trailing_comma = true
+skip = ["settings", "migrations"]
+
+[tool.pytest.ini_options]
+DJANGO_SETTINGS_MODULE = "enterprise_catalog.settings.test"
+addopts = "--cov enterprise_catalog --cov-report term-missing --cov-report xml"
+norecursedirs = [".*", "docs", "requirements"]
+filterwarnings = [
+    "ignore:.*urlresolvers is deprecated in favor of.*:DeprecationWarning:auth_backends.views:5",
+    "ignore:.*invalid escape sequence.*:DeprecationWarning:.*(newrelic|uritemplate|psutil).*",
+    "ignore:.*the imp module is deprecated in favour of importlib.*:DeprecationWarning:.*distutils.*",
+]