Modernise stack: drop foxglove for FastAPI + SQLModel + Celery + uv

.github/workflows/main.yml

@@ -14,7 +14,7 @@ jobs:
 
     services:
       postgres:
-        image: postgres:12
+        image: postgres:15
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -29,29 +29,32 @@ jobs:
         options: --entrypoint redis-server
 
     env:
-      TEST_DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/morpheus_test'
+      DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/morpheus_test'
+      REDIS_URL: 'redis://localhost:6379/4'
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
-    - name: set up python
-      uses: actions/setup-python@v1
+    - name: install uv
+      uses: astral-sh/setup-uv@v3
       with:
-        python-version: '3.9'
+        python-version: '3.12'
 
     - name: install dependencies
-      run: |
-        make install
-        pip freeze
+      run: uv sync --all-extras --dev
+
     - name: lint
       run: make lint
+
     - name: test
-      run: make test
+      run: uv run pytest tests/ --cov=app --cov-report=xml --cov-report=term
 
     - name: codecov
-      run: bash <(curl -s https://codecov.io/bash)
-      env:
-        CODECOV_TOKEN: '1b5eacd0-b422-4654-970d-84acdb03cf53'
+      uses: codecov/codecov-action@v4
+      with:
+        files: ./coverage.xml
+        token: ${{ secrets.CODECOV_TOKEN }}
+        fail_ci_if_error: false
 
   deploy:
     needs:
@@ -64,7 +67,7 @@ jobs:
       HEROKU_APP: tc-morpheus
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - run: git fetch --unshallow
       - run: git switch master
       - run: git remote add heroku https://heroku:$HEROKU_API_KEY@git.heroku.com/$HEROKU_APP.git
@@ -74,25 +77,37 @@ jobs:
     needs: test
     if: "success() && startsWith(github.ref, 'refs/tags/')"
     runs-on: ubuntu-latest
+
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
-      - name: set up python
-        uses: actions/setup-python@v1
+      - name: install uv
+        uses: astral-sh/setup-uv@v3
         with:
-          python-version: '3.9'
+          python-version: '3.12'
 
-      - name: install
-        run: |
-          make install
-          pip install -U wheel twine
+      - name: install build deps
+        run: uv pip install --system setuptools wheel twine
 
-      - name: build
+      - name: stage render module under morpheus/ namespace
+        working-directory: packaging/morpheus-mail
+        run: |
+          mkdir -p morpheus
+          cp -r ../../app/render morpheus/render
+          touch morpheus/__init__.py
+
+      - name: build morpheus-mail (render submodule)
+        # Builds from a subdirectory so the root pyproject.toml (which defines the main
+        # app) doesn't get auto-applied to this distribution.
+        working-directory: packaging/morpheus-mail
         run: python setup.py sdist bdist_wheel
 
-      - run: twine check dist/*
+      - name: twine check
+        working-directory: packaging/morpheus-mail
+        run: twine check dist/*
 
       - name: upload to pypi
+        working-directory: packaging/morpheus-mail
         run: twine upload dist/*
         env:
           TWINE_USERNAME: __token__

.gitignore

@@ -6,3 +6,5 @@ build/
 dist/
 .coverage
 .env
+.claude/
+packaging/morpheus-mail/morpheus/

.python-version

@@ -1 +1 @@
-3.9.14
+3.12

Makefile

@@ -1,38 +1,36 @@
-black = black -S -l 120 --target-version py38
-isort = isort -w 120
-
-PHONY: install
+.PHONY: install
 install:
-	pip install -r requirements.txt
-	pip install -r tests/requirements.txt
+	uv sync
 
 .PHONY: format
 format:
-	$(isort) src tests
-	$(black) src tests
+	uv run ruff format app tests
+	uv run ruff check --fix app tests
 
 .PHONY: lint
 lint:
-	flake8 src tests
-	$(isort) --check-only src tests
-	$(black) --check src tests
+	uv run ruff check app tests
+	uv run ruff format --check app tests
+	uv run ty check app
 
 .PHONY: test
 test:
-	pytest tests/ --cov=src
+	uv run pytest tests/ --cov=app
 
 .PHONY: reset-db
 reset-db:
 	psql -h localhost -U postgres -c "DROP DATABASE IF EXISTS morpheus"
 	psql -h localhost -U postgres -c "CREATE DATABASE morpheus"
-	psql -h localhost -U postgres -d morpheus -f src/models.sql
-	foxglove patch add_aggregation_view --live --patch-args ':'
-	foxglove patch add_spam_status_and_reason_to_messages --live --patch-args ':'
+	uv run python -c "from app.core.database import create_db_and_tables; create_db_and_tables()"
+
+.PHONY: dev
+dev:
+	uv run uvicorn app.main:app --reload
 
-# Run a specific patch by name:
-#   make run_patch PATCH=patch_function_name
-#   make run_patch PATCH=patch_function_name LIVE=1
-.PHONY: run_patch
-run_patch:
-	foxglove patch $(PATCH) $(if $(LIVE),--live,) --patch-args ':'
+.PHONY: worker
+worker:
+	uv run celery -A app.worker worker --loglevel=info
 
+.PHONY: beat
+beat:
+	uv run celery -A app.worker beat --loglevel=info

Procfile

@@ -1,2 +1,3 @@
-web: foxglove web
-worker: foxglove worker
+web: uvicorn app.main:app --host 0.0.0.0 --port $PORT
+worker: celery -A app.worker worker --loglevel=info
+beat: celery -A app.worker beat --loglevel=info

app/common/api/errors.py

@@ -0,0 +1,44 @@
+"""HTTP exceptions whose response bodies are `{'message': '...'}`.
+
+Mirrors the legacy foxglove behaviour so existing client integrations and tests keep working.
+"""
+
+from fastapi import HTTPException
+from fastapi.responses import JSONResponse
+from starlette.requests import Request
+from starlette.responses import Response
+
+
+class HttpMessageError(HTTPException):
+    def __init__(self, status_code: int, message: str) -> None:
+        self.message = message
+        super().__init__(status_code=status_code, detail=message)
+
+
+class HTTP400(HttpMessageError):
+    def __init__(self, message: str = 'Bad request') -> None:
+        super().__init__(400, message)
+
+
+class HTTP403(HttpMessageError):
+    def __init__(self, message: str = 'Forbidden') -> None:
+        super().__init__(403, message)
+
+
+class HTTP404(HttpMessageError):
+    def __init__(self, message: str = 'Not found') -> None:
+        super().__init__(404, message)
+
+
+class HTTP409(HttpMessageError):
+    def __init__(self, message: str = 'Conflict') -> None:
+        super().__init__(409, message)
+
+
+class HTTP422(HttpMessageError):
+    def __init__(self, message: str = 'Unprocessable entity') -> None:
+        super().__init__(422, message)
+
+
+async def http_message_error_handler(_: Request, exc: HttpMessageError) -> Response:
+    return JSONResponse({'message': exc.message}, status_code=exc.status_code)

app/common/auth.py

@@ -0,0 +1,66 @@
+import hashlib
+import hmac
+from datetime import datetime, timezone
+from typing import Optional
+
+from fastapi import Request
+from pydantic import BaseModel, ValidationError, field_validator
+
+from app.common.api.errors import HTTP403
+from app.core.config import settings
+
+
+class AdminAuth:
+    def __init__(self, request: Request):
+        if request.headers.get('Authorization', '') != settings.auth_key:
+            raise HTTP403('Invalid token')
+
+
+class _UserSessionData(BaseModel):
+    company: str
+    expires: datetime
+    signature: str
+
+    @field_validator('expires')
+    @classmethod
+    def add_tz(cls, v: datetime) -> datetime:
+        if v.tzinfo is None:  # pragma: no cover  -- pydantic v2 parses unix epoch strings as tz-aware
+            return v.replace(tzinfo=timezone.utc)
+        return v
+
+
+class UserSession:
+    """Validates a signed query-string session and exposes `company`/`expires`.
+
+    Implemented as a callable dependency rather than a Pydantic model so that
+    auth failures surface as a 403 with a `{'message': ...}` body, not a
+    Pydantic 422 ValidationError. Pydantic v2 catches HTTPException raised
+    inside validators and re-wraps as ValidationError, which is the wrong
+    failure mode here.
+    """
+
+    def __init__(
+        self,
+        company: Optional[str] = None,
+        expires: Optional[datetime] = None,
+        signature: Optional[str] = None,
+    ):
+        try:
+            data = _UserSessionData(company=company, expires=expires, signature=signature)  # ty:ignore[invalid-argument-type]
+        except ValidationError:
+            raise HTTP403('Invalid token')
+
+        if data.expires < datetime.now(tz=timezone.utc):
+            raise HTTP403('Token expired')
+
+        expected_sig = hmac.new(
+            settings.user_auth_key,
+            f'{data.company}:{data.expires.timestamp():.0f}'.encode(),
+            hashlib.sha256,
+        ).hexdigest()
+        if not hmac.compare_digest(data.signature, expected_sig):
+            raise HTTP403('Invalid token')
+
+        self.company = data.company
+        self.expires = data.expires
+        self.signature = data.signature

app/core/bootstrap.sql

@@ -0,0 +1,67 @@
+-- Pre-table SQL: extensions, enums, plpgsql functions.
+-- Runs before SQLModel.metadata.create_all. Idempotent.
+
+DO $do$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'send_methods') THEN
+    CREATE TYPE SEND_METHODS AS ENUM (
+      'email-mandrill', 'email-ses', 'email-test', 'sms-messagebird', 'sms-test'
+    );
+  END IF;
+END
+$do$;
+
+DO $do$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'message_statuses') THEN
+    CREATE TYPE MESSAGE_STATUSES AS ENUM (
+      'render_failed', 'send_request_failed', 'send', 'deferral', 'hard_bounce', 'soft_bounce',
+      'open', 'click', 'spam', 'unsub', 'reject', 'scheduled', 'buffered', 'delivered', 'expired',
+      'delivery_failed'
+    );
+  END IF;
+END
+$do$;
+
+CREATE OR REPLACE FUNCTION update_message() RETURNS trigger AS $update_message$
+  DECLARE
+    current_update_ts timestamptz;
+  BEGIN
+    select update_ts into current_update_ts from messages where id=new.message_id;
+    if new.ts > current_update_ts then
+      update messages set update_ts=new.ts, status=new.status where id=new.message_id;
+    end if;
+    return null;
+  END;
+$update_message$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION set_message_vector() RETURNS trigger AS $set_message_vector$
+  BEGIN
+    RAISE NOTICE '%', NEW.external_id;
+    NEW.vector := setweight(to_tsvector(coalesce(NEW.external_id, '')), 'A') ||
+                  setweight(to_tsvector(coalesce(NEW.to_first_name, '')), 'A') ||
+                  setweight(to_tsvector(coalesce(NEW.to_last_name, '')), 'A') ||
+                  setweight(to_tsvector(coalesce(NEW.to_address, '')), 'A') ||
+                  setweight(to_tsvector(coalesce(NEW.subject, '')), 'B') ||
+                  setweight(to_tsvector(coalesce(array_to_string(NEW.tags, ' '), '')), 'B') ||
+                  setweight(to_tsvector(coalesce(array_to_string(NEW.attachments, ' '), '')), 'C') ||
+                  setweight(to_tsvector(coalesce(NEW.body, '')), 'D');
+    return NEW;
+  END;
+$set_message_vector$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION iso_ts(v TIMESTAMPTZ, tz VARCHAR(63)) RETURNS VARCHAR(63) AS $iso_ts$
+  DECLARE
+  BEGIN
+    PERFORM set_config('timezone', tz, true);
+    return to_char(v, 'YYYY-MM-DD"T"HH24:MI:SSOF');
+  END;
+$iso_ts$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION pretty_ts(v TIMESTAMPTZ, tz VARCHAR(63)) RETURNS VARCHAR(63) AS $pretty_ts$
+  DECLARE
+  BEGIN
+    PERFORM set_config('timezone', tz, true);
+    return to_char(v, 'Dy YYYY-MM-DD HH24:MI TZ');
+  END;
+$pretty_ts$ LANGUAGE plpgsql;

app/core/celery.py

@@ -0,0 +1,35 @@
+from celery import Celery
+from celery.schedules import crontab
+
+from app.core.config import settings
+
+celery_app = Celery(
+    'morpheus',
+    broker=settings.redis_url,
+    backend=settings.redis_url,
+    include=['app.messages.tasks'],
+)
+
+celery_app.conf.update(
+    task_serializer='json',
+    accept_content=['json'],
+    result_serializer='json',
+    timezone='UTC',
+    enable_utc=True,
+    task_track_started=True,
+    task_acks_late=True,
+    worker_prefetch_multiplier=1,
+    broker_connection_retry_on_startup=True,
+)
+
+
+celery_app.conf.beat_schedule = {
+    'update-aggregation-view': {
+        'task': 'app.messages.tasks.update_aggregation_view',
+        'schedule': crontab(minute='12'),
+    },
+    'delete-old-emails': {
+        'task': 'app.messages.tasks.delete_old_emails',
+        'schedule': crontab(minute='30'),
+    },
+}