diff --git a/.github/scripts/resolve_plugins.py b/.github/scripts/resolve_plugins.py new file mode 100755 index 0000000000..1e5e9fc799 --- /dev/null +++ b/.github/scripts/resolve_plugins.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +import hashlib +import json +import os +import subprocess +import sys + +plugs_raw = os.environ.get("ADDITIONAL_PLUGS", "[]") +try: + plugs = json.loads(plugs_raw) +except json.JSONDecodeError: + sys.stdout.write("no-plugins") + sys.exit(0) + +if not plugs: + sys.stdout.write("no-plugins") + sys.exit(0) + +resolved = [] +for p in plugs: + ver = p.get("version") or "@main" + pkg = p.get("package_name") + if not pkg: + sys.stderr.write(f"plugin entry {p!r} is missing 'package_name'\n") + sys.exit(1) + if ver.startswith("@"): + ref = ver[1:] + git_url = pkg.removeprefix("git+") + try: + out = subprocess.run( # noqa: S603 + ["git", "ls-remote", git_url, ref], # noqa: S607 + check=False, + capture_output=True, + text=True, + timeout=15, + ) + sha = out.stdout.split()[0] if out.stdout else ref + except Exception as exc: + sys.stderr.write( + f"warning: git ls-remote failed for {git_url!r} ({exc}); using ref name as cache key\n" + ) + sha = ref + resolved.append(f"{pkg}@{sha}") + else: + resolved.append(f"{pkg}{ver}") + +h = hashlib.sha256(json.dumps(sorted(resolved)).encode()).hexdigest()[:16] +print(h, end="") # noqa: T201 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6fc84e07a1..96260ded21 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -73,18 +73,17 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Docker layers - uses: actions/cache@v4 - with: - path: ${{ runner.temp }}/.buildx-cache - key: ${{ runner.os }}-${{ runner.arch }}-buildx-prod-${{ hashFiles('Pipfile.lock', 'docker/prod.Dockerfile') }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-buildx-prod- + - name: Compute cache tag + id: cache-tag + run: | + echo "tag=buildcache-${{ env.PLATFORM_PAIR }}-$(date +%G-W%V)" >> "$GITHUB_OUTPUT" - - name: Create new cache + - name: Resolve plugin cache buster + id: plugin-hash run: | - mkdir -p ${{ runner.temp }}/.buildx-cache - mkdir -p ${{ runner.temp }}/.buildx-cache-new + echo "resolved_hash=$(python3 .github/scripts/resolve_plugins.py)" >> "$GITHUB_OUTPUT" + env: + ADDITIONAL_PLUGS: ${{ env.ADDITIONAL_PLUGS }} - name: Build and push by digest id: build @@ -98,8 +97,9 @@ jobs: build-args: | APP_VERSION=${{ github.sha }} ADDITIONAL_PLUGS=${{ env.ADDITIONAL_PLUGS }} - cache-from: type=local,src=${{ runner.temp }}/.buildx-cache - cache-to: type=local,dest=${{ runner.temp }}/.buildx-cache-new,mode=max + PLUGIN_RESOLVED_HASH=${{ steps.plugin-hash.outputs.resolved_hash }} + cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:${{ steps.cache-tag.outputs.tag }} + cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:${{ steps.cache-tag.outputs.tag }},mode=max - name: Export digest run: | @@ -115,11 +115,6 @@ jobs: if-no-files-found: error retention-days: 1 - - name: Move cache - run: | - rm -rf ${{ runner.temp }}/.buildx-cache - mv ${{ runner.temp }}/.buildx-cache-new ${{ runner.temp }}/.buildx-cache - merge-manifests: needs: build name: Merge & Push Manifests @@ -186,6 +181,43 @@ jobs: run: | echo "Release ${{ github.sha }} is ready to be deployed to production" + prune-cache: + if: github.ref == 'refs/heads/develop' && github.repository == 'ohcnetwork/care' + needs: merge-manifests + runs-on: ubuntu-24.04 + permissions: + packages: write + steps: + - name: Prune stale buildcache versions + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + OWNER="${{ github.repository_owner }}" + PACKAGE_NAME="${GITHUB_REPOSITORY#*/}" + CUTOFF=$(date -d "4 weeks ago" +%G-W%V) + + gh api "/orgs/$OWNER/packages/container/$PACKAGE_NAME/versions?per_page=100" \ + --paginate \ + --jq '.[] | select(any(.metadata.container.tags[]; startswith("buildcache-")))' \ + | python3 -c " + import json, re, sys + cutoff = '$CUTOFF' + for line in sys.stdin: + line = line.strip() + if not line: + continue + v = json.loads(line) + for tag in v['metadata']['container']['tags']: + m = re.match(r'^buildcache-.+-(\d{4}-W\d{2})$', tag) + if m and m.group(1) < cutoff: + print(v['id'], tag) + break + " | while read -r id tag; do + echo "Pruning $tag (version $id)" + gh api --method DELETE "/orgs/$OWNER/packages/container/$PACKAGE_NAME/versions/$id" \ + --silent || echo " (skipped)" + done + # deploy-staging-egov: # needs: merge-manifests # if: github.ref == 'refs/heads/develop' diff --git a/.github/workflows/reusable-test.yml b/.github/workflows/reusable-test.yml index 6bcd17bc8a..2a3686e35a 100644 --- a/.github/workflows/reusable-test.yml +++ b/.github/workflows/reusable-test.yml @@ -21,39 +21,39 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Restore Docker layers cache - id: cache-restore - uses: actions/cache/restore@v4 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 with: - path: | - ${{ runner.temp }}/.buildx-cache - ${{ runner.temp }}/.buildx-mounted-cache - key: ${{ runner.os }}-${{ runner.arch }}-buildx-dev-${{ hashFiles('Pipfile.lock', 'docker/dev.Dockerfile') }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-buildx-dev- + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: inject cache into docker - uses: reproducible-containers/buildkit-cache-dance@v3.3.0 - with: - cache-map: | - { - "${{ runner.temp }}/.buildx-mounted-cache": "/root/.cache/pip" - } - skip-extraction: ${{ steps.cache-restore.outputs.cache-hit }} + - name: Compute cache tag + run: | + echo "CACHE_TAG=buildcache-dev-$(date +%G-W%V)" >> "$GITHUB_ENV" - - name: Create new cache + - name: Resolve plugin cache buster + id: plugin-hash run: | - mkdir -p ${{ runner.temp }}/.buildx-cache - mkdir -p ${{ runner.temp }}/.buildx-cache-new - mkdir -p ${{ runner.temp }}/.buildx-mounted-cache + echo "resolved_hash=$(python3 .github/scripts/resolve_plugins.py)" >> "$GITHUB_OUTPUT" + env: + ADDITIONAL_PLUGS: ${{ env.ADDITIONAL_PLUGS }} - name: Build images + env: + ADDITIONAL_PLUGS: ${{ env.ADDITIONAL_PLUGS }} run: | + CACHE_TO="" + if [[ '${{ github.event_name }}' == 'push' ]]; then + CACHE_TO="--cache-to=type=registry,ref=ghcr.io/${{ github.repository }}:$CACHE_TAG,mode=max" + fi docker buildx build \ --file docker/dev.Dockerfile \ --tag care_local \ - --cache-from=type=local,src=${{ runner.temp }}/.buildx-cache \ - --cache-to=type=local,dest=${{ runner.temp }}/.buildx-cache-new,mode=max \ + --build-arg PLUGIN_RESOLVED_HASH=${{ steps.plugin-hash.outputs.resolved_hash }} \ + --build-arg "ADDITIONAL_PLUGS=$ADDITIONAL_PLUGS" \ + --cache-from=type=registry,ref=ghcr.io/${{ github.repository }}:$CACHE_TAG \ + $CACHE_TO \ --platform linux/arm64 \ --load \ . @@ -81,20 +81,6 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - name: Move cache - run: | - rm -rf ${{ runner.temp }}/.buildx-cache - mv ${{ runner.temp }}/.buildx-cache-new ${{ runner.temp }}/.buildx-cache - - - name: Save Docker layers cache - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' && steps.cache-restore.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v4 - with: - path: | - ${{ runner.temp }}/.buildx-cache - ${{ runner.temp }}/.buildx-mounted-cache - key: ${{ runner.os }}-${{ runner.arch }}-buildx-dev-${{ hashFiles('Pipfile.lock', 'docker/dev.Dockerfile') }} - # Upload dummy db as artifact so it can be used to speed up frontend tests - name: Dump db if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }} diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile index ee1737aa0e..c91ca8b96f 100644 --- a/docker/dev.Dockerfile +++ b/docker/dev.Dockerfile @@ -21,12 +21,13 @@ RUN --mount=type=cache,target=/root/.cache/pip pip install pipenv==2025.1.1 COPY Pipfile Pipfile.lock $APP_HOME/ RUN --mount=type=cache,target=/root/.cache/pip pipenv install --system --categories "packages dev-packages docs" +ARG PLUGIN_RESOLVED_HASH="" ARG ADDITIONAL_PLUGS="" ENV ADDITIONAL_PLUGS=$ADDITIONAL_PLUGS COPY . $APP_HOME/ -RUN --mount=type=cache,target=/root/.cache/pip python3 $APP_HOME/install_plugins.py +RUN --mount=type=cache,target=/root/.cache/pip PLUGIN_RESOLVED_HASH=$PLUGIN_RESOLVED_HASH python3 $APP_HOME/install_plugins.py HEALTHCHECK \ --interval=10s \ diff --git a/docker/prod.Dockerfile b/docker/prod.Dockerfile index 6aba95bdc8..bbb5c5f3e5 100644 --- a/docker/prod.Dockerfile +++ b/docker/prod.Dockerfile @@ -34,9 +34,10 @@ RUN pipenv install --deploy --categories "packages" COPY plugs/ $APP_HOME/plugs/ COPY install_plugins.py plug_config.py $APP_HOME/ +ARG PLUGIN_RESOLVED_HASH="" ARG ADDITIONAL_PLUGS="" ENV ADDITIONAL_PLUGS=$ADDITIONAL_PLUGS -RUN python3 $APP_HOME/install_plugins.py +RUN PLUGIN_RESOLVED_HASH=$PLUGIN_RESOLVED_HASH python3 $APP_HOME/install_plugins.py # --- FROM base AS runtime