From 43fe61b00bd20db3b6e753f93e57b1e81f90be99 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 28 Apr 2026 12:23:08 +0800 Subject: [PATCH] Drop blob branch and rebuild prebuilt on PRs Switch the kernel/rootfs prebuilt artifacts from the orphan blob branch to the fixed-tag prebuilt GitHub prerelease, and close the long-standing gap where a PR that touched configs/scripts/target would silently run make check against the stale prebuilt instead of its own change. --- .github/workflows/main.yml | 116 ++++++++++++++++++++++++++++----- .github/workflows/prebuilt.yml | 16 ++--- mk/external.mk | 50 +++++++++----- 3 files changed, 141 insertions(+), 41 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index df3338b1..dba2dbb6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,10 +8,80 @@ concurrency: permissions: contents: read - actions: write jobs: + # PR-only: rebuild Image/rootfs.cpio from source when local kernel + # and rootfs inputs have drifted from the prebuilt's recorded inputs. + # Without this, make check on a PR that touches configs, scripts, or + # target would silently exercise the stale prebuilt instead of the + # contributor's actual change. + # + # The artifacts produced here are published as a workflow artifact + # (reliable cross-job and cross-runner handoff) and consumed by the + # test jobs below. They are NOT published to the prebuilt GitHub + # release; that path is owned by .github/workflows/prebuilt.yml on + # master push. + # + # The job always runs but short-circuits unless this is a PR with + # drifted inputs, so non-drifting PRs and master pushes pay only the + # cost of a checkout plus drift check. + pr-prebuilt-build: + runs-on: ubuntu-24.04 + outputs: + should_build: ${{ steps.detect.outputs.should_build }} + steps: + - name: checkout code + uses: actions/checkout@v4 + - name: detect input drift + id: detect + shell: bash + run: | + if [ "${{ github.event_name }}" != "pull_request" ]; then + echo "should_build=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + expected=$(awk '/^PREBUILT_INPUTS_SHA1/ {print $3}' mk/external.mk) + live=$(cat \ + configs/linux.config \ + configs/busybox.config \ + configs/buildroot.config \ + scripts/build-image.sh \ + scripts/rootfs_ext4.sh \ + target/init \ + | sha1sum | awk '{print $1}') + if [ "$live" = "$expected" ]; then + echo "PR inputs match the prebuilt ($live); skipping rebuild" + echo "should_build=false" >> "$GITHUB_OUTPUT" + else + echo "PR inputs drifted ($live != $expected); will rebuild from source" + echo "should_build=true" >> "$GITHUB_OUTPUT" + fi + - name: install build dependencies + if: steps.detect.outputs.should_build == 'true' + run: | + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ + build-essential \ + bc bison flex cpio fakeroot e2fsprogs \ + git python3 libssl-dev libelf-dev wget + timeout-minutes: 5 + - name: build kernel and rootfs from source + if: steps.detect.outputs.should_build == 'true' + run: ./scripts/build-image.sh --all + timeout-minutes: 90 + - name: upload PR-built artifacts + if: steps.detect.outputs.should_build == 'true' + uses: actions/upload-artifact@v4 + with: + name: prebuilt-pr + path: | + Image + rootfs.cpio + retention-days: 1 + if-no-files-found: error + semu-linux: + needs: pr-prebuilt-build runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -25,17 +95,23 @@ jobs: uses: actions/checkout@v4 with: submodules: recursive + # Drift PR: pull the freshly-built Image/rootfs.cpio from + # pr-prebuilt-build via workflow artifact (reliable, not cache). + - name: download PR-built kernel/rootfs + if: needs.pr-prebuilt-build.outputs.should_build == 'true' + uses: actions/download-artifact@v4 + with: + name: prebuilt-pr + # Non-drift PR or master push: cache the release-downloaded artifacts + # across runs. Include mk/external.mk so checksum or input-pin bumps + # after a republish invalidate the old Image/rootfs.cpio pair. - name: cache external downloads + if: needs.pr-prebuilt-build.outputs.should_build != 'true' uses: actions/cache@v4 with: path: | Image rootfs.cpio - # Invalidate the cached prebuilts whenever any input that defines - # their content changes. Keep this list in sync with - # PREBUILT_INPUTS in mk/external.mk and the `paths:` filter in - # .github/workflows/prebuilt.yml; otherwise CI will silently - # restore stale Image / rootfs.cpio after a config bump. key: external-${{ hashFiles('mk/external.mk', 'configs/linux.config', 'configs/busybox.config', 'configs/buildroot.config', 'scripts/build-image.sh', 'scripts/rootfs_ext4.sh', 'target/**') }} - name: cache submodule builds uses: actions/cache@v4 @@ -85,23 +161,25 @@ jobs: # default boot mode is /dev/vda. Single slim job: fresh build with # ENABLE_EXTERNAL_ROOT=0, then run autorun. semu-linux-initramfs: + needs: pr-prebuilt-build runs-on: ubuntu-24.04 steps: - name: checkout code uses: actions/checkout@v4 with: submodules: recursive + - name: download PR-built kernel/rootfs + if: needs.pr-prebuilt-build.outputs.should_build == 'true' + uses: actions/download-artifact@v4 + with: + name: prebuilt-pr - name: cache external downloads + if: needs.pr-prebuilt-build.outputs.should_build != 'true' uses: actions/cache@v4 with: path: | Image rootfs.cpio - # Invalidate the cached prebuilts whenever any input that defines - # their content changes. Keep this list in sync with - # PREBUILT_INPUTS in mk/external.mk and the `paths:` filter in - # .github/workflows/prebuilt.yml; otherwise CI will silently - # restore stale Image / rootfs.cpio after a config bump. key: external-${{ hashFiles('mk/external.mk', 'configs/linux.config', 'configs/busybox.config', 'configs/buildroot.config', 'scripts/build-image.sh', 'scripts/rootfs_ext4.sh', 'target/**') }} - name: cache submodule builds uses: actions/cache@v4 @@ -131,22 +209,28 @@ jobs: timeout-minutes: 10 semu-macOS: + needs: pr-prebuilt-build runs-on: macos-latest steps: - uses: actions/checkout@v4 with: submodules: recursive + # macOS cannot run buildroot. On a drift PR it must consume + # pr-prebuilt-build's artifact (built on a linux runner) -- a cache + # miss here would silently fall back to downloading the stale + # release, defeating the whole point of the drift-detection logic. + - name: download PR-built kernel/rootfs + if: needs.pr-prebuilt-build.outputs.should_build == 'true' + uses: actions/download-artifact@v4 + with: + name: prebuilt-pr - name: cache external downloads + if: needs.pr-prebuilt-build.outputs.should_build != 'true' uses: actions/cache@v4 with: path: | Image rootfs.cpio - # Invalidate the cached prebuilts whenever any input that defines - # their content changes. Keep this list in sync with - # PREBUILT_INPUTS in mk/external.mk and the `paths:` filter in - # .github/workflows/prebuilt.yml; otherwise CI will silently - # restore stale Image / rootfs.cpio after a config bump. key: external-${{ hashFiles('mk/external.mk', 'configs/linux.config', 'configs/busybox.config', 'configs/buildroot.config', 'scripts/build-image.sh', 'scripts/rootfs_ext4.sh', 'target/**') }} - name: cache submodule builds uses: actions/cache@v4 diff --git a/.github/workflows/prebuilt.yml b/.github/workflows/prebuilt.yml index ef02f68f..5103613b 100644 --- a/.github/workflows/prebuilt.yml +++ b/.github/workflows/prebuilt.yml @@ -1,14 +1,14 @@ name: Publish prebuilt images -# Builds the Linux kernel and Buildroot rootfs that the rest of CI (and -# `make` on a fresh checkout) consumes, then publishes them as assets on a +# Builds the Linux kernel and Buildroot rootfs that the rest of CI and +# make on a fresh checkout consumes, then publishes them as assets on a # fixed-tag GitHub prerelease so the download URL stays stable across -# rebuilds. This replaces the old `blob` branch convention, which forced -# us to push large binary artifacts into the source tree. +# rebuilds, keeping large binary artifacts out of the source tree. # -# The workflow is manual by default (workflow_dispatch). Run it whenever -# the kernel config, buildroot config, or build-image.sh changes; the -# resulting SHA1 sums must be reflected in mk/external.mk. +# Triggers automatically on master pushes that touch any input listed +# in the paths filter below, and can be invoked manually via +# workflow_dispatch. The resulting SHA1 sums must be reflected in +# mk/external.mk (KERNEL_DATA_SHA1, INITRD_DATA_SHA1, PREBUILT_INPUTS_SHA1). on: workflow_dispatch: @@ -76,7 +76,7 @@ jobs: tag_name: prebuilt name: Prebuilt images (rolling) prerelease: true - # Replace existing assets at the `prebuilt` tag rather than + # Replace existing assets at the prebuilt tag rather than # appending duplicates with new names. fail_on_unmatched_files: true files: | diff --git a/mk/external.mk b/mk/external.mk index c4b28155..0c932100 100644 --- a/mk/external.mk +++ b/mk/external.mk @@ -3,32 +3,48 @@ # _DATA : the file to be read by specific executable. # _DATA_SHA1 : the checksum of the content in _DATA # -# Artifacts live on the orphan `blob` branch of this repository. The -# `Publish prebuilt images` workflow (see .github/workflows/prebuilt.yml) -# will eventually republish them to a fixed-tag GitHub prerelease; once -# that release exists, switch COMMON_URL to -# https://github.com/sysprog21/semu/releases/download/prebuilt and update -# KERNEL_DATA_SHA1, INITRD_DATA_SHA1, and PREBUILT_INPUTS_SHA1 from the -# release body. +# Artifacts are published as assets on the fixed-tag prebuilt GitHub +# prerelease by .github/workflows/prebuilt.yml. Update the SHA1 values +# below from the release body whenever the workflow republishes a new +# build. -COMMON_URL = https://github.com/sysprog21/semu/raw/blob +COMMON_URL = https://github.com/sysprog21/semu/releases/download/prebuilt # kernel KERNEL_DATA_URL = $(COMMON_URL)/Image.bz2 KERNEL_DATA = Image -KERNEL_DATA_SHA1 = f33badc277a88c17ce36b28d229a3f99cbccef91 +KERNEL_DATA_SHA1 = 39d273097f21a1bf38fd93b96a3d7459f843bc84 # initrd INITRD_DATA_URL = $(COMMON_URL)/rootfs.cpio.bz2 INITRD_DATA = rootfs.cpio -INITRD_DATA_SHA1 = a63336a28e484ed9cd560652c336b93affe50126 +INITRD_DATA_SHA1 = 9df154cdf58103e953ccdf0d40736cadf9318b12 define download +# Download to a .part file so an interrupted curl never lands a +# corrupt or incomplete .bz2 that a later run mistakes for valid input. +# Curl resume (-C -) is intentionally NOT used: a fully-downloaded .part +# left over from a previous run, e.g. interrupted before sha1 verify, +# would make curl request a byte range past EOF, the server replies +# HTTP 416, and curl exits non-zero, a permanent self-inflicted +# deadlock. These files are 5 to 7 MiB; a fresh GET is cheap. +# +# Decompress to a .tmp file and rename only on success, so an +# interrupted bunzip2 cannot leave a half-decompressed Image or +# rootfs.cpio that make would treat as a valid up-to-date target on the +# next invocation. $($(T)_DATA): $(VECHO) " GET\t$$@\n" - $(Q)curl --progress-bar -O -L -C - "$(strip $($(T)_DATA_URL))" - $(Q)echo "$(strip $$($(T)_DATA_SHA1)) $$@.bz2" | $(SHA1SUM) -c - - $(Q)bunzip2 $$@.bz2 + $(Q)curl --fail --retry 3 --retry-delay 1 --progress-bar \ + -L -o "$$@.bz2.part" "$(strip $($(T)_DATA_URL))" \ + || { rm -f "$$@.bz2.part"; exit 1; } + $(Q)echo "$(strip $$($(T)_DATA_SHA1)) $$@.bz2.part" | $(SHA1SUM) -c - \ + || { rm -f "$$@.bz2.part"; exit 1; } + $(Q)mv "$$@.bz2.part" "$$@.bz2" + $(Q)bunzip2 -c "$$@.bz2" > "$$@.tmp" \ + || { rm -f "$$@.tmp"; exit 1; } + $(Q)mv "$$@.tmp" "$$@" + $(Q)rm -f "$$@.bz2" endef EXTERNAL_DATA = KERNEL INITRD @@ -40,13 +56,13 @@ $(foreach T,$(EXTERNAL_DATA),$(eval $(download))) # input files (kernel/buildroot/busybox configs, the build script, and # the init stub). When any of those change locally the prebuilt may no # longer reflect the user's intent, so we compute the SHA1 of those -# inputs and compare against PREBUILT_INPUTS_SHA1 -- the value the -# `Publish prebuilt images` workflow recorded for the live release. +# inputs and compare against PREBUILT_INPUTS_SHA1, the value the +# Publish prebuilt images workflow recorded for the live release. # # Mismatch -> warn but do not auto-rebuild: a buildroot run takes the -# better part of an hour, so we let the user opt in via `make build-image`. +# better part of an hour, so we let the user opt in via make build-image. # Keep this list in sync with the INPUTS array in .ci/publish-prebuilt.sh -# and the `paths:` filter in .github/workflows/prebuilt.yml. +# and the paths filter in .github/workflows/prebuilt.yml. PREBUILT_INPUTS := \ configs/linux.config \ configs/busybox.config \