Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions .ci/publish-prebuilt.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
#!/usr/bin/env bash
#
# Compress the prebuilt Image and rootfs.cpio in cwd, write a sha1
# manifest, hash the input files that define the prebuilt's contents,
# and print all three sums in KEY=VAL form on stdout so callers can
# splice them into release notes, GITHUB_OUTPUT, or whatever else.
#
# Inputs (in cwd):
# Image
# rootfs.cpio
# plus the source inputs listed in INPUTS below (config + scripts +
# target/init that define the buildroot/kernel content)
#
# Outputs (in cwd):
# Image.bz2
# rootfs.cpio.bz2
# prebuilt.sha1 -- two-line manifest in standard `sha1sum` format
#
# Stdout (machine-readable, one assignment per line):
# kernel_sha1=<sha1 of Image.bz2>
# initrd_sha1=<sha1 of rootfs.cpio.bz2>
# inputs_sha1=<sha1 of the concatenated input files>

set -euo pipefail

# Pick a SHA1 tool. macOS dropped `sha1sum` from the base system; the
# coreutils-style `shasum -a 1` is the portable fallback.
if command -v sha1sum >/dev/null 2>&1; then
SHA1=(sha1sum)
elif command -v shasum >/dev/null 2>&1; then
SHA1=(shasum -a 1)
else
echo "[!] Need sha1sum (Linux) or shasum (macOS) on PATH" >&2
exit 1
fi

# Keep this list in sync with PREBUILT_INPUTS in mk/external.mk and the
# `paths:` filter in .github/workflows/prebuilt.yml.
INPUTS=(
configs/linux.config
configs/busybox.config
configs/buildroot.config
scripts/build-image.sh
scripts/rootfs_ext4.sh
target/init
)

for f in Image rootfs.cpio "${INPUTS[@]}"; do
if [ ! -f "$f" ]; then
echo "[!] Missing $f -- run scripts/build-image.sh --all first" >&2
exit 1
fi
done

bzip2 -k -f Image
bzip2 -k -f rootfs.cpio

KERNEL_SHA1=$("${SHA1[@]}" Image.bz2 | awk '{print $1}')
INITRD_SHA1=$("${SHA1[@]}" rootfs.cpio.bz2 | awk '{print $1}')
# Concatenate inputs in deterministic order and hash the stream. Matches
# the make-time computation in mk/external.mk so they compare directly.
INPUTS_SHA1=$(cat "${INPUTS[@]}" | "${SHA1[@]}" | awk '{print $1}')

# Write the human-friendly checksum manifest. Format matches `sha1sum -c`
# so the file works as input to that tool unchanged.
{
echo "$KERNEL_SHA1 Image.bz2"
echo "$INITRD_SHA1 rootfs.cpio.bz2"
} > prebuilt.sha1

# Echo the manifest + inputs hash to stderr for visibility in CI logs
# without polluting the parseable stdout block below.
{
cat prebuilt.sha1
echo "inputs_sha1: $INPUTS_SHA1"
} >&2

echo "kernel_sha1=$KERNEL_SHA1"
echo "initrd_sha1=$INITRD_SHA1"
echo "inputs_sha1=$INPUTS_SHA1"
44 changes: 44 additions & 0 deletions .ci/suggest-format.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
#!/usr/bin/env bash
#
# Post clang-format-20 diffs as PR review suggestions via reviewdog. Pairs
# with .ci/check-format.sh: this script *suggests* on pull requests; that
# one *enforces* by failing CI when violations land on master.
#
# Designed for the GitHub Actions `coding_style` job. The workflow must
# provide REVIEWDOG_GITHUB_API_TOKEN in the environment and install
# clang-format-20 + reviewdog beforehand.
#
# The script applies clang-format in-place to surface a diff, then always
# restores the working tree via an EXIT trap so a failed reviewdog run
# does not leave reformatted sources behind.

set -euo pipefail

# Collect all C/C++ sources tracked by git (POSIX-compatible array build
# for bash 3.2 compatibility on macOS runners).
SOURCES=()
while IFS= read -r file; do
SOURCES+=("$file")
done < <(git ls-files '*.c' '*.cxx' '*.cpp' '*.h' '*.hpp')

# Restore files on exit so a failed reviewdog run does not leave
# clang-format-modified sources in the working tree.
cleanup_files() {
if [ ${#SOURCES[@]} -gt 0 ]; then
echo "Restoring files to original state..."
git checkout -- "${SOURCES[@]}" 2>/dev/null || true
fi
}
trap cleanup_files EXIT INT TERM

# Apply formatting in-place to surface the diff against the working tree.
clang-format-20 -i "${SOURCES[@]}"

# Pipe the diff into reviewdog. `|| true` keeps the trap from being
# pre-empted by reviewdog's exit status when there are findings.
git diff -u --no-color | reviewdog -f=diff \
-name="clang-format" \
-reporter=github-pr-review \
-filter-mode=added \
-fail-on-error=false \
-level=warning || true
3 changes: 2 additions & 1 deletion .github/actions/setup-semu/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ runs:
build-essential \
device-tree-compiler \
expect \
fakeroot \
libasound2-dev \
libudev-dev \
libsdl2-dev
Expand All @@ -24,4 +25,4 @@ runs:
HOMEBREW_NO_AUTO_UPDATE: 1
HOMEBREW_NO_ANALYTICS: 1
run: |
brew install make dtc expect e2fsprogs sdl2
brew install make dtc expect fakeroot e2fsprogs sdl2
93 changes: 62 additions & 31 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,12 @@ jobs:
path: |
Image
rootfs.cpio
key: external-${{ hashFiles('mk/external.mk') }}
# Invalidate the cached prebuilts whenever any input that defines
# their content changes. Keep this list in sync with
# PREBUILT_INPUTS in mk/external.mk and the `paths:` filter in
# .github/workflows/prebuilt.yml; otherwise CI will silently
# restore stale Image / rootfs.cpio after a config bump.
key: external-${{ hashFiles('mk/external.mk', 'configs/linux.config', 'configs/busybox.config', 'configs/buildroot.config', 'scripts/build-image.sh', 'scripts/rootfs_ext4.sh', 'target/**') }}
- name: cache submodule builds
uses: actions/cache@v4
with:
Expand Down Expand Up @@ -76,6 +81,55 @@ jobs:
shell: bash
timeout-minutes: 5

# Guard the legacy initramfs path so it does not bitrot now that the
# default boot mode is /dev/vda. Single slim job: fresh build with
# ENABLE_EXTERNAL_ROOT=0, then run autorun.
semu-linux-initramfs:
runs-on: ubuntu-24.04
steps:
- name: checkout code
uses: actions/checkout@v4
with:
submodules: recursive
- name: cache external downloads
uses: actions/cache@v4
with:
path: |
Image
rootfs.cpio
# Invalidate the cached prebuilts whenever any input that defines
# their content changes. Keep this list in sync with
# PREBUILT_INPUTS in mk/external.mk and the `paths:` filter in
# .github/workflows/prebuilt.yml; otherwise CI will silently
# restore stale Image / rootfs.cpio after a config bump.
key: external-${{ hashFiles('mk/external.mk', 'configs/linux.config', 'configs/busybox.config', 'configs/buildroot.config', 'scripts/build-image.sh', 'scripts/rootfs_ext4.sh', 'target/**') }}
- name: cache submodule builds
uses: actions/cache@v4
with:
path: |
mini-gdbstub/build
minislirp/src/*.a
key: ${{ runner.os }}-submodules-${{ hashFiles('.gitmodules', 'mini-gdbstub/**', 'minislirp/**') }}
restore-keys: |
${{ runner.os }}-submodules-
- name: cache apt packages
uses: actions/cache@v4
with:
path: /var/cache/apt/archives
key: ${{ runner.os }}-apt-${{ hashFiles('.github/actions/setup-semu/action.yml') }}
restore-keys: |
${{ runner.os }}-apt-
- name: install-dependencies
uses: ./.github/actions/setup-semu
- name: legacy initramfs build
run: make ENABLE_EXTERNAL_ROOT=0
shell: bash
timeout-minutes: 5
- name: legacy initramfs autorun
run: ENABLE_EXTERNAL_ROOT=0 .ci/autorun.sh
shell: bash
timeout-minutes: 10

semu-macOS:
runs-on: macos-latest
steps:
Expand All @@ -88,7 +142,12 @@ jobs:
path: |
Image
rootfs.cpio
key: external-${{ hashFiles('mk/external.mk') }}
# Invalidate the cached prebuilts whenever any input that defines
# their content changes. Keep this list in sync with
# PREBUILT_INPUTS in mk/external.mk and the `paths:` filter in
# .github/workflows/prebuilt.yml; otherwise CI will silently
# restore stale Image / rootfs.cpio after a config bump.
key: external-${{ hashFiles('mk/external.mk', 'configs/linux.config', 'configs/busybox.config', 'configs/buildroot.config', 'scripts/build-image.sh', 'scripts/rootfs_ext4.sh', 'target/**') }}
- name: cache submodule builds
uses: actions/cache@v4
with:
Expand Down Expand Up @@ -163,35 +222,7 @@ jobs:
if: github.event_name == 'pull_request'
env:
REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail

# Get list of source files into array (POSIX-compatible for macOS bash 3.2)
SOURCES=()
while IFS= read -r file; do
SOURCES+=("$file")
done < <(git ls-files '*.c' '*.cxx' '*.cpp' '*.h' '*.hpp')

# Register cleanup function to restore files on exit
cleanup_files() {
if [ ${#SOURCES[@]} -gt 0 ]; then
echo "Restoring files to original state..."
git checkout -- "${SOURCES[@]}" 2>/dev/null || true
fi
}
trap cleanup_files EXIT INT TERM

# Apply clang-format in-place to generate diff
clang-format-20 -i "${SOURCES[@]}"

# Generate diff and pipe to reviewdog
# Note: reviewdog exit code doesn't affect cleanup due to trap
git diff -u --no-color | reviewdog -f=diff \
-name="clang-format" \
-reporter=github-pr-review \
-filter-mode=added \
-fail-on-error=false \
-level=warning || true
run: .ci/suggest-format.sh
shell: bash
timeout-minutes: 5
- name: Check formatting (fail on violations)
Expand Down
110 changes: 110 additions & 0 deletions .github/workflows/prebuilt.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
name: Publish prebuilt images

# Builds the Linux kernel and Buildroot rootfs that the rest of CI (and
# `make` on a fresh checkout) consumes, then publishes them as assets on a
# fixed-tag GitHub prerelease so the download URL stays stable across
# rebuilds. This replaces the old `blob` branch convention, which forced
# us to push large binary artifacts into the source tree.
#
# The workflow is manual by default (workflow_dispatch). Run it whenever
# the kernel config, buildroot config, or build-image.sh changes; the
# resulting SHA1 sums must be reflected in mk/external.mk.

on:
workflow_dispatch:
push:
branches: [master]
# Republish the prebuilt artifacts whenever any input that defines
# what gets baked into Image / rootfs.cpio actually changes. The list
# mirrors PREBUILT_INPUTS in mk/external.mk -- keep them in sync.
paths:
- 'configs/linux.config'
- 'configs/busybox.config'
- 'configs/buildroot.config'
- 'scripts/build-image.sh'
- 'scripts/rootfs_ext4.sh'
- 'target/**'
- '.github/workflows/prebuilt.yml'

permissions:
contents: write

concurrency:
group: prebuilt
cancel-in-progress: false

jobs:
build-and-publish:
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
with:
submodules: recursive

- name: Install build dependencies
run: |
sudo apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
build-essential \
bc \
bison \
flex \
cpio \
fakeroot \
e2fsprogs \
git \
python3 \
libssl-dev \
libelf-dev \
wget

- name: Build Buildroot and Linux
run: ./scripts/build-image.sh --all

- name: Compress and checksum artifacts
id: checksum
# The shell logic lives in .ci/publish-prebuilt.sh so it can be
# exercised locally and stays out of the YAML.
run: |
set -euo pipefail
.ci/publish-prebuilt.sh >> "$GITHUB_OUTPUT"
ls -la Image.bz2 rootfs.cpio.bz2 prebuilt.sha1

- name: Update prebuilt prerelease
uses: softprops/action-gh-release@v2
with:
tag_name: prebuilt
name: Prebuilt images (rolling)
prerelease: true
# Replace existing assets at the `prebuilt` tag rather than
# appending duplicates with new names.
fail_on_unmatched_files: true
files: |
Image.bz2
rootfs.cpio.bz2
prebuilt.sha1
body: |
Rolling prerelease of the Linux kernel and Buildroot rootfs
consumed by `mk/external.mk`. Re-published whenever any
input that defines the kernel/rootfs content changes.

## Update `mk/external.mk`

Paste these three lines into `mk/external.mk` to pin a fresh
`make` checkout to this build. `PREBUILT_INPUTS_SHA1` is what
local checkouts compare against to detect when their configs
have drifted from the prebuilt.

```make
KERNEL_DATA_SHA1 = ${{ steps.checksum.outputs.kernel_sha1 }}
INITRD_DATA_SHA1 = ${{ steps.checksum.outputs.initrd_sha1 }}
PREBUILT_INPUTS_SHA1 = ${{ steps.checksum.outputs.inputs_sha1 }}
```

## Raw checksums

```
${{ steps.checksum.outputs.kernel_sha1 }} Image.bz2
${{ steps.checksum.outputs.initrd_sha1 }} rootfs.cpio.bz2
${{ steps.checksum.outputs.inputs_sha1 }} inputs (configs + scripts + target/init, concatenated)
```
1 change: 0 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ rootfs.cpio
# intermediate
riscv-harts.dtsi
.smp_stamp
rootfs_full.cpio

# Build directories
buildroot/
Expand Down
Loading
Loading