update docstrings #67
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| env: | |
| BENCHLEDGER_VERSION: "0.7.0" | |
| TARGET_REPOSITORY: "" | |
| TARGET_REF: "" | |
| BENCH_DB_BRANCH: "gh-pages" | |
| BENCH_DB_FILE: "benchmarks/data/benchledger.sqlite" | |
| on: | |
| workflow_call: | |
| workflow_dispatch: | |
| inputs: | |
| target_mode: | |
| description: "Target selection mode" | |
| required: true | |
| default: current | |
| type: choice | |
| options: [current, ref, tags] | |
| target_repository: | |
| description: "Target repository as owner/repo; leave empty for this repository" | |
| required: false | |
| default: "" | |
| type: string | |
| target_ref: | |
| description: "Optional commit, tag, or branch when target_mode=ref; empty uses the repository default branch" | |
| required: false | |
| default: "" | |
| type: string | |
| tag_pattern: | |
| description: "Git tag pattern when target_mode=tags" | |
| required: false | |
| default: "*" | |
| type: string | |
| tag_start: | |
| description: "Optional first tag to benchmark, inclusive" | |
| required: false | |
| default: "" | |
| type: string | |
| tag_end: | |
| description: "Optional last tag to benchmark, inclusive" | |
| required: false | |
| default: "" | |
| type: string | |
| push: | |
| branches: [master] | |
| tags: ["v*"] | |
| permissions: | |
| contents: write | |
| concurrency: | |
| group: benchledger-database | |
| cancel-in-progress: false | |
| jobs: | |
| benchmark: | |
| name: Ubuntu 24.04 - x64 | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 360 | |
| steps: | |
| - name: Check out benchmark harness | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Prepare database branch | |
| env: | |
| REPO_URL: https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git | |
| run: | | |
| set -euo pipefail | |
| mkdir -p .benchledger-db | |
| git -C .benchledger-db init | |
| git -C .benchledger-db remote add origin "$REPO_URL" | |
| if git ls-remote --exit-code --heads "$REPO_URL" "$BENCH_DB_BRANCH" >/dev/null 2>&1; then | |
| git -C .benchledger-db fetch --depth=1 origin "$BENCH_DB_BRANCH" | |
| git -C .benchledger-db checkout -B "$BENCH_DB_BRANCH" FETCH_HEAD | |
| else | |
| git -C .benchledger-db checkout --orphan "$BENCH_DB_BRANCH" | |
| fi | |
| - name: Check out selected target | |
| if: (github.event_name == 'workflow_dispatch' && inputs.target_mode != 'current') || (github.event_name != 'workflow_dispatch' && env.TARGET_REPOSITORY != '') | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: ${{ github.event_name == 'workflow_dispatch' && inputs.target_repository != '' && inputs.target_repository || github.event_name != 'workflow_dispatch' && env.TARGET_REPOSITORY != '' && env.TARGET_REPOSITORY || github.repository }} | |
| ref: ${{ github.event_name != 'workflow_dispatch' && env.TARGET_REF || inputs.target_ref }} | |
| path: .bench-target | |
| fetch-depth: 0 | |
| - name: Resolve target branch provenance | |
| id: target_provenance | |
| env: | |
| TARGET_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.target_mode || env.TARGET_REPOSITORY != '' && 'ref' || 'current' }} | |
| REQUESTED_REF: ${{ github.event_name != 'workflow_dispatch' && env.TARGET_REF || inputs.target_ref }} | |
| CURRENT_REF: ${{ github.ref_name }} | |
| run: | | |
| set -euo pipefail | |
| if [ "$TARGET_MODE" = "tags" ]; then | |
| echo "branch=" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| if [ "$TARGET_MODE" = "current" ]; then | |
| TARGET_DIR="$GITHUB_WORKSPACE" | |
| else | |
| TARGET_DIR="$GITHUB_WORKSPACE/.bench-target" | |
| fi | |
| if [ "$TARGET_MODE" = "current" ]; then | |
| requested_ref="$CURRENT_REF" | |
| else | |
| requested_ref="$REQUESTED_REF" | |
| fi | |
| if [ -z "$requested_ref" ]; then | |
| requested_ref="$(git -C "$TARGET_DIR" ls-remote --symref origin HEAD 2>/dev/null | awk '/^ref:/ {sub("refs/heads/", "", $2); print $2; exit}' || true)" | |
| fi | |
| tag="$(git -C "$TARGET_DIR" tag --points-at HEAD | head -n 1 || true)" | |
| branch="" | |
| # actions/checkout commonly leaves detached HEAD, so prefer the requested ref when it is a branch. | |
| if [ -n "$requested_ref" ] && git -C "$TARGET_DIR" show-ref --verify --quiet "refs/remotes/origin/$requested_ref"; then | |
| branch="$requested_ref" | |
| elif [ -n "$requested_ref" ] && git -C "$TARGET_DIR" show-ref --verify --quiet "refs/heads/$requested_ref"; then | |
| branch="$requested_ref" | |
| else | |
| branch="$(git -C "$TARGET_DIR" branch --show-current || true)" | |
| fi | |
| if [ -n "$tag" ]; then | |
| branch="" | |
| fi | |
| echo "branch=$branch" >> "$GITHUB_OUTPUT" | |
| - name: Set up Julia | |
| uses: julia-actions/setup-julia@v2 | |
| with: | |
| version: "lts" | |
| - name: Cache Julia packages | |
| uses: julia-actions/cache@v2 | |
| - name: Instantiate benchmark project | |
| run: julia --project=benchmark -e 'using Pkg; Pkg.instantiate()' | |
| - name: Install matching BenchLedger probe | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| set -euo pipefail | |
| tag="v${BENCHLEDGER_VERSION}" | |
| asset="BenchLedger-probe-${tag}-linux-amd64.tar.gz" | |
| install_dir="$RUNNER_TEMP/benchledger-probe" | |
| rm -rf "$install_dir" | |
| mkdir -p "$install_dir/download" "$install_dir/runtime" | |
| gh release download "$tag" \ | |
| --repo abcdvvvv/BenchLedger \ | |
| --pattern "$asset" \ | |
| --pattern "$asset.sha256" \ | |
| --dir "$install_dir/download" | |
| ( | |
| cd "$install_dir/download" | |
| sha256sum -c "$asset.sha256" | |
| ) | |
| tar -xzf "$install_dir/download/$asset" -C "$install_dir/runtime" | |
| probe_path="$(find "$install_dir/runtime" -type f -name benchledger-probe -perm -111 -print -quit)" | |
| [ -n "$probe_path" ] || { echo "benchledger-probe was not found in $asset" >&2; exit 1; } | |
| echo "BENCH_PROBE_PATH=$probe_path" >> "$GITHUB_ENV" | |
| "$probe_path" --version | |
| "$probe_path" --pretty > "$RUNNER_TEMP/benchledger-probe-smoke.json" | |
| - name: Run benchmarks | |
| env: | |
| TARGET_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.target_mode || env.TARGET_REPOSITORY != '' && 'ref' || 'current' }} | |
| TARGET_REF: ${{ github.event_name != 'workflow_dispatch' && env.TARGET_REF || inputs.target_ref }} | |
| TAG_PATTERN: ${{ inputs.tag_pattern || '*' }} | |
| TAG_START: ${{ inputs.tag_start || '' }} | |
| TAG_END: ${{ inputs.tag_end || '' }} | |
| BENCH_RUN: >- | |
| {"notes":"GitHub Actions BenchLedger run","metadata":{"ci":{"provider":"GitHub Actions","workflow":${{ toJson(github.workflow) }},"job":${{ toJson(github.job) }},"run_id":${{ toJson(github.run_id) }},"run_attempt":${{ github.run_attempt }},"event":${{ toJson(github.event_name) }},"runner_name":${{ toJson(runner.name) }},"run_url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}","target_mode":${{ toJson(github.event_name == 'workflow_dispatch' && inputs.target_mode || env.TARGET_REPOSITORY != '' && 'ref' || 'current') }},"target_ref":${{ toJson(github.event_name != 'workflow_dispatch' && env.TARGET_REF || inputs.target_ref) }}},"source":{"repository":${{ toJson(github.event_name == 'workflow_dispatch' && inputs.target_repository != '' && inputs.target_repository || github.event_name != 'workflow_dispatch' && env.TARGET_REPOSITORY != '' && env.TARGET_REPOSITORY || github.repository) }},"branch":${{ toJson(steps.target_provenance.outputs.branch) }}}}} | |
| run: | | |
| set -euo pipefail | |
| DB="$GITHUB_WORKSPACE/.benchledger-db/$BENCH_DB_FILE" | |
| TARGET="$GITHUB_WORKSPACE/.bench-target" | |
| mkdir -p "$(dirname "$DB")" | |
| run_current() { | |
| BENCH_DB_PATH="$DB" julia --project=benchmark benchmark/runbench.jl | |
| } | |
| run_target() { | |
| BENCH_DB_PATH="$DB" BENCH_TARGET_PATH="$TARGET" julia --project=benchmark benchmark/runbench.jl | |
| } | |
| if [ "$TARGET_MODE" = "current" ]; then | |
| run_current | |
| exit 0 | |
| fi | |
| if [ "$TARGET_MODE" = "ref" ]; then | |
| run_target | |
| exit 0 | |
| fi | |
| mapfile -t TAGS < <(git -C "$TARGET" tag --list "$TAG_PATTERN" --sort=v:refname) | |
| [ "${#TAGS[@]}" -gt 0 ] || { echo "No tags match pattern: $TAG_PATTERN" >&2; exit 1; } | |
| SELECTED=() | |
| STARTED=false | |
| [ -z "$TAG_START" ] && STARTED=true | |
| FOUND_START=false | |
| FOUND_END=false | |
| for tag in "${TAGS[@]}"; do | |
| if [ "$STARTED" = false ]; then | |
| [ "$tag" = "$TAG_START" ] || continue | |
| STARTED=true | |
| FOUND_START=true | |
| fi | |
| SELECTED+=("$tag") | |
| if [ -n "$TAG_END" ] && [ "$tag" = "$TAG_END" ]; then | |
| FOUND_END=true | |
| break | |
| fi | |
| done | |
| [ -z "$TAG_START" ] || [ "$FOUND_START" = true ] || { echo "tag_start not found after applying tag_pattern: $TAG_START" >&2; exit 1; } | |
| [ -z "$TAG_END" ] || [ "$FOUND_END" = true ] || { echo "tag_end not found at or after tag_start: $TAG_END" >&2; exit 1; } | |
| [ "${#SELECTED[@]}" -gt 0 ] || { echo "The selected tag range is empty." >&2; exit 1; } | |
| printf 'Benchmarking %d tag(s):\n' "${#SELECTED[@]}" | |
| printf ' %s\n' "${SELECTED[@]}" | |
| FAILURE_LOG="$RUNNER_TEMP/benchledger-tag-failures.txt" | |
| success_count=0 | |
| failure_count=0 | |
| : > "$FAILURE_LOG" | |
| for tag in "${SELECTED[@]}"; do | |
| echo "Benchmarking $tag" | |
| git -C "$TARGET" checkout --detach "$tag" | |
| commit="$(git -C "$TARGET" rev-parse HEAD)" | |
| commit_timestamp="$(git -C "$TARGET" show -s --format=%ct HEAD)" | |
| code_date="$(date -u -d "@$commit_timestamp" '+%Y-%m-%dT%H:%M:%S.000Z')" | |
| if run_target; then | |
| success_count=$((success_count + 1)) | |
| echo "Benchmark succeeded for $tag" | |
| else | |
| failure_count=$((failure_count + 1)) | |
| echo "Benchmark failed for $tag, skipping to next tag." | |
| printf 'tag=%s commit=%s code_date=%s\n' "$tag" "$commit" "$code_date" >> "$FAILURE_LOG" | |
| fi | |
| done | |
| if [ "$failure_count" -gt 0 ]; then | |
| { | |
| echo "## BenchLedger skipped failed tags" | |
| echo | |
| echo "Successful tags: $success_count" | |
| echo "Failed tags: $failure_count" | |
| echo | |
| echo '```' | |
| cat "$FAILURE_LOG" | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| if [ "$success_count" -eq 0 ]; then | |
| echo "Tag benchmark did not produce any successful benchmark runs." >&2 | |
| exit 1 | |
| fi | |
| - name: Checkpoint benchmark database | |
| run: | | |
| DB="$GITHUB_WORKSPACE/.benchledger-db/$BENCH_DB_FILE" | |
| julia --project=benchmark -e 'using SQLite; db = SQLite.DB(ARGS[1]); SQLite.execute(db, "PRAGMA wal_checkpoint(TRUNCATE)"); close(db)' "$DB" | |
| rm -f "${DB}-wal" "${DB}-shm" | |
| - name: Install BenchLedger viewer | |
| if: env.BENCH_DB_BRANCH == 'gh-pages' | |
| run: | | |
| set -euo pipefail | |
| tag="v${BENCHLEDGER_VERSION}" | |
| asset="BenchLedger-${BENCHLEDGER_VERSION}-dist.tar.gz" | |
| base_url="https://github.com/abcdvvvv/BenchLedger/releases/download/${tag}" | |
| rm -rf /tmp/benchledger-dist /tmp/benchledger-dist-download | |
| mkdir -p /tmp/benchledger-dist /tmp/benchledger-dist-download | |
| curl -fsSL "${base_url}/${asset}" -o "/tmp/benchledger-dist-download/${asset}" | |
| curl -fsSL "${base_url}/${asset}.sha256" -o "/tmp/benchledger-dist-download/${asset}.sha256" | |
| ( | |
| cd /tmp/benchledger-dist-download | |
| sha256sum -c "${asset}.sha256" | |
| ) | |
| tar -xzf "/tmp/benchledger-dist-download/${asset}" -C /tmp/benchledger-dist | |
| mkdir -p .benchledger-db/benchmarks/data | |
| find .benchledger-db/benchmarks -mindepth 1 -maxdepth 1 ! -name data -exec rm -rf {} + | |
| cp -a /tmp/benchledger-dist/. .benchledger-db/benchmarks/ | |
| touch .benchledger-db/.nojekyll | |
| - name: Generate BenchLedger manifest | |
| if: env.BENCH_DB_BRANCH == 'gh-pages' | |
| run: | | |
| python3 - "$GITHUB_WORKSPACE/.benchledger-db/$BENCH_DB_FILE" ".benchledger-db/benchmarks/benchledger.json" "$BENCHLEDGER_VERSION" <<'PY' | |
| import hashlib | |
| import json | |
| import os | |
| import sqlite3 | |
| import sys | |
| from datetime import datetime, timezone | |
| db_path, manifest_path, web_version = sys.argv[1:] | |
| with sqlite3.connect(db_path) as db: | |
| metadata = dict(db.execute("SELECT key, value FROM benchledger_metadata")) | |
| name = metadata.get("name", "BenchLedger") | |
| description = metadata.get("description", "") | |
| project_url = metadata.get("project_url") or None | |
| logo_url = metadata.get("logo_url") or None | |
| generated_at = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") | |
| with open(db_path, "rb") as source: | |
| sha256 = hashlib.file_digest(source, "sha256").hexdigest() | |
| manifest_dir = os.path.dirname(os.path.abspath(manifest_path)) | |
| db_url = os.path.relpath(os.path.abspath(db_path), manifest_dir).replace(os.sep, "/") | |
| if not db_url.startswith("."): | |
| db_url = "./" + db_url | |
| manifest = { | |
| "benchledger_web_version": web_version, | |
| "generated_at": generated_at, | |
| "site": {"title": f"{name} BenchLedger", "description": description}, | |
| "databases": [{ | |
| "id": name, | |
| "name": name, | |
| "description": description, | |
| "url": db_url, | |
| "sha256": sha256, | |
| "size_bytes": os.path.getsize(db_path), | |
| "packed_at": generated_at, | |
| "metadata_preview": { | |
| "name": name, | |
| "project_url": project_url, | |
| "logo_url": logo_url, | |
| }, | |
| }], | |
| } | |
| with open(manifest_path, "w", encoding="utf-8") as target: | |
| json.dump(manifest, target, indent=2, ensure_ascii=False) | |
| target.write("\n") | |
| PY | |
| - name: Commit benchmark database | |
| working-directory: .benchledger-db | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add -- "$BENCH_DB_FILE" | |
| [ "$BENCH_DB_BRANCH" != "gh-pages" ] || git add -- benchmarks .nojekyll | |
| if git diff --cached --quiet; then | |
| echo "No benchmark database changes." | |
| else | |
| git commit -m "Update BenchLedger benchmark data [skip ci]" | |
| git push origin "HEAD:$BENCH_DB_BRANCH" | |
| fi |