chore(deps-dev): Bump the npm group with 2 updates (#93) #208
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| NODE_VERSION: "24" | |
| PYTHON_VERSION: "3.14" | |
| jobs: | |
| lint-typecheck-test: | |
| name: Lint, typecheck, unit tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 | |
| with: | |
| run_install: false | |
| - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 | |
| with: | |
| package-manager-cache: false | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| - run: pnpm run format:check | |
| - run: pnpm run lint | |
| - run: pnpm run typecheck | |
| - run: pnpm run test | |
| check-dist: | |
| name: Verify dist/ is up to date | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 | |
| with: | |
| run_install: false | |
| - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 | |
| with: | |
| package-manager-cache: false | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| - name: Rebuild bundle | |
| run: pnpm run build | |
| - name: Fail if dist/ differs from committed bundle | |
| run: | | |
| if ! git diff --exit-code -- dist; then | |
| echo "::error::dist/ is out of date — run 'pnpm build' and commit the result." | |
| exit 1 | |
| fi | |
| check-schema: | |
| name: Verify schemas/ is up to date | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 | |
| with: | |
| run_install: false | |
| - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 | |
| with: | |
| package-manager-cache: false | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| - name: Regenerate JSON Schema | |
| run: pnpm run check:schema | |
| e2e-langfuse: | |
| name: E2E — real Langfuse server | |
| runs-on: ubuntu-latest | |
| needs: [lint-typecheck-test, check-dist, check-schema] | |
| permissions: | |
| contents: read | |
| # Let the action post experiment result comments when the workflow runs | |
| # on a pull_request event. | |
| pull-requests: write | |
| # Lets the action resolve the current job URL via the REST API so | |
| # "View run" links to the specific job rather than the workflow run. | |
| actions: read | |
| env: | |
| LANGFUSE_BASE_URL: http://localhost:3000 | |
| LANGFUSE_PUBLIC_KEY: pk-lf-1234567890 | |
| LANGFUSE_SECRET_KEY: sk-lf-1234567890 | |
| E2E_DATASET_NAME: experiment-action-e2e-${{ github.run_id }} | |
| # "true" on pull_request, "false" elsewhere (push to main, schedule). | |
| COMMENT_ON_PR: ${{ github.event_name == 'pull_request' && 'true' || 'false' }} | |
| steps: | |
| # checkout stays in the job — a local composite action can only be | |
| # resolved after the repository is checked out. | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Set up toolchain, Langfuse server, and E2E dataset | |
| uses: ./.github/actions/e2e-setup | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| dataset-name: ${{ env.E2E_DATASET_NAME }} | |
| dataset-description: experiment-action e2e dataset | |
| dataset-items: | | |
| [ | |
| {"input": "hello", "expected_output": "HELLO"}, | |
| {"input": "world", "expected_output": "WORLD"}, | |
| {"input": "langfuse", "expected_output": "LANGFUSE"} | |
| ] | |
| # --- Scenario 1: single Python script ------------------------------ | |
| - name: Run action — single Python experiment | |
| id: py | |
| uses: ./ | |
| with: | |
| experiment_path: tests/fixtures/e2e/experiment.py | |
| langfuse_public_key: ${{ env.LANGFUSE_PUBLIC_KEY }} | |
| langfuse_secret_key: ${{ env.LANGFUSE_SECRET_KEY }} | |
| langfuse_base_url: ${{ env.LANGFUSE_BASE_URL }} | |
| dataset_name: ${{ env.E2E_DATASET_NAME }} | |
| github_token: ${{ github.token }} | |
| should_comment_on_pr: ${{ env.COMMENT_ON_PR }} | |
| - name: Assert single-python passed | |
| env: | |
| FAILED: ${{ steps.py.outputs.failed }} | |
| RESULT_JSON: ${{ steps.py.outputs.result_json }} | |
| run: | | |
| test "$FAILED" = "false" | |
| node --import tsx scripts/assert-result-shape.ts | |
| echo "$RESULT_JSON" | jq -e '.schema_version == "v1"' | |
| echo "$RESULT_JSON" | jq -e '.results | length == 1' | |
| echo "$RESULT_JSON" | jq -e '.results[0].runtime == "python"' | |
| echo "$RESULT_JSON" | jq -e '.results[0].status == "passed"' | |
| echo "$RESULT_JSON" | jq -e '.results[0].experiment_result.run_evaluations[0].name == "avg_accuracy"' | |
| echo "$RESULT_JSON" | jq -e '.results[0].experiment_result.run_evaluations[0].value == 1' | |
| echo "$RESULT_JSON" | jq -e ' | |
| .results[0].experiment_result.item_results | |
| | map(.expected_output) | |
| | sort | |
| | . == ["HELLO", "LANGFUSE", "WORLD"] | |
| ' | |
| # --- Scenario 2: single TypeScript script -------------------------- | |
| - name: Run action — single TypeScript experiment | |
| id: ts | |
| uses: ./ | |
| with: | |
| experiment_path: tests/fixtures/e2e/experiment.ts | |
| langfuse_public_key: ${{ env.LANGFUSE_PUBLIC_KEY }} | |
| langfuse_secret_key: ${{ env.LANGFUSE_SECRET_KEY }} | |
| langfuse_base_url: ${{ env.LANGFUSE_BASE_URL }} | |
| dataset_name: ${{ env.E2E_DATASET_NAME }} | |
| github_token: ${{ github.token }} | |
| should_comment_on_pr: ${{ env.COMMENT_ON_PR }} | |
| - name: Assert single-typescript passed | |
| env: | |
| FAILED: ${{ steps.ts.outputs.failed }} | |
| RESULT_JSON: ${{ steps.ts.outputs.result_json }} | |
| run: | | |
| test "$FAILED" = "false" | |
| node --import tsx scripts/assert-result-shape.ts | |
| echo "$RESULT_JSON" | jq -e '.results | length == 1' | |
| echo "$RESULT_JSON" | jq -e '.results[0].runtime == "node"' | |
| echo "$RESULT_JSON" | jq -e '.results[0].status == "passed"' | |
| echo "$RESULT_JSON" | jq -e '.results[0].experiment_result.run_evaluations[0].name == "avg_accuracy"' | |
| echo "$RESULT_JSON" | jq -e '.results[0].experiment_result.run_evaluations[0].value == 1' | |
| echo "$RESULT_JSON" | jq -e ' | |
| .results[0].experiment_result.item_results | |
| | map(.expected_output) | |
| | sort | |
| | . == ["HELLO", "LANGFUSE", "WORLD"] | |
| ' | |
| # --- Scenario 3: mixed-runtime directory --------------------------- | |
| - name: Run action — mixed Python+TS directory | |
| id: mixed | |
| uses: ./ | |
| with: | |
| experiment_path: tests/fixtures/e2e/mixed | |
| langfuse_public_key: ${{ env.LANGFUSE_PUBLIC_KEY }} | |
| langfuse_secret_key: ${{ env.LANGFUSE_SECRET_KEY }} | |
| langfuse_base_url: ${{ env.LANGFUSE_BASE_URL }} | |
| dataset_name: ${{ env.E2E_DATASET_NAME }} | |
| github_token: ${{ github.token }} | |
| should_comment_on_pr: ${{ env.COMMENT_ON_PR }} | |
| - name: Assert mixed directory ran both scripts | |
| env: | |
| FAILED: ${{ steps.mixed.outputs.failed }} | |
| RESULT_JSON: ${{ steps.mixed.outputs.result_json }} | |
| run: | | |
| test "$FAILED" = "false" | |
| node --import tsx scripts/assert-result-shape.ts | |
| echo "$RESULT_JSON" | jq -e '.results | length == 2' | |
| echo "$RESULT_JSON" | jq -e '.results | map(.runtime) | sort | . == ["node", "python"]' | |
| # --- Scenario 4: regression path (non-fatal) ----------------------- | |
| - name: Run action — regression fixture (should_fail_on_regression=false) | |
| id: regression | |
| uses: ./ | |
| with: | |
| experiment_path: tests/fixtures/e2e/regression/experiment.py | |
| langfuse_public_key: ${{ env.LANGFUSE_PUBLIC_KEY }} | |
| langfuse_secret_key: ${{ env.LANGFUSE_SECRET_KEY }} | |
| langfuse_base_url: ${{ env.LANGFUSE_BASE_URL }} | |
| dataset_name: ${{ env.E2E_DATASET_NAME }} | |
| github_token: ${{ github.token }} | |
| should_comment_on_pr: ${{ env.COMMENT_ON_PR }} | |
| should_fail_on_regression: "false" | |
| - name: Assert regression was captured (without failing the job) | |
| env: | |
| FAILED: ${{ steps.regression.outputs.failed }} | |
| RESULT_JSON: ${{ steps.regression.outputs.result_json }} | |
| run: | | |
| test "$FAILED" = "true" | |
| node --import tsx scripts/assert-result-shape.ts | |
| echo "$RESULT_JSON" | jq -e '.results[0].status == "regression"' | |
| echo "$RESULT_JSON" | jq -e '.results[0].error.name == "RegressionError"' | |
| echo "$RESULT_JSON" | jq -e '.results[0].error.is_regression == true' | |
| - name: Tear down Langfuse | |
| if: always() | |
| run: pnpm run dev:down | |
| # Regression test for the matrix comment collision | |
| # (https://github.com/langfuse/langfuse/issues/14907): two parallel legs run | |
| # the SAME experiment script; both sections must land in one shared PR | |
| # comment. The (script, job) section keying is asserted deterministically by | |
| # the follow-up job; the concurrent-upsert path is exercised whenever the | |
| # legs actually overlap. | |
| # | |
| # No `name:` on purpose — the action keys comment sections on the job | |
| # *display name*, and GitHub only appends matrix values to it (here: | |
| # "e2e-matrix (alpha)") when the name is derived from the job key. | |
| e2e-matrix: | |
| runs-on: ubuntu-latest | |
| needs: [lint-typecheck-test, check-dist, check-schema] | |
| # PR-only: this job exists solely as a regression test for the PR-comment | |
| # path, so spinning up its two full Langfuse stacks on pushes to main is | |
| # wasted compute. | |
| if: github.event_name == 'pull_request' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| leg: [alpha, beta] | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| # Lets the action resolve the job display name that tells matrix legs | |
| # apart in the PR comment (and link "View run" to the specific job). | |
| actions: read | |
| env: | |
| LANGFUSE_BASE_URL: http://localhost:3000 | |
| LANGFUSE_PUBLIC_KEY: pk-lf-1234567890 | |
| LANGFUSE_SECRET_KEY: sk-lf-1234567890 | |
| E2E_DATASET_NAME: experiment-action-e2e-matrix-${{ github.run_id }} | |
| # Only attempt to comment when e2e-matrix-comment will actually assert | |
| # it: on fork PRs and Dependabot PRs the GITHUB_TOKEN is read-only | |
| # regardless of the `permissions:` key, so the action can't post a | |
| # comment and would only emit a warning. Keep this expression in sync | |
| # with the e2e-matrix-comment `if:` below. | |
| COMMENT_ON_PR: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') && 'true' || 'false' }} | |
| steps: | |
| # checkout stays in the job — a local composite action can only be | |
| # resolved after the repository is checked out. | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Set up toolchain, Langfuse server, and E2E dataset | |
| uses: ./.github/actions/e2e-setup | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| dataset-name: ${{ env.E2E_DATASET_NAME }} | |
| dataset-description: experiment-action matrix e2e dataset | |
| dataset-items: | | |
| [ | |
| {"input": "hello", "expected_output": "HELLO"}, | |
| {"input": "world", "expected_output": "WORLD"} | |
| ] | |
| - name: Run action — same experiment_path on every leg | |
| id: leg | |
| uses: ./ | |
| with: | |
| experiment_path: tests/fixtures/e2e/experiment.ts | |
| langfuse_public_key: ${{ env.LANGFUSE_PUBLIC_KEY }} | |
| langfuse_secret_key: ${{ env.LANGFUSE_SECRET_KEY }} | |
| langfuse_base_url: ${{ env.LANGFUSE_BASE_URL }} | |
| dataset_name: ${{ env.E2E_DATASET_NAME }} | |
| github_token: ${{ github.token }} | |
| should_comment_on_pr: ${{ env.COMMENT_ON_PR }} | |
| - name: Assert leg passed | |
| env: | |
| FAILED: ${{ steps.leg.outputs.failed }} | |
| run: test "$FAILED" = "false" | |
| - name: Tear down Langfuse | |
| if: always() | |
| run: pnpm run dev:down | |
| # Comments only post on pull_request events from same-repo branches by a | |
| # non-Dependabot actor, so the assertion is skipped everywhere else (same | |
| # guard as COMMENT_ON_PR above): on fork PRs and Dependabot PRs the | |
| # GITHUB_TOKEN is read-only regardless of the `permissions:` key, the | |
| # action degrades to a warning without posting, and this job would go red | |
| # for every external contributor. Skipped counts as green for | |
| # all-tests-passed. | |
| e2e-matrix-comment: | |
| name: E2E — matrix legs share one PR comment | |
| runs-on: ubuntu-latest | |
| needs: [e2e-matrix] | |
| if: >- | |
| github.event_name == 'pull_request' && | |
| github.event.pull_request.head.repo.full_name == github.repository && | |
| github.actor != 'dependabot[bot]' | |
| permissions: | |
| pull-requests: read | |
| steps: | |
| - name: Assert one comment carrying both legs' sections | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| REPO: ${{ github.repository }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| RUN_ID: ${{ github.run_id }} | |
| run: | | |
| marker="<!-- langfuse-experiment-action run_id=${RUN_ID} -->" | |
| comments=$(gh api --paginate "/repos/${REPO}/issues/${PR_NUMBER}/comments" | jq -s 'add') | |
| count=$(jq --arg m "$marker" '[.[] | select(.body | contains($m))] | length' <<<"$comments") | |
| echo "Comments for run ${RUN_ID}: ${count}" | |
| # Exactly one: the creation race converged and duplicates were deleted. | |
| test "$count" = "1" | |
| body=$(jq -r --arg m "$marker" '[.[] | select(.body | contains($m))][0].body' <<<"$comments") | |
| # One section per leg, keyed on the job display name (URL-encoded). | |
| grep -F 'job=e2e-matrix%20(alpha)' <<<"$body" | |
| grep -F 'job=e2e-matrix%20(beta)' <<<"$body" | |
| # Single required-status-check target for branch protection. Gets the | |
| # green check if every upstream job passed (or was intentionally skipped). | |
| all-tests-passed: | |
| name: All tests passed | |
| runs-on: ubuntu-latest | |
| needs: | |
| [lint-typecheck-test, check-dist, check-schema, e2e-langfuse, e2e-matrix, e2e-matrix-comment] | |
| if: always() | |
| steps: | |
| - name: Succeed when no upstream failed | |
| if: ${{ !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} | |
| run: exit 0 | |
| - name: Fail when any upstream failed | |
| if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} | |
| run: exit 1 |