Mutation baseline and CI ledger (Phase 0)

.github/workflows/ci.yml

@@ -134,6 +134,9 @@ jobs:
           CODECOV: true
           NODE_ENV: development
 
+      - name: Migration ledger
+        run: npm run ledger
+
       - name: Webpack build
         run: node node_modules/webpack/bin/webpack.js --config webpack.config.prod.js --bail
 

.gitignore

@@ -137,3 +137,7 @@ ENV/
 
 # Docker Secrets
 secrets/
+
+# Stryker mutation testing reports
+reports/
+.stryker-tmp/

.stryker-baseline.json

@@ -0,0 +1,28 @@
+{
+  "_comment": "Mutation score baseline for the Enzyme->RTL migration (mitodl/hq#12637). Recorded on React 15 + Enzyme BEFORE any bulk conversion, so regressions during hq#12638 are attributable. scripts/test/ledger.sh fails when the computed score drops below mutationScore - tolerance. Raise these numbers only in a reviewed commit that explains why.",
+  "_howToReproduce": "npm run mutation   (~26 min, inPlace: never run on uncommitted work)",
+  "_scoreDefinition": "detected / valid, where detected = Killed + Timeout, and valid excludes Ignored, CompileError and RuntimeError. This matches Stryker's own reported figure exactly; ledger.sh uses the same arithmetic.",
+
+  "recordedOn": "2026-07-31",
+  "recordedOnCommit": "f2efc5b",
+  "reactVersion": "15.6.1",
+  "enzymeTestFiles": 31,
+  "passingTests": 492,
+
+  "mutationScore": 48.81,
+  "coveredScore": 56.8,
+  "tolerance": 2.0,
+
+  "files": 81,
+  "totalMutants": 2785,
+  "killed": 1178,
+  "timeout": 171,
+  "survived": 1026,
+  "noCoverage": 389,
+  "runtimeError": 21,
+  "runtimeSeconds": 1585,
+
+  "_interpretation": "48.81% is the fraction of all injected bugs the suite detects; 56.8% is the fraction it detects in code the tests actually execute. The 389 no-coverage mutants are the gap between them. This is a reference line, NOT a grade to improve -- its only job is to reveal a conversion that keeps the test count while weakening the assertions.",
+  "_expectDrift": "Converting .instance() assertions to behavioural ones can legitimately kill fewer mutants while being the better test, because RTL cannot reach private methods Enzyme could. Treat a drop as a prompt to look, not proof of a defect. That is what tolerance is for.",
+  "_weakestAreas": "store 0.00, data 3.85, constants.js 11.76, util 16.79, actions 31.88, components 44.58. Pre-existing gaps in the Enzyme suite, not in scope for the migration to fix."
+}

package.json

@@ -122,6 +122,8 @@
     "lint": "node ./node_modules/eslint/bin/eslint.js ./static/js",
     "scss_lint": "node ./node_modules/sass-lint/bin/sass-lint.js --verbose --no-exit",
     "test": "./scripts/test/js_test.sh",
+    "mutation": "stryker run",
+    "ledger": "./scripts/test/ledger.sh",
     "coverage": "COVERAGE=1 ./scripts/test/js_test.sh",
     "codecov": "CODECOV=1 ./scripts/test/js_test.sh",
     "watch": "WATCH=1 ./scripts/test/js_test.sh",
@@ -131,6 +133,8 @@
     "repl": "node --require ./scripts/repl.js"
   },
   "devDependencies": {
+    "@stryker-mutator/core": "9.6.1",
+    "@stryker-mutator/mocha-runner": "9.6.1",
     "@testing-library/dom": "8.20.1",
     "@testing-library/react": "12.1.5",
     "@testing-library/user-event": "14.6.1"

scripts/test/ledger.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+# Ledger for the Enzyme -> RTL migration (mitodl/hq#12637).
+#
+# Five metrics that erode silently during a large test migration, plus the
+# mutation score. Each threshold is a floor or ceiling recorded at a known-good
+# point, not an aspiration. A build that trips one of these has quietly lost
+# something -- most often assertions, which no other check can see.
+set -uo pipefail
+
+FAIL=0
+
+check() { # name actual op expected
+	local name=$1 actual=$2 op=$3 expected=$4 ok
+	case $op in
+	ge) [[ $actual -ge $expected ]] && ok=1 || ok=0 ;;
+	le) [[ $actual -le $expected ]] && ok=1 || ok=0 ;;
+	*)
+		echo "bad op $op"
+		exit 2
+		;;
+	esac
+	if [[ $ok -eq 1 ]]; then
+		printf "  PASS  %-34s %6s  (need %s %s)\n" "$name" "$actual" "$op" "$expected"
+	else
+		printf "  FAIL  %-34s %6s  (need %s %s)\n" "$name" "$actual" "$op" "$expected"
+		FAIL=1
+	fi
+}
+
+echo "=== migration ledger ==="
+
+# Mocha's reported count -- NOT grep -c "it(", which undercounts loop-generated
+# tests by 27% and which an agent can satisfy while deleting tests.
+TESTS=$(npm run test 2>&1 | grep -oE '[0-9]+ passing' | grep -oE '^[0-9]+' | tail -1)
+TESTS=${TESTS:-0}
+check "passing tests" "$TESTS" ge 492
+
+# May only decrease. This is the migration's actual progress metric.
+ENZYME=$(grep -rl 'from "enzyme"' static/js --include='*_test.js' 2>/dev/null | wc -l | tr -d ' ')
+check "enzyme test files" "$ENZYME" le 31
+
+# data-testid is the escape hatch that turns an RTL migration back into
+# implementation-coupled testing. 86 .find("ComponentName") selectors exist in
+# the suite; without a cap the frictionless path is 86 testids, which preserves
+# the exact brittleness the migration exists to remove.
+#
+# Counts PRODUCTION files only. A testid on an inline fixture inside a test file
+# is not sprawl -- the thing worth capping is testids added to real components
+# to make an RTL query easy.
+TESTIDS=$(grep -rho 'data-testid' static/js --include='*.js' \
+	--exclude='*_test.js' --exclude-dir=testUtils --exclude-dir=factories 2>/dev/null | wc -l | tr -d ' ')
+check "data-testid in components" "$TESTIDS" le 15
+
+# May only decrease.
+FLOWFIX=$(grep -rho 'FlowFixMe' static/js --include='*.js' 2>/dev/null | wc -l | tr -d ' ')
+check "FlowFixMe occurrences" "$FLOWFIX" le 40
+
+# Frozen. Adding a line here is how React 18 act() warnings get silenced --
+# turning a real signal about un-batched state updates into future flaky tests.
+ALLOWLIST=$(grep -c 'grep -v' scripts/test/js_test.sh)
+check "js_test.sh allowlist lines" "$ALLOWLIST" le 7
+
+# Mutation score, when a baseline has been recorded and a report exists.
+# The full run takes 30-90 minutes, so this is a per-phase or nightly check --
+# the ledger reads a report, it never runs Stryker itself.
+if [[ -f .stryker-baseline.json && -f reports/mutation/mutation.json ]]; then
+	# Must match Stryker's own arithmetic exactly, or the computed score drifts
+	# from the recorded baseline and the gate trips on nothing:
+	#   - Timeout counts as DETECTED (a mutant that hangs the suite was noticed)
+	#   - Ignored, CompileError and RuntimeError are excluded from the denominator
+	# Verified against Stryker's reported 48.81% on the baseline run.
+	SCORE=$(node -e 'const r=require("./reports/mutation/mutation.json");const skip=new Set(["Ignored","CompileError","RuntimeError"]);let k=0,t=0;for(const x of Object.values(r.files))for(const m of x.mutants){if(skip.has(m.status))continue;t++;if(m.status==="Killed"||m.status==="Timeout")k++}console.log(t?Math.round(k/t*1000)/10:0)')
+	FLOOR=$(node -e 'const b=require("./.stryker-baseline.json");console.log(Math.round((b.mutationScore-b.tolerance)*10)/10)')
+	if awk -v s="$SCORE" -v f="$FLOOR" 'BEGIN{exit !(s>=f)}'; then
+		printf "  PASS  %-34s %6s  (floor %s)\n" "mutation score" "$SCORE" "$FLOOR"
+	else
+		printf "  FAIL  %-34s %6s  (floor %s)\n" "mutation score" "$SCORE" "$FLOOR"
+		FAIL=1
+	fi
+else
+	printf "  SKIP  %-34s %s\n" "mutation score" "(no baseline or report)"
+fi
+
+echo
+if [[ $FAIL -ne 0 ]]; then
+	echo "LEDGER FAILED -- a migration metric moved the wrong way."
+	echo "If the change is intentional, update the threshold in this script in"
+	echo "the same commit, and say why in the commit message."
+	exit 1
+fi
+echo "ledger OK"

scripts/test/smoke_bundle.js

@@ -0,0 +1,112 @@
+// Boots the built production bundle inside jsdom to prove the app still
+// starts. The unit suite renders components in isolation and cannot catch a
+// bundle that fails to initialise -- a module-init error, a broken createRoot,
+// an unresolvable dependency -- which is the failure mode a React upgrade
+// produces.
+//
+// No new dependencies: jsdom is already present, and webpack-stats.json already
+// lists the entry chunks in load order -- the same list the Django template
+// uses to emit <script> tags.
+const fs = require("fs")
+const path = require("path")
+const vm = require("vm")
+const { JSDOM } = require("jsdom")
+
+const ROOT = path.resolve(__dirname, "../..")
+const STATS = path.join(ROOT, "webpack-stats.json")
+
+function readStats() {
+  return JSON.parse(fs.readFileSync(STATS, "utf8"))
+}
+
+// webpack-stats lists each chunk's files in load order -- the same order the
+// Django template emits <script> tags in. Follow it exactly.
+function rootChunkFiles() {
+  return readStats()
+    .chunks.root.map(name => (typeof name === "string" ? name : name.name))
+    .filter(name => name.endsWith(".js"))
+    .map(name => path.join(ROOT, "static", "bundles", path.basename(name)))
+    .filter(fs.existsSync)
+}
+
+// A dev build embeds webpack-hot-middleware, whose client opens a permanently
+// pending EventSource to the dev server. Booting that in jsdom hangs rather
+// than failing cleanly, so treat a dev build as "not built" and skip -- the
+// alternative is a confusing timeout whenever someone has `docker-compose up`
+// running locally.
+function isDevBuild() {
+  return readStats().chunks.root.some(name =>
+    String(typeof name === "string" ? name : name.name).includes(
+      "webpack-hot-middleware"
+    )
+  )
+}
+
+function bundleIsBuilt() {
+  if (!fs.existsSync(STATS)) return false
+  const stats = readStats()
+  if (stats.status !== "done") return false
+  if (!stats.chunks || !stats.chunks.root) return false
+  if (isDevBuild()) return false
+  return rootChunkFiles().length > 0
+}
+
+function bootBundle() {
+  const errors = []
+
+  const dom = new JSDOM(
+    '<!doctype html><html><body><div id="container"></div></body></html>',
+    { url: "http://fake/", runScripts: "outside-only", pretendToBeVisual: true }
+  )
+  const { window } = dom
+
+  // The bundle reads these globals; they normally come from Django.
+  window.SETTINGS = {
+    public_path: "/static/bundles/",
+    sentry_dsn: "",
+    release_version: "test",
+    environment: "test",
+    videoKey: "a_video_key",
+    user: "",
+    email: "",
+    is_app_admin: false,
+    is_edx_course_admin: false,
+    dropbox_key: "dropbox_key",
+    thumbnail_base_url: "http://fake/",
+    support_email_address: "support@example.com",
+    ga_dimension_camera: "dimension1",
+    FEATURES: { ENABLE_VIDEO_PERMISSIONS: false }
+  }
+  window.requestAnimationFrame = cb => window.setTimeout(cb, 0)
+  window.cancelAnimationFrame = id => window.clearTimeout(id)
+  window.HTMLCanvasElement.prototype.getContext = () => ({ drawImage() {} })
+  window.console.error = (...args) => errors.push(args.join(" "))
+
+  for (const file of rootChunkFiles()) {
+    const name = path.basename(file)
+
+    // Webpack's automatic publicPath runtime resolves its own URL from
+    // document.currentScript.src, falling back to the last <script> tag in the
+    // document. vm.runInContext provides neither, so the bundle throws
+    // "Automatic publicPath is not supported in this browser" before any app
+    // code runs. Appending a matching <script src> makes the fallback resolve,
+    // which is what a real browser would present.
+    const tag = window.document.createElement("script")
+    tag.src = `http://fake/static/bundles/${name}`
+    window.document.head.appendChild(tag)
+
+    try {
+      vm.runInContext(
+        fs.readFileSync(file, "utf8"),
+        dom.getInternalVMContext(),
+        { filename: file }
+      )
+    } catch (e) {
+      errors.push(`${name}: ${e.message}`)
+    }
+  }
+
+  return { container: window.document.getElementById("container"), errors }
+}
+
+module.exports = { bootBundle, bundleIsBuilt, rootChunkFiles, isDevBuild }

static/js/testUtils/smoke_bundle_test.js

@@ -0,0 +1,27 @@
+import { assert } from "chai"
+import { bootBundle, bundleIsBuilt } from "../../../scripts/test/smoke_bundle"
+
+describe("production bundle smoke test", function() {
+  // Booting the whole app bundle is far slower than a unit test.
+  this.timeout(60000)
+
+  it("boots and mounts React into #container", function() {
+    if (!bundleIsBuilt()) {
+      this.skip()
+      return
+    }
+
+    const { container, errors } = bootBundle()
+
+    assert.deepEqual(
+      errors,
+      [],
+      `bundle threw during boot: ${errors.join("; ")}`
+    )
+    assert.isAbove(
+      container.childNodes.length,
+      0,
+      "React did not mount anything into #container"
+    )
+  })
+})

stryker.conf.json

@@ -0,0 +1,44 @@
+{
+  "$schema": "./node_modules/@stryker-mutator/core/schema/stryker-schema.json",
+  "packageManager": "yarn",
+  "testRunner": "mocha",
+  "reporters": ["html", "json", "clear-text", "progress"],
+  "jsonReporter": { "fileName": "reports/mutation/mutation.json" },
+  "coverageAnalysis": "perTest",
+  "concurrency": 4,
+  "timeoutMS": 20000,
+  "inPlace": true,
+  "disableTypeChecks": false,
+  "ignorePatterns": [
+    ".claude/**",
+    ".stryker-tmp/**",
+    "node_modules/**",
+    "coverage/**",
+    "reports/**",
+    "static/bundles/**"
+  ],
+  "mutator": {
+    "plugins": [
+      "flow",
+      "jsx",
+      "classProperties",
+      "objectRestSpread",
+      "dynamicImport"
+    ]
+  },
+  "mutate": [
+    "static/js/**/*.js",
+    "!static/js/**/*_test.js",
+    "!static/js/testUtils/**",
+    "!static/js/factories/**",
+    "!static/js/flow/**",
+    "!static/js/entry/**",
+    "!static/js/global_init.js",
+    "!static/js/babelhook.js"
+  ],
+  "mochaOptions": {
+    "require": ["./static/js/babelhook.js"],
+    "file": ["static/js/global_init.js"],
+    "spec": ["static/**/*/*_test.js"]
+  }
+}