Add public benchmarks, starting with vscode and self tests (#5)

jakebailey · web-flow · commit 6488a3524684 · 2024-01-08T14:49:41.000-08:00
diff --git a/README.md b/README.md
@@ -3,6 +3,22 @@
 This repo contains the infrastructure for TypeScript's benchmarking system, including build scripts
 and the `ts-perf` CLI tool.
 
+## Adding a benchmark
+
+Public benchmarks are stored in the `cases` directory.
+
+The simplest thing to do is to copy another benchmark; `cases/scenarios/vscode` is a good template.
+
+Each benchmark consists of:
+
+- `cases/scenarios/<name>/scenario.json` - This is JSON file which describes what `ts-perf` will run.
+- `cases/scenarios/<name>/setup.sh` - This is an optional script that CI will run before benchmarking for setting up the benchmark.
+  This is where you would clone repos, install dependencies, etc, whatever is needed to run `tsc`.
+- `cases/solutions/<name>` (or an ignored dir of the same name in `cases/solutions/.gitignore`) - This is where the source code for the benchmark is stored.
+  This may be code that's checked in the repo, but preferably the directory is `.gitignore`'d and `setup.sh` script clones code here.
+
+After these files are created, they must be added to `scenarioConfig` in `scripts/src/generateMatrix.ts` to be run in CI.
+
 ## Contributing
 
 This project welcomes contributions and suggestions. Most contributions require you to agree to a
diff --git a/build/benchmark.yml b/build/benchmark.yml
@@ -63,6 +63,7 @@ parameters:
       - tsc-only
       - bun
       - vscode
+      - public
     default: full # Branch pushes use the defaults, so this is set to full.
 
   - name: HISTORICAL_RUN
@@ -148,6 +149,7 @@ jobs:
         displayName: Build PR TypeScript
         condition: and(succeeded(), eq(variables['IS_PR'], 'true'))
         workingDirectory: $(Pipeline.Workspace)/TypeScript
+        name: buildTypeScriptPR
 
       - bash: |
           set -eo pipefail
@@ -161,6 +163,7 @@ jobs:
           node $(BENCH_SCRIPTS)/buildTypeScript.js --baseline --outputDir $(ARTIFACTS_DIR)/baseline
         displayName: Build baseline TypeScript
         workingDirectory: $(Pipeline.Workspace)/TypeScript
+        name: buildTypeScriptBaseline
 
       - publish: $(ARTIFACTS_DIR)
         artifact: BuiltTypeScript
@@ -184,6 +187,8 @@ jobs:
         variables:
           BUILT_TYPESCRIPT_DIR: $(Pipeline.Workspace)/BuiltTypeScript
           ARTIFACTS_DIR: $(Pipeline.Workspace)/artifacts
+          TYPESCRIPT_COMMIT_PR: $[ dependencies.Setup.outputs['buildTypeScriptPR.TYPESCRIPT_COMMIT'] ]
+          TYPESCRIPT_COMMIT_BASELINE: $[ dependencies.Setup.outputs['buildTypeScriptBaseline.TYPESCRIPT_COMMIT'] ]
 
         steps:
           - task: AzureKeyVault@2
@@ -214,6 +219,18 @@ jobs:
               node $(BENCH_SCRIPTS)/runTsPerf.js install-hosts
             displayName: Install hosts
 
+          - bash: |
+              set -eo pipefail
+              # Special case for self benchmark
+              export TYPESCRIPT_COMMIT=$(TYPESCRIPT_COMMIT_PR)
+              SETUP=$(TSPERF_PUBLIC_SCENARIO_CONFIG_DIR)/$(TSPERF_JOB_SCENARIO)/setup.sh
+              if test -f $SETUP; then
+                echo "Running $SETUP"
+                bash $SETUP
+              fi
+            displayName: Set up scenario for PR benchmark
+            condition: and(succeeded(), eq(variables['IS_PR'], 'true'), eq(variables['TSPERF_JOB_LOCATION'], 'public'))
+
           - bash: |
               set -eo pipefail
               node $(BENCH_SCRIPTS)/runTsPerf.js benchmark-$(TSPERF_JOB_KIND) \
@@ -222,6 +239,18 @@ jobs:
             displayName: Run PR $(TSPERF_JOB_KIND) benchmark
             condition: and(succeeded(), eq(variables['IS_PR'], 'true'))
 
+          - bash: |
+              set -eo pipefail
+              # Special case for self benchmark
+              export TYPESCRIPT_COMMIT=$(TYPESCRIPT_COMMIT_BASELINE)
+              SETUP=$(TSPERF_PUBLIC_SCENARIO_CONFIG_DIR)/$(TSPERF_JOB_SCENARIO)/setup.sh
+              if test -f $SETUP; then
+                echo "Running $SETUP"
+                bash $SETUP
+              fi
+            displayName: Set up scenario for baseline benchmark
+            condition: and(succeeded(), eq(variables['TSPERF_JOB_LOCATION'], 'public'))
+
           - bash: |
               set -eo pipefail
               node $(BENCH_SCRIPTS)/runTsPerf.js benchmark-$(TSPERF_JOB_KIND) \
diff --git a/build/templates/cloneAndBuildBenchmarkRepo.yml b/build/templates/cloneAndBuildBenchmarkRepo.yml
@@ -25,6 +25,6 @@ steps:
       echo "##vso[task.setvariable variable=TSPERF_EXE]$(Pipeline.Workspace)/typescript-benchmarking/ts-perf/bin/ts-perf"
 
       # TODO(jakebailey): actually create these dirs, move them to more reasonable names, clone external repos as needed
-      echo "##vso[task.setvariable variable=TSPERF_PUBLIC_SCENARIO_CONFIG_DIR]$(Pipeline.Workspace)/typescript-benchmarking/cases/perf/scenarios"
-      echo "##vso[task.setvariable variable=TSPERF_PUBLIC_SUITE_DIR]$(Pipeline.Workspace)/typescript-benchmarking/cases/perf/solutions"
+      echo "##vso[task.setvariable variable=TSPERF_PUBLIC_SCENARIO_CONFIG_DIR]$(Pipeline.Workspace)/typescript-benchmarking/cases/scenarios"
+      echo "##vso[task.setvariable variable=TSPERF_PUBLIC_SUITE_DIR]$(Pipeline.Workspace)/typescript-benchmarking/cases/solutions"
     displayName: Set typescript-benchmarking variables
diff --git a/cases/scenarios/self-build-src/scenario.json b/cases/scenarios/self-build-src/scenario.json
@@ -0,0 +1,11 @@
+{
+    "name": "self-build-src",
+    "kind": "tsc",
+    "args": [
+        "-b",
+        "${suiteDirectory}/self-build-src/src"
+    ],
+    "platforms": [
+        "linux"
+    ]
+}
diff --git a/cases/scenarios/self-build-src/setup.sh b/cases/scenarios/self-build-src/setup.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+cd "${0%/*}"
+SCENARIO_NAME=$(basename $PWD)
+SOLUTION_DIR=../../solutions/$SCENARIO_NAME
+
+if ! test -d $SOLUTION_DIR; then
+  git clone --filter blob:none https://github.com/microsoft/TypeScript.git $SOLUTION_DIR
+fi
+
+cd $SOLUTION_DIR
+git clean -fdx
+git reset --hard HEAD
+git fetch origin ${TYPESCRIPT_COMMIT}
+git switch --detach FETCH_HEAD
+npm ci
+npx hereby generate-diagnostics
diff --git a/cases/scenarios/self-compiler/scenario.json b/cases/scenarios/self-compiler/scenario.json
@@ -0,0 +1,13 @@
+{
+    "name": "self-compiler",
+    "kind": "tsc",
+    "args": [
+        "-p",
+        "${suiteDirectory}/self-compiler/src/compiler",
+        "--outdir",
+        "${outDirectory}"
+    ],
+    "platforms": [
+        "linux"
+    ]
+}
diff --git a/cases/scenarios/self-compiler/setup.sh b/cases/scenarios/self-compiler/setup.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+cd "${0%/*}"
+SCENARIO_NAME=$(basename $PWD)
+SOLUTION_DIR=../../solutions/$SCENARIO_NAME
+
+if ! test -d $SOLUTION_DIR; then
+  git clone --filter blob:none https://github.com/microsoft/TypeScript.git $SOLUTION_DIR
+fi
+
+cd $SOLUTION_DIR
+git clean -fdx
+git reset --hard HEAD
+git fetch origin ${TYPESCRIPT_COMMIT}
+git switch --detach FETCH_HEAD
+npm ci
+npx hereby generate-diagnostics
diff --git a/cases/scenarios/vscode/scenario.json b/cases/scenarios/vscode/scenario.json
@@ -0,0 +1,13 @@
+{
+    "name": "vscode",
+    "kind": "tsc",
+    "args": [
+        "-p",
+        "${suiteDirectory}/vscode/src",
+        "--outdir",
+        "${outDirectory}"
+    ],
+    "platforms": [
+        "linux"
+    ]
+}
diff --git a/cases/scenarios/vscode/setup.sh b/cases/scenarios/vscode/setup.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+cd "${0%/*}"
+SCENARIO_NAME=$(basename $PWD)
+SOLUTION_DIR=../../solutions/$SCENARIO_NAME
+
+if ! test -d $SOLUTION_DIR; then
+  git clone --filter blob:none https://github.com/microsoft/vscode.git $SOLUTION_DIR
+fi
+
+cd $SOLUTION_DIR
+git clean -fdx
+git reset --hard HEAD
+git switch --detach f88bce8fe6a6d2ccd27cbd64bb26853cd8779afa
+COREPACK_ENABLE_STRICT=0 corepack yarn@1.22.21 install --ignore-scripts
diff --git a/cases/solutions/.gitignore b/cases/solutions/.gitignore
@@ -0,0 +1,3 @@
+/vscode/
+/self-compiler/
+/self-build-src/
diff --git a/scripts/src/buildTypeScript.ts b/scripts/src/buildTypeScript.ts
@@ -5,7 +5,7 @@ import path from "node:path";
 import { $ as _$ } from "execa";
 import minimist from "minimist";
 
-import { retry } from "./utils.js";
+import { retry, setOutputVariable } from "./utils.js";
 
 const $pipe = _$({ verbose: true });
 const $ = _$({ verbose: true, stdio: "inherit" });
@@ -79,3 +79,5 @@ const outputInfo = JSON.stringify(info, undefined, 4);
 
 console.log(`Writing ${outputInfoPath} with contents:\n${outputInfo}}`);
 await fs.promises.writeFile(outputInfoPath, outputInfo);
+
+setOutputVariable("TYPESCRIPT_COMMIT", commit);
diff --git a/scripts/src/checkLatestCommitForRef.ts b/scripts/src/checkLatestCommitForRef.ts
@@ -3,7 +3,7 @@ import assert from "node:assert";
 import { Octokit } from "@octokit/rest";
 import minimist from "minimist";
 
-import { getNonEmptyEnv, getRepoInfo } from "./utils.js";
+import { getNonEmptyEnv, getRepoInfo, setJobVariable } from "./utils.js";
 
 const auth = getNonEmptyEnv("GH_TOKEN");
 
@@ -49,4 +49,4 @@ const info = await getRepoInfo(args.builtDir);
 const ref = getNonEmptyEnv("REF");
 const isLatest = await isLatestCommitForRef(ref, info.commit);
 
-console.log(`##vso[task.setvariable variable=TSPERF_BLOB_LATEST]${isLatest}`);
+setJobVariable("TSPERF_BLOB_LATEST", isLatest);
diff --git a/scripts/src/generateMatrix.ts b/scripts/src/generateMatrix.ts
@@ -1,5 +1,7 @@
 import minimist from "minimist";
 
+import { setOutputVariable } from "./utils.js";
+
 // Keep in sync with inventory.yml and benchmark.yml.
 type AllAgents = "ts-perf1" | "ts-perf2" | "ts-perf3" | "ts-perf4";
 // We reserve some agents so that non-baseline jobs can make progress.
@@ -38,6 +40,9 @@ const scenarioConfig = {
         { name: "material-ui", agent: "ts-perf1", location: "internal" },
         { name: "Compiler-Unions", agent: "ts-perf2", location: "internal" },
         { name: "xstate", agent: "ts-perf3", location: "internal" },
+        { name: "vscode", agent: "ts-perf1", location: "public" },
+        { name: "self-compiler", agent: "ts-perf2", location: "public" },
+        { name: "self-build-src", agent: "ts-perf3", location: "public" },
     ],
     tsserver: [
         { name: "Compiler-UnionsTSServer", agent: "ts-perf1", location: "internal" },
@@ -67,76 +72,91 @@ type Preset = {
     };
 };
 
+function onlyInternal<T extends BaseScenario>(scenarios: readonly T[]): readonly T[] {
+    return scenarios.filter(s => s.location === "internal");
+}
+
+function onlyPublic<T extends BaseScenario>(scenarios: readonly T[]): readonly T[] {
+    return scenarios.filter(s => s.location === "public");
+}
+
 // Note: keep this up to date with TSPERF_PRESET and https://github.com/microsoft/typescript-bot-test-triggerer
 const presets: Record<string, Preset | undefined> = {
     "full": {
         tsc: {
             hosts: [hosts.node20, hosts.node18, hosts.node16],
             iterations: defaultIterations,
-            scenarios: scenarioConfig.tsc,
+            scenarios: onlyInternal(scenarioConfig.tsc),
         },
         tsserver: {
             hosts: [hosts.node16],
             iterations: defaultIterations,
-            scenarios: scenarioConfig.tsserver,
+            scenarios: onlyInternal(scenarioConfig.tsserver),
         },
         startup: {
             hosts: [hosts.node16],
             iterations: defaultIterations,
-            scenarios: scenarioConfig.startup,
+            scenarios: onlyInternal(scenarioConfig.startup),
         },
     },
     "regular": {
         tsc: {
             hosts: [hosts.node18],
             iterations: defaultIterations,
-            scenarios: scenarioConfig.tsc,
+            scenarios: onlyInternal(scenarioConfig.tsc),
         },
         tsserver: {
             hosts: [hosts.node18],
             iterations: defaultIterations,
-            scenarios: scenarioConfig.tsserver,
+            scenarios: onlyInternal(scenarioConfig.tsserver),
         },
         startup: {
             hosts: [hosts.node18],
             iterations: defaultIterations,
-            scenarios: scenarioConfig.startup,
+            scenarios: onlyInternal(scenarioConfig.startup),
         },
     },
     "tsc-only": {
         tsc: {
             hosts: [hosts.node18],
             iterations: defaultIterations,
-            scenarios: scenarioConfig.tsc,
+            scenarios: onlyInternal(scenarioConfig.tsc),
         },
     },
     "bun": {
         tsc: {
             hosts: [hosts.bun],
             iterations: defaultIterations * 2,
-            scenarios: scenarioConfig.tsc,
+            scenarios: onlyInternal(scenarioConfig.tsc),
         },
         startup: {
             hosts: [hosts.bun],
             iterations: defaultIterations,
-            scenarios: scenarioConfig.startup.filter(s => s.name !== "tsserver-startup"),
+            scenarios: onlyInternal(scenarioConfig.startup).filter(s => s.name !== "tsserver-startup"),
         },
     },
     "vscode": {
         tsc: {
             hosts: [hosts.vscode],
             iterations: defaultIterations,
-            scenarios: scenarioConfig.tsc,
+            scenarios: onlyInternal(scenarioConfig.tsc),
         },
         tsserver: {
             hosts: [hosts.vscode],
             iterations: defaultIterations,
-            scenarios: scenarioConfig.tsserver,
+            scenarios: onlyInternal(scenarioConfig.tsserver),
         },
         startup: {
             hosts: [hosts.vscode],
             iterations: defaultIterations,
-            scenarios: scenarioConfig.startup,
+            scenarios: onlyInternal(scenarioConfig.startup),
+        },
+    },
+    "public": {
+        tsc: {
+            hosts: [hosts.node20],
+            iterations: defaultIterations,
+            scenarios: onlyPublic(scenarioConfig.tsc),
         },
     },
 };
@@ -161,11 +181,6 @@ function sanitizeJobName(name: string): JobName {
     return name.replace(/[^a-zA-Z0-9_]/g, "_") as JobName;
 }
 
-function setVariable(name: string, value: string | number | boolean) {
-    console.log(`${name}=${value}`);
-    console.log(`##vso[task.setvariable variable=${name};isOutput=true]${value}`);
-}
-
 interface Job {
     TSPERF_JOB_KIND: JobKind;
     TSPERF_JOB_NAME: JobName;
@@ -217,12 +232,12 @@ for (const jobKind of allJobKinds) {
 }
 
 for (const [agent, value] of Object.entries(matrix)) {
-    setVariable(`MATRIX_${agent.replace(/-/g, "_")}`, JSON.stringify(value));
+    setOutputVariable(`MATRIX_${agent.replace(/-/g, "_")}`, JSON.stringify(value));
     console.log(JSON.stringify(value, undefined, 4));
 }
 
 // These are outputs for the ProcessResults job, specifying which results were
 // produced previously and need to be processed. This is a space separated list,
 // iterated in the pipeline in bash.
-setVariable(`TSPERF_PROCESS_KINDS`, [...processKinds].sort().join(" "));
-setVariable(`TSPERF_PROCESS_LOCATIONS`, [...processLocations].sort().join(","));
+setOutputVariable(`TSPERF_PROCESS_KINDS`, [...processKinds].sort().join(" "));
+setOutputVariable(`TSPERF_PROCESS_LOCATIONS`, [...processLocations].sort().join(","));
diff --git a/scripts/src/utils.ts b/scripts/src/utils.ts
@@ -58,3 +58,13 @@ export async function getRepoInfo(builtDir: string) {
     const parsed = JSON.parse(await fs.promises.readFile(repoInfoPath, { encoding: "utf8" }));
     return RepoInfo.parse(parsed);
 }
+
+export function setOutputVariable(name: string, value: string | number | boolean) {
+    console.log(`setting output ${name}=${value}`);
+    console.log(`##vso[task.setvariable variable=${name};isOutput=true]${value}`);
+}
+
+export function setJobVariable(name: string, value: string | number | boolean) {
+    console.log(`setting variable ${name}=${value}`);
+    console.log(`##vso[task.setvariable variable=${name}]${value}`);
+}

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+/vscode/`
	`2`	`+/self-compiler/`
	`3`	`+/self-build-src/`