diff --git a/Dockerfile.debug_test b/Dockerfile.debug_test new file mode 100644 index 000000000..076e91b1f --- /dev/null +++ b/Dockerfile.debug_test @@ -0,0 +1,99 @@ +# Copyright 2022 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +FROM debian:trixie-slim + +# Install Dependencies. +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + dnsutils \ + git \ + openssh-client \ + unzip \ + gpg \ + jq \ + procps \ + nodejs \ + npm \ + ; \ + rm -rf /var/lib/apt/lists/* + +# Create the `flutter` user and group. +RUN groupadd -r flutter && useradd -r -g flutter flutter + +# Create folders for home / flutter / test. + +ENV FLUTTER_ROOT="/opt/flutter" +ENV TEST_ANALYZER="/opt/testanalyzer" + +RUN mkdir -p /home/flutter && \ + chown -R flutter:flutter /home/flutter + +RUN mkdir -p "${FLUTTER_ROOT}" && \ + chown -R flutter:flutter "${FLUTTER_ROOT}" + +RUN mkdir -p "${TEST_ANALYZER}" && \ + chown -R flutter:flutter "${TEST_ANALYZER}" + +# Switch users for the rest of the file. +USER flutter + +# Clone flutter repository and set up the environment. +RUN git clone https://github.com/flutter/flutter "${FLUTTER_ROOT}" +ENV PATH="${FLUTTER_ROOT}/bin:${PATH}" + +WORKDIR "$FLUTTER_ROOT" + +# Disable analytics and crash reporting, disable all the other platforms except web. +RUN flutter config \ + --no-analytics \ + --enable-web \ + --no-enable-linux-desktop \ + --no-enable-macos-desktop \ + --no-enable-windows-desktop \ + --no-enable-android \ + --no-enable-ios \ + --no-enable-fuchsia + +# Only download the websdk +RUN flutter precache --web + +# Perform a doctor run. +RUN flutter doctor -v + +# Install Gemini CLI in the flutter home directory and update path. +WORKDIR /home/flutter +ENV PATH="/home/flutter/node_modules/.bin:$PATH" +RUN npm install @google/gemini-cli +RUN gemini extensions install --consent https://github.com/gemini-cli-extensions/flutter + +# Pre-setup Gemini to use API key authentication +RUN < /home/flutter/.gemini/settings.json +{ + "security": { + "auth": { + "selectedType": "gemini-api-key" + } + } +} +EOF + +# Copy the testanalyzer script +WORKDIR "${TEST_ANALYZER}" +COPY --chown=flutter:flutter packages/testanalyzer . + +# Make entrypoint script executable +RUN chmod +x "${TEST_ANALYZER}/bin/entrypoint.sh" + +# Set up environment variables placeholders +ENV GEMINI_API_KEY="" +ENV FAILING_TEST="" +ENV BUILD_NUMBER="" +ENV PR_NUMBER="" +ENV TEST_ENV="try" + +ENTRYPOINT [ "/opt/testanalyzer/bin/entrypoint.sh" ] diff --git a/packages/testanalyzer/TEST_PROMPT.md b/packages/testanalyzer/TEST_PROMPT.md new file mode 100644 index 000000000..50c6924c5 --- /dev/null +++ b/packages/testanalyzer/TEST_PROMPT.md @@ -0,0 +1,54 @@ +# Dart Log Failure Parser + +You are a helpful assistant that analyzes test failures on Flutter's infrastructure and provides helpful diagnostics and fixes. The failing test logs are provided after the `## Log Content` header. + +You are in a stand-alone checkout of the pull request. `dart` and `flutter` tools are in your PATH already. You can find the merge-base with `master` by running `git merge-base master HEAD`. You should use this diff to determine if the failures are related to the code changes in the pull request. + +You are in a linux docker container. You can `find` files with either `find . -name '` or `git ls-files | grep `. You can grep for strings or symbols with `git grep `. + +## Workflow + +### 1. Analyze Raw Log Output + +Analyze the raw log output for failure details. Do not skim the output; check the entire log. **The description of findings should include specific details for the failures (e.g., unformatted files, specific test names), not just the top-level command that failed.** + +### 2. Look for Failure Patterns + +#### Pattern A: Error Blocks (e.g., Linux Analyze) +Search for blocks starting with `╡ERROR #`. +Example: +``` +╔═╡ERROR #1╞════════════════════════════════════════════════════════════════════ +║ Command: bin/cache/dart-sdk/bin/dart --enable-asserts /b/s/w/ir/x/w/flutter/dev/bots/analyze_snippet_code.dart --verbose +║ Command exited with exit code 255 but expected zero exit code. +║ Working directory: /b/s/w/ir/x/w/flutter +╚═══════════════════════════════════════════════════════════════════════════════ +``` + +#### Pattern B: Task Result JSON +Search for "Task result:" followed by a JSON object. +Example: +```json +Task result: +{ + "success": false, + "reason": "Task failed: PathNotFoundException: Cannot open file..." +} +``` + +#### Pattern C: Failing Tests List +For general Dart tests, look for a list at the end of the log starting with "Failing tests:". +Example: +``` +Failing tests: + test/general.shard/cache_test.dart: FontSubset artifacts for all platforms on arm64 hosts + test/general.shard/cache_test.dart: FontSubset artifacts on arm64 linux +``` + +#### Pattern D: Build Failures +For build failures (e.g., engine tests failing at compile time), look for the following indicators in the logs or API summaries: +- Lines starting with `FAILED:` (indicates a Ninja target failed). +- Compiler error messages (e.g., `error:`, `fatal error:`). +- Linker error messages (e.g., `undefined reference to`). +- Summary messages in the check-runs API output like `1 build failed: []`. +` \ No newline at end of file diff --git a/packages/testanalyzer/bin/entrypoint.sh b/packages/testanalyzer/bin/entrypoint.sh new file mode 100644 index 000000000..b4b01be98 --- /dev/null +++ b/packages/testanalyzer/bin/entrypoint.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright 2026 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +set -e + +if [ -n "$PR_NUMBER" ]; then + echo "Creating worktree for PR $PR_NUMBER..." + cd "$FLUTTER_ROOT" + + # Ensure safe directory + git config --global --add safe.directory "$FLUTTER_ROOT" + + # Cleanup + git worktree remove -f pr_review 2>/dev/null || true + git worktree prune || true + + # Add worktree in detached state to avoid creating a branch + git worktree add pr_review + + cd pr_review + + echo "Checking out PR $PR_NUMBER in worktree..." + git fetch origin pull/$PR_NUMBER/head:pr-$PR_NUMBER + git checkout pr-$PR_NUMBER + + echo "Running testanalyzer..." + unset PR_NUMBER + # Run the script from the worktree directory + dart /opt/testanalyzer/bin/testanalyzer.dart + + if [ -f "failure_log.txt" ]; then + echo "Analyzing log with Gemini..." + + echo "Constructing prompt..." + echo "Analyze the following log failures based on the provided skill." > prompt.txt + echo "" >> prompt.txt + echo "### Skill Definition" >> prompt.txt + cat /opt/testanalyzer/TEST_PROMPT.md >> prompt.txt + echo "" >> prompt.txt + echo "## Log Content" >> prompt.txt + cat failure_log.txt >> prompt.txt + + gemini --skip-trust --yolo -p "Please read the following instructions and execute them: @prompt.txt" + else + echo "Failure log not found. Analysis skipped." + fi +fi diff --git a/packages/testanalyzer/bin/testanalyzer.dart b/packages/testanalyzer/bin/testanalyzer.dart new file mode 100644 index 000000000..175587498 --- /dev/null +++ b/packages/testanalyzer/bin/testanalyzer.dart @@ -0,0 +1,221 @@ +// Copyright 2026 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +void main() async { + final failingTest = Platform.environment['FAILING_TEST']; + final buildNumberStr = Platform.environment['BUILD_NUMBER']; + final commitSha = Platform.environment['COMMIT_SHA']; + final testEnv = Platform.environment['TEST_ENV'] ?? 'try'; + + if (failingTest == null) { + print('Error: FAILING_TEST environment variable is required.'); + exit(1); + } + + var buildNumber = buildNumberStr; + + if (buildNumber == null || buildNumber.isEmpty) { + print('BUILD_NUMBER not provided, looking for COMMIT_SHA'); + if (commitSha == null || commitSha.isEmpty) { + print('Error: Either BUILD_NUMBER or COMMIT_SHA must be provided.'); + exit(1); + } + + print('Looking up buildnumber for Commit $commitSha...'); + buildNumber = await lookupBuildNumberFromCommit(commitSha, failingTest); + } + + if (buildNumber == null) { + print('Error: Could not find build number or ID.'); + exit(1); + } + + print( + 'Fetching logs for test: $failingTest, build: $buildNumber, env: $testEnv', + ); + + final logUrl = await fetchLogUrl(failingTest, buildNumber, testEnv); + if (logUrl == null) { + print('Error: Could not find failure log URL.'); + exit(1); + } + + print('Found log URL: $logUrl'); + + final rawLog = await downloadRawLog(logUrl); + if (rawLog == null) { + print('Error: Could not download raw log.'); + exit(1); + } + + print('Downloaded raw log. Length: ${rawLog.length}'); + + final outputFile = File('failure_log.txt'); + await outputFile.writeAsString(rawLog); + print('Saved log to failure_log.txt'); +} + +Future lookupBuildNumberFromCommit( + String commitSha, + String failingTest, +) async { + try { + // gh api repos/flutter/flutter/commits//check-runs --jq '.check_runs[] | select(.name == "") | .details_url' + final uri = Uri.https( + 'api.github.com', + '/repos/flutter/flutter/commits/$commitSha/check-runs', + {'check_name': failingTest}, + ); + print('$uri'); + final result = await Process.run('curl', [ + '-H', + 'Accept: application/vnd.github+json', + '-H', + 'X-GitHub-Api-Version: 2022-11-28', + '$uri', + ]); + + if (result.exitCode != 0) { + print('curl failed: ${result.stderr}'); + return null; + } + + String? detailUrl; + final jsonObject = json.decode(result.stdout as String); + if (jsonObject case {'check_runs': final List checkRuns}) { + print('check runs found: ${checkRuns.length}'); + for (final check in checkRuns) { + if (check case { + 'name': final String testName, + 'details_url': final String url, + }) { + print('check $testName $url'); + if (testName != failingTest) continue; + detailUrl = url; + break; + } + } + } + if (detailUrl == null) { + throw Exception(''' +ERROR: missing detail_url + +The check run for the failing test "$failingTest" was either not found in the GitHub API response or did not contain a details_url. +'''); + } + + if (detailUrl.isNotEmpty) { + print('Found details URL: $detailUrl'); + // Extract ID from URL + final idRegExp = RegExp(r'/build/(\d+)'); + final idMatch = idRegExp.firstMatch(detailUrl); + if (idMatch != null) { + return idMatch.group(1); + } + } + } catch (e) { + print('Error running gh api: $e'); + } + return null; +} + +Future fetchLogUrl( + String failingTest, + String buildNumberOrId, + String testEnv, +) async { + final client = HttpClient(); + try { + final request = await client.postUrl( + Uri.parse( + 'https://cr-buildbucket.appspot.com/prpc/buildbucket.v2.Builds/GetBuild', + ), + ); + request.headers.set('accept', 'application/json'); + request.headers.set('content-type', 'application/json'); + + Map body; + if (buildNumberOrId.length > 10) { + body = { + 'id': buildNumberOrId, + 'mask': {'fields': 'steps,infra'}, + }; + } else { + body = { + 'builder': { + 'project': 'flutter', + 'bucket': testEnv, + 'builder': failingTest, + }, + 'buildNumber': int.parse(buildNumberOrId), + 'mask': {'fields': 'steps,infra'}, + }; + } + + request.add(utf8.encode(json.encode(body))); + final response = await request.close(); + // response stream of bytes -> stream of strings -> lines. + final lines = await response + .transform(utf8.decoder) + .transform(const LineSplitter()) + .toList(); + + if (lines.isEmpty) { + print('No response lines received'); + return null; + } + + if (lines[0].trim() == ")]}'") { + lines.removeAt(0); + } + + final data = json.decode(lines.join()); + final steps = data['steps'] as List?; + if (steps == null) return null; + + for (final step in steps) { + if (step['status'] == 'FAILURE') { + final logs = step['logs'] as List?; + if (logs != null) { + for (final log in logs) { + if (log['name'] == 'stdout') { + final viewUrl = log['viewUrl'] as String?; + if (viewUrl != null) { + print('Found stdout log URL: $viewUrl'); + return viewUrl; + } + } + } + } + } + } + } catch (e) { + print('Error fetching log URL: $e'); + } finally { + client.close(); + } + return null; +} + +Future downloadRawLog(String logUrl) async { + final client = HttpClient(); + try { + final rawUrl = '$logUrl?format=raw'; + final request = await client.getUrl(Uri.parse(rawUrl)); + final response = await request.close(); + if (response.statusCode == 200) { + return await response.transform(utf8.decoder).join(); + } else { + print('Failed to download log: ${response.statusCode}'); + } + } catch (e) { + print('Error downloading log: $e'); + } finally { + client.close(); + } + return null; +} diff --git a/packages/testanalyzer/pubspec.yaml b/packages/testanalyzer/pubspec.yaml new file mode 100644 index 000000000..56c241ecf --- /dev/null +++ b/packages/testanalyzer/pubspec.yaml @@ -0,0 +1,3 @@ +name: testanalyzer +environment: + sdk: '>=3.10.0 <4.0.0'