feat: flet test on-device integration testing + CI matrix

.fvmrc

@@ -1,3 +1,3 @@
 {
-  "flutter": "3.44.3"
+  "flutter": "3.44.4"
 }

.github/workflows/flet-test.yml

@@ -0,0 +1,213 @@
+name: Flet Test (on-device)
+
+# Runs `flet test` end-to-end on every target: provisions a per-platform test
+# host (build pipeline in test_mode), launches the built Counter app with embedded
+# Python, and drives it over the socket tester channel. Unlike
+# macos-integration-tests.yml (host-mode screenshots) and flet-build-test.yml
+# (compile only), this actually runs the app on the device/emulator/simulator.
+
+on:
+  push:
+    branches-ignore:
+      - main
+    paths:
+      - '.github/workflows/flet-test.yml'
+      - '.fvmrc'
+      - 'packages/flet/**'
+      - 'packages/flet_integration_test/**'
+      - 'sdk/python/packages/flet/**'
+      - 'sdk/python/packages/flet-cli/**'
+      - 'sdk/python/examples/apps/flet_test_counter/**'
+  pull_request:
+    paths:
+      - '.github/workflows/flet-test.yml'
+      - '.fvmrc'
+      - 'packages/flet/**'
+      - 'packages/flet_integration_test/**'
+      - 'sdk/python/packages/flet/**'
+      - 'sdk/python/packages/flet-cli/**'
+      - 'sdk/python/examples/apps/flet_test_counter/**'
+  workflow_dispatch:
+
+# Ensure only one run per branch (PR or push), cancel older ones
+concurrency:
+  group: flet-test-${{ github.workflow }}-${{ github.event.pull_request.head.ref || github.ref_name }}
+  cancel-in-progress: true
+
+env:
+  ROOT: "${{ github.workspace }}"
+  SDK_PYTHON: "${{ github.workspace }}/sdk/python"
+  SCRIPTS: "${{ github.workspace }}/.github/scripts"
+  APP_DIR: "sdk/python/examples/apps/flet_test_counter"
+
+  # Host venv (runs flet-cli + pytest). The *embedded* app runtime is built
+  # against the matrix python version via `--python-version`; the host stays on
+  # 3.13 for stable test-dependency wheels (numpy/pillow/scikit-image).
+  UV_PYTHON: "3.13"
+  PYTHONUTF8: 1
+
+  # https://flet.dev/docs/reference/environment-variables
+  FLET_CLI_NO_RICH_OUTPUT: 1
+  # Use the Flutter the setup action puts on PATH, not fvm.
+  FLET_TEST_DISABLE_FVM: 1
+
+jobs:
+  flet-test:
+    name: ${{ matrix.platform }} (py${{ matrix.python_version }})
+    runs-on: ${{ matrix.runner }}
+    # The embedded runtime is built against the matrix python version; the test
+    # also asserts the running app reports it (EXPECTED_PYTHON_VERSION).
+    env:
+      PYTHON_VERSION: ${{ matrix.python_version }}
+      EXPECTED_PYTHON_VERSION: ${{ matrix.python_version }}
+    # Cap the job so a wedged emulator/simulator (which can hang the on-device
+    # test indefinitely) auto-cancels instead of burning a full runner slot.
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        python_version: ["3.12", "3.13", "3.14"]
+        platform: [macos, ios, windows, linux, linux-arm64, android]
+        include:
+          - platform: macos
+            runner: macos-26
+            test_cmd: uv run flet test macos --python-version ${PYTHON_VERSION} --yes -v
+
+          - platform: ios
+            runner: macos-26
+            test_cmd: uv run flet test ios -d "$IOS_UDID" --python-version ${PYTHON_VERSION} --yes -v
+
+          - platform: windows
+            runner: windows-2025-vs2026
+            test_cmd: uv run flet test windows --python-version ${PYTHON_VERSION} --yes -v
+
+          - platform: linux
+            runner: ubuntu-latest
+            test_cmd: xvfb-run -a uv run flet test linux --python-version ${PYTHON_VERSION} --yes -v
+
+          # arm64 Linux: no kuhnroyal/Flutter action (Flutter ships no prebuilt
+          # arm64 Linux SDK) — `flet test` installs Flutter itself via a git
+          # clone of the SDK, exercising flet-cli's arm64 Linux install path.
+          - platform: linux-arm64
+            runner: ubuntu-24.04-arm
+            test_cmd: xvfb-run -a uv run flet test linux --python-version ${PYTHON_VERSION} --yes -v
+
+          - platform: android
+            runner: ubuntu-latest
+            # Run command lives in the emulator-runner step below.
+            test_cmd: ""
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v8.2.0
+
+      - name: Patch versions
+        shell: bash
+        run: |
+          source "${SCRIPTS}/update_build_version.sh"
+          source "${SCRIPTS}/common.sh"
+          patch_python_package_versions
+
+      # arm64 Linux gets no prebuilt Flutter SDK — `flet test` installs it.
+      - name: Setup Flutter
+        if: matrix.platform != 'linux-arm64'
+        uses: kuhnroyal/flutter-fvm-config-action/setup@v3
+        with:
+          path: '.fvmrc'
+          cache: true
+
+      - name: Show tool versions
+        shell: bash
+        run: |
+          uv --version
+          uv run --project sdk/python/packages/flet-cli python --version
+          flutter --version || echo "Flutter not on PATH yet (flet test will install it)."
+
+      # -------- Linux: build deps + virtual display (x64 and arm64) --------
+      - name: Install Linux dependencies
+        if: startsWith(matrix.platform, 'linux')
+        shell: bash
+        run: |
+          sudo apt-get update --allow-releaseinfo-change
+          LINUX_DEPS="$(uv run --project sdk/python/packages/flet-cli flet --version --json | jq -r '.linux_dependencies | join(" ")')"
+          # xvfb has no GPU; the Flutter GTK app needs Mesa's software GL
+          # (llvmpipe) or it crashes on GL context creation (exit 79).
+          sudo apt-get install -y --no-install-recommends \
+            $LINUX_DEPS xvfb libgl1-mesa-dri
+          sudo apt-get clean
+
+      # -------- iOS: boot a simulator, capture its UDID --------
+      - name: Boot iOS simulator
+        if: matrix.platform == 'ios'
+        shell: bash
+        run: |
+          UDID=$(xcrun simctl list devices available -j \
+            | jq -r '[.devices[][] | select(.name | startswith("iPhone"))][0].udid')
+          if [ -z "$UDID" ] || [ "$UDID" = "null" ]; then
+            echo "No available iPhone simulator found" >&2
+            xcrun simctl list devices available
+            exit 1
+          fi
+          echo "Using iOS simulator $UDID"
+          xcrun simctl boot "$UDID" || true
+          xcrun simctl bootstatus "$UDID" -b
+          echo "IOS_UDID=$UDID" >> "$GITHUB_ENV"
+
+      # -------- Android: enable KVM for a fast emulator --------
+      - name: Enable KVM
+        if: matrix.platform == 'android'
+        shell: bash
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
+            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+
+      # -------- Run: desktop + iOS (non-Android) --------
+      - name: Run flet test (${{ matrix.platform }})
+        if: matrix.platform != 'android'
+        shell: bash
+        working-directory: ${{ env.APP_DIR }}
+        # Force Mesa software GL on Linux (xvfb has no GPU); harmless elsewhere.
+        env:
+          LIBGL_ALWAYS_SOFTWARE: "true"
+          GALLIUM_DRIVER: llvmpipe
+        run: ${{ matrix.test_cmd }}
+
+      # -------- Run: Android (inside the emulator) --------
+      - name: Run flet test (android)
+        if: matrix.platform == 'android'
+        uses: reactivecircus/android-emulator-runner@v2
+        with:
+          api-level: 35
+          arch: x86_64
+          target: google_apis
+          force-avd-creation: true
+          disable-animations: true
+          emulator-options: -no-snapshot -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim
+          working-directory: ${{ env.APP_DIR }}
+          # Single folded line => one shell, so the test command is the last
+          # statement and ITS exit code is the job's (no false green). A trap
+          # writes a filtered device log (embedded Python + native crashes only,
+          # everything else silenced) to a file on exit — uploaded as an
+          # artifact below instead of flooding the console.
+          script: >-
+            adb logcat -G 16M || true ;
+            adb logcat -c || true ;
+            trap 'adb logcat -d -v time flet.python:V python:V AndroidRuntime:E DEBUG:F libc:F "*:S" > "$RUNNER_TEMP/android-logcat.txt" 2>&1 || true' EXIT ;
+            uv run flet test android -d emulator-5554 --python-version ${PYTHON_VERSION} --yes -v
+
+      # Upload the device log as an artifact (don't stream it to the console).
+      - name: Upload android logcat
+        if: always() && matrix.platform == 'android'
+        uses: actions/upload-artifact@v4
+        with:
+          name: android-logcat
+          path: ${{ runner.temp }}/android-logcat.txt
+          if-no-files-found: ignore

client/integration_test/app_test.dart

@@ -1,45 +1,7 @@
-import 'dart:io';
-
+import 'package:flet_integration_test/flet_integration_test.dart';
 import 'package:flet_client/main.dart' as app;
-import 'package:flutter/foundation.dart';
-import 'package:flutter_test/flutter_test.dart';
-import 'package:integration_test/integration_test.dart';
-
-import 'flutter_tester.dart';
-
-void main() {
-  var binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized();
-
-  group('end-to-end test', () {
-    testWidgets('test app', (tester) async {
-      var dir = Directory.current.path;
-      debugPrint("Current dir: $dir");
-
-      app.tester = FlutterWidgetTester(tester, binding);
-
-      List<String> args = [];
-      const fletTestAppUrl = String.fromEnvironment("FLET_TEST_APP_URL");
-      if (fletTestAppUrl != "") {
-        args.add(fletTestAppUrl);
-      }
-
-      const fletTestPidFile = String.fromEnvironment("FLET_TEST_PID_FILE_PATH");
-      if (fletTestPidFile != "") {
-        args.add(fletTestPidFile);
-      }
-
-      const fletTestAssetsDir = String.fromEnvironment("FLET_TEST_ASSETS_DIR");
-      if (fletTestAssetsDir != "") {
-        args.add(fletTestAssetsDir);
-      }
-
-      app.main(args);
 
-      await Future.delayed(const Duration(milliseconds: 500));
-      await app.tester?.pump(duration: const Duration(seconds: 1));
-      await app.tester
-          ?.pumpAndSettle(duration: const Duration(milliseconds: 100));
-      await app.tester?.waitForTeardown();
-    });
-  });
-}
+void main() => runFletHostTest(
+      appMain: app.main,
+      assignTester: (t) => app.tester = t,
+    );

client/linux/my_application.cc

@@ -59,6 +59,11 @@ static void my_application_activate(GApplication* application) {
   gtk_widget_show(GTK_WIDGET(view));
   gtk_container_add(GTK_CONTAINER(window), GTK_WIDGET(view));
 
+  // Realize the Flutter view while the top-level window is still hidden; this
+  // lets Flutter render its first frames and lets integration tests attach
+  // before Dart decides whether the app window should be shown.
+  gtk_widget_realize(GTK_WIDGET(view));
+
   fl_register_plugins(FL_PLUGIN_REGISTRY(view));
 
   gtk_widget_grab_focus(GTK_WIDGET(view));

client/pubspec.yaml

@@ -99,10 +99,21 @@ dependency_overrides:
     path: ../packages/flet
   screen_retriever: 0.2.1 # this one migrated to SPM, but 0.2.0 is used by some other packages
 
+  screen_brightness: 2.1.7
+  # screen_brightness_macos 2.1.3 ("Fix: swift package manager warning") ships a
+  # Package.swift declaring macOS 10.11, below FlutterFramework's 10.15 floor,
+  # which breaks Swift Package Manager resolution. Pin the last good impl until
+  # upstream fixes it: https://github.com/aaassseee/screen_brightness/issues/99
+  screen_brightness_macos: 2.1.2
+
 dev_dependencies:
   flutter_test:
     sdk: flutter
 
+  # Flet integration-test driver (used by integration_test/app_test.dart).
+  flet_integration_test:
+    path: ../packages/flet_integration_test
+
   # The "flutter_lints" package below contains a set of recommended lints to
   # encourage good coding practices. The lint set provided by the package is
   # activated in the `analysis_options.yaml` file located at the root of your

packages/flet/lib/src/flet_backend.dart

@@ -152,7 +152,7 @@ class FletBackend extends ChangeNotifier {
       "web": kIsWeb,
       "debug": kDebugMode,
       "wasm": const bool.fromEnvironment('dart.tool.dart2wasm'),
-      "test": tester != null,
+      "test": tester != null || const bool.fromEnvironment("FLET_TEST"),
       "multi_view": multiView,
       "pyodide": isPyodideMode(),
       "window": {

packages/flet/lib/src/utils.dart

@@ -6,7 +6,10 @@ import 'package:window_manager/window_manager.dart';
 
 import 'utils/platform.dart';
 
-Future setupDesktop({bool hideWindowOnStart = false}) async {
+Future setupDesktop({
+  bool hideWindowOnStart = false,
+  bool waitUntilReadyToShow = true,
+}) async {
   if (isDesktopPlatform()) {
     WidgetsFlutterBinding.ensureInitialized();
     await windowManager.ensureInitialized();
@@ -16,6 +19,10 @@ Future setupDesktop({bool hideWindowOnStart = false}) async {
     debugPrint("hideWindowOnStart: $hideWindowOnStart");
     debugPrint("hideWindowOnStartEnv: $hideWindowOnStartEnv");
 
+    if (!waitUntilReadyToShow) {
+      return;
+    }
+
     await windowManager.waitUntilReadyToShow(null, () async {
       if (hideWindowOnStartEnv == null && !hideWindowOnStart) {
         await windowManager.show();

packages/flet_integration_test/.gitignore

@@ -0,0 +1,32 @@
+# Miscellaneous
+*.class
+*.log
+*.pyc
+*.swp
+.DS_Store
+.atom/
+.buildlog/
+.history
+.svn/
+migrate_working_dir/
+
+# IntelliJ related
+*.iml
+*.ipr
+*.iws
+.idea/
+
+# The .vscode folder contains launch configuration and tasks you configure in
+# VS Code which you may wish to be included in version control, so this line
+# is commented out by default.
+#.vscode/
+
+# Flutter/Dart/Pub related
+# Libraries should not include pubspec.lock, per https://dart.dev/guides/libraries/private-files#pubspeclock.
+/pubspec.lock
+**/doc/api/
+.dart_tool/
+.packages
+build/
+.flutter-plugins
+.flutter-plugins-dependencies

packages/flet_integration_test/analysis_options.yaml

@@ -0,0 +1,4 @@
+include: package:flutter_lints/flutter.yaml
+
+# Additional information about this file can be found at
+# https://dart.dev/guides/language/analysis-options

packages/flet_integration_test/lib/flet_integration_test.dart

@@ -0,0 +1,14 @@
+/// Flet integration-testing driver.
+///
+/// This is a separate package (not part of `package:flet`) because it depends
+/// on the dev-only `flutter_test` and `integration_test` packages, which must
+/// never enter a normal Flet app's runtime dependency graph. Depend on it only
+/// from a Flutter integration test (under `integration_test/`), where those
+/// packages are available.
+library flet_integration_test;
+
+export 'src/device_test.dart';
+export 'src/flutter_test_finder.dart';
+export 'src/flutter_tester.dart';
+export 'src/host_test.dart';
+export 'src/remote_widget_tester.dart';