test(e2e): shard maestro flows across two devices per job

.changeset/maestro-ci-hardening.md

@@ -0,0 +1,2 @@
+---
+---

.github/workflows/expo-native-build.yml

@@ -1,4 +1,4 @@
-name: 'Expo native (@clerk/expo)'
+name: Expo
 
 on:
   pull_request:
@@ -27,12 +27,15 @@ env:
   SDK_PACK_DIR: /tmp/clerk-expo-pack
   E2E_INSTANCE_NAME: clerkstage-with-native-components
   BAPI_URL: https://api.clerkstage.dev
-  MAESTRO_VERSION: '2.8.0'
+  MAESTRO_VERSION: '2.10.0'
+  MAESTRO_CLI_NO_ANALYTICS: '1'
+  MAESTRO_SHARDS: '2'
+  MAESTRO_DRIVER_STARTUP_TIMEOUT: '180000'
 
 jobs:
   native-build:
     if: ${{ github.head_ref != 'changeset-release/main' }}
-    name: Expo ${{ matrix.expo-sdk }} ${{ matrix.run-e2e == true && 'Build + E2E' || 'Build' }} / ${{ matrix.platform }}
+    name: Native ${{ matrix.run-e2e == true && 'E2E' || 'Build' }} (${{ matrix.platform }}, sdk ${{ matrix.expo-sdk }})
     runs-on: ${{ matrix.runner }}
     timeout-minutes: ${{ matrix.run-e2e == true && 60 || 45 }}
     strategy:
@@ -223,6 +226,10 @@ jobs:
           path: ${{ steps.native-build-key.outputs.artifact }}
           key: ${{ steps.native-build-cache.outputs.cache-primary-key }}
 
+      - name: Boot iOS simulators
+        if: matrix.platform == 'ios' && matrix.run-e2e == true
+        run: ${{ env.E2E_DIR }}/boot-ios-simulators.sh boot "$MAESTRO_SHARDS"
+
       - name: Cache maestro CLI
         if: steps.keys.outputs.pk != ''
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
@@ -233,29 +240,17 @@ jobs:
       - name: Install maestro CLI
         if: steps.keys.outputs.pk != ''
         run: |
-          set -o pipefail
           if [ -x "$HOME/.maestro/bin/maestro" ]; then
             echo "Using cached Maestro"
           else
-            installed=0
-            for i in 1 2 3; do
-              if curl -fLs --retry 3 --retry-delay 5 "https://get.maestro.mobile.dev" | MAESTRO_VERSION="$MAESTRO_VERSION" bash; then
-                if [ -x "$HOME/.maestro/bin/maestro" ]; then installed=1; break; fi
-              fi
-              echo "Maestro install attempt $i failed (or binary missing); retrying"
-              sleep 5
-            done
-            [ "$installed" = 1 ] || { echo "::error::Maestro install failed after 3 attempts"; exit 1; }
+            archive="$RUNNER_TEMP/maestro.zip"
+            curl -fLs --retry 3 --retry-delay 5 -o "$archive" \
+              "https://github.com/mobile-dev-inc/Maestro/releases/download/cli-$MAESTRO_VERSION/maestro.zip"
+            unzip -q "$archive" -d "$RUNNER_TEMP/maestro-dist"
+            mv "$RUNNER_TEMP/maestro-dist/maestro" "$HOME/.maestro"
           fi
           echo "$HOME/.maestro/bin" >> "$GITHUB_PATH"
-          "$HOME/.maestro/bin/maestro" --version
-
-      - name: Boot iOS simulator
-        id: sim
-        if: matrix.platform == 'ios' && steps.keys.outputs.pk != ''
-        uses: futureware-tech/simulator-action@e89aa8f93d3aec35083ff49d2854d07f7186f7f5 # v5
-        with:
-          model: 'iPhone 17 Pro'
+          test "$("$HOME/.maestro/bin/maestro" --version)" = "$MAESTRO_VERSION"
 
       # Provisioned as late as possible so live credentials exist for the
       # shortest window.
@@ -293,35 +288,21 @@ jobs:
         env:
           CLERK_TEST_EMAIL: ${{ steps.user.outputs.email }}
           CLERK_TEST_PASSWORD: ${{ steps.user.outputs.password }}
-          SIM_UDID: ${{ steps.sim.outputs.udid }}
         run: |
-          echo "Using simulator $SIM_UDID"
-          # Kill animations + predictive keyboard: animations add latency to
-          # every tap; predictive text hijacks inputText targets.
-          xcrun simctl spawn "$SIM_UDID" defaults write com.apple.UIKit UIAnimationDragCoefficient -float 0.01 || true
-          xcrun simctl spawn "$SIM_UDID" defaults write -g ApplePersistenceIgnoreState -bool YES || true
-          xcrun simctl spawn "$SIM_UDID" defaults write com.apple.keyboard.ContinuousPath -bool NO || true
-          xcrun simctl spawn "$SIM_UDID" defaults write com.apple.keyboard.AutoCapitalization -bool NO || true
-          xcrun simctl spawn "$SIM_UDID" defaults write com.apple.keyboard.AutoCorrection -bool NO || true
-          xcrun simctl spawn "$SIM_UDID" defaults write com.apple.keyboard.Prediction -bool NO || true
-          # The one-time keyboard tutorial sheets carry their own Continue
-          # button, which can hijack taps on the AuthView's Continue.
-          for key in DidShowContinuousPathIntroduction DidShowGestureKeyboardIntroduction KeyboardDidShowProductivityTutorial UIKeyboardDidShowInternationalInfoIntroduction; do
-            xcrun simctl spawn "$SIM_UDID" defaults write com.apple.keyboard.preferences "$key" -bool YES || true
-          done
-          xcrun simctl install "$SIM_UDID" ios/build/Build/Products/Release-iphonesimulator/ClerkExpoNativeBuildFixture.app
-          # Stream the app's console output into the debug artifact so a hang has
-          # actionable evidence (keychain/network errors) instead of just screenshots.
+          "$GITHUB_WORKSPACE/$E2E_DIR/boot-ios-simulators.sh" wait
           mkdir -p "$RUNNER_TEMP/maestro-debug"
-          xcrun simctl spawn "$SIM_UDID" log stream --style compact \
-            --predicate 'processImagePath CONTAINS "ClerkExpoNativeBuildFixture"' \
-            > "$RUNNER_TEMP/maestro-debug/sim-console.log" 2>&1 &
-          LOG_PID=$!
-          cd "$GITHUB_WORKSPACE/$E2E_DIR"
+          log_pids=()
+          for udid in ${MAESTRO_UDID//,/ }; do
+            xcrun simctl install "$udid" ios/build/Build/Products/Release-iphonesimulator/ClerkExpoNativeBuildFixture.app
+            xcrun simctl spawn "$udid" log stream --style compact \
+              --predicate 'processImagePath CONTAINS "ClerkExpoNativeBuildFixture"' \
+              > "$RUNNER_TEMP/maestro-debug/$udid-console.log" 2>&1 &
+            log_pids+=($!)
+          done
           rc=0
-          MAESTRO_DEBUG_OUTPUT="$RUNNER_TEMP/maestro-debug" ./run-flows.sh \
-            xcrun simctl terminate "$SIM_UDID" com.clerk.exponativebuildfixture || rc=$?
-          kill "$LOG_PID" 2>/dev/null || true
+          PLATFORM=ios MAESTRO_DEBUG_OUTPUT="$RUNNER_TEMP/maestro-debug" \
+            "$GITHUB_WORKSPACE/$E2E_DIR/run-flows.sh" || rc=$?
+          kill "${log_pids[@]}" 2>/dev/null || true
           exit "$rc"
 
       - name: Enable KVM
@@ -365,7 +346,8 @@ jobs:
           target: google_apis
           arch: x86_64
           force-avd-creation: false
-          emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim
+          # read-only: a writable instance holds a lock the shard emulators wait on forever
+          emulator-options: -read-only -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim
           disable-animations: true
           # The action runs each script line in a separate sh -c; the folded
           # scalar (>-) plus && keeps everything in one shell invocation.
@@ -386,9 +368,11 @@ jobs:
             -H "Authorization: Bearer $CLERK_SECRET_KEY" || true
 
       # Test reports record flow env (and typed input) in plaintext;
-      # add-mask only covers step logs, not artifact contents.
+      # add-mask only covers step logs, not artifact contents. Must run whenever
+      # the upload does, or a passing run uploads the password.
       - name: Scrub test credentials from Maestro debug output
-        if: always() && (steps.run_e2e_ios.outcome == 'failure' || steps.run_e2e_android.outcome == 'failure')
+        id: scrub
+        if: always() && steps.user.outputs.user_id != ''
         env:
           CLERK_TEST_PASSWORD: ${{ steps.user.outputs.password }}
         run: |
@@ -399,12 +383,13 @@ jobs:
             \( -name '*.html' -o -name '*.json' -o -name '*.log' -o -name '*.txt' -o -name '*.xml' -o -name '*.yaml' \) \
             -exec perl -pi -e 's/\Q$ENV{CLERK_TEST_PASSWORD}\E/[REDACTED]/g' {} +
 
-      - name: Upload Maestro artifacts on e2e failure
-        if: always() && (steps.run_e2e_ios.outcome == 'failure' || steps.run_e2e_android.outcome == 'failure')
+      - name: Upload Maestro artifacts
+        if: always() && steps.scrub.outcome == 'success'
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: maestro-${{ matrix.platform }}
           path: ${{ runner.temp }}/maestro-debug
+          if-no-files-found: warn
           retention-days: 7
 
       - name: Report e2e outcome

integration/templates/expo-native/App.tsx

@@ -17,6 +17,7 @@ function NativeBuildFixture() {
   const { user } = useUser();
   const [isAuthOpen, setIsAuthOpen] = useState(false);
   const [isProfileOpen, setIsProfileOpen] = useState(false);
+  const [signOutResult, setSignOutResult] = useState<string | null>(null);
 
   if (isProfileOpen) {
     return (
@@ -56,6 +57,7 @@ function NativeBuildFixture() {
 
       <Text testID='auth-state'>{isLoaded ? `signed ${isSignedIn ? 'in' : 'out'}` : 'loading'}</Text>
       {user?.id && <Text testID='user-id'>{user.id}</Text>}
+      {signOutResult && <Text testID='sign-out-result'>{`sign out: ${signOutResult}`}</Text>}
       <Button
         testID='open-auth-view-button'
         title='Open native AuthView'
@@ -73,7 +75,14 @@ function NativeBuildFixture() {
         <Button
           testID='sign-out-button'
           title='Sign out'
-          onPress={() => void signOut()}
+          onPress={() => {
+            setSignOutResult('pending');
+            signOut().then(
+              () => setSignOutResult('ok'),
+              (error: unknown) =>
+                setSignOutResult((error instanceof Error ? error.message : String(error)).replace(/\s+/g, ' ')),
+            );
+          }}
         />
       )}
 

integration/tests/expo-native/boot-ios-simulators.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Usage: ./boot-ios-simulators.sh boot <count> [model] | wait
+set -euo pipefail
+
+boot() {
+  local count=$1 model=${2:-iPhone 17 Pro}
+  local runtime device_type udids=() i
+  # pinned to the SDK the app was built with; runners may also carry a newer beta runtime
+  sdk=$(xcrun --sdk iphonesimulator --show-sdk-version)
+  runtime=$(xcrun simctl list runtimes available -j | jq -r --arg v "$sdk" \
+    '[.runtimes[] | select(.platform == "iOS")] | (map(select(.version == $v)) + .) | first | .identifier')
+  device_type=$(xcrun simctl list devicetypes -j | jq -r --arg m "$model" '.devicetypes[] | select(.name == $m) | .identifier')
+  for i in $(seq 1 "$count"); do
+    udids+=("$(xcrun simctl create "$model $i" "$device_type" "$runtime")")
+    xcrun simctl boot "${udids[$((i - 1))]}"
+  done
+  MAESTRO_UDID=$(IFS=,; echo "${udids[*]}")
+  echo "Booting $count x $model on $runtime: $MAESTRO_UDID"
+  if [ -n "${GITHUB_ENV:-}" ]; then echo "MAESTRO_UDID=$MAESTRO_UDID" >> "$GITHUB_ENV"; fi
+}
+
+wait_ready() {
+  local udid key
+  IFS=, read -r -a udids <<< "${MAESTRO_UDID:?MAESTRO_UDID is required}"
+  for udid in "${udids[@]}"; do
+    xcrun simctl bootstatus "$udid" -b &
+    local pid=$! elapsed=0
+    while kill -0 "$pid" 2>/dev/null; do
+      if [ "$elapsed" -ge 240 ]; then
+        kill "$pid" 2>/dev/null || true
+        wait "$pid" 2>/dev/null || true
+        echo "::error::$udid did not finish booting within ${elapsed}s"
+        xcrun simctl list devices -j | jq -r --arg u "$udid" '.devices[][] | select(.udid == $u) | "\(.name): \(.state)"'
+        return 1
+      fi
+      sleep 5
+      elapsed=$((elapsed + 5))
+    done
+    wait "$pid"
+    xcrun simctl spawn "$udid" defaults write com.apple.UIKit UIAnimationDragCoefficient -float 0.01 || true
+    xcrun simctl spawn "$udid" defaults write -g ApplePersistenceIgnoreState -bool YES || true
+    xcrun simctl spawn "$udid" defaults write com.apple.keyboard.ContinuousPath -bool NO || true
+    xcrun simctl spawn "$udid" defaults write com.apple.keyboard.AutoCapitalization -bool NO || true
+    xcrun simctl spawn "$udid" defaults write com.apple.keyboard.AutoCorrection -bool NO || true
+    xcrun simctl spawn "$udid" defaults write com.apple.keyboard.Prediction -bool NO || true
+    # the keyboard tutorial sheets have their own Continue button that steals taps
+    for key in DidShowContinuousPathIntroduction DidShowGestureKeyboardIntroduction KeyboardDidShowProductivityTutorial UIKeyboardDidShowInternationalInfoIntroduction; do
+      xcrun simctl spawn "$udid" defaults write com.apple.keyboard.preferences "$key" -bool YES || true
+    done
+  done
+}
+
+case "${1:-}" in
+  boot) boot "${2:?count is required}" "${3:-}" ;;
+  wait) wait_ready ;;
+  *) echo "usage: $0 boot <count> [model] | wait" >&2; exit 2 ;;
+esac

integration/tests/expo-native/flows/sign-in.yaml

@@ -1,13 +1,19 @@
-# Native AuthView email+password sign-in, asserting the native->JS session
-# sync, then JS-side sign-out asserting the reverse direction.
+# Native sign-in observed by JS, kept across a restart without clearing state, then JS sign-out.
 appId: com.clerk.exponativebuildfixture
-name: Native AuthView sign-in syncs to JS
+name: Native AuthView sign-in syncs to JS and survives a restart
 ---
 - runFlow: subflows/open-app.yaml
 - tapOn:
     id: 'open-auth-view-button'
 - runFlow: subflows/sign-in-email-password.yaml
 - runFlow: subflows/assert-signed-in.yaml
+- stopApp
+- launchApp
+- extendedWaitUntil:
+    visible: 'signed in'
+    timeout: 45000
+- assertVisible:
+    id: 'user-id'
 - tapOn:
     id: 'sign-out-button'
 - runFlow: subflows/assert-signed-out.yaml

integration/tests/expo-native/flows/subflows/_warmup.yaml

@@ -3,10 +3,14 @@
 # flake its in-flow waits. Excluded from the per-flow loop (lives in subflows/).
 appId: com.clerk.exponativebuildfixture
 ---
-- launchApp:
-    clearState: true
-    clearKeychain: true
-- extendedWaitUntil:
-    visible:
-      id: 'auth-state'
-    timeout: 90000
+# Android can kill a fresh launch when pm clear's task removal lands after am start.
+- retry:
+    maxRetries: 1
+    commands:
+      - launchApp:
+          clearState: true
+          clearKeychain: true
+      - extendedWaitUntil:
+          visible:
+            id: 'auth-state'
+          timeout: 90000

integration/tests/expo-native/flows/subflows/open-app.yaml

@@ -1,13 +1,17 @@
 # Subflow: launch from a clean state and land signed out.
 appId: com.clerk.exponativebuildfixture
 ---
-- launchApp:
-    clearState: true
-    clearKeychain: true
-- extendedWaitUntil:
-    visible:
-      id: 'auth-state'
-    timeout: 30000
+# Android can kill a fresh launch when pm clear's task removal lands after am start.
+- retry:
+    maxRetries: 1
+    commands:
+      - launchApp:
+          clearState: true
+          clearKeychain: true
+      - extendedWaitUntil:
+          visible:
+            id: 'auth-state'
+          timeout: 30000
 # Text-only matching from here on: iOS failed combined id+text filters on RN
 # Text elements in CI while id-only and text-only both match. Also wait out
 # clerk-js init ('loading') with a retrying wait; assertVisible never retries.

integration/tests/expo-native/flows/subflows/sign-in-email-password.yaml

@@ -45,7 +45,7 @@ appId: com.clerk.exponativebuildfixture
                       commands:
                         - tapOn:
                             text: 'Select All'
-            - eraseText: 100
+            - eraseText: 60
             - inputText: ${CLERK_TEST_EMAIL}
       # Wait for the typed value to land before tapping: the tap can race the
       # recomposition that enables the button, and this also catches a mangled
@@ -79,7 +79,6 @@ appId: com.clerk.exponativebuildfixture
     commands:
       - tapOn:
           text: 'Enter your password'
-      - eraseText: 50
       - inputText: ${CLERK_TEST_PASSWORD}
       # The password is masked so its value can't be asserted; settle for the
       # screen going stable so the tap doesn't race the enabling recomposition.