fix(test-runner): make junit report delivery resilient on starved CI VMs

.github/actions/collect-test-diagnostics/action.yml

@@ -0,0 +1,50 @@
+name: Collect test diagnostics
+description: >
+  Gather what a failed simulator test run leaves outside the xcresult bundle:
+  host-side crash reports (.ips), CoreSimulator logs, and — when a device is
+  still booted — a targeted slice of the unified log. The xcresult itself
+  already carries the app's stdout/stderr and testmanagerd logs
+  (xcrun xcresulttool export diagnostics), so none of that is duplicated here.
+inputs:
+  test-folder:
+    description: Folder the diagnostics directory is created in
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Collect crash reports and simulator logs
+      shell: bash
+      run: |
+        DIAG="${{ inputs.test-folder }}/diagnostics"
+        mkdir -p "$DIAG"
+        # Crashes of simulator apps land in the *host's* DiagnosticReports.
+        cp -R ~/Library/Logs/DiagnosticReports/. "$DIAG/DiagnosticReports/" 2>/dev/null || true
+        cp -R ~/Library/Logs/CoreSimulator/. "$DIAG/CoreSimulator/" 2>/dev/null || true
+        # Only a booted device can serve its unified log; never boot one
+        # post-mortem — a fresh boot may pick the wrong device and stalls the
+        # job for minutes on an already-degraded VM. The pipeline must not leak
+        # grep's exit status: this shell runs with -e -o pipefail.
+        UDID="$(xcrun simctl list devices booted | grep -oE '[0-9A-Fa-f-]{36}' | head -1 || true)"
+        if [ -n "$UDID" ]; then
+          echo "Collecting unified log slice from booted simulator $UDID"
+          # simctl blocks forever if CoreSimulatorService is wedged — the very
+          # state this step tends to run in — so hard-cap it.
+          xcrun simctl spawn "$UDID" log show --style compact --last 45m \
+            --predicate 'process CONTAINS "TestRunner"' \
+            > "$DIAG/simulator-unified.log" 2>/dev/null &
+          LOG_PID=$!
+          ( sleep 120 && kill -9 "$LOG_PID" ) 2>/dev/null &
+          KILLER_PID=$!
+          wait "$LOG_PID" || echo "log show failed or was killed after 120s"
+          kill "$KILLER_PID" 2>/dev/null || true
+        else
+          echo "No booted simulator; relying on the xcresult's captured app output."
+        fi
+        echo "Collected diagnostics:"
+        du -sh "$DIAG"/* 2>/dev/null || true
+    - name: Upload test diagnostics
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+      with:
+        name: test-diagnostics
+        path: ${{ inputs.test-folder }}/diagnostics
+        if-no-files-found: ignore

.github/workflows/npm_release.yml

@@ -168,19 +168,30 @@ jobs:
           timeout_minutes: 40
           max_attempts: 2
           command: set -o pipefail && xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test | xcpretty
-          on_retry_command: rm -rf $TEST_FOLDER/test_results* && xcrun simctl shutdown all
+          # Keep the failed attempt's bundle — it holds the diagnostics of the
+          # failure being retried. Everything else at the result path must go
+          # (including extensionless staging leftovers), or the retry dies with
+          # "Existing file at -resultBundlePath".
+          on_retry_command: rm -rf $TEST_FOLDER/test_results_attempt1.xcresult; mv $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test_results_attempt1.xcresult 2>/dev/null; for f in $TEST_FOLDER/test_results*; do [ "$f" = "$TEST_FOLDER/test_results_attempt1.xcresult" ] || rm -rf "$f"; done; xcrun simctl shutdown all
           new_command_on_retry: xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test
       - name: Validate Test Results
         run: |
           xcparse attachments $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test-out
           find $TEST_FOLDER/test-out -name "*junit*.xml" -maxdepth 1 -print0 | xargs -n 1 -0 npx junit-cli-report-viewer
           find $TEST_FOLDER/test-out -name "*junit*.xml" -maxdepth 1 -print0 | xargs -n 1 -0 npx verify-junit-xml
+      - name: Collect test diagnostics
+        if: failure()
+        uses: ./.github/actions/collect-test-diagnostics
+        with:
+          test-folder: ${{env.TEST_FOLDER}}
       - name: Archive Test Result Data
         if: always()
         uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: test-results
-          path: ${{env.TEST_FOLDER}}/test_results.xcresult
+          path: |
+            ${{env.TEST_FOLDER}}/test_results.xcresult
+            ${{env.TEST_FOLDER}}/test_results_attempt1.xcresult
 
   # Publish the xcframework zips + dSYMs as a GitHub Release asset for EVERY
   # version (prerelease unless 'latest'), because the SwiftPM binaryTarget url

.github/workflows/pull_request.yml

@@ -87,16 +87,27 @@ jobs:
           timeout_minutes: 40
           max_attempts: 2
           command: set -o pipefail && xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test | xcpretty
-          on_retry_command: rm -rf $TEST_FOLDER/test_results* && xcrun simctl shutdown all
+          # Keep the failed attempt's bundle — it holds the diagnostics of the
+          # failure being retried. Everything else at the result path must go
+          # (including extensionless staging leftovers), or the retry dies with
+          # "Existing file at -resultBundlePath".
+          on_retry_command: rm -rf $TEST_FOLDER/test_results_attempt1.xcresult; mv $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test_results_attempt1.xcresult 2>/dev/null; for f in $TEST_FOLDER/test_results*; do [ "$f" = "$TEST_FOLDER/test_results_attempt1.xcresult" ] || rm -rf "$f"; done; xcrun simctl shutdown all
           new_command_on_retry: xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test
       - name: Validate Test Results
         run: |
           xcparse attachments $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test-out
           find $TEST_FOLDER/test-out -name "*junit*.xml" -maxdepth 1 -print0 | xargs -n 1 -0 npx junit-cli-report-viewer
           find $TEST_FOLDER/test-out -name "*junit*.xml" -maxdepth 1 -print0 | xargs -n 1 -0 npx verify-junit-xml
+      - name: Collect test diagnostics
+        if: failure()
+        uses: ./.github/actions/collect-test-diagnostics
+        with:
+          test-folder: ${{env.TEST_FOLDER}}
       - name: Archive Test Result Data
         if: always()
         uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: test-results
-          path: ${{env.TEST_FOLDER}}/test_results.xcresult
+          path: |
+            ${{env.TEST_FOLDER}}/test_results.xcresult
+            ${{env.TEST_FOLDER}}/test_results_attempt1.xcresult

TestRunner/app/tests/index.js

@@ -1,6 +1,9 @@
 // Inform the test results runner that the runtime is up.
 console.log('Application Start!');
 
+// The report delivery deadline (REPORT_DEADLINE_SECONDS) counts from launch.
+var appStartMs = Date.now();
+
 require("../Infrastructure/timers");
 require("../Infrastructure/simulator");
 global.utf8 = require("../Infrastructure/utf8")
@@ -22,17 +25,72 @@ global.__JUnitSaveResults = function (text) {
     }
 
     var reportUrl = NSProcessInfo.processInfo.environment.objectForKey("REPORT_BASEURL");
-    if (reportUrl) {
-        var urlRequest = NSMutableURLRequest.requestWithURL(NSURL.URLWithString(reportUrl));
+    if (!reportUrl) {
+        return;
+    }
+
+    // The results host (TestRunnerTests.swift) waits on this single POST; if it
+    // is lost the whole run times out. Retry hard, but stop early enough that a
+    // final delivery_failed sentinel can still reach the host before its wait
+    // budget (which also covers the suite itself) expires.
+    var maxAttempts = 10;
+    var retryDelayMs = 5000;
+    var requestTimeoutS = 30;
+    var deadlineSeconds = parseInt(NSProcessInfo.processInfo.environment.objectForKey("REPORT_DEADLINE_SECONDS"), 10) || 540;
+    var deadlineMs = appStartMs + deadlineSeconds * 1000;
+
+    var sessionConfig = NSURLSessionConfiguration.defaultSessionConfiguration;
+    sessionConfig.timeoutIntervalForRequest = requestTimeoutS;
+    var session = NSURLSession.sessionWithConfigurationDelegateDelegateQueue(sessionConfig, null, NSOperationQueue.mainQueue);
+
+    function post(url, body, onFailure) {
+        var urlRequest = NSMutableURLRequest.requestWithURL(NSURL.URLWithString(url));
         urlRequest.HTTPMethod = "POST";
         urlRequest.setValueForHTTPHeaderField("Content-Type", "application/xml");
-        urlRequest.HTTPBody = NSString.stringWithString(text).dataUsingEncoding(4);
-        var sessionConfig = NSURLSessionConfiguration.defaultSessionConfiguration;
-        var queue = NSOperationQueue.mainQueue;
-        var session = NSURLSession.sessionWithConfigurationDelegateDelegateQueue(sessionConfig, null, queue);
-        var dataTask = session.dataTaskWithRequestCompletionHandler(urlRequest, (data, response, error) => { });
+        urlRequest.HTTPBody = NSString.stringWithString(body).dataUsingEncoding(4);
+        var dataTask = session.dataTaskWithRequestCompletionHandler(urlRequest, (data, response, error) => {
+            if (error) {
+                onFailure("error: " + error.localizedDescription);
+            } else if (!response || response.statusCode < 200 || response.statusCode >= 300) {
+                onFailure("status: " + (response ? response.statusCode : "none"));
+            }
+        });
         dataTask.resume();
     }
+
+    // Best-effort sentinel so the host can fail immediately with a delivery
+    // diagnostic instead of waiting out its full timeout.
+    function sendDeliveryFailed(attempts, reason) {
+        post(reportUrl + "/delivery_failed",
+            "junit report delivery failed after " + attempts + " attempts; last failure " + reason,
+            function () { });
+    }
+
+    function attemptReport(attempt) {
+        post(reportUrl, text, function (reason) {
+            console.log("junit report POST failed (attempt " + attempt + "/" + maxAttempts + ", " + reason + ")");
+            if (attempt >= maxAttempts) {
+                sendDeliveryFailed(attempt, reason);
+                return;
+            }
+            setTimeout(function () {
+                // Deadline is checked when the timer fires, not when it is
+                // scheduled: under load timers run late, and a retry started
+                // past this window would eat the sentinel's slot.
+                var worstCaseRetryMs = 2 * requestTimeoutS * 1000;
+                if (Date.now() + worstCaseRetryMs <= deadlineMs) {
+                    attemptReport(attempt + 1);
+                } else {
+                    sendDeliveryFailed(attempt, reason);
+                }
+            }, retryDelayMs);
+        });
+    }
+
+    // The first attempt intentionally has no deadline guard: even past the
+    // deadline a one-shot report is still the most valuable outcome, and a
+    // late fulfill is harmless to the host.
+    attemptReport(1);
 };
 
 global.__approot = NSString.stringWithString(NSBundle.mainBundle.bundlePath).stringByResolvingSymlinksInPath;

TestRunnerTests/TestRunnerTests.swift

@@ -5,6 +5,7 @@ class TestRunnerTests: XCTestCase {
     private var loop: EventLoop!
     private var server: HTTPServer!
     private var runtimeUnitTestsExpectation: XCTestExpectation!
+    private var reportDeliveryFailureReason: String?
 
     override func setUp() {
         continueAfterFailure = false
@@ -29,6 +30,9 @@ class TestRunnerTests: XCTestCase {
                 sendBody(Data())
                 self.runtimeUnitTestsExpectation.fulfill()
             } else {
+                // The app POSTs here when it exhausted its report retries; the
+                // payload is a human-readable reason instead of junit XML.
+                let isDeliveryFailure = (environ["PATH_INFO"] as? String)?.hasSuffix("/delivery_failed") ?? false
                 var buffer = Data()
                 let input = environ["swsgi.input"] as! SWSGIInput
                 var finished = false
@@ -37,9 +41,13 @@ class TestRunnerTests: XCTestCase {
                     if data.isEmpty && !finished {
                         finished = true
 
-                        let report = XCTAttachment(uniformTypeIdentifier: "junit.xml", name: "junit.xml", payload: buffer, userInfo: nil)
-                        report.lifetime = .keepAlways
-                        self.add(report)
+                        if isDeliveryFailure {
+                            self.reportDeliveryFailureReason = String(data: buffer, encoding: .utf8) ?? "unknown"
+                        } else {
+                            let report = XCTAttachment(uniformTypeIdentifier: "junit.xml", name: "junit.xml", payload: buffer, userInfo: nil)
+                            report.lifetime = .keepAlways
+                            self.add(report)
+                        }
 
                         startResponse("204 No Content", [])
                         sendBody(Data())
@@ -51,7 +59,10 @@ class TestRunnerTests: XCTestCase {
 
         try! server.start()
 
-        DispatchQueue.global(qos: .background).async {
+        // Not .background: CI VMs throttle background-QoS threads hard enough
+        // to starve this accept loop, and it is the only channel through which
+        // results ever arrive.
+        DispatchQueue.global(qos: .userInitiated).async {
             self.loop.runForever()
         }
     }
@@ -72,6 +83,10 @@ class TestRunnerTests: XCTestCase {
 
         let app = XCUIApplication()
         app.launchEnvironment["REPORT_BASEURL"] = "http://[::1]:\(port)/junit_report"
+        // The app's report retries and delivery_failed sentinel count from its
+        // launch, which precedes the wait below — keep a margin so delivery
+        // gives up (and the sentinel lands) before our timeout fires.
+        app.launchEnvironment["REPORT_DEADLINE_SECONDS"] = String(Int(jasmineTestsTimeout) - 60)
         app.launch()
 
         // Watchdog: if the runtime crashes (e.g. EXC_BAD_ACCESS) it never
@@ -100,6 +115,8 @@ class TestRunnerTests: XCTestCase {
         case .completed:
             if didCrash {
                 XCTFail("TestRunner exited before reporting Jasmine results (likely crashed). Check ~/Library/Logs/DiagnosticReports/TestRunner-*.ips for the stack.")
+            } else if let reason = reportDeliveryFailureReason {
+                XCTFail("TestRunner finished the Jasmine suite but could not deliver the junit report: \(reason)")
             }
             return
         case .timedOut: