From cf3e3d97c51b1ed13f6d985e949aec490756c072 Mon Sep 17 00:00:00 2001 From: Eduardo Speroni Date: Thu, 30 Jul 2026 16:32:34 -0300 Subject: [PATCH 1/4] fix(test-runner): make junit report delivery resilient on starved CI VMs The Jasmine suite reports results through a single fire-and-forget POST to the Embassy server in the XCUITest host. On overloaded CI VMs that POST can be lost (the server's accept loop runs at .background QoS and gets starved), so the host waits out its full 600s budget and the job fails with the generic 'unfulfilled Jasmine tests expectation' error even though every spec passed. Same-VM step retries then fail the same way (run 30567093340). - Retry the report POST up to 10 times with a 30s per-request timeout, logging each failure to the app console so future flakes are diagnosable from the xcresult diagnostics. - Run the Embassy event loop at .userInitiated QoS instead of .background. - When the retry budget is exhausted, POST a best-effort sentinel to /delivery_failed so the host fails immediately with a delivery diagnostic instead of a 600s timeout. --- TestRunner/app/tests/index.js | 50 +++++++++++++++++++++++---- TestRunnerTests/TestRunnerTests.swift | 21 ++++++++--- 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/TestRunner/app/tests/index.js b/TestRunner/app/tests/index.js index 8fa28559..4c1720ee 100644 --- a/TestRunner/app/tests/index.js +++ b/TestRunner/app/tests/index.js @@ -22,17 +22,53 @@ global.__JUnitSaveResults = function (text) { } var reportUrl = NSProcessInfo.processInfo.environment.objectForKey("REPORT_BASEURL"); - if (reportUrl) { - var urlRequest = NSMutableURLRequest.requestWithURL(NSURL.URLWithString(reportUrl)); + if (!reportUrl) { + return; + } + + // The results host (TestRunnerTests.swift) waits on this single POST; if it + // is lost the whole run times out. Retry hard, and cap the per-request + // timeout so all attempts fit well inside the host's wait budget. + var maxAttempts = 10; + var retryDelayMs = 5000; + + var sessionConfig = NSURLSessionConfiguration.defaultSessionConfiguration; + sessionConfig.timeoutIntervalForRequest = 30; + var session = NSURLSession.sessionWithConfigurationDelegateDelegateQueue(sessionConfig, null, NSOperationQueue.mainQueue); + + function post(url, body, onFailure) { + var urlRequest = NSMutableURLRequest.requestWithURL(NSURL.URLWithString(url)); urlRequest.HTTPMethod = "POST"; urlRequest.setValueForHTTPHeaderField("Content-Type", "application/xml"); - urlRequest.HTTPBody = NSString.stringWithString(text).dataUsingEncoding(4); - var sessionConfig = NSURLSessionConfiguration.defaultSessionConfiguration; - var queue = NSOperationQueue.mainQueue; - var session = NSURLSession.sessionWithConfigurationDelegateDelegateQueue(sessionConfig, null, queue); - var dataTask = session.dataTaskWithRequestCompletionHandler(urlRequest, (data, response, error) => { }); + urlRequest.HTTPBody = NSString.stringWithString(body).dataUsingEncoding(4); + var dataTask = session.dataTaskWithRequestCompletionHandler(urlRequest, (data, response, error) => { + if (error) { + onFailure("error: " + error.localizedDescription); + } else if (!response || response.statusCode < 200 || response.statusCode >= 300) { + onFailure("status: " + (response ? response.statusCode : "none")); + } + }); dataTask.resume(); } + + function attemptReport(attempt) { + post(reportUrl, text, function (reason) { + console.log("junit report POST failed (attempt " + attempt + "/" + maxAttempts + ", " + reason + ")"); + if (attempt < maxAttempts) { + setTimeout(function () { + attemptReport(attempt + 1); + }, retryDelayMs); + } else { + // Best-effort sentinel so the host can fail immediately with a + // delivery diagnostic instead of waiting out its full timeout. + post(reportUrl + "/delivery_failed", + "junit report delivery failed after " + maxAttempts + " attempts; last failure " + reason, + function () { }); + } + }); + } + + attemptReport(1); }; global.__approot = NSString.stringWithString(NSBundle.mainBundle.bundlePath).stringByResolvingSymlinksInPath; diff --git a/TestRunnerTests/TestRunnerTests.swift b/TestRunnerTests/TestRunnerTests.swift index 37862a12..ee881571 100644 --- a/TestRunnerTests/TestRunnerTests.swift +++ b/TestRunnerTests/TestRunnerTests.swift @@ -5,6 +5,7 @@ class TestRunnerTests: XCTestCase { private var loop: EventLoop! private var server: HTTPServer! private var runtimeUnitTestsExpectation: XCTestExpectation! + private var reportDeliveryFailureReason: String? override func setUp() { continueAfterFailure = false @@ -29,6 +30,9 @@ class TestRunnerTests: XCTestCase { sendBody(Data()) self.runtimeUnitTestsExpectation.fulfill() } else { + // The app POSTs here when it exhausted its report retries; the + // payload is a human-readable reason instead of junit XML. + let isDeliveryFailure = (environ["PATH_INFO"] as? String)?.hasSuffix("/delivery_failed") ?? false var buffer = Data() let input = environ["swsgi.input"] as! SWSGIInput var finished = false @@ -37,9 +41,13 @@ class TestRunnerTests: XCTestCase { if data.isEmpty && !finished { finished = true - let report = XCTAttachment(uniformTypeIdentifier: "junit.xml", name: "junit.xml", payload: buffer, userInfo: nil) - report.lifetime = .keepAlways - self.add(report) + if isDeliveryFailure { + self.reportDeliveryFailureReason = String(data: buffer, encoding: .utf8) ?? "unknown" + } else { + let report = XCTAttachment(uniformTypeIdentifier: "junit.xml", name: "junit.xml", payload: buffer, userInfo: nil) + report.lifetime = .keepAlways + self.add(report) + } startResponse("204 No Content", []) sendBody(Data()) @@ -51,7 +59,10 @@ class TestRunnerTests: XCTestCase { try! server.start() - DispatchQueue.global(qos: .background).async { + // Not .background: CI VMs throttle background-QoS threads hard enough + // to starve this accept loop, and it is the only channel through which + // results ever arrive. + DispatchQueue.global(qos: .userInitiated).async { self.loop.runForever() } } @@ -100,6 +111,8 @@ class TestRunnerTests: XCTestCase { case .completed: if didCrash { XCTFail("TestRunner exited before reporting Jasmine results (likely crashed). Check ~/Library/Logs/DiagnosticReports/TestRunner-*.ips for the stack.") + } else if let reason = reportDeliveryFailureReason { + XCTFail("TestRunner finished the Jasmine suite but could not deliver the junit report: \(reason)") } return case .timedOut: From 6ee563c0b2e9c34ddd53b63bc2d068abcaecc2d9 Mon Sep 17 00:00:00 2001 From: Eduardo Speroni Date: Thu, 30 Jul 2026 17:00:52 -0300 Subject: [PATCH 2/4] fix(test-runner): bound report retries by a delivery deadline The host's 600s wait covers the suite run plus report delivery, so an unbounded retry ladder (up to ~345s) could still push the delivery_failed sentinel past the host timeout after a slow suite. The host now passes REPORT_DEADLINE_SECONDS (wait budget minus a 60s margin, counted from app launch) and the app stops retrying while there is still room for one final sentinel request. --- TestRunner/app/tests/index.js | 20 +++++++++++++++----- TestRunnerTests/TestRunnerTests.swift | 4 ++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/TestRunner/app/tests/index.js b/TestRunner/app/tests/index.js index 4c1720ee..7a947c52 100644 --- a/TestRunner/app/tests/index.js +++ b/TestRunner/app/tests/index.js @@ -1,6 +1,9 @@ // Inform the test results runner that the runtime is up. console.log('Application Start!'); +// The report delivery deadline (REPORT_DEADLINE_SECONDS) counts from launch. +var appStartMs = Date.now(); + require("../Infrastructure/timers"); require("../Infrastructure/simulator"); global.utf8 = require("../Infrastructure/utf8") @@ -27,13 +30,17 @@ global.__JUnitSaveResults = function (text) { } // The results host (TestRunnerTests.swift) waits on this single POST; if it - // is lost the whole run times out. Retry hard, and cap the per-request - // timeout so all attempts fit well inside the host's wait budget. + // is lost the whole run times out. Retry hard, but stop early enough that a + // final delivery_failed sentinel can still reach the host before its wait + // budget (which also covers the suite itself) expires. var maxAttempts = 10; var retryDelayMs = 5000; + var requestTimeoutS = 30; + var deadlineSeconds = parseInt(NSProcessInfo.processInfo.environment.objectForKey("REPORT_DEADLINE_SECONDS"), 10) || 540; + var deadlineMs = appStartMs + deadlineSeconds * 1000; var sessionConfig = NSURLSessionConfiguration.defaultSessionConfiguration; - sessionConfig.timeoutIntervalForRequest = 30; + sessionConfig.timeoutIntervalForRequest = requestTimeoutS; var session = NSURLSession.sessionWithConfigurationDelegateDelegateQueue(sessionConfig, null, NSOperationQueue.mainQueue); function post(url, body, onFailure) { @@ -54,7 +61,10 @@ global.__JUnitSaveResults = function (text) { function attemptReport(attempt) { post(reportUrl, text, function (reason) { console.log("junit report POST failed (attempt " + attempt + "/" + maxAttempts + ", " + reason + ")"); - if (attempt < maxAttempts) { + // Worst case a retry costs delay + full request timeout, and the + // sentinel afterwards needs another request timeout. + var worstCaseRetryMs = retryDelayMs + 2 * requestTimeoutS * 1000; + if (attempt < maxAttempts && Date.now() + worstCaseRetryMs <= deadlineMs) { setTimeout(function () { attemptReport(attempt + 1); }, retryDelayMs); @@ -62,7 +72,7 @@ global.__JUnitSaveResults = function (text) { // Best-effort sentinel so the host can fail immediately with a // delivery diagnostic instead of waiting out its full timeout. post(reportUrl + "/delivery_failed", - "junit report delivery failed after " + maxAttempts + " attempts; last failure " + reason, + "junit report delivery failed after " + attempt + " attempts; last failure " + reason, function () { }); } }); diff --git a/TestRunnerTests/TestRunnerTests.swift b/TestRunnerTests/TestRunnerTests.swift index ee881571..70ba8eb1 100644 --- a/TestRunnerTests/TestRunnerTests.swift +++ b/TestRunnerTests/TestRunnerTests.swift @@ -83,6 +83,10 @@ class TestRunnerTests: XCTestCase { let app = XCUIApplication() app.launchEnvironment["REPORT_BASEURL"] = "http://[::1]:\(port)/junit_report" + // The app's report retries and delivery_failed sentinel count from its + // launch, which precedes the wait below — keep a margin so delivery + // gives up (and the sentinel lands) before our timeout fires. + app.launchEnvironment["REPORT_DEADLINE_SECONDS"] = String(Int(jasmineTestsTimeout) - 60) app.launch() // Watchdog: if the runtime crashes (e.g. EXC_BAD_ACCESS) it never From b1138b7caa4c0d7c9f93958964c5b0a3613ebd31 Mon Sep 17 00:00:00 2001 From: Eduardo Speroni Date: Thu, 30 Jul 2026 17:00:52 -0300 Subject: [PATCH 3/4] ci: preserve first-attempt xcresult and upload failure diagnostics The in-step retry used to rm -rf the failed attempt's result bundle, so the evidence of the very failure being retried never reached the uploaded artifact. Rename it aside instead and include it in the test-results upload. On failure, a new collect-test-diagnostics composite action gathers what the xcresult does not carry: host-side crash reports (.ips), CoreSimulator logs, and a process-filtered unified-log slice from a still-booted simulator. It deliberately never boots a device post-mortem and never collects a full logarchive - the xcresult's exported diagnostics already contain the app's stdout/stderr and testmanagerd logs. --- .../collect-test-diagnostics/action.yml | 42 +++++++++++++++++++ .github/workflows/npm_release.yml | 13 +++++- .github/workflows/pull_request.yml | 13 +++++- 3 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 .github/actions/collect-test-diagnostics/action.yml diff --git a/.github/actions/collect-test-diagnostics/action.yml b/.github/actions/collect-test-diagnostics/action.yml new file mode 100644 index 00000000..e44fdc5e --- /dev/null +++ b/.github/actions/collect-test-diagnostics/action.yml @@ -0,0 +1,42 @@ +name: Collect test diagnostics +description: > + Gather what a failed simulator test run leaves outside the xcresult bundle: + host-side crash reports (.ips), CoreSimulator logs, and — when a device is + still booted — a targeted slice of the unified log. The xcresult itself + already carries the app's stdout/stderr and testmanagerd logs + (xcrun xcresulttool export diagnostics), so none of that is duplicated here. +inputs: + test-folder: + description: Folder the diagnostics directory is created in + required: true +runs: + using: composite + steps: + - name: Collect crash reports and simulator logs + shell: bash + run: | + DIAG="${{ inputs.test-folder }}/diagnostics" + mkdir -p "$DIAG" + # Crashes of simulator apps land in the *host's* DiagnosticReports. + cp -R ~/Library/Logs/DiagnosticReports/. "$DIAG/DiagnosticReports/" 2>/dev/null || true + cp -R ~/Library/Logs/CoreSimulator/. "$DIAG/CoreSimulator/" 2>/dev/null || true + # Only a booted device can serve its unified log; never boot one + # post-mortem — a fresh boot may pick the wrong device and stalls the + # job for minutes on an already-degraded VM. + UDID="$(xcrun simctl list devices booted | grep -oE '[0-9A-Fa-f-]{36}' | head -1)" + if [ -n "$UDID" ]; then + echo "Collecting unified log slice from booted simulator $UDID" + xcrun simctl spawn "$UDID" log show --style compact --last 45m \ + --predicate 'process CONTAINS "TestRunner"' \ + > "$DIAG/simulator-unified.log" 2>/dev/null || true + else + echo "No booted simulator; relying on the xcresult's captured app output." + fi + echo "Collected diagnostics:" + du -sh "$DIAG"/* 2>/dev/null || true + - name: Upload test diagnostics + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: test-diagnostics + path: ${{ inputs.test-folder }}/diagnostics + if-no-files-found: ignore diff --git a/.github/workflows/npm_release.yml b/.github/workflows/npm_release.yml index a6752b99..c2b243b1 100644 --- a/.github/workflows/npm_release.yml +++ b/.github/workflows/npm_release.yml @@ -168,19 +168,28 @@ jobs: timeout_minutes: 40 max_attempts: 2 command: set -o pipefail && xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test | xcpretty - on_retry_command: rm -rf $TEST_FOLDER/test_results* && xcrun simctl shutdown all + # Keep the failed attempt's bundle — it holds the diagnostics of the + # failure being retried, and the retry needs the original path free. + on_retry_command: rm -rf $TEST_FOLDER/test_results_attempt1.xcresult && mv $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test_results_attempt1.xcresult || true; xcrun simctl shutdown all new_command_on_retry: xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test - name: Validate Test Results run: | xcparse attachments $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test-out find $TEST_FOLDER/test-out -name "*junit*.xml" -maxdepth 1 -print0 | xargs -n 1 -0 npx junit-cli-report-viewer find $TEST_FOLDER/test-out -name "*junit*.xml" -maxdepth 1 -print0 | xargs -n 1 -0 npx verify-junit-xml + - name: Collect test diagnostics + if: failure() + uses: ./.github/actions/collect-test-diagnostics + with: + test-folder: ${{env.TEST_FOLDER}} - name: Archive Test Result Data if: always() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: test-results - path: ${{env.TEST_FOLDER}}/test_results.xcresult + path: | + ${{env.TEST_FOLDER}}/test_results.xcresult + ${{env.TEST_FOLDER}}/test_results_attempt1.xcresult # Publish the xcframework zips + dSYMs as a GitHub Release asset for EVERY # version (prerelease unless 'latest'), because the SwiftPM binaryTarget url diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 7db3a4b1..92a8b364 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -87,16 +87,25 @@ jobs: timeout_minutes: 40 max_attempts: 2 command: set -o pipefail && xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test | xcpretty - on_retry_command: rm -rf $TEST_FOLDER/test_results* && xcrun simctl shutdown all + # Keep the failed attempt's bundle — it holds the diagnostics of the + # failure being retried, and the retry needs the original path free. + on_retry_command: rm -rf $TEST_FOLDER/test_results_attempt1.xcresult && mv $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test_results_attempt1.xcresult || true; xcrun simctl shutdown all new_command_on_retry: xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test - name: Validate Test Results run: | xcparse attachments $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test-out find $TEST_FOLDER/test-out -name "*junit*.xml" -maxdepth 1 -print0 | xargs -n 1 -0 npx junit-cli-report-viewer find $TEST_FOLDER/test-out -name "*junit*.xml" -maxdepth 1 -print0 | xargs -n 1 -0 npx verify-junit-xml + - name: Collect test diagnostics + if: failure() + uses: ./.github/actions/collect-test-diagnostics + with: + test-folder: ${{env.TEST_FOLDER}} - name: Archive Test Result Data if: always() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: test-results - path: ${{env.TEST_FOLDER}}/test_results.xcresult + path: | + ${{env.TEST_FOLDER}}/test_results.xcresult + ${{env.TEST_FOLDER}}/test_results_attempt1.xcresult From f6cc6e08513b3814fad54ba241e8c45f3547671b Mon Sep 17 00:00:00 2001 From: Eduardo Speroni Date: Thu, 30 Jul 2026 17:19:29 -0300 Subject: [PATCH 4/4] fix(ci): survive empty simulator lists and staging leftovers in diagnostics The first real-world run of the diagnostics path exposed three holes: - The collect step runs under bash -e -o pipefail, so the UDID lookup aborted the whole script when no device was booted (grep exit 1 propagates through the command substitution into the assignment). - xcodebuild leaves an extensionless staging path at -resultBundlePath when it dies early (e.g. 'Unable to find a device'), and the retry then fails with 'Existing file at -resultBundlePath'. The retry cleanup now removes everything at the result path except the preserved test_results_attempt1.xcresult. - simctl blocks indefinitely when CoreSimulatorService is wedged - the very state the diagnostics step tends to run in - so log show now runs under a 120s watchdog. Also re-check the report retry deadline when the timer fires rather than when it is scheduled: under load timers run late, and a retry started past the window would consume the delivery_failed sentinel's slot. --- .../collect-test-diagnostics/action.yml | 14 +++++-- .github/workflows/npm_release.yml | 6 ++- .github/workflows/pull_request.yml | 6 ++- TestRunner/app/tests/index.js | 38 ++++++++++++------- 4 files changed, 44 insertions(+), 20 deletions(-) diff --git a/.github/actions/collect-test-diagnostics/action.yml b/.github/actions/collect-test-diagnostics/action.yml index e44fdc5e..08ce858b 100644 --- a/.github/actions/collect-test-diagnostics/action.yml +++ b/.github/actions/collect-test-diagnostics/action.yml @@ -22,13 +22,21 @@ runs: cp -R ~/Library/Logs/CoreSimulator/. "$DIAG/CoreSimulator/" 2>/dev/null || true # Only a booted device can serve its unified log; never boot one # post-mortem — a fresh boot may pick the wrong device and stalls the - # job for minutes on an already-degraded VM. - UDID="$(xcrun simctl list devices booted | grep -oE '[0-9A-Fa-f-]{36}' | head -1)" + # job for minutes on an already-degraded VM. The pipeline must not leak + # grep's exit status: this shell runs with -e -o pipefail. + UDID="$(xcrun simctl list devices booted | grep -oE '[0-9A-Fa-f-]{36}' | head -1 || true)" if [ -n "$UDID" ]; then echo "Collecting unified log slice from booted simulator $UDID" + # simctl blocks forever if CoreSimulatorService is wedged — the very + # state this step tends to run in — so hard-cap it. xcrun simctl spawn "$UDID" log show --style compact --last 45m \ --predicate 'process CONTAINS "TestRunner"' \ - > "$DIAG/simulator-unified.log" 2>/dev/null || true + > "$DIAG/simulator-unified.log" 2>/dev/null & + LOG_PID=$! + ( sleep 120 && kill -9 "$LOG_PID" ) 2>/dev/null & + KILLER_PID=$! + wait "$LOG_PID" || echo "log show failed or was killed after 120s" + kill "$KILLER_PID" 2>/dev/null || true else echo "No booted simulator; relying on the xcresult's captured app output." fi diff --git a/.github/workflows/npm_release.yml b/.github/workflows/npm_release.yml index c2b243b1..905a6720 100644 --- a/.github/workflows/npm_release.yml +++ b/.github/workflows/npm_release.yml @@ -169,8 +169,10 @@ jobs: max_attempts: 2 command: set -o pipefail && xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test | xcpretty # Keep the failed attempt's bundle — it holds the diagnostics of the - # failure being retried, and the retry needs the original path free. - on_retry_command: rm -rf $TEST_FOLDER/test_results_attempt1.xcresult && mv $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test_results_attempt1.xcresult || true; xcrun simctl shutdown all + # failure being retried. Everything else at the result path must go + # (including extensionless staging leftovers), or the retry dies with + # "Existing file at -resultBundlePath". + on_retry_command: rm -rf $TEST_FOLDER/test_results_attempt1.xcresult; mv $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test_results_attempt1.xcresult 2>/dev/null; for f in $TEST_FOLDER/test_results*; do [ "$f" = "$TEST_FOLDER/test_results_attempt1.xcresult" ] || rm -rf "$f"; done; xcrun simctl shutdown all new_command_on_retry: xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test - name: Validate Test Results run: | diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 92a8b364..76dce974 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -88,8 +88,10 @@ jobs: max_attempts: 2 command: set -o pipefail && xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test | xcpretty # Keep the failed attempt's bundle — it holds the diagnostics of the - # failure being retried, and the retry needs the original path free. - on_retry_command: rm -rf $TEST_FOLDER/test_results_attempt1.xcresult && mv $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test_results_attempt1.xcresult || true; xcrun simctl shutdown all + # failure being retried. Everything else at the result path must go + # (including extensionless staging leftovers), or the retry dies with + # "Existing file at -resultBundlePath". + on_retry_command: rm -rf $TEST_FOLDER/test_results_attempt1.xcresult; mv $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test_results_attempt1.xcresult 2>/dev/null; for f in $TEST_FOLDER/test_results*; do [ "$f" = "$TEST_FOLDER/test_results_attempt1.xcresult" ] || rm -rf "$f"; done; xcrun simctl shutdown all new_command_on_retry: xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test - name: Validate Test Results run: | diff --git a/TestRunner/app/tests/index.js b/TestRunner/app/tests/index.js index 7a947c52..9871a181 100644 --- a/TestRunner/app/tests/index.js +++ b/TestRunner/app/tests/index.js @@ -58,26 +58,38 @@ global.__JUnitSaveResults = function (text) { dataTask.resume(); } + // Best-effort sentinel so the host can fail immediately with a delivery + // diagnostic instead of waiting out its full timeout. + function sendDeliveryFailed(attempts, reason) { + post(reportUrl + "/delivery_failed", + "junit report delivery failed after " + attempts + " attempts; last failure " + reason, + function () { }); + } + function attemptReport(attempt) { post(reportUrl, text, function (reason) { console.log("junit report POST failed (attempt " + attempt + "/" + maxAttempts + ", " + reason + ")"); - // Worst case a retry costs delay + full request timeout, and the - // sentinel afterwards needs another request timeout. - var worstCaseRetryMs = retryDelayMs + 2 * requestTimeoutS * 1000; - if (attempt < maxAttempts && Date.now() + worstCaseRetryMs <= deadlineMs) { - setTimeout(function () { - attemptReport(attempt + 1); - }, retryDelayMs); - } else { - // Best-effort sentinel so the host can fail immediately with a - // delivery diagnostic instead of waiting out its full timeout. - post(reportUrl + "/delivery_failed", - "junit report delivery failed after " + attempt + " attempts; last failure " + reason, - function () { }); + if (attempt >= maxAttempts) { + sendDeliveryFailed(attempt, reason); + return; } + setTimeout(function () { + // Deadline is checked when the timer fires, not when it is + // scheduled: under load timers run late, and a retry started + // past this window would eat the sentinel's slot. + var worstCaseRetryMs = 2 * requestTimeoutS * 1000; + if (Date.now() + worstCaseRetryMs <= deadlineMs) { + attemptReport(attempt + 1); + } else { + sendDeliveryFailed(attempt, reason); + } + }, retryDelayMs); }); } + // The first attempt intentionally has no deadline guard: even past the + // deadline a one-shot report is still the most valuable outcome, and a + // late fulfill is harmless to the host. attemptReport(1); };