diff --git a/.github/actions/collect-test-diagnostics/action.yml b/.github/actions/collect-test-diagnostics/action.yml new file mode 100644 index 00000000..08ce858b --- /dev/null +++ b/.github/actions/collect-test-diagnostics/action.yml @@ -0,0 +1,50 @@ +name: Collect test diagnostics +description: > + Gather what a failed simulator test run leaves outside the xcresult bundle: + host-side crash reports (.ips), CoreSimulator logs, and — when a device is + still booted — a targeted slice of the unified log. The xcresult itself + already carries the app's stdout/stderr and testmanagerd logs + (xcrun xcresulttool export diagnostics), so none of that is duplicated here. +inputs: + test-folder: + description: Folder the diagnostics directory is created in + required: true +runs: + using: composite + steps: + - name: Collect crash reports and simulator logs + shell: bash + run: | + DIAG="${{ inputs.test-folder }}/diagnostics" + mkdir -p "$DIAG" + # Crashes of simulator apps land in the *host's* DiagnosticReports. + cp -R ~/Library/Logs/DiagnosticReports/. "$DIAG/DiagnosticReports/" 2>/dev/null || true + cp -R ~/Library/Logs/CoreSimulator/. "$DIAG/CoreSimulator/" 2>/dev/null || true + # Only a booted device can serve its unified log; never boot one + # post-mortem — a fresh boot may pick the wrong device and stalls the + # job for minutes on an already-degraded VM. The pipeline must not leak + # grep's exit status: this shell runs with -e -o pipefail. + UDID="$(xcrun simctl list devices booted | grep -oE '[0-9A-Fa-f-]{36}' | head -1 || true)" + if [ -n "$UDID" ]; then + echo "Collecting unified log slice from booted simulator $UDID" + # simctl blocks forever if CoreSimulatorService is wedged — the very + # state this step tends to run in — so hard-cap it. + xcrun simctl spawn "$UDID" log show --style compact --last 45m \ + --predicate 'process CONTAINS "TestRunner"' \ + > "$DIAG/simulator-unified.log" 2>/dev/null & + LOG_PID=$! + ( sleep 120 && kill -9 "$LOG_PID" ) 2>/dev/null & + KILLER_PID=$! + wait "$LOG_PID" || echo "log show failed or was killed after 120s" + kill "$KILLER_PID" 2>/dev/null || true + else + echo "No booted simulator; relying on the xcresult's captured app output." + fi + echo "Collected diagnostics:" + du -sh "$DIAG"/* 2>/dev/null || true + - name: Upload test diagnostics + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: test-diagnostics + path: ${{ inputs.test-folder }}/diagnostics + if-no-files-found: ignore diff --git a/.github/workflows/npm_release.yml b/.github/workflows/npm_release.yml index a6752b99..905a6720 100644 --- a/.github/workflows/npm_release.yml +++ b/.github/workflows/npm_release.yml @@ -168,19 +168,30 @@ jobs: timeout_minutes: 40 max_attempts: 2 command: set -o pipefail && xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test | xcpretty - on_retry_command: rm -rf $TEST_FOLDER/test_results* && xcrun simctl shutdown all + # Keep the failed attempt's bundle — it holds the diagnostics of the + # failure being retried. Everything else at the result path must go + # (including extensionless staging leftovers), or the retry dies with + # "Existing file at -resultBundlePath". + on_retry_command: rm -rf $TEST_FOLDER/test_results_attempt1.xcresult; mv $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test_results_attempt1.xcresult 2>/dev/null; for f in $TEST_FOLDER/test_results*; do [ "$f" = "$TEST_FOLDER/test_results_attempt1.xcresult" ] || rm -rf "$f"; done; xcrun simctl shutdown all new_command_on_retry: xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test - name: Validate Test Results run: | xcparse attachments $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test-out find $TEST_FOLDER/test-out -name "*junit*.xml" -maxdepth 1 -print0 | xargs -n 1 -0 npx junit-cli-report-viewer find $TEST_FOLDER/test-out -name "*junit*.xml" -maxdepth 1 -print0 | xargs -n 1 -0 npx verify-junit-xml + - name: Collect test diagnostics + if: failure() + uses: ./.github/actions/collect-test-diagnostics + with: + test-folder: ${{env.TEST_FOLDER}} - name: Archive Test Result Data if: always() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: test-results - path: ${{env.TEST_FOLDER}}/test_results.xcresult + path: | + ${{env.TEST_FOLDER}}/test_results.xcresult + ${{env.TEST_FOLDER}}/test_results_attempt1.xcresult # Publish the xcframework zips + dSYMs as a GitHub Release asset for EVERY # version (prerelease unless 'latest'), because the SwiftPM binaryTarget url diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 7db3a4b1..76dce974 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -87,16 +87,27 @@ jobs: timeout_minutes: 40 max_attempts: 2 command: set -o pipefail && xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test | xcpretty - on_retry_command: rm -rf $TEST_FOLDER/test_results* && xcrun simctl shutdown all + # Keep the failed attempt's bundle — it holds the diagnostics of the + # failure being retried. Everything else at the result path must go + # (including extensionless staging leftovers), or the retry dies with + # "Existing file at -resultBundlePath". + on_retry_command: rm -rf $TEST_FOLDER/test_results_attempt1.xcresult; mv $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test_results_attempt1.xcresult 2>/dev/null; for f in $TEST_FOLDER/test_results*; do [ "$f" = "$TEST_FOLDER/test_results_attempt1.xcresult" ] || rm -rf "$f"; done; xcrun simctl shutdown all new_command_on_retry: xcodebuild -project v8ios.xcodeproj -scheme TestRunner -resultBundlePath $TEST_FOLDER/test_results -destination platform\=iOS\ Simulator,OS\=latest,name\=iPhone\ 16\ Pro build test - name: Validate Test Results run: | xcparse attachments $TEST_FOLDER/test_results.xcresult $TEST_FOLDER/test-out find $TEST_FOLDER/test-out -name "*junit*.xml" -maxdepth 1 -print0 | xargs -n 1 -0 npx junit-cli-report-viewer find $TEST_FOLDER/test-out -name "*junit*.xml" -maxdepth 1 -print0 | xargs -n 1 -0 npx verify-junit-xml + - name: Collect test diagnostics + if: failure() + uses: ./.github/actions/collect-test-diagnostics + with: + test-folder: ${{env.TEST_FOLDER}} - name: Archive Test Result Data if: always() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: test-results - path: ${{env.TEST_FOLDER}}/test_results.xcresult + path: | + ${{env.TEST_FOLDER}}/test_results.xcresult + ${{env.TEST_FOLDER}}/test_results_attempt1.xcresult diff --git a/TestRunner/app/tests/index.js b/TestRunner/app/tests/index.js index 8fa28559..9871a181 100644 --- a/TestRunner/app/tests/index.js +++ b/TestRunner/app/tests/index.js @@ -1,6 +1,9 @@ // Inform the test results runner that the runtime is up. console.log('Application Start!'); +// The report delivery deadline (REPORT_DEADLINE_SECONDS) counts from launch. +var appStartMs = Date.now(); + require("../Infrastructure/timers"); require("../Infrastructure/simulator"); global.utf8 = require("../Infrastructure/utf8") @@ -22,17 +25,72 @@ global.__JUnitSaveResults = function (text) { } var reportUrl = NSProcessInfo.processInfo.environment.objectForKey("REPORT_BASEURL"); - if (reportUrl) { - var urlRequest = NSMutableURLRequest.requestWithURL(NSURL.URLWithString(reportUrl)); + if (!reportUrl) { + return; + } + + // The results host (TestRunnerTests.swift) waits on this single POST; if it + // is lost the whole run times out. Retry hard, but stop early enough that a + // final delivery_failed sentinel can still reach the host before its wait + // budget (which also covers the suite itself) expires. + var maxAttempts = 10; + var retryDelayMs = 5000; + var requestTimeoutS = 30; + var deadlineSeconds = parseInt(NSProcessInfo.processInfo.environment.objectForKey("REPORT_DEADLINE_SECONDS"), 10) || 540; + var deadlineMs = appStartMs + deadlineSeconds * 1000; + + var sessionConfig = NSURLSessionConfiguration.defaultSessionConfiguration; + sessionConfig.timeoutIntervalForRequest = requestTimeoutS; + var session = NSURLSession.sessionWithConfigurationDelegateDelegateQueue(sessionConfig, null, NSOperationQueue.mainQueue); + + function post(url, body, onFailure) { + var urlRequest = NSMutableURLRequest.requestWithURL(NSURL.URLWithString(url)); urlRequest.HTTPMethod = "POST"; urlRequest.setValueForHTTPHeaderField("Content-Type", "application/xml"); - urlRequest.HTTPBody = NSString.stringWithString(text).dataUsingEncoding(4); - var sessionConfig = NSURLSessionConfiguration.defaultSessionConfiguration; - var queue = NSOperationQueue.mainQueue; - var session = NSURLSession.sessionWithConfigurationDelegateDelegateQueue(sessionConfig, null, queue); - var dataTask = session.dataTaskWithRequestCompletionHandler(urlRequest, (data, response, error) => { }); + urlRequest.HTTPBody = NSString.stringWithString(body).dataUsingEncoding(4); + var dataTask = session.dataTaskWithRequestCompletionHandler(urlRequest, (data, response, error) => { + if (error) { + onFailure("error: " + error.localizedDescription); + } else if (!response || response.statusCode < 200 || response.statusCode >= 300) { + onFailure("status: " + (response ? response.statusCode : "none")); + } + }); dataTask.resume(); } + + // Best-effort sentinel so the host can fail immediately with a delivery + // diagnostic instead of waiting out its full timeout. + function sendDeliveryFailed(attempts, reason) { + post(reportUrl + "/delivery_failed", + "junit report delivery failed after " + attempts + " attempts; last failure " + reason, + function () { }); + } + + function attemptReport(attempt) { + post(reportUrl, text, function (reason) { + console.log("junit report POST failed (attempt " + attempt + "/" + maxAttempts + ", " + reason + ")"); + if (attempt >= maxAttempts) { + sendDeliveryFailed(attempt, reason); + return; + } + setTimeout(function () { + // Deadline is checked when the timer fires, not when it is + // scheduled: under load timers run late, and a retry started + // past this window would eat the sentinel's slot. + var worstCaseRetryMs = 2 * requestTimeoutS * 1000; + if (Date.now() + worstCaseRetryMs <= deadlineMs) { + attemptReport(attempt + 1); + } else { + sendDeliveryFailed(attempt, reason); + } + }, retryDelayMs); + }); + } + + // The first attempt intentionally has no deadline guard: even past the + // deadline a one-shot report is still the most valuable outcome, and a + // late fulfill is harmless to the host. + attemptReport(1); }; global.__approot = NSString.stringWithString(NSBundle.mainBundle.bundlePath).stringByResolvingSymlinksInPath; diff --git a/TestRunnerTests/TestRunnerTests.swift b/TestRunnerTests/TestRunnerTests.swift index 37862a12..70ba8eb1 100644 --- a/TestRunnerTests/TestRunnerTests.swift +++ b/TestRunnerTests/TestRunnerTests.swift @@ -5,6 +5,7 @@ class TestRunnerTests: XCTestCase { private var loop: EventLoop! private var server: HTTPServer! private var runtimeUnitTestsExpectation: XCTestExpectation! + private var reportDeliveryFailureReason: String? override func setUp() { continueAfterFailure = false @@ -29,6 +30,9 @@ class TestRunnerTests: XCTestCase { sendBody(Data()) self.runtimeUnitTestsExpectation.fulfill() } else { + // The app POSTs here when it exhausted its report retries; the + // payload is a human-readable reason instead of junit XML. + let isDeliveryFailure = (environ["PATH_INFO"] as? String)?.hasSuffix("/delivery_failed") ?? false var buffer = Data() let input = environ["swsgi.input"] as! SWSGIInput var finished = false @@ -37,9 +41,13 @@ class TestRunnerTests: XCTestCase { if data.isEmpty && !finished { finished = true - let report = XCTAttachment(uniformTypeIdentifier: "junit.xml", name: "junit.xml", payload: buffer, userInfo: nil) - report.lifetime = .keepAlways - self.add(report) + if isDeliveryFailure { + self.reportDeliveryFailureReason = String(data: buffer, encoding: .utf8) ?? "unknown" + } else { + let report = XCTAttachment(uniformTypeIdentifier: "junit.xml", name: "junit.xml", payload: buffer, userInfo: nil) + report.lifetime = .keepAlways + self.add(report) + } startResponse("204 No Content", []) sendBody(Data()) @@ -51,7 +59,10 @@ class TestRunnerTests: XCTestCase { try! server.start() - DispatchQueue.global(qos: .background).async { + // Not .background: CI VMs throttle background-QoS threads hard enough + // to starve this accept loop, and it is the only channel through which + // results ever arrive. + DispatchQueue.global(qos: .userInitiated).async { self.loop.runForever() } } @@ -72,6 +83,10 @@ class TestRunnerTests: XCTestCase { let app = XCUIApplication() app.launchEnvironment["REPORT_BASEURL"] = "http://[::1]:\(port)/junit_report" + // The app's report retries and delivery_failed sentinel count from its + // launch, which precedes the wait below — keep a margin so delivery + // gives up (and the sentinel lands) before our timeout fires. + app.launchEnvironment["REPORT_DEADLINE_SECONDS"] = String(Int(jasmineTestsTimeout) - 60) app.launch() // Watchdog: if the runtime crashes (e.g. EXC_BAD_ACCESS) it never @@ -100,6 +115,8 @@ class TestRunnerTests: XCTestCase { case .completed: if didCrash { XCTFail("TestRunner exited before reporting Jasmine results (likely crashed). Check ~/Library/Logs/DiagnosticReports/TestRunner-*.ips for the stack.") + } else if let reason = reportDeliveryFailureReason { + XCTFail("TestRunner finished the Jasmine suite but could not deliver the junit report: \(reason)") } return case .timedOut: