From 9c162ac85b9472c755332408384216060253d950 Mon Sep 17 00:00:00 2001 From: Cameron Cooke Date: Fri, 1 May 2026 18:46:27 +0100 Subject: [PATCH 1/2] fix(cli): Suppress false compiler errors in test summaries Tighten xcodebuild diagnostic detection so Objective-C selector labels in NSError dump lines are not treated as compiler errors. Align CLI streaming finalization so buffered compiler diagnostics do not print after a successful summary. Add compiler-error snapshot coverage for simulator, device, and macOS build, build-and-run, and test flows across CLI, MCP, and JSON output. Fixes #383 Co-Authored-By: Codex --- CHANGELOG.md | 1 + .../cli/device/build--error-compiler.txt | 16 ++++ .../device/build-and-run--error-compiler.txt | 17 +++++ .../cli/device/test--error-compiler.txt | 22 ++++++ .../cli/macos/build--error-compiler.txt | 16 ++++ .../macos/build-and-run--error-compiler.txt | 16 ++++ .../cli/macos/test--error-compiler.txt | 23 ++++++ .../cli/simulator/build--error-compiler.txt | 17 +++++ .../build-and-run--error-compiler.txt | 17 +++++ .../cli/simulator/test--error-compiler.txt | 22 ++++++ .../json/device/build--error-compiler.json | 33 ++++++++ .../device/build-and-run--error-compiler.json | 35 +++++++++ .../json/device/test--error-compiler.json | 51 +++++++++++++ .../json/macos/build--error-compiler.json | 33 ++++++++ .../macos/build-and-run--error-compiler.json | 33 ++++++++ .../json/macos/test--error-compiler.json | 51 +++++++++++++ .../json/simulator/build--error-compiler.json | 33 ++++++++ .../build-and-run--error-compiler.json | 33 ++++++++ .../json/simulator/test--error-compiler.json | 49 ++++++++++++ .../mcp/device/build--error-compiler.txt | 16 ++++ .../device/build-and-run--error-compiler.txt | 17 +++++ .../mcp/device/test--error-compiler.txt | 22 ++++++ .../mcp/macos/build--error-compiler.txt | 16 ++++ .../macos/build-and-run--error-compiler.txt | 16 ++++ .../mcp/macos/test--error-compiler.txt | 23 ++++++ .../mcp/simulator/build--error-compiler.txt | 17 +++++ .../build-and-run--error-compiler.txt | 17 +++++ .../mcp/simulator/test--error-compiler.txt | 22 ++++++ src/snapshot-tests/suites/device-suite.ts | 42 ++++++++++- src/snapshot-tests/suites/helpers.ts | 44 +++++++++++ src/snapshot-tests/suites/macos-suite.ts | 43 ++++++++++- src/snapshot-tests/suites/simulator-suite.ts | 55 +++++++++++++- .../__tests__/xcodebuild-event-parser.test.ts | 28 +++++++ .../__tests__/xcodebuild-line-parsers.test.ts | 43 +++++++++++ .../__tests__/cli-text-renderer.test.ts | 75 +++++++++++++++++++ src/utils/renderers/cli-text-renderer.ts | 8 +- src/utils/xcodebuild-line-parsers.ts | 2 +- 37 files changed, 1019 insertions(+), 5 deletions(-) create mode 100644 src/snapshot-tests/__fixtures__/cli/device/build--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/cli/device/build-and-run--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/cli/device/test--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/cli/macos/build--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/cli/macos/build-and-run--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/cli/macos/test--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/cli/simulator/build--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/cli/simulator/build-and-run--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/cli/simulator/test--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/json/device/build--error-compiler.json create mode 100644 src/snapshot-tests/__fixtures__/json/device/build-and-run--error-compiler.json create mode 100644 src/snapshot-tests/__fixtures__/json/device/test--error-compiler.json create mode 100644 src/snapshot-tests/__fixtures__/json/macos/build--error-compiler.json create mode 100644 src/snapshot-tests/__fixtures__/json/macos/build-and-run--error-compiler.json create mode 100644 src/snapshot-tests/__fixtures__/json/macos/test--error-compiler.json create mode 100644 src/snapshot-tests/__fixtures__/json/simulator/build--error-compiler.json create mode 100644 src/snapshot-tests/__fixtures__/json/simulator/build-and-run--error-compiler.json create mode 100644 src/snapshot-tests/__fixtures__/json/simulator/test--error-compiler.json create mode 100644 src/snapshot-tests/__fixtures__/mcp/device/build--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/mcp/device/build-and-run--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/mcp/device/test--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/mcp/macos/build--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/mcp/macos/build-and-run--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/mcp/macos/test--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/mcp/simulator/build--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/mcp/simulator/build-and-run--error-compiler.txt create mode 100644 src/snapshot-tests/__fixtures__/mcp/simulator/test--error-compiler.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 374fa26e6..e5d47063a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed +- Fixed CLI test summaries showing false-positive compiler errors from xcodebuild NSError dump lines, and added compiler-error snapshot coverage for simulator, device, and macOS build-style flows ([#383](https://github.com/getsentry/XcodeBuildMCP/issues/383)). - Fixed simulator OSLog helper cleanup so server and daemon startup reconcile same-workspace orphaned log streams without stopping helpers owned by live sessions in other workspaces ([#382](https://github.com/getsentry/XcodeBuildMCP/issues/382)). ## [2.5.0-beta.1] diff --git a/src/snapshot-tests/__fixtures__/cli/device/build--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/device/build--error-compiler.txt new file mode 100644 index 000000000..5fd8ca357 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/cli/device/build--error-compiler.txt @@ -0,0 +1,16 @@ + +๐Ÿ”จ Build + + Scheme: CalculatorApp + Workspace: example_projects/iOS_Calculator/CalculatorApp.xcworkspace + Configuration: Debug + Platform: iOS + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp- + +Compiler Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32:42 + +โŒ Build failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_device__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/device/build-and-run--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/device/build-and-run--error-compiler.txt new file mode 100644 index 000000000..7fb598d46 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/cli/device/build-and-run--error-compiler.txt @@ -0,0 +1,17 @@ + +๐Ÿš€ Build & Run + + Scheme: CalculatorApp + Workspace: example_projects/iOS_Calculator/CalculatorApp.xcworkspace + Configuration: Debug + Platform: iOS + Device: () + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp- + +Compiler Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32:42 + +โŒ Build failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_run_device__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/device/test--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/device/test--error-compiler.txt new file mode 100644 index 000000000..e2e358ab0 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/cli/device/test--error-compiler.txt @@ -0,0 +1,22 @@ + +๐Ÿงช Test + + Scheme: CalculatorApp + Workspace: example_projects/iOS_Calculator/CalculatorApp.xcworkspace + Configuration: Debug + Platform: iOS + Device: () + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp- + Selective Testing: + CalculatorAppTests/CalculatorAppTests/testAddition + +Discovered 1 test(s): + CalculatorAppTests/CalculatorAppTests/testAddition + +Compiler Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32:42 + +โŒ Test failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/test_device__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/macos/build--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/macos/build--error-compiler.txt new file mode 100644 index 000000000..6d46f365d --- /dev/null +++ b/src/snapshot-tests/__fixtures__/cli/macos/build--error-compiler.txt @@ -0,0 +1,16 @@ + +๐Ÿ”จ Build + + Scheme: MCPTest + Project: example_projects/macOS/MCPTest.xcodeproj + Configuration: Debug + Platform: macOS + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/MCPTest- + +Compiler Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + example_projects/macOS/MCPTest/MCPTestApp.swift:19:42 + +โŒ Build failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_macos__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/macos/build-and-run--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/macos/build-and-run--error-compiler.txt new file mode 100644 index 000000000..a466239b0 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/cli/macos/build-and-run--error-compiler.txt @@ -0,0 +1,16 @@ + +๐Ÿš€ Build & Run + + Scheme: MCPTest + Project: example_projects/macOS/MCPTest.xcodeproj + Configuration: Debug + Platform: macOS + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/MCPTest- + +Compiler Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + example_projects/macOS/MCPTest/MCPTestApp.swift:19:42 + +โŒ Build failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_run_macos__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/macos/test--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/macos/test--error-compiler.txt new file mode 100644 index 000000000..6a96ee0e4 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/cli/macos/test--error-compiler.txt @@ -0,0 +1,23 @@ + +๐Ÿงช Test + + Scheme: MCPTest + Project: example_projects/macOS/MCPTest.xcodeproj + Configuration: Debug + Platform: macOS + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/MCPTest- + Selective Testing: + MCPTestTests/MCPTestTests/appNameIsCorrect() + MCPTestTests/MCPTestsXCTests/testAppNameIsCorrect + +Discovered 2 test(s): + MCPTestTests/MCPTestTests/appNameIsCorrect + MCPTestTests/MCPTestsXCTests/testAppNameIsCorrect + +Compiler Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + example_projects/macOS/MCPTest/MCPTestApp.swift:19:42 + +โŒ Test failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/test_macos__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/simulator/build--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/simulator/build--error-compiler.txt new file mode 100644 index 000000000..c82c53b7c --- /dev/null +++ b/src/snapshot-tests/__fixtures__/cli/simulator/build--error-compiler.txt @@ -0,0 +1,17 @@ + +๐Ÿ”จ Build + + Scheme: CalculatorApp + Workspace: example_projects/iOS_Calculator/CalculatorApp.xcworkspace + Configuration: Debug + Platform: iOS Simulator + Simulator: iPhone 17 + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp- + +Compiler Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32:42 + +โŒ Build failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_sim__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/simulator/build-and-run--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/simulator/build-and-run--error-compiler.txt new file mode 100644 index 000000000..93a88ee22 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/cli/simulator/build-and-run--error-compiler.txt @@ -0,0 +1,17 @@ + +๐Ÿš€ Build & Run + + Scheme: CalculatorApp + Workspace: example_projects/iOS_Calculator/CalculatorApp.xcworkspace + Configuration: Debug + Platform: iOS Simulator + Simulator: iPhone 17 + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp- + +Compiler Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32:42 + +โŒ Build failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_run_sim__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/simulator/test--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/simulator/test--error-compiler.txt new file mode 100644 index 000000000..90f984dd1 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/cli/simulator/test--error-compiler.txt @@ -0,0 +1,22 @@ + +๐Ÿงช Test + + Scheme: CalculatorApp + Workspace: example_projects/iOS_Calculator/CalculatorApp.xcworkspace + Configuration: Debug + Platform: iOS Simulator + Simulator: iPhone 17 + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp- + Selective Testing: + CalculatorAppTests/CalculatorAppTests/testAddition + +Discovered 1 test(s): + CalculatorAppTests/CalculatorAppTests/testAddition + +Compiler Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32:42 + +โŒ Test failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/test_sim__pid.log diff --git a/src/snapshot-tests/__fixtures__/json/device/build--error-compiler.json b/src/snapshot-tests/__fixtures__/json/device/build--error-compiler.json new file mode 100644 index 000000000..a8a4d7ca7 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/json/device/build--error-compiler.json @@ -0,0 +1,33 @@ +{ + "schema": "xcodebuildmcp.output.build-result", + "schemaVersion": "1", + "didError": true, + "error": "Build failed", + "data": { + "request": { + "scheme": "CalculatorApp", + "workspacePath": "example_projects/iOS_Calculator/CalculatorApp.xcworkspace", + "derivedDataPath": "/Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp-", + "configuration": "Debug", + "platform": "iOS", + "target": "device" + }, + "summary": { + "status": "FAILED", + "durationMs": 1234, + "target": "device" + }, + "artifacts": { + "buildLogPath": "/Library/Developer/XcodeBuildMCP/logs/build_device__pid.log" + }, + "diagnostics": { + "warnings": [], + "errors": [ + { + "message": "cannot convert value of type 'String' to specified type 'Int'", + "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32" + } + ] + } + } +} diff --git a/src/snapshot-tests/__fixtures__/json/device/build-and-run--error-compiler.json b/src/snapshot-tests/__fixtures__/json/device/build-and-run--error-compiler.json new file mode 100644 index 000000000..07a72f579 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/json/device/build-and-run--error-compiler.json @@ -0,0 +1,35 @@ +{ + "schema": "xcodebuildmcp.output.build-run-result", + "schemaVersion": "1", + "didError": true, + "error": "Build failed", + "data": { + "request": { + "scheme": "CalculatorApp", + "workspacePath": "example_projects/iOS_Calculator/CalculatorApp.xcworkspace", + "derivedDataPath": "/Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp-", + "configuration": "Debug", + "platform": "iOS", + "deviceId": "", + "target": "device" + }, + "summary": { + "status": "FAILED", + "durationMs": 1234, + "target": "device" + }, + "artifacts": { + "deviceId": "", + "buildLogPath": "/Library/Developer/XcodeBuildMCP/logs/build_run_device__pid.log" + }, + "diagnostics": { + "warnings": [], + "errors": [ + { + "message": "cannot convert value of type 'String' to specified type 'Int'", + "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32" + } + ] + } + } +} diff --git a/src/snapshot-tests/__fixtures__/json/device/test--error-compiler.json b/src/snapshot-tests/__fixtures__/json/device/test--error-compiler.json new file mode 100644 index 000000000..1eb1e8073 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/json/device/test--error-compiler.json @@ -0,0 +1,51 @@ +{ + "schema": "xcodebuildmcp.output.test-result", + "schemaVersion": "1", + "didError": true, + "error": "Tests failed", + "data": { + "request": { + "scheme": "CalculatorApp", + "workspacePath": "example_projects/iOS_Calculator/CalculatorApp.xcworkspace", + "derivedDataPath": "/Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp-", + "configuration": "Debug", + "platform": "iOS", + "deviceId": "", + "target": "device", + "onlyTesting": [ + "CalculatorAppTests/CalculatorAppTests/testAddition" + ], + "skipTesting": [] + }, + "summary": { + "status": "FAILED", + "durationMs": 1234, + "target": "device" + }, + "artifacts": { + "deviceId": "", + "buildLogPath": "/Library/Developer/XcodeBuildMCP/logs/test_device__pid.log" + }, + "tests": { + "selected": [ + "CalculatorAppTests/CalculatorAppTests/testAddition" + ], + "discovered": { + "total": 1, + "items": [ + "CalculatorAppTests/CalculatorAppTests/testAddition" + ] + } + }, + "diagnostics": { + "warnings": [], + "errors": [ + { + "message": "cannot convert value of type 'String' to specified type 'Int'", + "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32" + } + ], + "testFailures": [] + } + } +} diff --git a/src/snapshot-tests/__fixtures__/json/macos/build--error-compiler.json b/src/snapshot-tests/__fixtures__/json/macos/build--error-compiler.json new file mode 100644 index 000000000..b04f9fe58 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/json/macos/build--error-compiler.json @@ -0,0 +1,33 @@ +{ + "schema": "xcodebuildmcp.output.build-result", + "schemaVersion": "1", + "didError": true, + "error": "Build failed", + "data": { + "request": { + "scheme": "MCPTest", + "projectPath": "example_projects/macOS/MCPTest.xcodeproj", + "derivedDataPath": "/Library/Developer/XcodeBuildMCP/DerivedData/MCPTest-", + "configuration": "Debug", + "platform": "macOS", + "target": "macos" + }, + "summary": { + "status": "FAILED", + "durationMs": 1234, + "target": "macos" + }, + "artifacts": { + "buildLogPath": "/Library/Developer/XcodeBuildMCP/logs/build_macos__pid.log" + }, + "diagnostics": { + "warnings": [], + "errors": [ + { + "message": "cannot convert value of type 'String' to specified type 'Int'", + "location": "/example_projects/macOS/MCPTest/MCPTestApp.swift:19" + } + ] + } + } +} diff --git a/src/snapshot-tests/__fixtures__/json/macos/build-and-run--error-compiler.json b/src/snapshot-tests/__fixtures__/json/macos/build-and-run--error-compiler.json new file mode 100644 index 000000000..216ad1131 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/json/macos/build-and-run--error-compiler.json @@ -0,0 +1,33 @@ +{ + "schema": "xcodebuildmcp.output.build-run-result", + "schemaVersion": "1", + "didError": true, + "error": "Build failed", + "data": { + "request": { + "scheme": "MCPTest", + "projectPath": "example_projects/macOS/MCPTest.xcodeproj", + "derivedDataPath": "/Library/Developer/XcodeBuildMCP/DerivedData/MCPTest-", + "configuration": "Debug", + "platform": "macOS", + "target": "macos" + }, + "summary": { + "status": "FAILED", + "durationMs": 1234, + "target": "macos" + }, + "artifacts": { + "buildLogPath": "/Library/Developer/XcodeBuildMCP/logs/build_run_macos__pid.log" + }, + "diagnostics": { + "warnings": [], + "errors": [ + { + "message": "cannot convert value of type 'String' to specified type 'Int'", + "location": "/example_projects/macOS/MCPTest/MCPTestApp.swift:19" + } + ] + } + } +} diff --git a/src/snapshot-tests/__fixtures__/json/macos/test--error-compiler.json b/src/snapshot-tests/__fixtures__/json/macos/test--error-compiler.json new file mode 100644 index 000000000..282123bcf --- /dev/null +++ b/src/snapshot-tests/__fixtures__/json/macos/test--error-compiler.json @@ -0,0 +1,51 @@ +{ + "schema": "xcodebuildmcp.output.test-result", + "schemaVersion": "1", + "didError": true, + "error": "Tests failed", + "data": { + "request": { + "scheme": "MCPTest", + "projectPath": "example_projects/macOS/MCPTest.xcodeproj", + "derivedDataPath": "/Library/Developer/XcodeBuildMCP/DerivedData/MCPTest-", + "configuration": "Debug", + "platform": "macOS", + "onlyTesting": [ + "MCPTestTests/MCPTestTests/appNameIsCorrect()", + "MCPTestTests/MCPTestsXCTests/testAppNameIsCorrect" + ], + "skipTesting": [] + }, + "summary": { + "status": "FAILED", + "durationMs": 1234, + "target": "macos" + }, + "artifacts": { + "buildLogPath": "/Library/Developer/XcodeBuildMCP/logs/test_macos__pid.log" + }, + "tests": { + "selected": [ + "MCPTestTests/MCPTestTests/appNameIsCorrect", + "MCPTestTests/MCPTestsXCTests/testAppNameIsCorrect" + ], + "discovered": { + "total": 2, + "items": [ + "MCPTestTests/MCPTestTests/appNameIsCorrect", + "MCPTestTests/MCPTestsXCTests/testAppNameIsCorrect" + ] + } + }, + "diagnostics": { + "warnings": [], + "errors": [ + { + "message": "cannot convert value of type 'String' to specified type 'Int'", + "location": "/example_projects/macOS/MCPTest/MCPTestApp.swift:19" + } + ], + "testFailures": [] + } + } +} diff --git a/src/snapshot-tests/__fixtures__/json/simulator/build--error-compiler.json b/src/snapshot-tests/__fixtures__/json/simulator/build--error-compiler.json new file mode 100644 index 000000000..850006e7f --- /dev/null +++ b/src/snapshot-tests/__fixtures__/json/simulator/build--error-compiler.json @@ -0,0 +1,33 @@ +{ + "schema": "xcodebuildmcp.output.build-result", + "schemaVersion": "1", + "didError": true, + "error": "Build failed", + "data": { + "request": { + "scheme": "CalculatorApp", + "workspacePath": "example_projects/iOS_Calculator/CalculatorApp.xcworkspace", + "derivedDataPath": "/Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp-", + "configuration": "Debug", + "platform": "iOS Simulator", + "simulatorName": "iPhone 17" + }, + "summary": { + "status": "FAILED", + "durationMs": 1234, + "target": "simulator" + }, + "artifacts": { + "buildLogPath": "/Library/Developer/XcodeBuildMCP/logs/build_sim__pid.log" + }, + "diagnostics": { + "warnings": [], + "errors": [ + { + "message": "cannot convert value of type 'String' to specified type 'Int'", + "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32" + } + ] + } + } +} diff --git a/src/snapshot-tests/__fixtures__/json/simulator/build-and-run--error-compiler.json b/src/snapshot-tests/__fixtures__/json/simulator/build-and-run--error-compiler.json new file mode 100644 index 000000000..bbe287cdb --- /dev/null +++ b/src/snapshot-tests/__fixtures__/json/simulator/build-and-run--error-compiler.json @@ -0,0 +1,33 @@ +{ + "schema": "xcodebuildmcp.output.build-run-result", + "schemaVersion": "1", + "didError": true, + "error": "Build failed", + "data": { + "request": { + "scheme": "CalculatorApp", + "workspacePath": "example_projects/iOS_Calculator/CalculatorApp.xcworkspace", + "derivedDataPath": "/Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp-", + "configuration": "Debug", + "platform": "iOS Simulator", + "simulatorName": "iPhone 17" + }, + "summary": { + "status": "FAILED", + "durationMs": 1234, + "target": "simulator" + }, + "artifacts": { + "buildLogPath": "/Library/Developer/XcodeBuildMCP/logs/build_run_sim__pid.log" + }, + "diagnostics": { + "warnings": [], + "errors": [ + { + "message": "cannot convert value of type 'String' to specified type 'Int'", + "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32" + } + ] + } + } +} diff --git a/src/snapshot-tests/__fixtures__/json/simulator/test--error-compiler.json b/src/snapshot-tests/__fixtures__/json/simulator/test--error-compiler.json new file mode 100644 index 000000000..2941c31c1 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/json/simulator/test--error-compiler.json @@ -0,0 +1,49 @@ +{ + "schema": "xcodebuildmcp.output.test-result", + "schemaVersion": "1", + "didError": true, + "error": "Tests failed", + "data": { + "request": { + "scheme": "CalculatorApp", + "workspacePath": "example_projects/iOS_Calculator/CalculatorApp.xcworkspace", + "derivedDataPath": "/Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp-", + "configuration": "Debug", + "platform": "iOS Simulator", + "simulatorName": "iPhone 17", + "onlyTesting": [ + "CalculatorAppTests/CalculatorAppTests/testAddition" + ], + "skipTesting": [] + }, + "summary": { + "status": "FAILED", + "durationMs": 1234, + "target": "simulator" + }, + "artifacts": { + "buildLogPath": "/Library/Developer/XcodeBuildMCP/logs/test_sim__pid.log" + }, + "tests": { + "selected": [ + "CalculatorAppTests/CalculatorAppTests/testAddition" + ], + "discovered": { + "total": 1, + "items": [ + "CalculatorAppTests/CalculatorAppTests/testAddition" + ] + } + }, + "diagnostics": { + "warnings": [], + "errors": [ + { + "message": "cannot convert value of type 'String' to specified type 'Int'", + "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32" + } + ], + "testFailures": [] + } + } +} diff --git a/src/snapshot-tests/__fixtures__/mcp/device/build--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/device/build--error-compiler.txt new file mode 100644 index 000000000..61e55c15f --- /dev/null +++ b/src/snapshot-tests/__fixtures__/mcp/device/build--error-compiler.txt @@ -0,0 +1,16 @@ + +๐Ÿ”จ Build + + Scheme: CalculatorApp + Workspace: example_projects/iOS_Calculator/CalculatorApp.xcworkspace + Configuration: Debug + Platform: iOS + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp- + +Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32 + +โŒ Build failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_device__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/device/build-and-run--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/device/build-and-run--error-compiler.txt new file mode 100644 index 000000000..b8bd68c1c --- /dev/null +++ b/src/snapshot-tests/__fixtures__/mcp/device/build-and-run--error-compiler.txt @@ -0,0 +1,17 @@ + +๐Ÿš€ Build & Run + + Scheme: CalculatorApp + Workspace: example_projects/iOS_Calculator/CalculatorApp.xcworkspace + Configuration: Debug + Platform: iOS + Device: () + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp- + +Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32 + +โŒ Build failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_run_device__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/device/test--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/device/test--error-compiler.txt new file mode 100644 index 000000000..9677eaab1 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/mcp/device/test--error-compiler.txt @@ -0,0 +1,22 @@ + +๐Ÿงช Test + + Scheme: CalculatorApp + Workspace: example_projects/iOS_Calculator/CalculatorApp.xcworkspace + Configuration: Debug + Platform: iOS + Device: () + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp- + Selective Testing: + CalculatorAppTests/CalculatorAppTests/testAddition + +Discovered 1 test(s): + CalculatorAppTests/CalculatorAppTests/testAddition + +Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32 + +โŒ Test failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/test_device__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/macos/build--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/macos/build--error-compiler.txt new file mode 100644 index 000000000..a73d026cf --- /dev/null +++ b/src/snapshot-tests/__fixtures__/mcp/macos/build--error-compiler.txt @@ -0,0 +1,16 @@ + +๐Ÿ”จ Build + + Scheme: MCPTest + Project: example_projects/macOS/MCPTest.xcodeproj + Configuration: Debug + Platform: macOS + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/MCPTest- + +Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + /example_projects/macOS/MCPTest/MCPTestApp.swift:19 + +โŒ Build failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_macos__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/macos/build-and-run--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/macos/build-and-run--error-compiler.txt new file mode 100644 index 000000000..3839ac1d5 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/mcp/macos/build-and-run--error-compiler.txt @@ -0,0 +1,16 @@ + +๐Ÿš€ Build & Run + + Scheme: MCPTest + Project: example_projects/macOS/MCPTest.xcodeproj + Configuration: Debug + Platform: macOS + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/MCPTest- + +Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + /example_projects/macOS/MCPTest/MCPTestApp.swift:19 + +โŒ Build failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_run_macos__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/macos/test--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/macos/test--error-compiler.txt new file mode 100644 index 000000000..e309e4ee2 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/mcp/macos/test--error-compiler.txt @@ -0,0 +1,23 @@ + +๐Ÿงช Test + + Scheme: MCPTest + Project: example_projects/macOS/MCPTest.xcodeproj + Configuration: Debug + Platform: macOS + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/MCPTest- + Selective Testing: + MCPTestTests/MCPTestTests/appNameIsCorrect() + MCPTestTests/MCPTestsXCTests/testAppNameIsCorrect + +Discovered 2 test(s): + MCPTestTests/MCPTestTests/appNameIsCorrect + MCPTestTests/MCPTestsXCTests/testAppNameIsCorrect + +Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + /example_projects/macOS/MCPTest/MCPTestApp.swift:19 + +โŒ Test failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/test_macos__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/simulator/build--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/simulator/build--error-compiler.txt new file mode 100644 index 000000000..1a8eff2a5 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/mcp/simulator/build--error-compiler.txt @@ -0,0 +1,17 @@ + +๐Ÿ”จ Build + + Scheme: CalculatorApp + Workspace: example_projects/iOS_Calculator/CalculatorApp.xcworkspace + Configuration: Debug + Platform: iOS Simulator + Simulator: iPhone 17 + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp- + +Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32 + +โŒ Build failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_sim__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/simulator/build-and-run--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/simulator/build-and-run--error-compiler.txt new file mode 100644 index 000000000..bcc09d3e1 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/mcp/simulator/build-and-run--error-compiler.txt @@ -0,0 +1,17 @@ + +๐Ÿš€ Build & Run + + Scheme: CalculatorApp + Workspace: example_projects/iOS_Calculator/CalculatorApp.xcworkspace + Configuration: Debug + Platform: iOS Simulator + Simulator: iPhone 17 + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp- + +Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32 + +โŒ Build failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_run_sim__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/simulator/test--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/simulator/test--error-compiler.txt new file mode 100644 index 000000000..b2db282f1 --- /dev/null +++ b/src/snapshot-tests/__fixtures__/mcp/simulator/test--error-compiler.txt @@ -0,0 +1,22 @@ + +๐Ÿงช Test + + Scheme: CalculatorApp + Workspace: example_projects/iOS_Calculator/CalculatorApp.xcworkspace + Configuration: Debug + Platform: iOS Simulator + Simulator: iPhone 17 + Derived Data: /Library/Developer/XcodeBuildMCP/DerivedData/CalculatorApp- + Selective Testing: + CalculatorAppTests/CalculatorAppTests/testAddition + +Discovered 1 test(s): + CalculatorAppTests/CalculatorAppTests/testAddition + +Errors (1): + + โœ— cannot convert value of type 'String' to specified type 'Int' + /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32 + +โŒ Test failed. (โฑ๏ธ ) + โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/test_sim__pid.log diff --git a/src/snapshot-tests/suites/device-suite.ts b/src/snapshot-tests/suites/device-suite.ts index f5912ec31..c0770050a 100644 --- a/src/snapshot-tests/suites/device-suite.ts +++ b/src/snapshot-tests/suites/device-suite.ts @@ -5,7 +5,11 @@ import { extractAppPathFromSnapshotOutput, extractProcessIdFromSnapshotOutput, } from '../output-parsers.ts'; -import { createHarnessForRuntime, createWorkflowFixtureMatcher } from './helpers.ts'; +import { + createHarnessForRuntime, + createWorkflowFixtureMatcher, + withCalculatorAppCompilerError, +} from './helpers.ts'; const WORKSPACE = 'example_projects/iOS_Calculator/CalculatorApp.xcworkspace'; const BUNDLE_ID = 'io.sentry.calculatorapp'; @@ -61,6 +65,17 @@ export function registerDeviceSnapshotSuite(runtime: SnapshotRuntime): void { expect(isError).toBe(true); expectFixture(text, 'build--error-wrong-scheme'); }); + + it('error - compiler error', async () => { + const { text, isError } = await withCalculatorAppCompilerError(() => + harness.invoke('device', 'build', { + workspacePath: WORKSPACE, + scheme: 'CalculatorApp', + }), + ); + expect(isError).toBe(true); + expectFixture(text, 'build--error-compiler'); + }); }); describe('get-app-path', () => { @@ -139,6 +154,18 @@ export function registerDeviceSnapshotSuite(runtime: SnapshotRuntime): void { expect(isError).toBe(true); expectFixture(text, 'build-and-run--error-wrong-scheme'); }); + + it('error - compiler error', async () => { + const { text, isError } = await withCalculatorAppCompilerError(() => + harness.invoke('device', 'build-and-run', { + workspacePath: WORKSPACE, + scheme: 'CalculatorApp', + deviceId: DEVICE_ID, + }), + ); + expect(isError).toBe(true); + expectFixture(text, 'build-and-run--error-compiler'); + }); }); describe.runIf(DEVICE_READY)('install (requires device)', () => { @@ -216,6 +243,19 @@ export function registerDeviceSnapshotSuite(runtime: SnapshotRuntime): void { expect(text.length).toBeGreaterThan(10); expectFixture(text, 'test--failure'); }, 300_000); + + it('error - compiler error', async () => { + const { text, isError } = await withCalculatorAppCompilerError(() => + harness.invoke('device', 'test', { + workspacePath: WORKSPACE, + scheme: 'CalculatorApp', + deviceId: DEVICE_ID, + extraArgs: ['-only-testing:CalculatorAppTests/CalculatorAppTests/testAddition'], + }), + ); + expect(isError).toBe(true); + expectFixture(text, 'test--error-compiler'); + }, 300_000); }); }); } diff --git a/src/snapshot-tests/suites/helpers.ts b/src/snapshot-tests/suites/helpers.ts index d5cc096ec..fc6b31fe1 100644 --- a/src/snapshot-tests/suites/helpers.ts +++ b/src/snapshot-tests/suites/helpers.ts @@ -1,9 +1,21 @@ +import fs from 'node:fs'; +import path from 'node:path'; import type { SnapshotRuntime, WorkflowSnapshotHarness } from '../contracts.ts'; import { createFixtureMatcher, type FixtureMatchOptions } from '../fixture-io.ts'; import { createSnapshotHarness } from '../harness.ts'; import { createJsonSnapshotHarness } from '../json-harness.ts'; import { createMcpSnapshotHarness } from '../mcp-harness.ts'; +const CALCULATOR_APP_SOURCE_PATH = path.resolve( + process.cwd(), + 'example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift', +); +const MACOS_APP_SOURCE_PATH = path.resolve( + process.cwd(), + 'example_projects/macOS/MCPTest/MCPTestApp.swift', +); +const COMPILER_ERROR_SNIPPET = 'private let snapshotCompilerError: Int = "not an int"'; + export function createHarnessForRuntime( runtime: SnapshotRuntime, ): Promise { @@ -22,6 +34,38 @@ export interface WorkflowFixtureMatcherOptions extends FixtureMatchOptions { fixtureRuntime?: SnapshotRuntime; } +export async function withCalculatorAppCompilerError(action: () => Promise): Promise { + const originalSource = fs.readFileSync(CALCULATOR_APP_SOURCE_PATH, 'utf8'); + + try { + fs.writeFileSync( + CALCULATOR_APP_SOURCE_PATH, + `${originalSource}\n${COMPILER_ERROR_SNIPPET}\n`, + 'utf8', + ); + + return await action(); + } finally { + fs.writeFileSync(CALCULATOR_APP_SOURCE_PATH, originalSource, 'utf8'); + } +} + +export async function withMacosAppCompilerError(action: () => Promise): Promise { + const originalSource = fs.readFileSync(MACOS_APP_SOURCE_PATH, 'utf8'); + + try { + fs.writeFileSync( + MACOS_APP_SOURCE_PATH, + `${originalSource}\n${COMPILER_ERROR_SNIPPET}\n`, + 'utf8', + ); + + return await action(); + } finally { + fs.writeFileSync(MACOS_APP_SOURCE_PATH, originalSource, 'utf8'); + } +} + export function createWorkflowFixtureMatcher( runtime: SnapshotRuntime, workflow: string, diff --git a/src/snapshot-tests/suites/macos-suite.ts b/src/snapshot-tests/suites/macos-suite.ts index 6b533e8cb..8c0aa46fd 100644 --- a/src/snapshot-tests/suites/macos-suite.ts +++ b/src/snapshot-tests/suites/macos-suite.ts @@ -4,7 +4,11 @@ import os from 'node:os'; import path from 'node:path'; import type { SnapshotRuntime, WorkflowSnapshotHarness } from '../contracts.ts'; import { extractAppPathFromSnapshotOutput } from '../output-parsers.ts'; -import { createHarnessForRuntime, createWorkflowFixtureMatcher } from './helpers.ts'; +import { + createHarnessForRuntime, + createWorkflowFixtureMatcher, + withMacosAppCompilerError, +} from './helpers.ts'; const PROJECT = 'example_projects/macOS/MCPTest.xcodeproj'; @@ -68,6 +72,17 @@ export function registerMacosSnapshotSuite(runtime: SnapshotRuntime): void { expect(isError).toBe(true); expectFixture(text, 'build--error-wrong-scheme'); }); + + it('error - compiler error', { timeout: 120000 }, async () => { + const { text, isError } = await withMacosAppCompilerError(() => + harness.invoke('macos', 'build', { + projectPath: PROJECT, + scheme: 'MCPTest', + }), + ); + expect(isError).toBe(true); + expectFixture(text, 'build--error-compiler'); + }); }); describe('build-and-run', () => { @@ -89,6 +104,17 @@ export function registerMacosSnapshotSuite(runtime: SnapshotRuntime): void { expect(isError).toBe(true); expectFixture(text, 'build-and-run--error-wrong-scheme'); }); + + it('error - compiler error', { timeout: 120000 }, async () => { + const { text, isError } = await withMacosAppCompilerError(() => + harness.invoke('macos', 'build-and-run', { + projectPath: PROJECT, + scheme: 'MCPTest', + }), + ); + expect(isError).toBe(true); + expectFixture(text, 'build-and-run--error-compiler'); + }); }); describe('test', () => { @@ -124,6 +150,21 @@ export function registerMacosSnapshotSuite(runtime: SnapshotRuntime): void { expect(isError).toBe(true); expectFixture(text, 'test--error-wrong-scheme'); }); + + it('error - compiler error', { timeout: 120000 }, async () => { + const { text, isError } = await withMacosAppCompilerError(() => + harness.invoke('macos', 'test', { + projectPath: PROJECT, + scheme: 'MCPTest', + extraArgs: [ + '-only-testing:MCPTestTests/MCPTestTests/appNameIsCorrect()', + '-only-testing:MCPTestTests/MCPTestsXCTests/testAppNameIsCorrect', + ], + }), + ); + expect(isError).toBe(true); + expectFixture(text, 'test--error-compiler'); + }); }); describe('get-app-path', () => { diff --git a/src/snapshot-tests/suites/simulator-suite.ts b/src/snapshot-tests/suites/simulator-suite.ts index 9762af8fd..bdcfe7f13 100644 --- a/src/snapshot-tests/suites/simulator-suite.ts +++ b/src/snapshot-tests/suites/simulator-suite.ts @@ -5,7 +5,11 @@ import path from 'node:path'; import { ensureSimulatorBooted } from '../harness.ts'; import type { SnapshotRuntime, WorkflowSnapshotHarness } from '../contracts.ts'; import { extractAppPathFromSnapshotOutput } from '../output-parsers.ts'; -import { createHarnessForRuntime, createWorkflowFixtureMatcher } from './helpers.ts'; +import { + createHarnessForRuntime, + createWorkflowFixtureMatcher, + withCalculatorAppCompilerError, +} from './helpers.ts'; const TEST_TIMEOUT_MS = 120_000; const WORKSPACE = 'example_projects/iOS_Calculator/CalculatorApp.xcworkspace'; @@ -63,6 +67,22 @@ export function registerSimulatorSnapshotSuite(runtime: SnapshotRuntime): void { }, TEST_TIMEOUT_MS, ); + + it( + 'error - compiler error', + async () => { + const { text, isError } = await withCalculatorAppCompilerError(() => + harness.invoke('simulator', 'build', { + workspacePath: WORKSPACE, + scheme: SCHEME, + simulatorName: SIMULATOR_NAME, + }), + ); + expect(isError).toBe(true); + expectFixture(text, 'build--error-compiler'); + }, + TEST_TIMEOUT_MS, + ); }); describe('build-and-run', () => { @@ -94,6 +114,22 @@ export function registerSimulatorSnapshotSuite(runtime: SnapshotRuntime): void { }, TEST_TIMEOUT_MS, ); + + it( + 'error - compiler error', + async () => { + const { text, isError } = await withCalculatorAppCompilerError(() => + harness.invoke('simulator', 'build-and-run', { + workspacePath: WORKSPACE, + scheme: SCHEME, + simulatorName: SIMULATOR_NAME, + }), + ); + expect(isError).toBe(true); + expectFixture(text, 'build-and-run--error-compiler'); + }, + TEST_TIMEOUT_MS, + ); }); describe('test', () => { @@ -141,6 +177,23 @@ export function registerSimulatorSnapshotSuite(runtime: SnapshotRuntime): void { }, TEST_TIMEOUT_MS, ); + + it( + 'error - compiler error', + async () => { + const { text, isError } = await withCalculatorAppCompilerError(() => + harness.invoke('simulator', 'test', { + workspacePath: WORKSPACE, + scheme: SCHEME, + simulatorName: SIMULATOR_NAME, + extraArgs: ['-only-testing:CalculatorAppTests/CalculatorAppTests/testAddition'], + }), + ); + expect(isError).toBe(true); + expectFixture(text, 'test--error-compiler'); + }, + TEST_TIMEOUT_MS, + ); }); describe('get-app-path', () => { diff --git a/src/utils/__tests__/xcodebuild-event-parser.test.ts b/src/utils/__tests__/xcodebuild-event-parser.test.ts index 6eb454a41..a49ed3328 100644 --- a/src/utils/__tests__/xcodebuild-event-parser.test.ts +++ b/src/utils/__tests__/xcodebuild-event-parser.test.ts @@ -262,6 +262,34 @@ describe('xcodebuild-event-parser', () => { }); }); + it('does not emit compiler errors for NSError selector dump lines', () => { + const events = collectEvents('TEST', [ + { source: 'stderr', text: 'pid:error:,\n' }, + { + source: 'stderr', + text: '} (error = Error Domain=FBSOpenApplicationServiceErrorDomain Code=1 "The request was denied" UserInfo={BSErrorCodeDescription=RequestDenied, SimCallingSelector=launchApplicationWithID:options:pid:error:, NSLocalizedDescription=The request was denied})\n', + }, + { + source: 'stdout', + text: "Test Case '-[WeatherTests.WeatherTests testLoadsForecast]' passed (0.002 seconds)\n", + }, + ]); + + const errors = events.filter( + (e) => e.fragment === 'compiler-diagnostic' && e.severity === 'error', + ); + const cases = events.filter((e) => e.fragment === 'test-case-result'); + + expect(errors).toHaveLength(0); + expect(cases).toHaveLength(1); + expect(cases[0]).toMatchObject({ + fragment: 'test-case-result', + status: 'passed', + suite: 'WeatherTests', + test: 'testLoadsForecast', + }); + }); + it('emits swift-testing issue fallbacks as test failures with the full raw line', () => { const line = 'โœ˜ Test "Parameterized failure" recorded an issue with 1 argument value โ†’ key:value: opaque failure'; diff --git a/src/utils/__tests__/xcodebuild-line-parsers.test.ts b/src/utils/__tests__/xcodebuild-line-parsers.test.ts index 1ca7ca29b..2b76b96af 100644 --- a/src/utils/__tests__/xcodebuild-line-parsers.test.ts +++ b/src/utils/__tests__/xcodebuild-line-parsers.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { + isBuildErrorDiagnosticLine, parseBuildErrorDiagnostic, parseDurationMs, parseRawTestName, @@ -18,6 +19,35 @@ describe('parseDurationMs', () => { }); describe('parseBuildErrorDiagnostic', () => { + it('parses structured compiler and xcodebuild errors', () => { + expect( + parseBuildErrorDiagnostic( + "/tmp/App.swift:8:17: error: cannot convert value of type 'String' to specified type 'Int'", + ), + ).toEqual({ + location: '/tmp/App.swift:8', + message: "cannot convert value of type 'String' to specified type 'Int'", + renderedLine: + "/tmp/App.swift:8:17: error: cannot convert value of type 'String' to specified type 'Int'", + }); + + expect(parseBuildErrorDiagnostic('/tmp/MyApp.xcodeproj: error: No such project')).toEqual({ + location: '/tmp/MyApp.xcodeproj', + message: 'No such project', + renderedLine: '/tmp/MyApp.xcodeproj: error: No such project', + }); + + expect(parseBuildErrorDiagnostic('xcodebuild: error: Unable to find destination')).toEqual({ + message: 'Unable to find destination', + renderedLine: 'xcodebuild: error: Unable to find destination', + }); + + expect(parseBuildErrorDiagnostic('error: emit-module command failed')).toEqual({ + message: 'emit-module command failed', + renderedLine: 'error: emit-module command failed', + }); + }); + it('preserves the full raw line for diagnostic-looking errors without a known structure', () => { const line = '2026-04-23 12:00:00.000 xcodebuild[123:456] error: IDE operation failed'; @@ -26,6 +56,19 @@ describe('parseBuildErrorDiagnostic', () => { renderedLine: line, }); }); + + it('does not classify Objective-C selector fragments or NSError dump lines as build errors', () => { + const selectorLine = 'pid:error:,'; + const nserrorLine = + '} (error = Error Domain=FBSOpenApplicationServiceErrorDomain Code=1 "The request was denied" UserInfo={BSErrorCodeDescription=RequestDenied, SimCallingSelector=launchApplicationWithID:options:pid:error:, NSLocalizedDescription=The request was denied})'; + const nsMachLine = + '} (error = Error Domain=NSMachErrorDomain Code=3 "No such process" UserInfo={NSLocalizedDescription=No such process})'; + + for (const line of [selectorLine, nserrorLine, nsMachLine]) { + expect(isBuildErrorDiagnosticLine(line)).toBe(false); + expect(parseBuildErrorDiagnostic(line)).toBeNull(); + } + }); }); describe('parseRawTestName', () => { diff --git a/src/utils/renderers/__tests__/cli-text-renderer.test.ts b/src/utils/renderers/__tests__/cli-text-renderer.test.ts index 8ba6a13f2..d112d7ea6 100644 --- a/src/utils/renderers/__tests__/cli-text-renderer.test.ts +++ b/src/utils/renderers/__tests__/cli-text-renderer.test.ts @@ -191,6 +191,81 @@ describe('cli-text-renderer', () => { expect(output).toContain('\u{274C} Build failed. (\u{23F1}\u{FE0F} 1.2s)'); }); + it('does not flush buffered compiler errors after a successful final summary', () => { + const output = renderCliTextTranscript({ + items: [ + { + kind: 'test-result', + fragment: 'compiler-diagnostic', + severity: 'error', + operation: 'TEST', + message: 'SimCallingSelector=launchApplicationWithID:options:pid:error:,', + rawLine: 'SimCallingSelector=launchApplicationWithID:options:pid:error:,', + }, + { + kind: 'test-result', + fragment: 'build-summary', + operation: 'TEST', + status: 'SUCCEEDED', + totalTests: 1, + passedTests: 1, + failedTests: 0, + skippedTests: 0, + }, + ], + }); + + expect(output).toContain('โœ… 1 test passed, 0 skipped'); + expect(output).not.toContain('Compiler Errors (1):'); + expect(output).not.toContain('SimCallingSelector=launchApplicationWithID:options:pid:error:,'); + }); + + it('flushes buffered compiler errors after a failed final summary', () => { + const output = renderCliTextTranscript({ + items: [ + { + kind: 'test-result', + fragment: 'compiler-diagnostic', + severity: 'error', + operation: 'TEST', + message: 'unterminated string literal', + rawLine: '/tmp/MCPTest/ContentView.swift:16:18: error: unterminated string literal', + }, + { + kind: 'test-result', + fragment: 'build-summary', + operation: 'TEST', + status: 'FAILED', + totalTests: 1, + passedTests: 0, + failedTests: 1, + skippedTests: 0, + }, + ], + }); + + expect(output).toContain('Compiler Errors (1):'); + expect(output).toContain('unterminated string literal'); + }); + + it('flushes buffered compiler errors when final status is unknown', () => { + const output = renderCliTextTranscript({ + items: [ + { + kind: 'build-result', + fragment: 'compiler-diagnostic', + severity: 'error', + operation: 'BUILD', + message: 'unknown build failure', + rawLine: 'error: unknown build failure', + }, + ], + }); + + expect(output).toContain('Errors (1):'); + expect(output).toContain('unknown build failure'); + }); + it('groups compiler diagnostics under a nested failure header before the failed summary', () => { const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); const renderer = createCliTextRenderer({ interactive: false }); diff --git a/src/utils/renderers/cli-text-renderer.ts b/src/utils/renderers/cli-text-renderer.ts index fd1c23b58..897df0cc2 100644 --- a/src/utils/renderers/cli-text-renderer.ts +++ b/src/utils/renderers/cli-text-renderer.ts @@ -105,6 +105,7 @@ function createCliTextProcessor(options: CliTextProcessorOptions): TranscriptRen let hasDurableRuntimeContent = false; let lastVisibleEventType: TextRenderableItem['type'] | null = null; let lastStatusLineLevel: StatusRenderItem['level'] | null = null; + let lastSummaryStatus: 'SUCCEEDED' | 'FAILED' | null = null; let structuredOutput: StructuredToolOutput | undefined; let sawIncomingHeaderEvent = false; let sawIncomingNonHeaderEvent = false; @@ -281,6 +282,7 @@ function createCliTextProcessor(options: CliTextProcessorOptions): TranscriptRen } case 'summary': { + lastSummaryStatus = item.status; const renderedDiagnostics = flushGroupedDiagnostics(item.status === 'FAILED'); if (!renderedDiagnostics && item.status === 'FAILED') { @@ -417,7 +419,10 @@ function createCliTextProcessor(options: CliTextProcessorOptions): TranscriptRen } } } - flushGroupedDiagnostics(true); + flushGroupedDiagnostics(lastSummaryStatus !== 'SUCCEEDED'); + groupedCompilerErrors.length = 0; + groupedTestFailures.length = 0; + groupedWarnings.length = 0; const nextStepsBlock = createNextStepsBlock(nextSteps, nextStepsRuntime); if (nextStepsBlock && !sawProgressNextSteps) { processItem(nextStepsBlock); @@ -428,6 +433,7 @@ function createCliTextProcessor(options: CliTextProcessorOptions): TranscriptRen hasDurableRuntimeContent = false; lastVisibleEventType = null; lastStatusLineLevel = null; + lastSummaryStatus = null; structuredOutput = undefined; sawIncomingHeaderEvent = false; sawIncomingNonHeaderEvent = false; diff --git a/src/utils/xcodebuild-line-parsers.ts b/src/utils/xcodebuild-line-parsers.ts index 9ad6a99d6..3e81fb406 100644 --- a/src/utils/xcodebuild-line-parsers.ts +++ b/src/utils/xcodebuild-line-parsers.ts @@ -49,7 +49,7 @@ export interface ParsedBuildError { renderedLine: string; } -const BUILD_ERROR_DIAGNOSTIC_PATTERN = /(?:^|[\s:])(?:fatal error|error):\s*\S/iu; +const BUILD_ERROR_DIAGNOSTIC_PATTERN = /(?:^|\s)(?:fatal error|error):\s+\S/iu; export function isBuildErrorDiagnosticLine(line: string): boolean { return BUILD_ERROR_DIAGNOSTIC_PATTERN.test(line); From b7e981389ea84fd62f37db0cf0faac5193f36458 Mon Sep 17 00:00:00 2001 From: Cameron Cooke Date: Fri, 1 May 2026 22:01:30 +0100 Subject: [PATCH 2/2] test(snapshot): Avoid mutating example app sources Trigger compiler-error snapshot cases with a guarded Swift compile flag instead of patching example project files during test execution. This keeps snapshot tests deterministic and avoids leaving tracked source files corrupted if a run is interrupted. Refs GH-387 Co-Authored-By: Codex --- .../CalculatorApp/CalculatorApp.swift | 4 ++ .../macOS/MCPTest/MCPTestApp.swift | 4 ++ .../cli/device/build--error-compiler.txt | 2 +- .../device/build-and-run--error-compiler.txt | 2 +- .../cli/device/test--error-compiler.txt | 2 +- .../cli/macos/build--error-compiler.txt | 2 +- .../macos/build-and-run--error-compiler.txt | 2 +- .../cli/macos/test--error-compiler.txt | 2 +- .../cli/simulator/build--error-compiler.txt | 2 +- .../build-and-run--error-compiler.txt | 2 +- .../cli/simulator/test--error-compiler.txt | 2 +- .../json/device/build--error-compiler.json | 2 +- .../device/build-and-run--error-compiler.json | 2 +- .../json/device/test--error-compiler.json | 2 +- .../json/macos/build--error-compiler.json | 2 +- .../macos/build-and-run--error-compiler.json | 2 +- .../json/macos/test--error-compiler.json | 2 +- .../json/simulator/build--error-compiler.json | 2 +- .../build-and-run--error-compiler.json | 2 +- .../json/simulator/test--error-compiler.json | 2 +- .../mcp/device/build--error-compiler.txt | 2 +- .../device/build-and-run--error-compiler.txt | 2 +- .../mcp/device/test--error-compiler.txt | 2 +- .../mcp/macos/build--error-compiler.txt | 2 +- .../macos/build-and-run--error-compiler.txt | 2 +- .../mcp/macos/test--error-compiler.txt | 2 +- .../mcp/simulator/build--error-compiler.txt | 2 +- .../build-and-run--error-compiler.txt | 2 +- .../mcp/simulator/test--error-compiler.txt | 2 +- src/snapshot-tests/suites/device-suite.ts | 42 +++++++++--------- src/snapshot-tests/suites/helpers.ts | 44 ++----------------- src/snapshot-tests/suites/macos-suite.ts | 42 ++++++++---------- src/snapshot-tests/suites/simulator-suite.ts | 44 +++++++++---------- 33 files changed, 98 insertions(+), 136 deletions(-) diff --git a/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift b/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift index 08e33364f..6125b83cd 100644 --- a/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift +++ b/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift @@ -28,3 +28,7 @@ struct CalculatorApp: App { #Preview { ContentView() } + +#if SNAPSHOT_COMPILER_ERROR +private let snapshotCompilerError: Int = "not an int" +#endif diff --git a/example_projects/macOS/MCPTest/MCPTestApp.swift b/example_projects/macOS/MCPTest/MCPTestApp.swift index ad27efb26..615dd984a 100644 --- a/example_projects/macOS/MCPTest/MCPTestApp.swift +++ b/example_projects/macOS/MCPTest/MCPTestApp.swift @@ -15,3 +15,7 @@ struct MCPTestApp: App { } } } + +#if SNAPSHOT_COMPILER_ERROR +private let snapshotCompilerError: Int = "not an int" +#endif diff --git a/src/snapshot-tests/__fixtures__/cli/device/build--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/device/build--error-compiler.txt index 5fd8ca357..d43c9fac5 100644 --- a/src/snapshot-tests/__fixtures__/cli/device/build--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/cli/device/build--error-compiler.txt @@ -10,7 +10,7 @@ Compiler Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32:42 + example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33:42 โŒ Build failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_device__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/device/build-and-run--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/device/build-and-run--error-compiler.txt index 7fb598d46..2f77b9a25 100644 --- a/src/snapshot-tests/__fixtures__/cli/device/build-and-run--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/cli/device/build-and-run--error-compiler.txt @@ -11,7 +11,7 @@ Compiler Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32:42 + example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33:42 โŒ Build failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_run_device__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/device/test--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/device/test--error-compiler.txt index e2e358ab0..abfb256d6 100644 --- a/src/snapshot-tests/__fixtures__/cli/device/test--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/cli/device/test--error-compiler.txt @@ -16,7 +16,7 @@ Discovered 1 test(s): Compiler Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32:42 + example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33:42 โŒ Test failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/test_device__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/macos/build--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/macos/build--error-compiler.txt index 6d46f365d..b44a2ac3a 100644 --- a/src/snapshot-tests/__fixtures__/cli/macos/build--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/cli/macos/build--error-compiler.txt @@ -10,7 +10,7 @@ Compiler Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - example_projects/macOS/MCPTest/MCPTestApp.swift:19:42 + example_projects/macOS/MCPTest/MCPTestApp.swift:20:42 โŒ Build failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_macos__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/macos/build-and-run--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/macos/build-and-run--error-compiler.txt index a466239b0..ffebe9491 100644 --- a/src/snapshot-tests/__fixtures__/cli/macos/build-and-run--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/cli/macos/build-and-run--error-compiler.txt @@ -10,7 +10,7 @@ Compiler Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - example_projects/macOS/MCPTest/MCPTestApp.swift:19:42 + example_projects/macOS/MCPTest/MCPTestApp.swift:20:42 โŒ Build failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_run_macos__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/macos/test--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/macos/test--error-compiler.txt index 6a96ee0e4..a78ec0cdb 100644 --- a/src/snapshot-tests/__fixtures__/cli/macos/test--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/cli/macos/test--error-compiler.txt @@ -17,7 +17,7 @@ Discovered 2 test(s): Compiler Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - example_projects/macOS/MCPTest/MCPTestApp.swift:19:42 + example_projects/macOS/MCPTest/MCPTestApp.swift:20:42 โŒ Test failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/test_macos__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/simulator/build--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/simulator/build--error-compiler.txt index c82c53b7c..a387be9e0 100644 --- a/src/snapshot-tests/__fixtures__/cli/simulator/build--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/cli/simulator/build--error-compiler.txt @@ -11,7 +11,7 @@ Compiler Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32:42 + example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33:42 โŒ Build failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_sim__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/simulator/build-and-run--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/simulator/build-and-run--error-compiler.txt index 93a88ee22..d05a3de4b 100644 --- a/src/snapshot-tests/__fixtures__/cli/simulator/build-and-run--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/cli/simulator/build-and-run--error-compiler.txt @@ -11,7 +11,7 @@ Compiler Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32:42 + example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33:42 โŒ Build failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_run_sim__pid.log diff --git a/src/snapshot-tests/__fixtures__/cli/simulator/test--error-compiler.txt b/src/snapshot-tests/__fixtures__/cli/simulator/test--error-compiler.txt index 90f984dd1..b899ab955 100644 --- a/src/snapshot-tests/__fixtures__/cli/simulator/test--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/cli/simulator/test--error-compiler.txt @@ -16,7 +16,7 @@ Discovered 1 test(s): Compiler Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32:42 + example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33:42 โŒ Test failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/test_sim__pid.log diff --git a/src/snapshot-tests/__fixtures__/json/device/build--error-compiler.json b/src/snapshot-tests/__fixtures__/json/device/build--error-compiler.json index a8a4d7ca7..0ffabd315 100644 --- a/src/snapshot-tests/__fixtures__/json/device/build--error-compiler.json +++ b/src/snapshot-tests/__fixtures__/json/device/build--error-compiler.json @@ -25,7 +25,7 @@ "errors": [ { "message": "cannot convert value of type 'String' to specified type 'Int'", - "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32" + "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33" } ] } diff --git a/src/snapshot-tests/__fixtures__/json/device/build-and-run--error-compiler.json b/src/snapshot-tests/__fixtures__/json/device/build-and-run--error-compiler.json index 07a72f579..8235c836d 100644 --- a/src/snapshot-tests/__fixtures__/json/device/build-and-run--error-compiler.json +++ b/src/snapshot-tests/__fixtures__/json/device/build-and-run--error-compiler.json @@ -27,7 +27,7 @@ "errors": [ { "message": "cannot convert value of type 'String' to specified type 'Int'", - "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32" + "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33" } ] } diff --git a/src/snapshot-tests/__fixtures__/json/device/test--error-compiler.json b/src/snapshot-tests/__fixtures__/json/device/test--error-compiler.json index 1eb1e8073..9e9bc2887 100644 --- a/src/snapshot-tests/__fixtures__/json/device/test--error-compiler.json +++ b/src/snapshot-tests/__fixtures__/json/device/test--error-compiler.json @@ -42,7 +42,7 @@ "errors": [ { "message": "cannot convert value of type 'String' to specified type 'Int'", - "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32" + "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33" } ], "testFailures": [] diff --git a/src/snapshot-tests/__fixtures__/json/macos/build--error-compiler.json b/src/snapshot-tests/__fixtures__/json/macos/build--error-compiler.json index b04f9fe58..c182f632a 100644 --- a/src/snapshot-tests/__fixtures__/json/macos/build--error-compiler.json +++ b/src/snapshot-tests/__fixtures__/json/macos/build--error-compiler.json @@ -25,7 +25,7 @@ "errors": [ { "message": "cannot convert value of type 'String' to specified type 'Int'", - "location": "/example_projects/macOS/MCPTest/MCPTestApp.swift:19" + "location": "/example_projects/macOS/MCPTest/MCPTestApp.swift:20" } ] } diff --git a/src/snapshot-tests/__fixtures__/json/macos/build-and-run--error-compiler.json b/src/snapshot-tests/__fixtures__/json/macos/build-and-run--error-compiler.json index 216ad1131..4d4e836fe 100644 --- a/src/snapshot-tests/__fixtures__/json/macos/build-and-run--error-compiler.json +++ b/src/snapshot-tests/__fixtures__/json/macos/build-and-run--error-compiler.json @@ -25,7 +25,7 @@ "errors": [ { "message": "cannot convert value of type 'String' to specified type 'Int'", - "location": "/example_projects/macOS/MCPTest/MCPTestApp.swift:19" + "location": "/example_projects/macOS/MCPTest/MCPTestApp.swift:20" } ] } diff --git a/src/snapshot-tests/__fixtures__/json/macos/test--error-compiler.json b/src/snapshot-tests/__fixtures__/json/macos/test--error-compiler.json index 282123bcf..a10b16e2b 100644 --- a/src/snapshot-tests/__fixtures__/json/macos/test--error-compiler.json +++ b/src/snapshot-tests/__fixtures__/json/macos/test--error-compiler.json @@ -42,7 +42,7 @@ "errors": [ { "message": "cannot convert value of type 'String' to specified type 'Int'", - "location": "/example_projects/macOS/MCPTest/MCPTestApp.swift:19" + "location": "/example_projects/macOS/MCPTest/MCPTestApp.swift:20" } ], "testFailures": [] diff --git a/src/snapshot-tests/__fixtures__/json/simulator/build--error-compiler.json b/src/snapshot-tests/__fixtures__/json/simulator/build--error-compiler.json index 850006e7f..44ab7ffc6 100644 --- a/src/snapshot-tests/__fixtures__/json/simulator/build--error-compiler.json +++ b/src/snapshot-tests/__fixtures__/json/simulator/build--error-compiler.json @@ -25,7 +25,7 @@ "errors": [ { "message": "cannot convert value of type 'String' to specified type 'Int'", - "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32" + "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33" } ] } diff --git a/src/snapshot-tests/__fixtures__/json/simulator/build-and-run--error-compiler.json b/src/snapshot-tests/__fixtures__/json/simulator/build-and-run--error-compiler.json index bbe287cdb..d9f65bc94 100644 --- a/src/snapshot-tests/__fixtures__/json/simulator/build-and-run--error-compiler.json +++ b/src/snapshot-tests/__fixtures__/json/simulator/build-and-run--error-compiler.json @@ -25,7 +25,7 @@ "errors": [ { "message": "cannot convert value of type 'String' to specified type 'Int'", - "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32" + "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33" } ] } diff --git a/src/snapshot-tests/__fixtures__/json/simulator/test--error-compiler.json b/src/snapshot-tests/__fixtures__/json/simulator/test--error-compiler.json index 2941c31c1..5e8518d64 100644 --- a/src/snapshot-tests/__fixtures__/json/simulator/test--error-compiler.json +++ b/src/snapshot-tests/__fixtures__/json/simulator/test--error-compiler.json @@ -40,7 +40,7 @@ "errors": [ { "message": "cannot convert value of type 'String' to specified type 'Int'", - "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32" + "location": "/example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33" } ], "testFailures": [] diff --git a/src/snapshot-tests/__fixtures__/mcp/device/build--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/device/build--error-compiler.txt index 61e55c15f..20bc32b7a 100644 --- a/src/snapshot-tests/__fixtures__/mcp/device/build--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/mcp/device/build--error-compiler.txt @@ -10,7 +10,7 @@ Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32 + /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33 โŒ Build failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_device__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/device/build-and-run--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/device/build-and-run--error-compiler.txt index b8bd68c1c..e273fb5ac 100644 --- a/src/snapshot-tests/__fixtures__/mcp/device/build-and-run--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/mcp/device/build-and-run--error-compiler.txt @@ -11,7 +11,7 @@ Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32 + /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33 โŒ Build failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_run_device__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/device/test--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/device/test--error-compiler.txt index 9677eaab1..0b3acf227 100644 --- a/src/snapshot-tests/__fixtures__/mcp/device/test--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/mcp/device/test--error-compiler.txt @@ -16,7 +16,7 @@ Discovered 1 test(s): Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32 + /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33 โŒ Test failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/test_device__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/macos/build--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/macos/build--error-compiler.txt index a73d026cf..66803c8de 100644 --- a/src/snapshot-tests/__fixtures__/mcp/macos/build--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/mcp/macos/build--error-compiler.txt @@ -10,7 +10,7 @@ Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - /example_projects/macOS/MCPTest/MCPTestApp.swift:19 + /example_projects/macOS/MCPTest/MCPTestApp.swift:20 โŒ Build failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_macos__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/macos/build-and-run--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/macos/build-and-run--error-compiler.txt index 3839ac1d5..b3491412a 100644 --- a/src/snapshot-tests/__fixtures__/mcp/macos/build-and-run--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/mcp/macos/build-and-run--error-compiler.txt @@ -10,7 +10,7 @@ Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - /example_projects/macOS/MCPTest/MCPTestApp.swift:19 + /example_projects/macOS/MCPTest/MCPTestApp.swift:20 โŒ Build failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_run_macos__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/macos/test--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/macos/test--error-compiler.txt index e309e4ee2..60ac06f9b 100644 --- a/src/snapshot-tests/__fixtures__/mcp/macos/test--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/mcp/macos/test--error-compiler.txt @@ -17,7 +17,7 @@ Discovered 2 test(s): Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - /example_projects/macOS/MCPTest/MCPTestApp.swift:19 + /example_projects/macOS/MCPTest/MCPTestApp.swift:20 โŒ Test failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/test_macos__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/simulator/build--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/simulator/build--error-compiler.txt index 1a8eff2a5..230c8f5f9 100644 --- a/src/snapshot-tests/__fixtures__/mcp/simulator/build--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/mcp/simulator/build--error-compiler.txt @@ -11,7 +11,7 @@ Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32 + /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33 โŒ Build failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_sim__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/simulator/build-and-run--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/simulator/build-and-run--error-compiler.txt index bcc09d3e1..ddc35f169 100644 --- a/src/snapshot-tests/__fixtures__/mcp/simulator/build-and-run--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/mcp/simulator/build-and-run--error-compiler.txt @@ -11,7 +11,7 @@ Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32 + /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33 โŒ Build failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/build_run_sim__pid.log diff --git a/src/snapshot-tests/__fixtures__/mcp/simulator/test--error-compiler.txt b/src/snapshot-tests/__fixtures__/mcp/simulator/test--error-compiler.txt index b2db282f1..ccae37aec 100644 --- a/src/snapshot-tests/__fixtures__/mcp/simulator/test--error-compiler.txt +++ b/src/snapshot-tests/__fixtures__/mcp/simulator/test--error-compiler.txt @@ -16,7 +16,7 @@ Discovered 1 test(s): Errors (1): โœ— cannot convert value of type 'String' to specified type 'Int' - /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:32 + /example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift:33 โŒ Test failed. (โฑ๏ธ ) โ”” Build Logs: /Library/Developer/XcodeBuildMCP/logs/test_sim__pid.log diff --git a/src/snapshot-tests/suites/device-suite.ts b/src/snapshot-tests/suites/device-suite.ts index c0770050a..cc73b60f7 100644 --- a/src/snapshot-tests/suites/device-suite.ts +++ b/src/snapshot-tests/suites/device-suite.ts @@ -6,9 +6,9 @@ import { extractProcessIdFromSnapshotOutput, } from '../output-parsers.ts'; import { + compilerErrorExtraArgs, createHarnessForRuntime, createWorkflowFixtureMatcher, - withCalculatorAppCompilerError, } from './helpers.ts'; const WORKSPACE = 'example_projects/iOS_Calculator/CalculatorApp.xcworkspace'; @@ -67,12 +67,11 @@ export function registerDeviceSnapshotSuite(runtime: SnapshotRuntime): void { }); it('error - compiler error', async () => { - const { text, isError } = await withCalculatorAppCompilerError(() => - harness.invoke('device', 'build', { - workspacePath: WORKSPACE, - scheme: 'CalculatorApp', - }), - ); + const { text, isError } = await harness.invoke('device', 'build', { + workspacePath: WORKSPACE, + scheme: 'CalculatorApp', + extraArgs: compilerErrorExtraArgs(), + }); expect(isError).toBe(true); expectFixture(text, 'build--error-compiler'); }); @@ -156,13 +155,12 @@ export function registerDeviceSnapshotSuite(runtime: SnapshotRuntime): void { }); it('error - compiler error', async () => { - const { text, isError } = await withCalculatorAppCompilerError(() => - harness.invoke('device', 'build-and-run', { - workspacePath: WORKSPACE, - scheme: 'CalculatorApp', - deviceId: DEVICE_ID, - }), - ); + const { text, isError } = await harness.invoke('device', 'build-and-run', { + workspacePath: WORKSPACE, + scheme: 'CalculatorApp', + deviceId: DEVICE_ID, + extraArgs: compilerErrorExtraArgs(), + }); expect(isError).toBe(true); expectFixture(text, 'build-and-run--error-compiler'); }); @@ -245,14 +243,14 @@ export function registerDeviceSnapshotSuite(runtime: SnapshotRuntime): void { }, 300_000); it('error - compiler error', async () => { - const { text, isError } = await withCalculatorAppCompilerError(() => - harness.invoke('device', 'test', { - workspacePath: WORKSPACE, - scheme: 'CalculatorApp', - deviceId: DEVICE_ID, - extraArgs: ['-only-testing:CalculatorAppTests/CalculatorAppTests/testAddition'], - }), - ); + const { text, isError } = await harness.invoke('device', 'test', { + workspacePath: WORKSPACE, + scheme: 'CalculatorApp', + deviceId: DEVICE_ID, + extraArgs: compilerErrorExtraArgs([ + '-only-testing:CalculatorAppTests/CalculatorAppTests/testAddition', + ]), + }); expect(isError).toBe(true); expectFixture(text, 'test--error-compiler'); }, 300_000); diff --git a/src/snapshot-tests/suites/helpers.ts b/src/snapshot-tests/suites/helpers.ts index fc6b31fe1..cfcf8de8a 100644 --- a/src/snapshot-tests/suites/helpers.ts +++ b/src/snapshot-tests/suites/helpers.ts @@ -1,20 +1,10 @@ -import fs from 'node:fs'; -import path from 'node:path'; import type { SnapshotRuntime, WorkflowSnapshotHarness } from '../contracts.ts'; import { createFixtureMatcher, type FixtureMatchOptions } from '../fixture-io.ts'; import { createSnapshotHarness } from '../harness.ts'; import { createJsonSnapshotHarness } from '../json-harness.ts'; import { createMcpSnapshotHarness } from '../mcp-harness.ts'; -const CALCULATOR_APP_SOURCE_PATH = path.resolve( - process.cwd(), - 'example_projects/iOS_Calculator/CalculatorApp/CalculatorApp.swift', -); -const MACOS_APP_SOURCE_PATH = path.resolve( - process.cwd(), - 'example_projects/macOS/MCPTest/MCPTestApp.swift', -); -const COMPILER_ERROR_SNIPPET = 'private let snapshotCompilerError: Int = "not an int"'; +const COMPILER_ERROR_EXTRA_ARGS = ['OTHER_SWIFT_FLAGS=$(inherited) -D SNAPSHOT_COMPILER_ERROR']; export function createHarnessForRuntime( runtime: SnapshotRuntime, @@ -34,36 +24,8 @@ export interface WorkflowFixtureMatcherOptions extends FixtureMatchOptions { fixtureRuntime?: SnapshotRuntime; } -export async function withCalculatorAppCompilerError(action: () => Promise): Promise { - const originalSource = fs.readFileSync(CALCULATOR_APP_SOURCE_PATH, 'utf8'); - - try { - fs.writeFileSync( - CALCULATOR_APP_SOURCE_PATH, - `${originalSource}\n${COMPILER_ERROR_SNIPPET}\n`, - 'utf8', - ); - - return await action(); - } finally { - fs.writeFileSync(CALCULATOR_APP_SOURCE_PATH, originalSource, 'utf8'); - } -} - -export async function withMacosAppCompilerError(action: () => Promise): Promise { - const originalSource = fs.readFileSync(MACOS_APP_SOURCE_PATH, 'utf8'); - - try { - fs.writeFileSync( - MACOS_APP_SOURCE_PATH, - `${originalSource}\n${COMPILER_ERROR_SNIPPET}\n`, - 'utf8', - ); - - return await action(); - } finally { - fs.writeFileSync(MACOS_APP_SOURCE_PATH, originalSource, 'utf8'); - } +export function compilerErrorExtraArgs(extraArgs: string[] = []): string[] { + return [...extraArgs, ...COMPILER_ERROR_EXTRA_ARGS]; } export function createWorkflowFixtureMatcher( diff --git a/src/snapshot-tests/suites/macos-suite.ts b/src/snapshot-tests/suites/macos-suite.ts index 8c0aa46fd..ac6fe1ab1 100644 --- a/src/snapshot-tests/suites/macos-suite.ts +++ b/src/snapshot-tests/suites/macos-suite.ts @@ -5,9 +5,9 @@ import path from 'node:path'; import type { SnapshotRuntime, WorkflowSnapshotHarness } from '../contracts.ts'; import { extractAppPathFromSnapshotOutput } from '../output-parsers.ts'; import { + compilerErrorExtraArgs, createHarnessForRuntime, createWorkflowFixtureMatcher, - withMacosAppCompilerError, } from './helpers.ts'; const PROJECT = 'example_projects/macOS/MCPTest.xcodeproj'; @@ -74,12 +74,11 @@ export function registerMacosSnapshotSuite(runtime: SnapshotRuntime): void { }); it('error - compiler error', { timeout: 120000 }, async () => { - const { text, isError } = await withMacosAppCompilerError(() => - harness.invoke('macos', 'build', { - projectPath: PROJECT, - scheme: 'MCPTest', - }), - ); + const { text, isError } = await harness.invoke('macos', 'build', { + projectPath: PROJECT, + scheme: 'MCPTest', + extraArgs: compilerErrorExtraArgs(), + }); expect(isError).toBe(true); expectFixture(text, 'build--error-compiler'); }); @@ -106,12 +105,11 @@ export function registerMacosSnapshotSuite(runtime: SnapshotRuntime): void { }); it('error - compiler error', { timeout: 120000 }, async () => { - const { text, isError } = await withMacosAppCompilerError(() => - harness.invoke('macos', 'build-and-run', { - projectPath: PROJECT, - scheme: 'MCPTest', - }), - ); + const { text, isError } = await harness.invoke('macos', 'build-and-run', { + projectPath: PROJECT, + scheme: 'MCPTest', + extraArgs: compilerErrorExtraArgs(), + }); expect(isError).toBe(true); expectFixture(text, 'build-and-run--error-compiler'); }); @@ -152,16 +150,14 @@ export function registerMacosSnapshotSuite(runtime: SnapshotRuntime): void { }); it('error - compiler error', { timeout: 120000 }, async () => { - const { text, isError } = await withMacosAppCompilerError(() => - harness.invoke('macos', 'test', { - projectPath: PROJECT, - scheme: 'MCPTest', - extraArgs: [ - '-only-testing:MCPTestTests/MCPTestTests/appNameIsCorrect()', - '-only-testing:MCPTestTests/MCPTestsXCTests/testAppNameIsCorrect', - ], - }), - ); + const { text, isError } = await harness.invoke('macos', 'test', { + projectPath: PROJECT, + scheme: 'MCPTest', + extraArgs: compilerErrorExtraArgs([ + '-only-testing:MCPTestTests/MCPTestTests/appNameIsCorrect()', + '-only-testing:MCPTestTests/MCPTestsXCTests/testAppNameIsCorrect', + ]), + }); expect(isError).toBe(true); expectFixture(text, 'test--error-compiler'); }); diff --git a/src/snapshot-tests/suites/simulator-suite.ts b/src/snapshot-tests/suites/simulator-suite.ts index bdcfe7f13..c083f8bc1 100644 --- a/src/snapshot-tests/suites/simulator-suite.ts +++ b/src/snapshot-tests/suites/simulator-suite.ts @@ -6,9 +6,9 @@ import { ensureSimulatorBooted } from '../harness.ts'; import type { SnapshotRuntime, WorkflowSnapshotHarness } from '../contracts.ts'; import { extractAppPathFromSnapshotOutput } from '../output-parsers.ts'; import { + compilerErrorExtraArgs, createHarnessForRuntime, createWorkflowFixtureMatcher, - withCalculatorAppCompilerError, } from './helpers.ts'; const TEST_TIMEOUT_MS = 120_000; @@ -71,13 +71,12 @@ export function registerSimulatorSnapshotSuite(runtime: SnapshotRuntime): void { it( 'error - compiler error', async () => { - const { text, isError } = await withCalculatorAppCompilerError(() => - harness.invoke('simulator', 'build', { - workspacePath: WORKSPACE, - scheme: SCHEME, - simulatorName: SIMULATOR_NAME, - }), - ); + const { text, isError } = await harness.invoke('simulator', 'build', { + workspacePath: WORKSPACE, + scheme: SCHEME, + simulatorName: SIMULATOR_NAME, + extraArgs: compilerErrorExtraArgs(), + }); expect(isError).toBe(true); expectFixture(text, 'build--error-compiler'); }, @@ -118,13 +117,12 @@ export function registerSimulatorSnapshotSuite(runtime: SnapshotRuntime): void { it( 'error - compiler error', async () => { - const { text, isError } = await withCalculatorAppCompilerError(() => - harness.invoke('simulator', 'build-and-run', { - workspacePath: WORKSPACE, - scheme: SCHEME, - simulatorName: SIMULATOR_NAME, - }), - ); + const { text, isError } = await harness.invoke('simulator', 'build-and-run', { + workspacePath: WORKSPACE, + scheme: SCHEME, + simulatorName: SIMULATOR_NAME, + extraArgs: compilerErrorExtraArgs(), + }); expect(isError).toBe(true); expectFixture(text, 'build-and-run--error-compiler'); }, @@ -181,14 +179,14 @@ export function registerSimulatorSnapshotSuite(runtime: SnapshotRuntime): void { it( 'error - compiler error', async () => { - const { text, isError } = await withCalculatorAppCompilerError(() => - harness.invoke('simulator', 'test', { - workspacePath: WORKSPACE, - scheme: SCHEME, - simulatorName: SIMULATOR_NAME, - extraArgs: ['-only-testing:CalculatorAppTests/CalculatorAppTests/testAddition'], - }), - ); + const { text, isError } = await harness.invoke('simulator', 'test', { + workspacePath: WORKSPACE, + scheme: SCHEME, + simulatorName: SIMULATOR_NAME, + extraArgs: compilerErrorExtraArgs([ + '-only-testing:CalculatorAppTests/CalculatorAppTests/testAddition', + ]), + }); expect(isError).toBe(true); expectFixture(text, 'test--error-compiler'); },