E2E Smoke Tests (Public APIs) #967
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # E2E Smoke Tests - Real Public API Compatibility Testing | |
| # | |
| # This workflow tests uxc against real public APIs to verify compatibility. | |
| # These tests are primarily for compatibility monitoring and are non-blocking | |
| # for PRs (they run but don't block merging). | |
| # | |
| # For local deterministic E2E tests, see the local-e2e job in ci.yml. | |
| name: E2E Smoke Tests (Public APIs) | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| types: [opened, synchronize, reopened, labeled] | |
| schedule: | |
| # Run smoke tests daily at 00:00 UTC | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: # Allow manual triggering | |
| concurrency: | |
| group: e2e-smoke-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUST_BACKTRACE: 1 | |
| jobs: | |
| changes: | |
| name: Detect Changes | |
| runs-on: ubuntu-latest | |
| outputs: | |
| e2e_relevant: ${{ steps.filter.outputs.e2e_relevant }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6.0.2 | |
| - name: Detect changed files | |
| id: filter | |
| if: github.event_name == 'push' || github.event_name == 'pull_request' | |
| uses: dorny/paths-filter@v3 | |
| with: | |
| filters: | | |
| e2e_relevant: | |
| - 'src/**' | |
| - 'tests/**' | |
| - 'Cargo.toml' | |
| - 'Cargo.lock' | |
| - 'build.rs' | |
| - '.github/workflows/e2e-smoke.yml' | |
| smoke: | |
| name: Protocol E2E Smoke | |
| runs-on: ubuntu-latest | |
| needs: changes | |
| if: | | |
| github.event_name == 'schedule' || | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'push' && needs.changes.outputs.e2e_relevant == 'true') || | |
| (github.event_name == 'pull_request' && | |
| contains(github.event.pull_request.labels.*.name, 'ci/e2e')) | |
| # Make smoke tests non-blocking for PRs (they still run but don't block merge) | |
| continue-on-error: ${{ github.event_name == 'pull_request' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6.0.2 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install jq | |
| run: sudo apt-get update && sudo apt-get install -y jq | |
| - name: Setup Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.22" | |
| - name: Install grpcurl | |
| run: | | |
| go install github.com/fullstorydev/grpcurl/cmd/grpcurl@v1.9.1 | |
| echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" | |
| - name: Setup Node | |
| uses: actions/setup-node@v6.4.0 | |
| with: | |
| node-version: "20" | |
| - name: Build | |
| run: cargo build --release | |
| - name: OpenAPI smoke | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| UXC=target/release/uxc | |
| echo "=== Testing OpenAPI endpoint: https://petstore3.swagger.io/api/v3 ===" | |
| # Host help (operation discovery) with retry | |
| for attempt in {1..3}; do | |
| echo "Attempt $attempt/3: Fetching host help..." | |
| if OUTPUT=$($UXC https://petstore3.swagger.io/api/v3 -h 2>&1); then | |
| if echo "$OUTPUT" | jq -e '.ok == true and .kind == "host_help" and any(.data.operations[]; .operation_id == "get:/store/inventory")' >/dev/null; then | |
| echo "✓ Host help succeeded" | |
| break | |
| else | |
| echo "✗ Host help returned unexpected output:" | |
| echo "$OUTPUT" | jq '.' | |
| fi | |
| else | |
| echo "✗ Host help failed with exit code $?:" | |
| echo "$OUTPUT" | |
| fi | |
| if [[ "$attempt" -eq 3 ]]; then | |
| echo "=== OpenAPI host help test failed after 3 attempts ===" | |
| echo "Last output:" | |
| echo "$OUTPUT" | jq '.' || echo "$OUTPUT" | |
| exit 1 | |
| fi | |
| echo "Retrying in 3 seconds..." | |
| sleep 3 | |
| done | |
| # Inspect operation help with retry | |
| for attempt in {1..3}; do | |
| echo "Attempt $attempt/3: Inspecting get:/store/inventory..." | |
| if OUTPUT=$($UXC https://petstore3.swagger.io/api/v3 get:/store/inventory -h 2>&1); then | |
| if echo "$OUTPUT" | jq -e '.ok == true and .protocol == "openapi" and .kind == "operation_detail" and .operation == "get:/store/inventory"' >/dev/null; then | |
| echo "✓ Operation help succeeded" | |
| break | |
| else | |
| echo "✗ Operation help returned unexpected output:" | |
| echo "$OUTPUT" | jq '.' | |
| fi | |
| else | |
| echo "✗ Operation help failed with exit code $?:" | |
| echo "$OUTPUT" | |
| fi | |
| if [[ "$attempt" -eq 3 ]]; then | |
| echo "=== OpenAPI operation help test failed after 3 attempts ===" | |
| echo "Last output:" | |
| echo "$OUTPUT" | jq '.' || echo "$OUTPUT" | |
| exit 1 | |
| fi | |
| echo "Retrying in 3 seconds..." | |
| sleep 3 | |
| done | |
| echo "=== OpenAPI smoke tests passed ===" | |
| - name: GraphQL smoke | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| UXC=target/release/uxc | |
| echo "=== Testing GraphQL endpoint: https://countries.trevorblades.com/ ===" | |
| # Host help (operation discovery) with retry | |
| for attempt in {1..3}; do | |
| echo "Attempt $attempt/3: Fetching host help..." | |
| if OUTPUT=$($UXC https://countries.trevorblades.com/ -h 2>&1); then | |
| if echo "$OUTPUT" | jq -e '.ok == true and .kind == "host_help" and any(.data.operations[]; .operation_id == "query/country")' >/dev/null; then | |
| echo "✓ Host help succeeded" | |
| break | |
| else | |
| echo "✗ Host help returned unexpected output:" | |
| echo "$OUTPUT" | jq '.' | |
| fi | |
| else | |
| echo "✗ Host help failed with exit code $?:" | |
| echo "$OUTPUT" | |
| fi | |
| if [[ "$attempt" -eq 3 ]]; then | |
| echo "=== GraphQL host help test failed after 3 attempts ===" | |
| echo "Last output:" | |
| echo "$OUTPUT" | jq '.' || echo "$OUTPUT" | |
| exit 1 | |
| fi | |
| echo "Retrying in 3 seconds..." | |
| sleep 3 | |
| done | |
| # Call operation with retry | |
| for attempt in {1..3}; do | |
| echo "Attempt $attempt/3: Calling query/country..." | |
| if OUTPUT=$($UXC https://countries.trevorblades.com/ query/country '{"code":"US"}' 2>&1); then | |
| if echo "$OUTPUT" | jq -e '.ok == true and .protocol == "graphql" and .kind == "call_result"' >/dev/null; then | |
| echo "✓ Call operation succeeded" | |
| echo "Country data:" | |
| echo "$OUTPUT" | jq '.data.data.country | {name, code, capital}' | |
| break | |
| else | |
| echo "✗ Call operation returned unexpected output:" | |
| echo "$OUTPUT" | jq '.' | |
| fi | |
| else | |
| echo "✗ Call operation failed with exit code $?:" | |
| echo "$OUTPUT" | |
| fi | |
| if [[ "$attempt" -eq 3 ]]; then | |
| echo "=== GraphQL call test failed after 3 attempts ===" | |
| echo "Last output:" | |
| echo "$OUTPUT" | jq '.' || echo "$OUTPUT" | |
| exit 1 | |
| fi | |
| echo "Retrying in 3 seconds..." | |
| sleep 3 | |
| done | |
| echo "=== GraphQL smoke tests passed ===" | |
| - name: gRPC smoke | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| UXC=target/release/uxc | |
| echo "=== Testing gRPC endpoint: grpcb.in:9000 ===" | |
| # Host help (operation discovery) with retry | |
| for attempt in {1..3}; do | |
| echo "Attempt $attempt/3: Fetching host help..." | |
| if OUTPUT=$("$UXC" grpcb.in:9000 -h 2>&1); then | |
| if echo "$OUTPUT" | jq -e '.ok == true and .kind == "host_help" and any(.data.operations[]; .operation_id == "addsvc.Add/Sum")' >/dev/null; then | |
| echo "✓ Host help succeeded" | |
| break | |
| else | |
| echo "✗ Host help returned unexpected output:" | |
| echo "$OUTPUT" | jq '.' | |
| fi | |
| else | |
| echo "✗ Host help failed with exit code $?:" | |
| echo "$OUTPUT" | |
| fi | |
| if [[ "$attempt" -eq 3 ]]; then | |
| echo "=== gRPC host help test failed after 3 attempts ===" | |
| echo "Last output:" | |
| echo "$OUTPUT" | jq '.' || echo "$OUTPUT" | |
| exit 1 | |
| fi | |
| echo "Retrying in 3 seconds..." | |
| sleep 3 | |
| done | |
| # Call operation with retry | |
| for attempt in {1..3}; do | |
| echo "Attempt $attempt/3: Calling addsvc.Add/Sum..." | |
| if OUTPUT=$("$UXC" grpcb.in:9000 addsvc.Add/Sum '{"a":1,"b":2}' 2>&1); then | |
| if echo "$OUTPUT" | jq -e '.ok == true and .protocol == "grpc" and .data.v == "3"' >/dev/null; then | |
| echo "✓ Call operation succeeded" | |
| echo "Result: 1 + 2 = $(jq -r '.data.v' <<<"$OUTPUT")" | |
| break | |
| else | |
| echo "✗ Call operation returned unexpected output:" | |
| echo "$OUTPUT" | jq '.' | |
| fi | |
| else | |
| echo "✗ Call operation failed with exit code $?:" | |
| echo "$OUTPUT" | |
| fi | |
| if [[ "$attempt" -eq 3 ]]; then | |
| echo "=== gRPC call test failed after 3 attempts ===" | |
| echo "Last output:" | |
| echo "$OUTPUT" | jq '.' || echo "$OUTPUT" | |
| exit 1 | |
| fi | |
| echo "Retrying in 3 seconds..." | |
| sleep 3 | |
| done | |
| echo "=== gRPC smoke tests passed ===" | |
| - name: MCP HTTP smoke | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| UXC=target/release/uxc | |
| echo "=== Testing MCP HTTP endpoint: https://mcp.deepwiki.com/mcp ===" | |
| # Host help (operation discovery) with retry | |
| for attempt in {1..3}; do | |
| echo "Attempt $attempt/3: Fetching host help..." | |
| if OUTPUT=$($UXC https://mcp.deepwiki.com/mcp -h 2>&1); then | |
| if echo "$OUTPUT" | jq -e '.ok == true and .kind == "host_help" and any(.data.operations[]; .operation_id == "read_wiki_structure")' >/dev/null; then | |
| echo "✓ Host help succeeded" | |
| break | |
| else | |
| echo "✗ Host help returned unexpected output:" | |
| echo "$OUTPUT" | jq '.' | |
| fi | |
| else | |
| echo "✗ Host help failed with exit code $?:" | |
| echo "$OUTPUT" | |
| fi | |
| if [[ "$attempt" -eq 3 ]]; then | |
| echo "=== MCP HTTP host help test failed after 3 attempts ===" | |
| echo "Last output:" | |
| echo "$OUTPUT" | jq '.' || echo "$OUTPUT" | |
| exit 1 | |
| fi | |
| echo "Retrying in 3 seconds..." | |
| sleep 3 | |
| done | |
| # Call operation with retry | |
| for attempt in {1..3}; do | |
| echo "Attempt $attempt/3: Calling read_wiki_structure..." | |
| if OUTPUT=$($UXC https://mcp.deepwiki.com/mcp read_wiki_structure '{"repoName":"holon-run/uxc"}' 2>&1); then | |
| if echo "$OUTPUT" | jq -e '.ok == true and .protocol == "mcp"' >/dev/null; then | |
| echo "✓ Call operation succeeded" | |
| break | |
| else | |
| echo "✗ Call operation returned unexpected output:" | |
| echo "$OUTPUT" | jq '.' | |
| fi | |
| else | |
| echo "✗ Call operation failed with exit code $?:" | |
| echo "$OUTPUT" | |
| fi | |
| if [[ "$attempt" -eq 3 ]]; then | |
| echo "=== MCP HTTP call test failed after 3 attempts ===" | |
| echo "Last output:" | |
| echo "$OUTPUT" | jq '.' || echo "$OUTPUT" | |
| exit 1 | |
| fi | |
| echo "Retrying in 3 seconds..." | |
| sleep 3 | |
| done | |
| echo "=== MCP HTTP smoke tests passed ===" | |
| - name: MCP stdio smoke | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| export npm_config_loglevel=error | |
| UXC=target/release/uxc | |
| MCP_STDIO='npx -y @modelcontextprotocol/server-filesystem /tmp' | |
| echo "=== Testing MCP stdio with @modelcontextprotocol/server-filesystem ===" | |
| # Host help (operation discovery) with retry | |
| for attempt in {1..3}; do | |
| echo "Attempt $attempt/3: Fetching host help..." | |
| if OUTPUT=$("$UXC" "$MCP_STDIO" -h); then | |
| if echo "$OUTPUT" | jq -e '.ok == true and .kind == "host_help" and any(.data.operations[]; .operation_id == "list_directory")' >/dev/null; then | |
| echo "✓ Host help succeeded" | |
| break | |
| else | |
| echo "✗ Host help returned unexpected output:" | |
| echo "$OUTPUT" | jq '.' | |
| fi | |
| else | |
| echo "✗ Host help failed with exit code $?:" | |
| echo "${OUTPUT:-<empty stdout>}" | |
| fi | |
| if [[ "$attempt" -eq 3 ]]; then | |
| echo "=== MCP stdio host help test failed after 3 attempts ===" | |
| echo "Last output:" | |
| echo "$OUTPUT" | jq '.' || echo "$OUTPUT" | |
| exit 1 | |
| fi | |
| echo "Retrying in 3 seconds..." | |
| sleep 3 | |
| done | |
| # Call operation with retry | |
| for attempt in {1..3}; do | |
| echo "Attempt $attempt/3: Calling list_directory..." | |
| if OUTPUT=$("$UXC" "$MCP_STDIO" list_directory '{"path":"/tmp"}'); then | |
| if echo "$OUTPUT" | jq -e '.ok == true and .protocol == "mcp"' >/dev/null; then | |
| echo "✓ Call operation succeeded" | |
| echo "Found $(echo "$OUTPUT" | jq '.data.data | length') entries in /tmp" | |
| break | |
| else | |
| echo "✗ Call operation returned unexpected output:" | |
| echo "$OUTPUT" | jq '.' | |
| fi | |
| else | |
| echo "✗ Call operation failed with exit code $?:" | |
| echo "${OUTPUT:-<empty stdout>}" | |
| fi | |
| if [[ "$attempt" -eq 3 ]]; then | |
| echo "=== MCP stdio call test failed after 3 attempts ===" | |
| echo "Last output:" | |
| echo "$OUTPUT" | jq '.' || echo "$OUTPUT" | |
| exit 1 | |
| fi | |
| echo "Retrying in 3 seconds..." | |
| sleep 3 | |
| done | |
| echo "=== MCP stdio smoke tests passed ===" | |
| - name: Smoke tests summary | |
| if: always() | |
| run: | | |
| echo "=== Smoke Tests Summary ===" | |
| echo "All external service smoke tests have completed." | |
| echo "These tests verify compatibility with real public APIs:" | |
| echo " - OpenAPI: https://petstore3.swagger.io/api/v3" | |
| echo " - GraphQL: https://countries.trevorblades.com/" | |
| echo " - gRPC: grpcb.in:9000" | |
| echo " - MCP HTTP: https://mcp.deepwiki.com/mcp" | |
| echo " - MCP stdio: @modelcontextprotocol/server-filesystem" | |
| echo "" | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| echo "ℹ️ Running on PR - failures are non-blocking (for compatibility monitoring only)" | |
| else | |
| echo "✅ Running on main branch - failures will block merge" | |
| fi |