Refactor example CI setup. (#437)

This change:
- Adds many more `build_test` targets to ensure non-tests will still be
built and checked.
  - Some of these are not working and are currently disabled.
- Splits each top-level package into a separate CI action so that it's
easier to spot failures and ensure other tests continue
- Adds a new tag-based way to disable tests:
  - `no-ci` will skip the test in CI entirely
  - `no-remote-ci` and `no-local-ci` will skip based on remote execution
  - `no-linux-ci` and `no-macos-ci` will skip based on the OS

Author: katre

.github/workflows/main.yml

@@ -21,32 +21,32 @@ env:
   OPAL_RPC_CREDENTIALS: ${{ secrets.OPAL_RPC_CREDENTIALS }}
 
 jobs:
-  test-matrix:
+  macos-test-matrix:
     runs-on:
       - self-hosted
-      - "os=${{ matrix.os }}"
-      - "arch=${{ matrix.arch }}"
-      - "os_distribution=${{ matrix.os_distribution }}"
-      - "os_version=${{ matrix.os_version }}"
-      - "revision=${{ matrix.revision }}"
+      - "os=macos"
+      - "arch=arm64"
+      - "os_distribution=sonoma"
+      - "os_version=14"
+      - "revision=4edc10aa8f8ca239699aaf1078832208fb3b7efe"
     timeout-minutes: 10
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: macos
-            arch: arm64
-            os_distribution: sonoma
-            os_version: "14"
-            revision: 4edc10aa8f8ca239699aaf1078832208fb3b7efe
-            remote_execution: 'false'
-
-          - os: macos
-            arch: arm64
-            os_distribution: sonoma
-            os_version: "14"
-            revision: 4edc10aa8f8ca239699aaf1078832208fb3b7efe
-            remote_execution: 'true'
+        test_package:
+          - cpp
+          - csharp
+          - docker
+          - genrules
+          - go
+          #- ios # TODO: fix ios builds
+          - java
+          - kotlin
+          - perl
+          - scala
+          - swift
+          - typescript
+        execution_type: [remote, local]
 
     steps:
       - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd"
@@ -61,36 +61,81 @@ jobs:
 
       - name: Build and test
         env:
-          ARCH: ${{ matrix.arch }}
-          OS: ${{ matrix.os }}
-          REMOTE_EXECUTION: ${{ matrix.remote_execution }}
-        run: python3 infra/test-all.py
+          ARCH: arm64
+          OS: macos
+          EXECUTION_TYPE: ${{ matrix.execution_type }}
+        run: python3 infra/test-all.py --package=${{ matrix.test_package }}
 
-  ci-runners-test-matrix:
+  linux-test-matrix:
     runs-on:
       - self-hosted
       - "os=linux"
       - "arch=x64"
-      - "remote_execution=${{ matrix.remote_execution }}"
       - "engflow-bento-name=example-gh-x64"
       - "engflow-cluster=glass"
-      - "engflow-job-name=ci-runners-test-matrix-${{matrix.remote_execution}}_${{matrix.allow_bento_snapshot}}"
+      - "engflow-job-name=linux-test-matrix-${{matrix.execution_type}}"
+      - "engflow-job-type=${{github.event_name}}-${{github.ref_name}}"
+      - "engflow-run-id=${{ github.run_id }}"
+      - "engflow-bento-allow-save=false"
+
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        test_package:
+          - cpp
+          - csharp
+          - docker
+          - genrules
+          - go
+          #- ios # TODO: fix ios builds
+          - java
+          - kotlin
+          - perl
+          - scala
+          - swift
+          - typescript
+        execution_type: [remote, local]
+
+    steps:
+      - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd"
+
+      - name: Set up authentication
+        shell: bash
+        run: cp infra/bazelrc .bazelrc.user
+
+      - name: Generate python requirements lock
+        shell: bash
+        run: bazel run //python:requirements.update
+
+      - name: Build and test
+        env:
+          ARCH: x64
+          OS: linux
+          EXECUTION_TYPE: ${{ matrix.execution_type }}
+        run: python3 infra/test-all.py --package=${{ matrix.test_package }}
+
+  bento-snapshot:
+    runs-on:
+      - self-hosted
+      - "os=linux"
+      - "arch=x64"
+      - "remote_execution=true"
+      - "engflow-bento-name=example-gh-x64"
+      - "engflow-cluster=glass"
+      - "engflow-job-name=bento-snapshot"
       - "engflow-job-type=${{github.event_name}}-${{github.ref_name}}"
       - "engflow-run-id=${{ github.run_id }}"
       # To avoid a race condition, among all the jobs that use the same Bento in the same repo, only
       # one should specify "engflow-bento-allow-save=true".
-      - "engflow-bento-allow-save=${{ matrix.allow_bento_snapshot }}"
+      - "engflow-bento-allow-save=true"
 
     timeout-minutes: 10
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - remote_execution: 'false'
-            allow_bento_snapshot: 'false'
-
-          - remote_execution: 'true'
-            allow_bento_snapshot: 'true'
+        test_package: [cpp]
+        execution_type: [remote]
 
     steps:
       - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd"
@@ -107,8 +152,8 @@ jobs:
         env:
           ARCH: x64
           OS: linux
-          REMOTE_EXECUTION: ${{ matrix.remote_execution }}
-        run: python3 infra/test-all.py
+          EXECUTION_TYPE: ${{ matrix.execution_type }}
+        run: python3 infra/test-all.py --package=${{ matrix.test_package }}
 
   buck2-test:
     runs-on:

cpp/BUILD

@@ -1,3 +1,4 @@
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
 load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
 load("@rules_cc//cc:cc_library.bzl", "cc_library")
 load("@rules_cc//cc:cc_test.bzl", "cc_test")
@@ -16,6 +17,13 @@ cc_library(
     ],
 )
 
+build_test(
+    name = "cpp_build_test",
+    targets = [
+        ":cpp",
+    ],
+)
+
 cc_test(
     name = "cpp_test",
     size = "small",

csharp/BUILD

@@ -1,3 +1,4 @@
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
 load("@rules_dotnet//dotnet:defs.bzl", "csharp_binary")
 
 csharp_binary(
@@ -9,3 +10,10 @@ csharp_binary(
         "net8.0",
     ],
 )
+
+build_test(
+    name = "csharp_test",
+    targets = [
+        ":csharp.exe",
+    ],
+)

docker/BUILD

@@ -22,7 +22,10 @@ DOCKER_JDK_PSQL = "docker://gcr.io/YOUR-IMAGE-NAME@sha256:b6ed8f68abd6e23409ee7a
 java_test(
     name = "docker-network-test",
     srcs = ["DockerNetworkTest.java"],
-    tags = ["manual"],
+    tags = [
+        "manual",
+        "no-ci",  # Consistent NPE
+    ],
     test_class = "docker.network.DockerNetworkTest",
 )
 

docker/network/BUILD

@@ -1,3 +1,4 @@
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
 load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
 load("@rules_java//java:java_binary.bzl", "java_binary")
 
@@ -25,3 +26,11 @@ cc_binary(
     name = "hello-cc",
     srcs = ["hello.cc"],
 )
+
+build_test(
+    name = "hello_test",
+    tags = ["no-macos-ci"],
+    targets = [
+        ":hello",
+    ],
+)

docker/sandbox/BUILD

@@ -30,5 +30,9 @@ sh_test(
         # DO NOT ENABLE!!! "dockerPrivileged": "True",
         # sysbox containers will refuse to start
     },
-    tags = ["manual"],
+    tags = [
+        "manual",
+        "no-local-ci",
+        "no-macos-ci",
+    ],
 )

docker/sysbox/dind_test/BUILD

@@ -16,6 +16,8 @@
 # on the user's input.
 # This can be used for stress testing your build infrastructure.
 # Change the upper bound of the range and enjoy!
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
+
 UPPER_BOUND = 10
 
 [genrule(
@@ -55,3 +57,11 @@ UPPER_BOUND = 10
     tags = ["manual"],
     tools = ["@ubuntu_20.04_1.3GB//file"],
 ) for x in range(1, 2)]
+
+# Only test a few to ensure they build, without creating undue load.
+build_test(
+    name = "foo_TXT_test",
+    targets = [
+        ":foo_TXT_4KB_1",
+    ],
+)

genrules/BUILD

@@ -1,3 +1,4 @@
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
 load("@rules_go//go:def.bzl", "go_binary", "go_library")
 
 go_binary(
@@ -12,3 +13,10 @@ go_library(
     importpath = "github.com/EngFlow/example/go",
     visibility = ["//visibility:private"],
 )
+
+build_test(
+    name = "go_build_test",
+    targets = [
+        ":go",
+    ],
+)

go/BUILD

@@ -1,25 +1,57 @@
 #!/usr/bin/env python3
 
+import argparse
 import os
 import subprocess
 import sys
 
-def main():
-    # All targets that can run in any environment, except ios examples.
-    targets = [
-        "//...",
-        # TODO: Add testing configuration for ios tests.
-        "-//ios/...",
-    ]
-
-    for key in ("ARCH", "OPAL_RPC_CREDENTIALS", "OS", "REMOTE_EXECUTION"):
+EXTRA_FLAGS = {
+    "ios": ["--config=ios"],
+    "swift": ["--config=clang"],
+}
+
+
+def report_error(msg):
+    print(msg, file=sys.stderr)
+    sys.exit(1)
+
+
+def validate_env():
+    for key in ("ARCH", "OS", "EXECUTION_TYPE", "OPAL_RPC_CREDENTIALS"):
         if not os.getenv(key):
-            sys.stderr.write(f"{key} not set\n")
-            sys.exit(1)
+            report_error(f"{key} not set")
+
+
+def find_tests(package):
+    os_name = os.getenv("OS")
+    remote = os.getenv("EXECUTION_TYPE")
+
+    # Remove tests based on tags and environment
+    query = f"""
+    let t = tests(//{package}/...) in
+    $t 
+    - attr(tags, no-ci, $t)
+    - attr(tags, no-{os_name}-ci, $t)
+    - attr(tags, no-{remote}-ci, $t)
+    """.strip()
+
+    print(f"Executing query to find test targets:\n{query}")
+    args = ["bazel", "query", "--output=label", "--", query]
+    result = subprocess.run(args, capture_output=True)
+    output = result.stdout.decode()
+    return [t for t in output.split("\n") if t]
+
+
+def run_tests(package):
+    targets = find_tests(package)
+    if not targets:
+        print(f"No targets found for {package}, skipping.")
+        return 0
+    print(f"Executing tests {targets}")
 
     os_arch = os.getenv("OS") + "_" + os.getenv("ARCH")
     flags = ["--config=ci"]
-    if os.getenv("REMOTE_EXECUTION") == "true":
+    if os.getenv("EXECUTION_TYPE") == "remote":
         flags += [
             "--config=remote_" + os_arch,
             "--config=opal",
@@ -29,13 +61,44 @@ def main():
             "--config=opal_bes",
             "--config=opal_auth",
         ]
-    # The //docker/sysbox/... targets should only run in linux + remote
-    if os.getenv("REMOTE_EXECUTION") == "true" and os.getenv("OS") == "linux":
-        targets += ["//docker/sysbox/dind_test:check_docker",]
+
+    if package in EXTRA_FLAGS:
+        flags += EXTRA_FLAGS[package]
+
+    print(f"---------------\nTesting {package}...")
+    sys.stdout.flush()
+
     args = ["bazel", "test"] + flags + ["--"] + targets
+    result = subprocess.run(args, check=False)
+    sys.stdout.flush()
+    sys.stderr.flush()
+    return result.returncode
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Runs example test suites", fromfile_prefix_chars="@"
+    )
+    parser.add_argument(
+        "--package",
+        required=False,
+        help="Package to test.",
+    )
+    parser.add_argument(
+        "--validate-env",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to validate environment variables",
+    )
+    opts = parser.parse_args()
+
+    if opts.validate_env:
+        validate_env()
+
+    package = opts.package
 
-    result = subprocess.run(args)
-    sys.exit(result.returncode)
+    returncode = run_tests(opts.package)
+    if returncode != 0:
+        sys.exit(returncode)
 
 
 if __name__ == "__main__":