From d9d3e2240b1a7b07a0c01c0faa5a56965321825f Mon Sep 17 00:00:00 2001 From: Bikash Shaw Date: Tue, 2 Jun 2026 23:18:30 +0530 Subject: [PATCH 1/2] OCPBUGS-84961: append suite names to external extension binary tests TestBinaries.ListTests() loads tests from external extension binaries but never calls appendSuiteNames() on them. Without [Suite:openshift/conformance/...] tags, the conformance suite filter in NewQualifiersFilter excludes these tests, causing the openshift/conformance suite to drop from ~4000 to ~190 tests on OCP 4.21+. Call appendSuiteNames() on the aggregated external binary test specs after loading them. The function already guards against double-tagging tests that already have [Suite:] in their name. Tested on OCP 4.21.8 GA cluster: - Before fix: 184 tests in openshift/conformance suite - After fix: 3895 tests in openshift/conformance suite Co-Authored-By: Claude Opus 4.6 --- pkg/test/extensions/binary.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/test/extensions/binary.go b/pkg/test/extensions/binary.go index 1fdf4a13b607..54d62ac9e4b1 100644 --- a/pkg/test/extensions/binary.go +++ b/pkg/test/extensions/binary.go @@ -947,6 +947,12 @@ func (binaries TestBinaries) ListTests(ctx context.Context, parallelism int, env return nil, fmt.Errorf("encountered errors while listing tests: %s", strings.Join(errs, ";")) } + var baseSpecs extensiontests.ExtensionTestSpecs + for _, spec := range allTests { + baseSpecs = append(baseSpecs, spec.ExtensionTestSpec) + } + appendSuiteNames(baseSpecs) + return allTests, nil } From 14c7947fe7c18bb7257b685bef3a8b5c64ffe31e Mon Sep 17 00:00:00 2001 From: Bikash Shaw Date: Wed, 3 Jun 2026 13:25:39 +0530 Subject: [PATCH 2/2] OCPBUGS-84961: apply full test processing pipeline to external binary tests External binary tests loaded via TestBinaries.ListTests() were missing the processing pipeline that origin built-in tests go through in InitializeOpenShiftTestsExtensionFramework(). This caused external binary tests to lack [Suite:openshift/conformance/...] tags, dropping the openshift/conformance suite from ~4000 to ~190 tests on OCP 4.21+. Apply the same processing steps to external binary test specs: - filterOutDisabledSpecs(): removes tests known to be broken - addEnvironmentSelectors(): adds [Skipped:] markers for platform/ network/topology-specific tests - addLabelsToSpecs(): adds labels like [Serial] where needed - appendSuiteNames(): adds [Suite:openshift/conformance/...] tags All four functions have guards to skip tests that already have the relevant annotations, so they are safe to call on specs that may already be partially processed by their source binary. Tested on OCP 4.21.8 GA cluster: - Before fix: 184 tests in openshift/conformance suite - After fix: 3895 tests in openshift/conformance suite Co-Authored-By: Claude Opus 4.6 --- pkg/test/extensions/binary.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/test/extensions/binary.go b/pkg/test/extensions/binary.go index 54d62ac9e4b1..66471856d7a6 100644 --- a/pkg/test/extensions/binary.go +++ b/pkg/test/extensions/binary.go @@ -951,6 +951,9 @@ func (binaries TestBinaries) ListTests(ctx context.Context, parallelism int, env for _, spec := range allTests { baseSpecs = append(baseSpecs, spec.ExtensionTestSpec) } + baseSpecs = filterOutDisabledSpecs(baseSpecs) + addEnvironmentSelectors(baseSpecs) + addLabelsToSpecs(baseSpecs) appendSuiteNames(baseSpecs) return allTests, nil