From 7d1c60c1473e12845617dbecd1200bbb56c57b73 Mon Sep 17 00:00:00 2001 From: Guy Daich Date: Thu, 25 Jun 2026 17:42:38 -0500 Subject: [PATCH] use shared system trust store sds secret Signed-off-by: Guy Daich --- internal/xds/translator/globalresources.go | 40 +++- ...http-route-multiple-system-truststore.yaml | 73 ++++++++ ...ute-extension-translate-error.secrets.yaml | 4 + .../xds-ir/backend-tls-settings.clusters.yaml | 6 +- .../xds-ir/backend-tls-settings.secrets.yaml | 10 +- ...resolver-with-host-rewriting.clusters.yaml | 2 +- ...-resolver-with-host-rewriting.secrets.yaml | 2 +- ...e-multiple-system-truststore.clusters.yaml | 171 ++++++++++++++++++ ...-multiple-system-truststore.endpoints.yaml | 48 +++++ ...-multiple-system-truststore.listeners.yaml | 35 ++++ ...ute-multiple-system-truststore.routes.yaml | 28 +++ ...te-multiple-system-truststore.secrets.yaml | 4 + ...e-with-tls-system-truststore.clusters.yaml | 2 +- ...te-with-tls-system-truststore.secrets.yaml | 2 +- .../httproute-with-tls-and-http.clusters.yaml | 2 +- .../httproute-with-tls-and-http.secrets.yaml | 2 +- .../out/xds-ir/tcproute-mtls.clusters.yaml | 2 +- .../out/xds-ir/tcproute-mtls.secrets.yaml | 2 +- ...backend-force-http1-upstream.clusters.yaml | 4 +- ...-backend-force-http1-upstream.secrets.yaml | 6 +- internal/xds/translator/translator.go | 42 +++-- ...57-system-trust-store-single-sds-secret.md | 1 + ...57-system-trust-store-single-sds-secret.md | 1 + 23 files changed, 442 insertions(+), 47 deletions(-) create mode 100644 internal/xds/translator/testdata/in/xds-ir/http-route-multiple-system-truststore.yaml create mode 100644 internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.secrets.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.clusters.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.endpoints.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.listeners.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.routes.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.secrets.yaml create mode 100644 release-notes/current/breaking_changes/9357-system-trust-store-single-sds-secret.md create mode 100644 release-notes/current/performance_improvements/9357-system-trust-store-single-sds-secret.md diff --git a/internal/xds/translator/globalresources.go b/internal/xds/translator/globalresources.go index 32a67d5203..ea5897f9d6 100644 --- a/internal/xds/translator/globalresources.go +++ b/internal/xds/translator/globalresources.go @@ -16,6 +16,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/envoyproxy/gateway/internal/ir" + "github.com/envoyproxy/gateway/internal/utils/cert" "github.com/envoyproxy/gateway/internal/xds/types" ) @@ -25,15 +26,17 @@ const ( wasmHTTPServiceClusterName = "wasm_cluster" wasmHTTPServiceHost = "envoy-gateway" wasmHTTPServicePort = 18002 + + // SystemTrustStoreSecretName is the shared SDS secret name for the system CA trust store. + SystemTrustStoreSecretName = "system_ca_certificates" //nolint:gosec // not a credential ) -// patchGlobalResources builds and appends the global resources that are shared across listeners and routes. -// for example, the envoy client certificate and the OIDC HMAC secret. +// patchGlobalResources builds and appends global resources shared across listeners and routes. func (t *Translator) patchGlobalResources(tCtx *types.ResourceVersionTable, irXds *ir.Xds) error { var errs error if irXds.GlobalResources != nil && irXds.GlobalResources.EnvoyClientCertificate != nil { - // Create the envoy client TLS secret. It is used for envoy to establish a TLS connection with control plane components. + // Create the envoy client TLS secret for control plane connections (rate limit, wasm). if err := createEnvoyClientTLSCertSecret(tCtx, irXds.GlobalResources); err != nil { errs = errors.Join(errs, err) } @@ -66,6 +69,37 @@ func containsGlobalRateLimit(httpListeners []*ir.HTTPListener) bool { return false } +// ensureSystemTrustStoreSecret creates the shared system trust store SDS secret if not yet present. +func ensureSystemTrustStoreSecret(tCtx *types.ResourceVersionTable) error { + if findXdsSecret(tCtx, SystemTrustStoreSecretName) != nil { + return nil + } + return tCtx.AddXdsResource(resourcev3.SecretType, &tlsv3.Secret{ + Name: SystemTrustStoreSecretName, + Type: &tlsv3.Secret_ValidationContext{ + ValidationContext: &tlsv3.CertificateValidationContext{ + TrustedCa: &corev3.DataSource{ + Specifier: &corev3.DataSource_Filename{Filename: cert.SystemCertPath}, + }, + }, + }, + }) +} + +// validateSystemTrustStoreSecret verifies system_ca_certificates has expected content if present. +// Called once at end of Translate() to catch conflicts from EnvoyPatchPolicy or extensions. +func validateSystemTrustStoreSecret(tCtx *types.ResourceVersionTable) error { + existing := findXdsSecret(tCtx, SystemTrustStoreSecretName) + if existing == nil { + return nil + } + vc := existing.GetValidationContext() + if vc == nil || vc.GetTrustedCa().GetFilename() != cert.SystemCertPath { + return fmt.Errorf("secret name %q is reserved for the system trust store and cannot be used by other resources", SystemTrustStoreSecretName) + } + return nil +} + func createEnvoyClientTLSCertSecret(tCtx *types.ResourceVersionTable, globalResources *ir.GlobalResources) error { if err := tCtx.AddXdsResource( resourcev3.SecretType, diff --git a/internal/xds/translator/testdata/in/xds-ir/http-route-multiple-system-truststore.yaml b/internal/xds/translator/testdata/in/xds-ir/http-route-multiple-system-truststore.yaml new file mode 100644 index 0000000000..8c00d99aaa --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/http-route-multiple-system-truststore.yaml @@ -0,0 +1,73 @@ +http: + - address: 0.0.0.0 + hostnames: + - '*' + name: envoy-gateway/gateway-btls/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - destination: + name: httproute/envoy-gateway/httproute-btls-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 10.244.0.11 + port: 8080 + protocol: HTTP + tls: + CACertificate: + name: policy-btls-1/policies-ca + sni: backend-1.example.com + useSystemTrustStore: true + weight: 1 + name: httproute/envoy-gateway/httproute-btls-1/rule/0/backend/0 + hostname: '*' + name: httproute/envoy-gateway/httproute-btls-1/rule/0/match/0/* + pathMatch: + distinct: false + exact: /backend-1 + name: "" + - destination: + name: httproute/envoy-gateway/httproute-btls-2/rule/0 + settings: + - addressType: IP + endpoints: + - host: 10.244.0.12 + port: 8080 + protocol: HTTP + tls: + CACertificate: + name: policy-btls-2/policies-ca + sni: backend-2.example.com + useSystemTrustStore: true + weight: 1 + name: httproute/envoy-gateway/httproute-btls-2/rule/0/backend/0 + hostname: '*' + name: httproute/envoy-gateway/httproute-btls-2/rule/0/match/0/* + pathMatch: + distinct: false + exact: /backend-2 + name: "" + - destination: + name: httproute/envoy-gateway/httproute-btls-3/rule/0 + settings: + - addressType: IP + endpoints: + - host: 10.244.0.13 + port: 8080 + protocol: HTTP + tls: + CACertificate: + name: policy-btls-3/policies-ca + sni: backend-3.example.com + useSystemTrustStore: true + weight: 1 + name: httproute/envoy-gateway/httproute-btls-3/rule/0/backend/0 + hostname: '*' + name: httproute/envoy-gateway/httproute-btls-3/rule/0/match/0/* + pathMatch: + distinct: false + exact: /backend-3 + name: "" diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.secrets.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.secrets.yaml new file mode 100644 index 0000000000..b522884fef --- /dev/null +++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.secrets.yaml @@ -0,0 +1,4 @@ +- name: system_ca_certificates + validationContext: + trustedCa: + filename: /etc/ssl/certs/ca-certificates.crt diff --git a/internal/xds/translator/testdata/out/xds-ir/backend-tls-settings.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/backend-tls-settings.clusters.yaml index 4a24c657eb..052e3037e8 100644 --- a/internal/xds/translator/testdata/out/xds-ir/backend-tls-settings.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/backend-tls-settings.clusters.yaml @@ -51,7 +51,7 @@ exact: subdomain.secondexample.com sanType: DNS validationContextSdsSecretConfig: - name: policy-btls-for-backend-1/default-ca + name: system_ca_certificates sdsConfig: ads: {} resourceApiVersion: V3 @@ -210,7 +210,7 @@ exact: example.com sanType: DNS validationContextSdsSecretConfig: - name: policy-btls-for-backend-3/default-ca + name: system_ca_certificates sdsConfig: ads: {} resourceApiVersion: V3 @@ -289,7 +289,7 @@ exact: example.com sanType: DNS validationContextSdsSecretConfig: - name: policy-btls-for-backend-4/default-ca + name: system_ca_certificates sdsConfig: ads: {} resourceApiVersion: V3 diff --git a/internal/xds/translator/testdata/out/xds-ir/backend-tls-settings.secrets.yaml b/internal/xds/translator/testdata/out/xds-ir/backend-tls-settings.secrets.yaml index 39e4aabcbe..3294dc12dc 100644 --- a/internal/xds/translator/testdata/out/xds-ir/backend-tls-settings.secrets.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/backend-tls-settings.secrets.yaml @@ -22,7 +22,7 @@ inlineBytes: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURKRENDQWd5Z0F3SUJBZ0lVU3JTYktMZjBiTEVHb2dXeC9nQ3cyR0N0dnhFd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0V6RVJNQThHQTFVRUF3d0lWR1Z6ZENCSmJtTXdIaGNOTWpRd01qSTVNRGt6TURFd1doY05NelF3TWpJMgpNRGt6TURFd1dqQVRNUkV3RHdZRFZRUUREQWhVWlhOMElFbHVZekNDQVNJd0RRWUpLb1pJaHZjTkFRRUJCUUFECmdnRVBBRENDQVFvQ2dnRUJBSzFKempQSWlXZzNxb0hTckFkZGtlSmphTVA5aXlNVGkvQlBvOWNKUG9SRThaaTcKV2FwVXJYTC85QTlyK2pITXlHSVpOWk5kY1o1Y1kyWHYwTFA4WnhWeTJsazArM3d0WXpIbnBHWUdWdHlxMnRldApEaEZzaVBsODJZUmpDMG16V2E0UU16NFNYekZITmdJRHBSZGhmcm92bXNldVdHUUU4cFY0VWQ5VUsvU0tpbE1PCnF0QjVKaXJMUDJWczVUMW9XaWNXTFF2ZmJHd3Y3c0ZEZHI5YkcwWHRTUXAxN0hTZ281MFNERTUrQmpTbXB0RncKMVZjS0xscWFoTVhCRERpb3Jnd2hJaEdHS3BFU2VNMFA3YkZoVm1rTTNhc2gyeFNUQnVGVUJEbEU0Sk9haHp3cwpEWHJ1cFVoRGRTMWhkYzJmUHJqaEZBbEpmV0VZWjZCbFpqeXNpVlVDQXdFQUFhTndNRzR3SFFZRFZSME9CQllFCkZCUXVmSzFMaWJ1Vm05VHMvVmpCeDhMM3VpTmVNQjhHQTFVZEl3UVlNQmFBRkJRdWZLMUxpYnVWbTlUcy9WakIKeDhMM3VpTmVNQThHQTFVZEV3RUIvd1FGTUFNQkFmOHdHd1lEVlIwUkJCUXdFb0lCS29JTktpNWxlR0Z0Y0d4bApMbU52YlRBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQWZQUzQxYWdldldNVjNaWHQwQ09GRzN1WWZQRlhuVnc2ClA0MXA5TzZHa2RZc3VxRnZQZVR5eUgyL2RBSUtLd1N6TS9wdGhnOEtuOExabG1KeUZObkExc3RKeG41WGRiVjEKcFBxajhVdllDQnp5ak1JcW1SeW9peUxpUWxib2hNYTBVZEVCS2NIL1BkTEU5SzhUR0pyWmdvR1hxcTFXbWl0RAozdmNQalNlUEtFaVVKVlM5bENoeVNzMEtZNUIraFVRRDBKajZucEZENFprMHhxZHhoMHJXdWVDcXE3dmpxRVl6CnBqNFB3cnVmbjFQQlRtZnhNdVYvVUpWNWViaWtldVpQMzVrV3pMUjdaV0FMN3d1RGRXcC82bzR5azNRTGFuRFEKQ3dnQ0ZjWCtzcyswVnl1TTNZZXJUT1VVOFFWSkp4NFVaQU5aeDYrNDNwZEpaT2NudFBaNENBPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= privateKey: inlineBytes: W3JlZGFjdGVkXQ== -- name: policy-btls-for-backend-1/default-ca +- name: system_ca_certificates validationContext: trustedCa: filename: /etc/ssl/certs/ca-certificates.crt @@ -30,11 +30,3 @@ validationContext: trustedCa: inlineBytes: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURKekNDQWcrZ0F3SUJBZ0lVQWw2VUtJdUttenRlODFjbGx6NVBmZE4ySWxJd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0l6RVFNQTRHQTFVRUF3d0hiWGxqYVdWdWRERVBNQTBHQTFVRUNnd0dhM1ZpWldSaU1CNFhEVEl6TVRBdwpNakExTkRFMU4xb1hEVEkwTVRBd01UQTFOREUxTjFvd0l6RVFNQTRHQTFVRUF3d0hiWGxqYVdWdWRERVBNQTBHCkExVUVDZ3dHYTNWaVpXUmlNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXdTVGMKMXlqOEhXNjJueW5rRmJYbzRWWEt2MmpDMFBNN2RQVmt5ODdGd2VaY1RLTG9XUVZQUUUycDJrTERLNk9Fc3ptTQp5eXIreHhXdHlpdmVyZW1yV3FuS2tOVFloTGZZUGhnUWtjemliN2VVYWxtRmpVYmhXZEx2SGFrYkVnQ29kbjNiCmt6NTdtSW5YMlZwaURPS2c0a3lIZml1WFdwaUJxckN4MEtOTHB4bzNERVFjRmNzUVRlVEh6aDQ3NTJHVjA0UlUKVGkvR0VXeXpJc2w0Umc3dEd0QXdtY0lQZ1VOVWZZMlEzOTBGR3FkSDRhaG4rbXcvNmFGYlczMVc2M2Q5WUpWcQppb3lPVmNhTUlwTTVCL2M3UWM4U3VoQ0kxWUdoVXlnNGNSSExFdzVWdGlraW95RTNYMDRrbmEzalFBajU0WWJSCmJwRWhjMzVhcEtMQjIxSE9VUUlEQVFBQm8xTXdVVEFkQmdOVkhRNEVGZ1FVeXZsMFZJNXZKVlN1WUZYdTdCNDgKNlBiTUVBb3dId1lEVlIwakJCZ3dGb0FVeXZsMFZJNXZKVlN1WUZYdTdCNDg2UGJNRUFvd0R3WURWUjBUQVFILwpCQVV3QXdFQi96QU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFNTHhyZ0ZWTXVOUnEyd0F3Y0J0N1NuTlI1Q2Z6CjJNdlhxNUVVbXVhd0lVaTlrYVlqd2RWaURSRUdTams3SlcxN3ZsNTc2SGpEa2RmUndpNEUyOFN5ZFJJblpmNkoKaThIWmNaN2NhSDZEeFIzMzVmZ0hWekxpNU5pVGNlL09qTkJRelEyTUpYVkRkOERCbUc1ZnlhdEppT0pRNGJXRQpBN0ZsUDBSZFAzQ08zR1dFME01aVhPQjJtMXFXa0UyZXlPNFVIdndUcU5RTGRyZEFYZ0RRbGJhbTllNEJHM0dnCmQvNnRoQWtXRGJ0L1FOVCtFSkhEQ3ZoRFJLaDFSdUdIeWcrWSsvbmViVFdXckZXc2t0UnJiT29IQ1ppQ3BYSTEKM2VYRTZudDBZa2d0RHhHMjJLcW5ocEFnOWdVU3MyaGxob3h5dmt6eUYwbXU2TmhQbHdBZ25xNysvUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K -- name: policy-btls-for-backend-3/default-ca - validationContext: - trustedCa: - filename: /etc/ssl/certs/ca-certificates.crt -- name: policy-btls-for-backend-4/default-ca - validationContext: - trustedCa: - filename: /etc/ssl/certs/ca-certificates.crt diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver-with-host-rewriting.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver-with-host-rewriting.clusters.yaml index e0a6ae10d2..23d13f6467 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver-with-host-rewriting.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver-with-host-rewriting.clusters.yaml @@ -66,7 +66,7 @@ combinedValidationContext: defaultValidationContext: {} validationContextSdsSecretConfig: - name: backend-with-system-ca/default-ca + name: system_ca_certificates sdsConfig: ads: {} resourceApiVersion: V3 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver-with-host-rewriting.secrets.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver-with-host-rewriting.secrets.yaml index 78ae34806b..b522884fef 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver-with-host-rewriting.secrets.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver-with-host-rewriting.secrets.yaml @@ -1,4 +1,4 @@ -- name: backend-with-system-ca/default-ca +- name: system_ca_certificates validationContext: trustedCa: filename: /etc/ssl/certs/ca-certificates.crt diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.clusters.yaml new file mode 100644 index 0000000000..6f130d2e01 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.clusters.yaml @@ -0,0 +1,171 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: httproute/envoy-gateway/httproute-btls-1/rule/0 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + name: httproute/envoy-gateway/httproute-btls-1/rule/0 + perConnectionBufferLimitBytes: 32768 + transportSocket: + name: dummy.transport_socket + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: {} + transportSocketMatches: + - match: + name: httproute/envoy-gateway/httproute-btls-1/rule/0/tls/0 + name: httproute/envoy-gateway/httproute-btls-1/rule/0/tls/0 + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: + combinedValidationContext: + defaultValidationContext: + matchTypedSubjectAltNames: + - matcher: + exact: backend-1.example.com + sanType: DNS + validationContextSdsSecretConfig: + name: system_ca_certificates + sdsConfig: + ads: {} + resourceApiVersion: V3 + sni: backend-1.example.com + type: EDS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + autoConfig: + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + httpProtocolOptions: {} +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: httproute/envoy-gateway/httproute-btls-2/rule/0 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + name: httproute/envoy-gateway/httproute-btls-2/rule/0 + perConnectionBufferLimitBytes: 32768 + transportSocket: + name: dummy.transport_socket + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: {} + transportSocketMatches: + - match: + name: httproute/envoy-gateway/httproute-btls-2/rule/0/tls/0 + name: httproute/envoy-gateway/httproute-btls-2/rule/0/tls/0 + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: + combinedValidationContext: + defaultValidationContext: + matchTypedSubjectAltNames: + - matcher: + exact: backend-2.example.com + sanType: DNS + validationContextSdsSecretConfig: + name: system_ca_certificates + sdsConfig: + ads: {} + resourceApiVersion: V3 + sni: backend-2.example.com + type: EDS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + autoConfig: + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + httpProtocolOptions: {} +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: httproute/envoy-gateway/httproute-btls-3/rule/0 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + name: httproute/envoy-gateway/httproute-btls-3/rule/0 + perConnectionBufferLimitBytes: 32768 + transportSocket: + name: dummy.transport_socket + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: {} + transportSocketMatches: + - match: + name: httproute/envoy-gateway/httproute-btls-3/rule/0/tls/0 + name: httproute/envoy-gateway/httproute-btls-3/rule/0/tls/0 + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: + combinedValidationContext: + defaultValidationContext: + matchTypedSubjectAltNames: + - matcher: + exact: backend-3.example.com + sanType: DNS + validationContextSdsSecretConfig: + name: system_ca_certificates + sdsConfig: + ads: {} + resourceApiVersion: V3 + sni: backend-3.example.com + type: EDS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + autoConfig: + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + httpProtocolOptions: {} diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.endpoints.yaml new file mode 100644 index 0000000000..47805a81cb --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.endpoints.yaml @@ -0,0 +1,48 @@ +- clusterName: httproute/envoy-gateway/httproute-btls-1/rule/0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 10.244.0.11 + portValue: 8080 + loadBalancingWeight: 1 + metadata: + filterMetadata: + envoy.transport_socket_match: + name: httproute/envoy-gateway/httproute-btls-1/rule/0/tls/0 + loadBalancingWeight: 1 + locality: + region: httproute/envoy-gateway/httproute-btls-1/rule/0/backend/0 +- clusterName: httproute/envoy-gateway/httproute-btls-2/rule/0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 10.244.0.12 + portValue: 8080 + loadBalancingWeight: 1 + metadata: + filterMetadata: + envoy.transport_socket_match: + name: httproute/envoy-gateway/httproute-btls-2/rule/0/tls/0 + loadBalancingWeight: 1 + locality: + region: httproute/envoy-gateway/httproute-btls-2/rule/0/backend/0 +- clusterName: httproute/envoy-gateway/httproute-btls-3/rule/0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 10.244.0.13 + portValue: 8080 + loadBalancingWeight: 1 + metadata: + filterMetadata: + envoy.transport_socket_match: + name: httproute/envoy-gateway/httproute-btls-3/rule/0/tls/0 + loadBalancingWeight: 1 + locality: + region: httproute/envoy-gateway/httproute-btls-3/rule/0/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.listeners.yaml new file mode 100644 index 0000000000..86036a19d5 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.listeners.yaml @@ -0,0 +1,35 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: envoy-gateway/gateway-btls/http + serverHeaderTransformation: PASS_THROUGH + statPrefix: http-10080 + useRemoteAddress: true + name: envoy-gateway/gateway-btls/http + maxConnectionsToAcceptPerSocketEvent: 1 + name: envoy-gateway/gateway-btls/http + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.routes.yaml new file mode 100644 index 0000000000..6c66166d8e --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.routes.yaml @@ -0,0 +1,28 @@ +- ignorePortInHostMatching: true + name: envoy-gateway/gateway-btls/http + virtualHosts: + - domains: + - '*' + name: envoy-gateway/gateway-btls/http/* + routes: + - match: + path: /backend-1 + name: httproute/envoy-gateway/httproute-btls-1/rule/0/match/0/* + route: + cluster: httproute/envoy-gateway/httproute-btls-1/rule/0 + upgradeConfigs: + - upgradeType: websocket + - match: + path: /backend-2 + name: httproute/envoy-gateway/httproute-btls-2/rule/0/match/0/* + route: + cluster: httproute/envoy-gateway/httproute-btls-2/rule/0 + upgradeConfigs: + - upgradeType: websocket + - match: + path: /backend-3 + name: httproute/envoy-gateway/httproute-btls-3/rule/0/match/0/* + route: + cluster: httproute/envoy-gateway/httproute-btls-3/rule/0 + upgradeConfigs: + - upgradeType: websocket diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.secrets.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.secrets.yaml new file mode 100644 index 0000000000..b522884fef --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-system-truststore.secrets.yaml @@ -0,0 +1,4 @@ +- name: system_ca_certificates + validationContext: + trustedCa: + filename: /etc/ssl/certs/ca-certificates.crt diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-with-tls-system-truststore.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-with-tls-system-truststore.clusters.yaml index f08220608d..cf86938840 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-with-tls-system-truststore.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-with-tls-system-truststore.clusters.yaml @@ -41,7 +41,7 @@ exact: example.com sanType: DNS validationContextSdsSecretConfig: - name: policy-btls/policies-ca + name: system_ca_certificates sdsConfig: ads: {} resourceApiVersion: V3 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-with-tls-system-truststore.secrets.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-with-tls-system-truststore.secrets.yaml index 313d6f8906..b522884fef 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-with-tls-system-truststore.secrets.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-with-tls-system-truststore.secrets.yaml @@ -1,4 +1,4 @@ -- name: policy-btls/policies-ca +- name: system_ca_certificates validationContext: trustedCa: filename: /etc/ssl/certs/ca-certificates.crt diff --git a/internal/xds/translator/testdata/out/xds-ir/httproute-with-tls-and-http.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/httproute-with-tls-and-http.clusters.yaml index 1ae97c4dce..e97b7f60a9 100644 --- a/internal/xds/translator/testdata/out/xds-ir/httproute-with-tls-and-http.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/httproute-with-tls-and-http.clusters.yaml @@ -46,7 +46,7 @@ exact: subdomain.secondexample.com sanType: DNS validationContextSdsSecretConfig: - name: policy-btls-for-backend-1/default-ca + name: system_ca_certificates sdsConfig: ads: {} resourceApiVersion: V3 diff --git a/internal/xds/translator/testdata/out/xds-ir/httproute-with-tls-and-http.secrets.yaml b/internal/xds/translator/testdata/out/xds-ir/httproute-with-tls-and-http.secrets.yaml index a348298f78..b522884fef 100644 --- a/internal/xds/translator/testdata/out/xds-ir/httproute-with-tls-and-http.secrets.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/httproute-with-tls-and-http.secrets.yaml @@ -1,4 +1,4 @@ -- name: policy-btls-for-backend-1/default-ca +- name: system_ca_certificates validationContext: trustedCa: filename: /etc/ssl/certs/ca-certificates.crt diff --git a/internal/xds/translator/testdata/out/xds-ir/tcproute-mtls.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/tcproute-mtls.clusters.yaml index e0fb3cf616..ca3be65c98 100644 --- a/internal/xds/translator/testdata/out/xds-ir/tcproute-mtls.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/tcproute-mtls.clusters.yaml @@ -47,7 +47,7 @@ exact: subdomain.secondexample.com sanType: DNS validationContextSdsSecretConfig: - name: policy-btls-for-service-1/default-ca + name: system_ca_certificates sdsConfig: ads: {} resourceApiVersion: V3 diff --git a/internal/xds/translator/testdata/out/xds-ir/tcproute-mtls.secrets.yaml b/internal/xds/translator/testdata/out/xds-ir/tcproute-mtls.secrets.yaml index f5e6180467..02a60c5e5e 100644 --- a/internal/xds/translator/testdata/out/xds-ir/tcproute-mtls.secrets.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/tcproute-mtls.secrets.yaml @@ -1,4 +1,4 @@ -- name: policy-btls-for-service-1/default-ca +- name: system_ca_certificates validationContext: trustedCa: filename: /etc/ssl/certs/ca-certificates.crt diff --git a/internal/xds/translator/testdata/out/xds-ir/websocket-backend-force-http1-upstream.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/websocket-backend-force-http1-upstream.clusters.yaml index 09df7715b7..0c3ffa6803 100644 --- a/internal/xds/translator/testdata/out/xds-ir/websocket-backend-force-http1-upstream.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/websocket-backend-force-http1-upstream.clusters.yaml @@ -41,7 +41,7 @@ exact: websocket.example.com sanType: DNS validationContextSdsSecretConfig: - name: websocket-wss-route/ca + name: system_ca_certificates sdsConfig: ads: {} resourceApiVersion: V3 @@ -95,7 +95,7 @@ exact: ordinary.example.com sanType: DNS validationContextSdsSecretConfig: - name: ordinary-tls-http-route/ca + name: system_ca_certificates sdsConfig: ads: {} resourceApiVersion: V3 diff --git a/internal/xds/translator/testdata/out/xds-ir/websocket-backend-force-http1-upstream.secrets.yaml b/internal/xds/translator/testdata/out/xds-ir/websocket-backend-force-http1-upstream.secrets.yaml index 4abecf47fc..b522884fef 100644 --- a/internal/xds/translator/testdata/out/xds-ir/websocket-backend-force-http1-upstream.secrets.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/websocket-backend-force-http1-upstream.secrets.yaml @@ -1,8 +1,4 @@ -- name: websocket-wss-route/ca - validationContext: - trustedCa: - filename: /etc/ssl/certs/ca-certificates.crt -- name: ordinary-tls-http-route/ca +- name: system_ca_certificates validationContext: trustedCa: filename: /etc/ssl/certs/ca-certificates.crt diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go index ec4272e49d..7663f00d53 100644 --- a/internal/xds/translator/translator.go +++ b/internal/xds/translator/translator.go @@ -33,7 +33,6 @@ import ( "github.com/envoyproxy/gateway/internal/ir" "github.com/envoyproxy/gateway/internal/logging" "github.com/envoyproxy/gateway/internal/utils" - "github.com/envoyproxy/gateway/internal/utils/cert" "github.com/envoyproxy/gateway/internal/utils/proto" "github.com/envoyproxy/gateway/internal/xds/types" ) @@ -182,6 +181,11 @@ func (t *Translator) Translate(xdsIR *ir.Xds) (*types.ResourceVersionTable, erro errs = errors.Join(errs, err) } + // Single check: verify system_ca_certificates content wasn't altered by patches or extensions. + if err := validateSystemTrustStoreSecret(tCtx); err != nil { + errs = errors.Join(errs, err) + } + return tCtx, errs } @@ -1121,12 +1125,19 @@ func addXdsCluster(tCtx *types.ResourceVersionTable, args *xdsClusterArgs) error for _, ds := range args.settings { shouldValidateTLS := ds.TLS != nil && !ds.TLS.InsecureSkipVerify if shouldValidateTLS { - // Create an SDS secret for the CA certificate - either with inline bytes or with a filesystem ref - secret := buildXdsUpstreamTLSCASecret(ds.TLS) - if secret != nil { - if err := tCtx.AddXdsResource(resourcev3.SecretType, secret); err != nil { + if ds.TLS.UseSystemTrustStore { + // Lazily create the shared secret; content is validated once at end of Translate(). + if err := ensureSystemTrustStoreSecret(tCtx); err != nil { return err } + } else { + // Create an SDS secret for the CA certificate — inline bytes or filesystem ref. + secret := buildXdsUpstreamTLSCASecret(ds.TLS) + if secret != nil { + if err := tCtx.AddXdsResource(resourcev3.SecretType, secret); err != nil { + return err + } + } } } } @@ -1184,16 +1195,8 @@ const ( func buildXdsUpstreamTLSCASecret(tlsConfig *ir.TLSUpstreamConfig) *tlsv3.Secret { if tlsConfig.UseSystemTrustStore { - return &tlsv3.Secret{ - Name: tlsConfig.CACertificate.Name, - Type: &tlsv3.Secret_ValidationContext{ - ValidationContext: &tlsv3.CertificateValidationContext{ - TrustedCa: &corev3.DataSource{ - Specifier: &corev3.DataSource_Filename{Filename: cert.SystemCertPath}, - }, - }, - }, - } + // No per-cluster secret needed — all clusters share SystemTrustStoreSecretName. + return nil } // For SDS based CA certificate, the secret will be generated by SDS server, so we can skip adding the secret here @@ -1214,14 +1217,19 @@ func buildXdsUpstreamTLSCASecret(tlsConfig *ir.TLSUpstreamConfig) *tlsv3.Secret } func buildValidationContext(tlsConfig *ir.TLSUpstreamConfig) (*tlsv3.CommonTlsContext_CombinedValidationContext, bool) { + secretName := tlsConfig.CACertificate.Name + if tlsConfig.UseSystemTrustStore { + secretName = SystemTrustStoreSecretName + } + validationContext := &tlsv3.CommonTlsContext_CombinedCertificateValidationContext{ ValidationContextSdsSecretConfig: &tlsv3.SdsSecretConfig{ - Name: tlsConfig.CACertificate.Name, + Name: secretName, SdsConfig: makeConfigSource(), }, DefaultValidationContext: &tlsv3.CertificateValidationContext{}, } - if tlsConfig.CACertificate.SDS != nil { + if !tlsConfig.UseSystemTrustStore && tlsConfig.CACertificate.SDS != nil { // get secret from SDS sds := tlsConfig.CACertificate.SDS clusterName := sdsClusterNameFromURL(sds.URL) diff --git a/release-notes/current/breaking_changes/9357-system-trust-store-single-sds-secret.md b/release-notes/current/breaking_changes/9357-system-trust-store-single-sds-secret.md new file mode 100644 index 0000000000..426bdf8180 --- /dev/null +++ b/release-notes/current/breaking_changes/9357-system-trust-store-single-sds-secret.md @@ -0,0 +1 @@ +All `BackendTLSPolicy` resources using `WellKnownCACertificates: System` now share a single SDS secret named `system_ca_certificates` instead of one per-policy secret named `/-ca`. EnvoyPatchPolicies or extension servers referencing the old per-policy secret names or the cluster's `validationContextSdsSecretConfig` field must be updated accordingly. Patching `system_ca_certificates` is not supported; users requiring a custom CA bundle per backend should use `CACertificateRefs` instead of `WellKnownCACertificates: System`. diff --git a/release-notes/current/performance_improvements/9357-system-trust-store-single-sds-secret.md b/release-notes/current/performance_improvements/9357-system-trust-store-single-sds-secret.md new file mode 100644 index 0000000000..7d25106200 --- /dev/null +++ b/release-notes/current/performance_improvements/9357-system-trust-store-single-sds-secret.md @@ -0,0 +1 @@ +Reduced inotify watch usage for `BackendTLSPolicy` with `WellKnownCACertificates: System` by sharing a single SDS secret across all policies instead of creating one per policy.