Skip to content

fix: invalid telemetry backendref degrades gateway #9406

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vishwas-bm wants to merge 3 commits into
base: main
Choose a base branch
from
Open

fix: invalid telemetry backendref degrades gateway #9406

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0c478aa
fix: invalid telemetry backendRef no longer degrades the Gateway
vishwas-bm Jul 1, 2026
7507206
Merge branch 'envoyproxy:main' into fix/9229-invalid-telemetry-backen…
vishwas-bm Jul 2, 2026
79e1321
fix: ignore invalid access log CEL matches instead of skipping access…
vishwas-bm Jul 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 51 additions & 11 deletions internal/gatewayapi/listener.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -702,31 +702,67 @@ func (t *Translator) processProxyReadyListener(xdsIR *ir.Xds, envoyProxy *egv1a1
}

func (t *Translator) processProxyObservability(gwCtx *GatewayContext, xdsIR *ir.Xds, proxyInfra *ir.ProxyInfra, resources *resource.Resources) {
var err error
var (
err error
warnings []error
)
envoyProxy := proxyInfra.Config

// Invalid telemetry backendRefs must not degrade the Gateway. Instead, the affected telemetry feature is skipped and a warning msg is added
// on the EnvoyProxy status.
xdsIR.AccessLog, err = t.processAccessLog(gwCtx, envoyProxy, resources)
if err != nil {
status.UpdateGatewayStatusNotAccepted(gwCtx.Gateway, gwapiv1.GatewayReasonInvalidParameters,
fmt.Sprintf("Invalid access log backendRefs in the referenced EnvoyProxy: %v", err))
return
warnings = append(warnings, fmt.Errorf("invalid access log backendRefs in the referenced EnvoyProxy: %w", err))
Comment on lines 713 to +715

@chatgpt-codex-connector chatgpt-codex-connector Bot Jul 2, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve access logs when only CEL matches are invalid

When an access log setting contains any invalid matches CEL expression, processAccessLog returns that error before building the IR; this new blanket handling treats it as an invalid backendRef warning and leaves xdsIR.AccessLog nil, so a config with valid file/OTel sinks plus one bad match now keeps the Gateway accepted but removes all access logging. The access-log API documents invalid CEL expressions as ignored, so this path should drop/report only the bad match and reserve feature skipping for actual backendRef resolution failures.

Useful? React with 👍 / 👎.

}

xdsIR.Tracing, err = t.processTracing(gwCtx, envoyProxy, t.MergeGateways, resources)
if err != nil {
status.UpdateGatewayStatusNotAccepted(gwCtx.Gateway, gwapiv1.GatewayReasonInvalidParameters,
fmt.Sprintf("Invalid tracing backendRefs in the referenced EnvoyProxy: %v", err))
return
warnings = append(warnings, fmt.Errorf("invalid tracing backendRefs in the referenced EnvoyProxy: %w", err))
}

var resolvedSinks []ir.ResolvedMetricSink
xdsIR.Metrics, resolvedSinks, err = t.processMetrics(gwCtx, envoyProxy, resources)
if err != nil {
status.UpdateGatewayStatusNotAccepted(gwCtx.Gateway, gwapiv1.GatewayReasonInvalidParameters,
fmt.Sprintf("Invalid metrics backendRefs in the referenced EnvoyProxy: %v", err))
warnings = append(warnings, fmt.Errorf("invalid metrics backendRefs in the referenced EnvoyProxy: %w", err))
} else {
proxyInfra.ResolvedMetricSinks = resolvedSinks
}

if len(warnings) > 0 {
msg := utilerrors.NewAggregate(warnings).Error()
t.Logger.Info("skipping invalid telemetry configuration in the referenced EnvoyProxy",
"namespace", gwCtx.Namespace, "name", gwCtx.Name, "warning", msg)
t.setEnvoyProxyObservabilityWarning(gwCtx, resources, msg)
}
}

// setEnvoyProxyObservabilityWarning adds telemetry misconfigurations as a warning on the
// EnvoyProxy status. The EnvoyProxy remains Accepted so that the Gateway keeps running,
// only the affected telemetry feature is skipped.
func (t *Translator) setEnvoyProxyObservabilityWarning(gwCtx *GatewayContext, resources *resource.Resources, msg string) {
warning := fmt.Sprintf("EnvoyProxy has been accepted, but the following telemetry configuration was ignored: %s", msg)

// EnvoyProxy attached directly to the Gateway via its infrastructure parametersRef.
if gwCtx.envoyProxyFromGateway && gwCtx.envoyProxy != nil {
ancestor := &gwapiv1.ParentReference{
Group: GroupPtr(gwapiv1.GroupName),
Kind: KindPtr(resource.KindGateway),
Name: gwapiv1.ObjectName(gwCtx.Name),
Namespace: NamespacePtr(gwCtx.Namespace),
}
status.UpdateEnvoyProxyStatusAccepted(gwCtx.envoyProxy, ancestor, egv1a1.EnvoyProxyReasonAccepted, warning)
return
}
proxyInfra.ResolvedMetricSinks = resolvedSinks

// EnvoyProxy inherited from the GatewayClass parametersRef.
if ep := resources.EnvoyProxyForGatewayClass; ep != nil && resources.GatewayClass != nil {
ancestor := &gwapiv1.ParentReference{
Group: GroupPtr(gwapiv1.GroupName),
Kind: KindPtr(resource.KindGatewayClass),
Name: gwapiv1.ObjectName(resources.GatewayClass.Name),
}
status.UpdateEnvoyProxyStatusAccepted(ep, ancestor, egv1a1.EnvoyProxyReasonAccepted, warning)
}
}

func (t *Translator) processInfraIRListener(listener *ListenerContext, infraIR resource.InfraIRMap, irKey string, servicePort *protocolPort, containerPort int32) {
Expand Down Expand Up @@ -815,7 +851,11 @@ func (t *Translator) processAccessLog(gwCtx *GatewayContext, envoyproxy *egv1a1.
validExprs = append(validExprs, expr)
}
if len(errs) > 0 {
return nil, utilerrors.NewAggregate(errs)
// Per the AccessLog API, invalid CEL match expressions are ignored:
// only the invalid ones are dropped, the rest of the config is kept.
t.Logger.Info("ignoring invalid CEL match expressions in the EnvoyProxy access log configuration",
"namespace", envoyproxy.Namespace, "name", envoyproxy.Name,
"error", utilerrors.NewAggregate(errs).Error())
}

if len(accessLog.Sinks) == 0 {
Expand Down
12 changes: 12 additions & 0 deletions internal/gatewayapi/testdata/envoyproxy-accesslog-backend-invalid.in.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
gatewayClass:
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
name: envoy-gateway-class
spec:
controllerName: gateway.envoyproxy.io/gatewayclass-controller
parametersRef:
group: gateway.envoyproxy.io
kind: EnvoyProxy
name: test
namespace: envoy-gateway-system
envoyProxyForGatewayClass:
apiVersion: gateway.envoyproxy.io/v1alpha1
kind: EnvoyProxy
Expand Down
130 changes: 122 additions & 8 deletions internal/gatewayapi/testdata/envoyproxy-accesslog-backend-invalid.out.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,111 @@
envoyProxyForGatewayClass:
apiVersion: gateway.envoyproxy.io/v1alpha1
kind: EnvoyProxy
metadata:
name: test
namespace: envoy-gateway-system
spec:
logging: {}
provider:
kubernetes:
envoyDeployment:
container:
env:
- name: env_a
value: env_a_value
- name: env_b
value: env_b_name
image: envoyproxy/envoy:distroless-dev
resources:
requests:
cpu: 100m
memory: 512Mi
securityContext:
allowPrivilegeEscalation: false
runAsUser: 2000
pod:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: cloud.google.com/gke-nodepool
operator: In
values:
- router-node
annotations:
key1: val1
key2: val2
securityContext:
fsGroup: 2000
fsGroupChangePolicy: OnRootMismatch
runAsGroup: 3000
runAsUser: 1000
tolerations:
- effect: NoSchedule
key: node-type
operator: Exists
value: router
volumes:
- name: certs
secret:
secretName: envoy-cert
replicas: 2
envoyService:
type: LoadBalancer
type: Kubernetes
telemetry:
accessLog:
settings:
- format:
text: |
[%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n
type: Text
sinks:
- file:
path: /dev/stdout
type: File
- openTelemetry:
backendRefs:
- name: service-not-found
namespace: monitoring
port: 4317
resourceAttributes:
k8s.cluster.name: cluster-1
type: OpenTelemetry
status:
ancestors:
- ancestorRef:
group: gateway.networking.k8s.io
kind: GatewayClass
name: envoy-gateway-class
conditions:
- lastTransitionTime: null
message: 'EnvoyProxy has been accepted, but the following telemetry configuration
was ignored: invalid access log backendRefs in the referenced EnvoyProxy:
service monitoring/service-not-found not found'
reason: Accepted
status: "True"
type: Accepted
gatewayClass:
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
name: envoy-gateway-class
spec:
controllerName: gateway.envoyproxy.io/gatewayclass-controller
parametersRef:
group: gateway.envoyproxy.io
kind: EnvoyProxy
name: test
namespace: envoy-gateway-system
status:
conditions:
- lastTransitionTime: null
message: Valid GatewayClass
reason: Accepted
status: "True"
type: Accepted
gateways:
- apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
Expand All @@ -14,13 +122,6 @@ gateways:
port: 80
protocol: HTTP
status:
conditions:
- lastTransitionTime: null
message: 'Invalid access log backendRefs in the referenced EnvoyProxy: service
monitoring/service-not-found not found'
reason: InvalidParameters
status: "False"
type: Accepted
listeners:
- attachedRoutes: 0
conditions:
Expand Down Expand Up @@ -123,7 +224,20 @@ infraIR:
resourceAttributes:
k8s.cluster.name: cluster-1
type: OpenTelemetry
status: {}
status:
ancestors:
- ancestorRef:
group: gateway.networking.k8s.io
kind: GatewayClass
name: envoy-gateway-class
conditions:
- lastTransitionTime: null
message: 'EnvoyProxy has been accepted, but the following telemetry
configuration was ignored: invalid access log backendRefs in the referenced
EnvoyProxy: service monitoring/service-not-found not found'
reason: Accepted
status: "True"
type: Accepted
listeners:
- name: envoy-gateway/gateway-1/http
ports:
Expand Down
12 changes: 12 additions & 0 deletions internal/gatewayapi/testdata/envoyproxy-accesslog-cel-with-invalid.in.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
gatewayClass:
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
name: envoy-gateway-class
spec:
controllerName: gateway.envoyproxy.io/gatewayclass-controller
parametersRef:
group: gateway.envoyproxy.io
kind: EnvoyProxy
name: test
namespace: envoy-gateway-system
envoyProxyForGatewayClass:
apiVersion: gateway.envoyproxy.io/v1alpha1
kind: EnvoyProxy
Expand Down
Loading