From 5c7338e6ff873af1fc9bffbc2c005dd168ae4f15 Mon Sep 17 00:00:00 2001 From: kahirokunn Date: Mon, 13 Feb 2023 14:55:11 +0900 Subject: [PATCH 1/3] Add `--envoy-ingress-name`, and `--envoy-ingress-namespace` flags. (#4952) Signed-off-by: kahirokunn --- apis/projectcontour/v1alpha1/contourconfig.go | 6 ++ cmd/contour/serve.go | 46 ++++++++++++++ cmd/contour/servecontext.go | 4 ++ internal/k8s/statusaddress.go | 60 +++++++++++++++++++ internal/provisioner/model/names.go | 5 ++ .../objects/contourconfig/contourconfig.go | 4 ++ .../objects/deployment/deployment.go | 1 + pkg/config/parameters.go | 8 +++ 8 files changed, 134 insertions(+) diff --git a/apis/projectcontour/v1alpha1/contourconfig.go b/apis/projectcontour/v1alpha1/contourconfig.go index 15467c25de0..6f7836b8ddc 100644 --- a/apis/projectcontour/v1alpha1/contourconfig.go +++ b/apis/projectcontour/v1alpha1/contourconfig.go @@ -239,6 +239,12 @@ type EnvoyConfig struct { // +optional Service *NamespacedName `json:"service,omitempty"` + // Ingress holds Envoy service parameters for setting Ingress status. + // + // Contour's default is { namespace: "projectcontour", name: "envoy" }. + // +optional + Ingress *NamespacedName `json:"ingress,omitempty"` + // Defines the HTTP Listener for Envoy. // // Contour's default is { address: "0.0.0.0", port: 8080, accessLog: "/dev/stdout" }. diff --git a/cmd/contour/serve.go b/cmd/contour/serve.go index 6ac8898475f..10a6b4df17a 100644 --- a/cmd/contour/serve.go +++ b/cmd/contour/serve.go @@ -147,6 +147,8 @@ func registerServe(app *kingpin.Application) (*kingpin.CmdClause, *serveContext) serve.Flag("envoy-service-https-port", "Kubernetes Service port for HTTPS requests.").PlaceHolder("").IntVar(&ctx.httpsPort) serve.Flag("envoy-service-name", "Name of the Envoy service to inspect for Ingress status details.").PlaceHolder("").StringVar(&ctx.Config.EnvoyServiceName) serve.Flag("envoy-service-namespace", "Envoy Service Namespace.").PlaceHolder("").StringVar(&ctx.Config.EnvoyServiceNamespace) + serve.Flag("envoy-ingress-name", "Name of the Envoy ingress to inspect for Ingress status details.").PlaceHolder("").StringVar(&ctx.Config.EnvoyIngressName) + serve.Flag("envoy-ingress-namespace", "Envoy Ingress Namespace.").PlaceHolder("").StringVar(&ctx.Config.EnvoyIngressNamespace) serve.Flag("health-address", "Address the health HTTP endpoint will bind to.").PlaceHolder("").StringVar(&ctx.healthAddr) serve.Flag("health-port", "Port the health HTTP endpoint will bind to.").PlaceHolder("").IntVar(&ctx.healthPort) @@ -691,6 +693,50 @@ func (s *Server) doServe() error { return err } + // Register an informer to watch envoy's service if we haven't been given static details. + if lbAddress := contourConfiguration.Ingress.StatusAddress; len(lbAddress) > 0 { + s.log.WithField("loadbalancer-address", lbAddress).Info("Using supplied information for Ingress status") + lbsw.lbStatus <- parseStatusFlag(lbAddress) + } else { + serviceHandler := &k8s.ServiceStatusLoadBalancerWatcher{ + ServiceName: contourConfiguration.Envoy.Service.Name, + LBStatus: lbsw.lbStatus, + Log: s.log.WithField("context", "serviceStatusLoadBalancerWatcher"), + } + + var handler cache.ResourceEventHandler = serviceHandler + if contourConfiguration.Envoy.Service.Namespace != "" { + handler = k8s.NewNamespaceFilter([]string{contourConfiguration.Envoy.Service.Namespace}, handler) + } + + if err := s.informOnResource(&corev1.Service{}, handler); err != nil { + s.log.WithError(err).WithField("resource", "services").Fatal("failed to create informer") + } + + ingressHandler := &k8s.IngressStatusLoadBalancerWatcher{ + ServiceName: contourConfiguration.Envoy.Service.Name, + LBStatus: lbsw.lbStatus, + Log: s.log.WithField("context", "ingressStatusLoadBalancerWatcher"), + } + + var ingressEventHandler cache.ResourceEventHandler = ingressHandler + if contourConfiguration.Envoy.Ingress.Namespace != "" { + handler = k8s.NewNamespaceFilter([]string{contourConfiguration.Envoy.Ingress.Namespace}, handler) + } + + if err := informOnResource(&networking_v1.Ingress{}, ingressEventHandler, s.mgr.GetCache()); err != nil { + s.log.WithError(err).WithField("resource", "ingresses").Fatal("failed to create ingresses informer") + } + + s.log.WithField("envoy-service-name", contourConfiguration.Envoy.Service.Name). + WithField("envoy-service-namespace", contourConfiguration.Envoy.Service.Namespace). + Info("Watching Service for Ingress status") + + s.log.WithField("envoy-ingress-name", contourConfiguration.Envoy.Ingress.Name). + WithField("envoy-ingress-namespace", contourConfiguration.Envoy.Ingress.Namespace). + Info("Watching Ingress for Ingress status") + } + xdsServer := &xdsServer{ log: s.log, registry: s.registry, diff --git a/cmd/contour/servecontext.go b/cmd/contour/servecontext.go index d87657dd975..6644a4ee19b 100644 --- a/cmd/contour/servecontext.go +++ b/cmd/contour/servecontext.go @@ -589,6 +589,10 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_v1alpha1.Co Name: ctx.Config.EnvoyServiceName, Namespace: ctx.Config.EnvoyServiceNamespace, }, + Ingress: &contour_v1alpha1.NamespacedName{ + Name: ctx.Config.EnvoyIngressName, + Namespace: ctx.Config.EnvoyIngressNamespace, + }, HTTPListener: &contour_v1alpha1.EnvoyListener{ Address: ctx.httpAddr, Port: ctx.httpPort, diff --git a/internal/k8s/statusaddress.go b/internal/k8s/statusaddress.go index 0cb5cc1daa2..b7263c3af2f 100644 --- a/internal/k8s/statusaddress.go +++ b/internal/k8s/statusaddress.go @@ -280,3 +280,63 @@ func lbStatusToGatewayAddresses(lbs core_v1.LoadBalancerStatus) []gatewayapi_v1. return addrs } + +// IngressStatusLoadBalancerWatcher implements ResourceEventHandler and +// watches for changes to the status.loadbalancer field +// Note that we specifically *don't* inspect inside the struct, as sending empty values +// is desirable to clear the status. +type IngressStatusLoadBalancerWatcher struct { + IngressName string + LBStatus chan networking_v1.IngressLoadBalancerStatus + Log logrus.FieldLogger +} + +func (s *IngressStatusLoadBalancerWatcher) OnAdd(obj interface{}) { + ingress, ok := obj.(*networking_v1.Ingress) + if !ok { + // not a service + return + } + if ingress.Name != s.IngressName { + return + } + s.Log.WithField("name", ingress.Name). + WithField("namespace", ingress.Namespace). + Debug("received new service address") + + s.notify(ingress.Status.LoadBalancer) +} + +func (s *IngressStatusLoadBalancerWatcher) OnUpdate(oldObj, newObj interface{}) { + ingress, ok := newObj.(*networking_v1.Ingress) + if !ok { + // not a service + return + } + if ingress.Name != s.IngressName { + return + } + s.Log.WithField("name", ingress.Name). + WithField("namespace", ingress.Namespace). + Debug("received new service address") + + s.notify(ingress.Status.LoadBalancer) +} + +func (s *IngressStatusLoadBalancerWatcher) OnDelete(obj interface{}) { + ingress, ok := obj.(*networking_v1.Ingress) + if !ok { + // not a service + return + } + if ingress.Name != s.IngressName { + return + } + s.notify(networking_v1.IngressLoadBalancerStatus{ + Ingress: nil, + }) +} + +func (s *IngressStatusLoadBalancerWatcher) notify(lbstatus networking_v1.IngressLoadBalancerStatus) { + s.LBStatus <- lbstatus +} diff --git a/internal/provisioner/model/names.go b/internal/provisioner/model/names.go index 081c25c0205..58706d83931 100644 --- a/internal/provisioner/model/names.go +++ b/internal/provisioner/model/names.go @@ -33,6 +33,11 @@ func (c *Contour) EnvoyServiceName() string { return "envoy-" + c.Name } +// EnvoyIngressName returns the name of the Envoy Ingress resource. +func (c *Contour) EnvoyIngressName() string { + return "envoy-" + c.Name +} + // ContourDeploymentName returns the name of the Contour Deployment resource. func (c *Contour) ContourDeploymentName() string { return "contour-" + c.Name diff --git a/internal/provisioner/objects/contourconfig/contourconfig.go b/internal/provisioner/objects/contourconfig/contourconfig.go index 1591125851e..497cdac5247 100644 --- a/internal/provisioner/objects/contourconfig/contourconfig.go +++ b/internal/provisioner/objects/contourconfig/contourconfig.go @@ -73,6 +73,10 @@ func setGatewayConfig(config *contour_v1alpha1.ContourConfiguration, contour *mo Namespace: contour.Namespace, Name: contour.EnvoyServiceName(), } + config.Spec.Envoy.Ingress = &contour_api_v1alpha1.NamespacedName{ + Namespace: contour.Namespace, + Name: contour.EnvoyIngressName(), + } } // EnsureContourConfigDeleted deletes a ContourConfig for the provided contour, if the configured owner labels exist. diff --git a/internal/provisioner/objects/deployment/deployment.go b/internal/provisioner/objects/deployment/deployment.go index 21c23531283..956b7e5b8a9 100644 --- a/internal/provisioner/objects/deployment/deployment.go +++ b/internal/provisioner/objects/deployment/deployment.go @@ -96,6 +96,7 @@ func DesiredDeployment(contour *model.Contour, image string) *apps_v1.Deployment fmt.Sprintf("--contour-config-name=%s", contour.ContourConfigurationName()), fmt.Sprintf("--leader-election-resource-name=%s", contour.LeaderElectionLeaseName()), fmt.Sprintf("--envoy-service-name=%s", contour.EnvoyServiceName()), + fmt.Sprintf("--envoy-ingress-name=%s", contour.EnvoyIngressName()), fmt.Sprintf("--kubernetes-debug=%d", contour.Spec.KubernetesLogLevel), } diff --git a/pkg/config/parameters.go b/pkg/config/parameters.go index 9e9953c6098..9494e152b99 100644 --- a/pkg/config/parameters.go +++ b/pkg/config/parameters.go @@ -682,6 +682,12 @@ type Parameters struct { // Name of the envoy service to inspect for Ingress status details. EnvoyServiceName string `yaml:"envoy-service-name,omitempty"` + // Namespace of the envoy ingress to inspect for Ingress status details. + EnvoyIngressNamespace string `yaml:"envoy-ingress-namespace,omitempty"` + + // Name of the envoy ingress to inspect for Ingress status details. + EnvoyIngressName string `yaml:"envoy-ingress-name,omitempty"` + // DefaultHTTPVersions defines the default set of HTTPS // versions the proxy should accept. HTTP versions are // strings of the form "HTTP/xx". Supported versions are @@ -1126,6 +1132,8 @@ func Defaults() Parameters { }, EnvoyServiceName: "envoy", EnvoyServiceNamespace: contourNamespace, + EnvoyIngressName: "envoy", + EnvoyIngressNamespace: contourNamespace, DefaultHTTPVersions: []HTTPVersionType{}, Cluster: ClusterParameters{ DNSLookupFamily: AutoClusterDNSFamily, From 045756a5b3fe66ebb676c189dac966f1f331abf1 Mon Sep 17 00:00:00 2001 From: Haitao Li Date: Fri, 24 Nov 2023 22:57:22 +1100 Subject: [PATCH 2/3] Add a new load-balancer-status flag for setting ingress details Signed-off-by: Haitao Li --- apis/projectcontour/v1alpha1/contourconfig.go | 10 +- cmd/contour/serve.go | 154 +++++++++++++----- cmd/contour/serve_test.go | 106 ++++++++++++ cmd/contour/servecontext.go | 5 +- examples/contour/01-crds.yaml | 18 ++ examples/render/contour-deployment.yaml | 18 ++ .../render/contour-gateway-provisioner.yaml | 18 ++ examples/render/contour-gateway.yaml | 18 ++ examples/render/contour.yaml | 18 ++ internal/k8s/statusaddress.go | 32 +++- .../objects/contourconfig/contourconfig.go | 4 - pkg/config/parameters.go | 10 +- .../docs/main/config/api-reference.html | 18 ++ site/content/docs/main/configuration.md | 93 +++++------ 14 files changed, 414 insertions(+), 108 deletions(-) diff --git a/apis/projectcontour/v1alpha1/contourconfig.go b/apis/projectcontour/v1alpha1/contourconfig.go index 6f7836b8ddc..103b31781fc 100644 --- a/apis/projectcontour/v1alpha1/contourconfig.go +++ b/apis/projectcontour/v1alpha1/contourconfig.go @@ -239,11 +239,15 @@ type EnvoyConfig struct { // +optional Service *NamespacedName `json:"service,omitempty"` - // Ingress holds Envoy service parameters for setting Ingress status. + // LoadBalancer specifies how Contour should set the ingress status address. + // If provided, the value can be in one of the formats: + // - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. + // - service:/: Contour will use the address of the designated service. + // - ingress:/: Contour will use the address of the designated ingress. // - // Contour's default is { namespace: "projectcontour", name: "envoy" }. + // Contour's default is an empty string. // +optional - Ingress *NamespacedName `json:"ingress,omitempty"` + LoadBalancer string `json:"loadBalancer,omitempty"` // Defines the HTTP Listener for Envoy. // diff --git a/cmd/contour/serve.go b/cmd/contour/serve.go index 10a6b4df17a..2c83082916c 100644 --- a/cmd/contour/serve.go +++ b/cmd/contour/serve.go @@ -21,6 +21,7 @@ import ( "net/http" "os" "strconv" + "strings" "time" "github.com/alecthomas/kingpin/v2" @@ -147,8 +148,6 @@ func registerServe(app *kingpin.Application) (*kingpin.CmdClause, *serveContext) serve.Flag("envoy-service-https-port", "Kubernetes Service port for HTTPS requests.").PlaceHolder("").IntVar(&ctx.httpsPort) serve.Flag("envoy-service-name", "Name of the Envoy service to inspect for Ingress status details.").PlaceHolder("").StringVar(&ctx.Config.EnvoyServiceName) serve.Flag("envoy-service-namespace", "Envoy Service Namespace.").PlaceHolder("").StringVar(&ctx.Config.EnvoyServiceNamespace) - serve.Flag("envoy-ingress-name", "Name of the Envoy ingress to inspect for Ingress status details.").PlaceHolder("").StringVar(&ctx.Config.EnvoyIngressName) - serve.Flag("envoy-ingress-namespace", "Envoy Ingress Namespace.").PlaceHolder("").StringVar(&ctx.Config.EnvoyIngressNamespace) serve.Flag("health-address", "Address the health HTTP endpoint will bind to.").PlaceHolder("").StringVar(&ctx.healthAddr) serve.Flag("health-port", "Port the health HTTP endpoint will bind to.").PlaceHolder("").IntVar(&ctx.healthPort) @@ -171,6 +170,8 @@ func registerServe(app *kingpin.Application) (*kingpin.CmdClause, *serveContext) serve.Flag("leader-election-resource-namespace", "The namespace of the resource (Lease) leader election will lease.").Default(config.GetenvOr("CONTOUR_NAMESPACE", "projectcontour")).StringVar(&ctx.LeaderElection.Namespace) serve.Flag("leader-election-retry-period", "The interval which Contour will attempt to acquire leadership lease.").Default("2s").DurationVar(&ctx.LeaderElection.RetryPeriod) + serve.Flag("load-balancer-status", "Address to set or the source to inspect for ingress status.").PlaceHolder("").StringVar(&ctx.Config.LoadBalancerStatus) + serve.Flag("root-namespaces", "Restrict contour to searching these namespaces for root ingress routes.").PlaceHolder("").StringVar(&ctx.rootNamespaces) serve.Flag("stats-address", "Envoy /stats interface address.").PlaceHolder("").StringVar(&ctx.statsAddr) @@ -678,13 +679,46 @@ func (s *Server) doServe() error { } // Set up ingress load balancer status writer. + if err := s.setupIngressLoadBalancerStatusWriter(contourConfiguration, ingressClassNames, gatewayRef, sh.Writer()); err != nil { + return err + } + + xdsServer := &xdsServer{ + log: s.log, + registry: s.registry, + config: *contourConfiguration.XDSServer, + snapshotHandler: snapshotHandler, + resources: resources, + initialDagBuilt: contourHandler.HasBuiltInitialDag, + } + if err := s.mgr.Add(xdsServer); err != nil { + return err + } + + notifier := &leadership.Notifier{ + ToNotify: []leadership.NeedLeaderElectionNotification{contourHandler, observer}, + } + if err := s.mgr.Add(notifier); err != nil { + return err + } + + // GO! + return s.mgr.Start(signals.SetupSignalHandler()) +} + +func (s *Server) setupIngressLoadBalancerStatusWriter( + contourConfiguration contour_v1alpha1.ContourConfigurationSpec, + ingressClassNames []string, + gatewayRef *types.NamespacedName, + statusUpdater k8s.StatusUpdater, +) error { lbsw := &loadBalancerStatusWriter{ log: s.log.WithField("context", "loadBalancerStatusWriter"), cache: s.mgr.GetCache(), lbStatus: make(chan core_v1.LoadBalancerStatus, 1), ingressClassNames: ingressClassNames, gatewayRef: gatewayRef, - statusUpdater: sh.Writer(), + statusUpdater: statusUpdater, statusAddress: contourConfiguration.Ingress.StatusAddress, serviceName: contourConfiguration.Envoy.Service.Name, serviceNamespace: contourConfiguration.Envoy.Service.Namespace, @@ -693,71 +727,113 @@ func (s *Server) doServe() error { return err } - // Register an informer to watch envoy's service if we haven't been given static details. + elbs := &envoyLoadBalancerStatus{} if lbAddress := contourConfiguration.Ingress.StatusAddress; len(lbAddress) > 0 { - s.log.WithField("loadbalancer-address", lbAddress).Info("Using supplied information for Ingress status") - lbsw.lbStatus <- parseStatusFlag(lbAddress) + elbs.Kind = "hostname" + elbs.FQDNs = lbAddress + } else if contourConfiguration.Envoy.LoadBalancer != "" { + status, err := parseEnvoyLoadBalancerStatus(contourConfiguration.Envoy.LoadBalancer) + if err != nil { + return err + } + elbs = status } else { + elbs.Kind = "service" + elbs.Namespace = contourConfiguration.Envoy.Service.Namespace + elbs.Name = contourConfiguration.Envoy.Service.Name + } + switch strings.ToLower(elbs.Kind) { + case "hostname": + s.log.WithField("loadbalancer-fqdns", lbAddress).Info("Using supplied hostname for Ingress status") + lbsw.lbStatus <- parseStatusFlag(elbs.FQDNs) + case "service": + // Register an informer to watch supplied service serviceHandler := &k8s.ServiceStatusLoadBalancerWatcher{ - ServiceName: contourConfiguration.Envoy.Service.Name, + ServiceName: elbs.Name, LBStatus: lbsw.lbStatus, Log: s.log.WithField("context", "serviceStatusLoadBalancerWatcher"), } var handler cache.ResourceEventHandler = serviceHandler - if contourConfiguration.Envoy.Service.Namespace != "" { - handler = k8s.NewNamespaceFilter([]string{contourConfiguration.Envoy.Service.Namespace}, handler) + if elbs.Namespace != "" { + handler = k8s.NewNamespaceFilter([]string{elbs.Namespace}, handler) } - if err := s.informOnResource(&corev1.Service{}, handler); err != nil { - s.log.WithError(err).WithField("resource", "services").Fatal("failed to create informer") + if err := s.informOnResource(&core_v1.Service{}, handler); err != nil { + s.log.WithError(err).WithField("resource", "services").Fatal("failed to create services informer") } - + s.log.Infof("Watching %s for Ingress status", elbs) + case "ingress": + // Register an informer to watch supplied ingress ingressHandler := &k8s.IngressStatusLoadBalancerWatcher{ - ServiceName: contourConfiguration.Envoy.Service.Name, + IngressName: elbs.Name, LBStatus: lbsw.lbStatus, Log: s.log.WithField("context", "ingressStatusLoadBalancerWatcher"), } - var ingressEventHandler cache.ResourceEventHandler = ingressHandler - if contourConfiguration.Envoy.Ingress.Namespace != "" { - handler = k8s.NewNamespaceFilter([]string{contourConfiguration.Envoy.Ingress.Namespace}, handler) + var handler cache.ResourceEventHandler = ingressHandler + if elbs.Namespace != "" { + handler = k8s.NewNamespaceFilter([]string{elbs.Namespace}, handler) } - if err := informOnResource(&networking_v1.Ingress{}, ingressEventHandler, s.mgr.GetCache()); err != nil { + if err := s.informOnResource(&networking_v1.Ingress{}, handler); err != nil { s.log.WithError(err).WithField("resource", "ingresses").Fatal("failed to create ingresses informer") } + s.log.Infof("Watching %s for Ingress status", elbs) + default: + return fmt.Errorf("unsupported ingress kind: %s", elbs.Kind) + } - s.log.WithField("envoy-service-name", contourConfiguration.Envoy.Service.Name). - WithField("envoy-service-namespace", contourConfiguration.Envoy.Service.Namespace). - Info("Watching Service for Ingress status") + return nil +} - s.log.WithField("envoy-ingress-name", contourConfiguration.Envoy.Ingress.Name). - WithField("envoy-ingress-namespace", contourConfiguration.Envoy.Ingress.Namespace). - Info("Watching Ingress for Ingress status") - } +type envoyLoadBalancerStatus struct { + Kind string + FQDNs string + config.NamespacedName +} - xdsServer := &xdsServer{ - log: s.log, - registry: s.registry, - config: *contourConfiguration.XDSServer, - snapshotHandler: snapshotHandler, - resources: resources, - initialDagBuilt: contourHandler.HasBuiltInitialDag, +func (elbs *envoyLoadBalancerStatus) String() string { + if elbs.Kind == "hostname" { + return fmt.Sprintf("%s:%s", elbs.Kind, elbs.FQDNs) } - if err := s.mgr.Add(xdsServer); err != nil { - return err + return fmt.Sprintf("%s:%s/%s", elbs.Kind, elbs.Namespace, elbs.Name) +} + +func parseEnvoyLoadBalancerStatus(s string) (*envoyLoadBalancerStatus, error) { + parts := strings.SplitN(s, ":", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid load-balancer-status: %s", s) } - notifier := &leadership.Notifier{ - ToNotify: []leadership.NeedLeaderElectionNotification{contourHandler, observer}, + if parts[1] == "" { + return nil, fmt.Errorf("invalid load-balancer-status: empty object reference") } - if err := s.mgr.Add(notifier); err != nil { - return err + + elbs := envoyLoadBalancerStatus{} + + elbs.Kind = strings.ToLower(parts[0]) + switch elbs.Kind { + case "ingress", "service": + parts = strings.Split(parts[1], "/") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid load-balancer-status: %s is not in the format of /", s) + } + + if parts[0] == "" || parts[1] == "" { + return nil, fmt.Errorf("invalid load-balancer-status: or is empty") + } + elbs.Namespace = parts[0] + elbs.Name = parts[1] + case "hostname": + elbs.FQDNs = parts[1] + case "": + return nil, fmt.Errorf("invalid load-balancer-status: kind is empty") + default: + return nil, fmt.Errorf("invalid load-balancer-status: unsupported kind: %s", elbs.Kind) } - // GO! - return s.mgr.Start(signals.SetupSignalHandler()) + return &elbs, nil } func (s *Server) getExtensionSvcConfig(name, namespace string) (xdscache_v3.ExtensionServiceConfig, error) { diff --git a/cmd/contour/serve_test.go b/cmd/contour/serve_test.go index 4a60f15dd1d..ac00bbdf1be 100644 --- a/cmd/contour/serve_test.go +++ b/cmd/contour/serve_test.go @@ -24,6 +24,7 @@ import ( contour_v1alpha1 "github.com/projectcontour/contour/apis/projectcontour/v1alpha1" "github.com/projectcontour/contour/internal/dag" + "github.com/projectcontour/contour/pkg/config" ) func TestGetDAGBuilder(t *testing.T) { @@ -307,3 +308,108 @@ func mustGetIngressProcessor(t *testing.T, builder *dag.Builder) *dag.IngressPro require.FailNow(t, "IngressProcessor not found in list of DAG builder's processors") return nil } + +func TestParseEnvoyLoadBalancerStatus(t *testing.T) { + tests := []struct { + name string + status string + want envoyLoadBalancerStatus + }{ + { + name: "Service", + status: "service:namespace-1/name-1", + want: envoyLoadBalancerStatus{ + Kind: "service", + NamespacedName: config.NamespacedName{ + Name: "name-1", + Namespace: "namespace-1", + }, + }, + }, + { + name: "Ingress", + status: "ingress:namespace-1/name-1", + want: envoyLoadBalancerStatus{ + Kind: "ingress", + NamespacedName: config.NamespacedName{ + Name: "name-1", + Namespace: "namespace-1", + }, + }, + }, + { + name: "hostname", + status: "hostname:example.com", + want: envoyLoadBalancerStatus{ + Kind: "hostname", + FQDNs: "example.com", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r, err := parseEnvoyLoadBalancerStatus(tt.status) + require.NoError(t, err) + assert.Equal(t, tt.want, *r) + }) + } + + tests2 := []struct { + name string + status string + error string + }{ + { + name: "Empty", + status: "", + error: "invalid", + }, + { + name: "No kind", + status: ":n", + error: "kind is empty", + }, + { + name: "Invalid kind", + status: "test:n", + error: "unsupported kind", + }, + { + name: "No reference", + status: "service:", + error: "empty object reference", + }, + { + name: "No colon", + status: "service", + error: "invalid", + }, + { + name: "No slash", + status: "service:name-1", + error: "not in the format", + }, + { + name: "starts with slash", + status: "service:/name-1", + error: "is empty", + }, + { + name: "ends with slash", + status: "service:name-1/", + error: "is empty", + }, + { + name: "two many slashes", + status: "service:name/x/y", + error: "not in the format", + }, + } + for _, tt := range tests2 { + t.Run(tt.name, func(t *testing.T) { + _, err := parseEnvoyLoadBalancerStatus(tt.status) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.error) + }) + } +} diff --git a/cmd/contour/servecontext.go b/cmd/contour/servecontext.go index 6644a4ee19b..91bf90ed7b8 100644 --- a/cmd/contour/servecontext.go +++ b/cmd/contour/servecontext.go @@ -589,10 +589,6 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_v1alpha1.Co Name: ctx.Config.EnvoyServiceName, Namespace: ctx.Config.EnvoyServiceNamespace, }, - Ingress: &contour_v1alpha1.NamespacedName{ - Name: ctx.Config.EnvoyIngressName, - Namespace: ctx.Config.EnvoyIngressNamespace, - }, HTTPListener: &contour_v1alpha1.EnvoyListener{ Address: ctx.httpAddr, Port: ctx.httpPort, @@ -634,6 +630,7 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_v1alpha1.Co EnvoyStripTrailingHostDot: &ctx.Config.Network.EnvoyStripTrailingHostDot, }, OMEnforcedHealth: envoyOMEnforcedHealthListenerConfig, + LoadBalancer: ctx.Config.LoadBalancerStatus, }, Gateway: gatewayConfig, HTTPProxy: &contour_v1alpha1.HTTPProxyConfig{ diff --git a/examples/contour/01-crds.yaml b/examples/contour/01-crds.yaml index 3fbbc7fabb8..3aff8eaf35c 100644 --- a/examples/contour/01-crds.yaml +++ b/examples/contour/01-crds.yaml @@ -475,6 +475,15 @@ spec: Contour's default is false. type: boolean type: object + loadBalancer: + description: |- + LoadBalancer specifies how Contour should set the ingress status address. + If provided, the value can be in one of the formats: + - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. + - service:/: Contour will use the address of the designated service. + - ingress:/: Contour will use the address of the designated ingress. + Contour's default is an empty string. + type: string logging: description: Logging defines how Envoy's logs can be configured. properties: @@ -4561,6 +4570,15 @@ spec: Contour's default is false. type: boolean type: object + loadBalancer: + description: |- + LoadBalancer specifies how Contour should set the ingress status address. + If provided, the value can be in one of the formats: + - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. + - service:/: Contour will use the address of the designated service. + - ingress:/: Contour will use the address of the designated ingress. + Contour's default is an empty string. + type: string logging: description: Logging defines how Envoy's logs can be configured. properties: diff --git a/examples/render/contour-deployment.yaml b/examples/render/contour-deployment.yaml index 64637cd16a3..1d168425f63 100644 --- a/examples/render/contour-deployment.yaml +++ b/examples/render/contour-deployment.yaml @@ -694,6 +694,15 @@ spec: Contour's default is false. type: boolean type: object + loadBalancer: + description: |- + LoadBalancer specifies how Contour should set the ingress status address. + If provided, the value can be in one of the formats: + - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. + - service:/: Contour will use the address of the designated service. + - ingress:/: Contour will use the address of the designated ingress. + Contour's default is an empty string. + type: string logging: description: Logging defines how Envoy's logs can be configured. properties: @@ -4780,6 +4789,15 @@ spec: Contour's default is false. type: boolean type: object + loadBalancer: + description: |- + LoadBalancer specifies how Contour should set the ingress status address. + If provided, the value can be in one of the formats: + - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. + - service:/: Contour will use the address of the designated service. + - ingress:/: Contour will use the address of the designated ingress. + Contour's default is an empty string. + type: string logging: description: Logging defines how Envoy's logs can be configured. properties: diff --git a/examples/render/contour-gateway-provisioner.yaml b/examples/render/contour-gateway-provisioner.yaml index 5872297a433..751164811c7 100644 --- a/examples/render/contour-gateway-provisioner.yaml +++ b/examples/render/contour-gateway-provisioner.yaml @@ -486,6 +486,15 @@ spec: Contour's default is false. type: boolean type: object + loadBalancer: + description: |- + LoadBalancer specifies how Contour should set the ingress status address. + If provided, the value can be in one of the formats: + - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. + - service:/: Contour will use the address of the designated service. + - ingress:/: Contour will use the address of the designated ingress. + Contour's default is an empty string. + type: string logging: description: Logging defines how Envoy's logs can be configured. properties: @@ -4572,6 +4581,15 @@ spec: Contour's default is false. type: boolean type: object + loadBalancer: + description: |- + LoadBalancer specifies how Contour should set the ingress status address. + If provided, the value can be in one of the formats: + - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. + - service:/: Contour will use the address of the designated service. + - ingress:/: Contour will use the address of the designated ingress. + Contour's default is an empty string. + type: string logging: description: Logging defines how Envoy's logs can be configured. properties: diff --git a/examples/render/contour-gateway.yaml b/examples/render/contour-gateway.yaml index f63ad7b9f00..604e3cd5696 100644 --- a/examples/render/contour-gateway.yaml +++ b/examples/render/contour-gateway.yaml @@ -511,6 +511,15 @@ spec: Contour's default is false. type: boolean type: object + loadBalancer: + description: |- + LoadBalancer specifies how Contour should set the ingress status address. + If provided, the value can be in one of the formats: + - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. + - service:/: Contour will use the address of the designated service. + - ingress:/: Contour will use the address of the designated ingress. + Contour's default is an empty string. + type: string logging: description: Logging defines how Envoy's logs can be configured. properties: @@ -4597,6 +4606,15 @@ spec: Contour's default is false. type: boolean type: object + loadBalancer: + description: |- + LoadBalancer specifies how Contour should set the ingress status address. + If provided, the value can be in one of the formats: + - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. + - service:/: Contour will use the address of the designated service. + - ingress:/: Contour will use the address of the designated ingress. + Contour's default is an empty string. + type: string logging: description: Logging defines how Envoy's logs can be configured. properties: diff --git a/examples/render/contour.yaml b/examples/render/contour.yaml index 4ba9bf44efe..8fcb311041f 100644 --- a/examples/render/contour.yaml +++ b/examples/render/contour.yaml @@ -694,6 +694,15 @@ spec: Contour's default is false. type: boolean type: object + loadBalancer: + description: |- + LoadBalancer specifies how Contour should set the ingress status address. + If provided, the value can be in one of the formats: + - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. + - service:/: Contour will use the address of the designated service. + - ingress:/: Contour will use the address of the designated ingress. + Contour's default is an empty string. + type: string logging: description: Logging defines how Envoy's logs can be configured. properties: @@ -4780,6 +4789,15 @@ spec: Contour's default is false. type: boolean type: object + loadBalancer: + description: |- + LoadBalancer specifies how Contour should set the ingress status address. + If provided, the value can be in one of the formats: + - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. + - service:/: Contour will use the address of the designated service. + - ingress:/: Contour will use the address of the designated ingress. + Contour's default is an empty string. + type: string logging: description: Logging defines how Envoy's logs can be configured. properties: diff --git a/internal/k8s/statusaddress.go b/internal/k8s/statusaddress.go index b7263c3af2f..fed0f6eaf9b 100644 --- a/internal/k8s/statusaddress.go +++ b/internal/k8s/statusaddress.go @@ -281,17 +281,39 @@ func lbStatusToGatewayAddresses(lbs core_v1.LoadBalancerStatus) []gatewayapi_v1. return addrs } +func networkingToCoreLBStatus(lbs networking_v1.IngressLoadBalancerStatus) core_v1.LoadBalancerStatus { + ingress := make([]core_v1.LoadBalancerIngress, len(lbs.Ingress)) + for i, ing := range lbs.Ingress { + ports := make([]core_v1.PortStatus, len(ing.Ports)) + for j, ps := range ing.Ports { + ports[j] = core_v1.PortStatus{ + Port: ps.Port, + Protocol: ps.Protocol, + Error: ps.Error, + } + } + ingress[i] = core_v1.LoadBalancerIngress{ + IP: ing.IP, + Hostname: ing.Hostname, + Ports: ports, + } + } + return core_v1.LoadBalancerStatus{ + Ingress: ingress, + } +} + // IngressStatusLoadBalancerWatcher implements ResourceEventHandler and // watches for changes to the status.loadbalancer field // Note that we specifically *don't* inspect inside the struct, as sending empty values // is desirable to clear the status. type IngressStatusLoadBalancerWatcher struct { IngressName string - LBStatus chan networking_v1.IngressLoadBalancerStatus + LBStatus chan core_v1.LoadBalancerStatus Log logrus.FieldLogger } -func (s *IngressStatusLoadBalancerWatcher) OnAdd(obj interface{}) { +func (s *IngressStatusLoadBalancerWatcher) OnAdd(obj any, _ bool) { ingress, ok := obj.(*networking_v1.Ingress) if !ok { // not a service @@ -307,7 +329,7 @@ func (s *IngressStatusLoadBalancerWatcher) OnAdd(obj interface{}) { s.notify(ingress.Status.LoadBalancer) } -func (s *IngressStatusLoadBalancerWatcher) OnUpdate(oldObj, newObj interface{}) { +func (s *IngressStatusLoadBalancerWatcher) OnUpdate(_, newObj any) { ingress, ok := newObj.(*networking_v1.Ingress) if !ok { // not a service @@ -323,7 +345,7 @@ func (s *IngressStatusLoadBalancerWatcher) OnUpdate(oldObj, newObj interface{}) s.notify(ingress.Status.LoadBalancer) } -func (s *IngressStatusLoadBalancerWatcher) OnDelete(obj interface{}) { +func (s *IngressStatusLoadBalancerWatcher) OnDelete(obj any) { ingress, ok := obj.(*networking_v1.Ingress) if !ok { // not a service @@ -338,5 +360,5 @@ func (s *IngressStatusLoadBalancerWatcher) OnDelete(obj interface{}) { } func (s *IngressStatusLoadBalancerWatcher) notify(lbstatus networking_v1.IngressLoadBalancerStatus) { - s.LBStatus <- lbstatus + s.LBStatus <- networkingToCoreLBStatus(lbstatus) } diff --git a/internal/provisioner/objects/contourconfig/contourconfig.go b/internal/provisioner/objects/contourconfig/contourconfig.go index 497cdac5247..1591125851e 100644 --- a/internal/provisioner/objects/contourconfig/contourconfig.go +++ b/internal/provisioner/objects/contourconfig/contourconfig.go @@ -73,10 +73,6 @@ func setGatewayConfig(config *contour_v1alpha1.ContourConfiguration, contour *mo Namespace: contour.Namespace, Name: contour.EnvoyServiceName(), } - config.Spec.Envoy.Ingress = &contour_api_v1alpha1.NamespacedName{ - Namespace: contour.Namespace, - Name: contour.EnvoyIngressName(), - } } // EnsureContourConfigDeleted deletes a ContourConfig for the provided contour, if the configured owner labels exist. diff --git a/pkg/config/parameters.go b/pkg/config/parameters.go index 9494e152b99..bef3d0db550 100644 --- a/pkg/config/parameters.go +++ b/pkg/config/parameters.go @@ -682,11 +682,9 @@ type Parameters struct { // Name of the envoy service to inspect for Ingress status details. EnvoyServiceName string `yaml:"envoy-service-name,omitempty"` - // Namespace of the envoy ingress to inspect for Ingress status details. - EnvoyIngressNamespace string `yaml:"envoy-ingress-namespace,omitempty"` - - // Name of the envoy ingress to inspect for Ingress status details. - EnvoyIngressName string `yaml:"envoy-ingress-name,omitempty"` + // Identifier of ingress object for load balancer address in the format of + // (ingress|service):/, or hostname:fqdn1[,fqdn2]. + LoadBalancerStatus string `yaml:"load-balancer-status,omitempty"` // DefaultHTTPVersions defines the default set of HTTPS // versions the proxy should accept. HTTP versions are @@ -1132,8 +1130,6 @@ func Defaults() Parameters { }, EnvoyServiceName: "envoy", EnvoyServiceNamespace: contourNamespace, - EnvoyIngressName: "envoy", - EnvoyIngressNamespace: contourNamespace, DefaultHTTPVersions: []HTTPVersionType{}, Cluster: ClusterParameters{ DNSLookupFamily: AutoClusterDNSFamily, diff --git a/site/content/docs/main/config/api-reference.html b/site/content/docs/main/config/api-reference.html index 5d232e083d7..2e73a4a33ef 100644 --- a/site/content/docs/main/config/api-reference.html +++ b/site/content/docs/main/config/api-reference.html @@ -6978,6 +6978,24 @@

EnvoyConfig +loadBalancer +
+ +string + + + +(Optional) +

LoadBalancer specifies how Contour should set the ingress status address. +If provided, the value can be in one of the formats: +- address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. +- service:/: Contour will use the address of the designated service. +- ingress:/: Contour will use the address of the designated ingress.

+

Contour’s default is an empty string.

+ + + + http
diff --git a/site/content/docs/main/configuration.md b/site/content/docs/main/configuration.md index 5b2bf578c1b..111d3c9e1fe 100644 --- a/site/content/docs/main/configuration.md +++ b/site/content/docs/main/configuration.md @@ -16,52 +16,53 @@ The `contour serve` command is the main command which is used to watch for Kuber There are a number of flags that can be passed to this command which further configures how Contour operates. Many of these flags are mirrored in the [Contour Configuration File](#configuration-file). -| Flag Name | Description | -| --------------------------------------------------------------- | --------------------------------------------------------------------------------------- | -| `--config-path` | Path to base configuration | -| `--contour-config-name` | Name of the ContourConfiguration resource to use | -| `--incluster` | Use in cluster configuration | -| `--kubeconfig=` | Path to kubeconfig (if not in running inside a cluster) | -| `--xds-address=` | xDS gRPC API address | -| `--xds-port=` | xDS gRPC API port | -| `--stats-address=` | Envoy /stats interface address | -| `--stats-port=` | Envoy /stats interface port | -| `--debug-http-address=
` | Address the debug http endpoint will bind to. | -| `--debug-http-port=` | Port the debug http endpoint will bind to | -| `--http-address=` | Address the metrics HTTP endpoint will bind to | -| `--http-port=` | Port the metrics HTTP endpoint will bind to. | -| `--health-address=` | Address the health HTTP endpoint will bind to | -| `--health-port=` | Port the health HTTP endpoint will bind to | -| `--contour-cafile=` | CA bundle file name for serving gRPC with TLS | -| `--contour-cert-file=` | Contour certificate file name for serving gRPC over TLS | -| `--contour-key-file=` | Contour key file name for serving gRPC over TLS | -| `--insecure` | Allow serving without TLS secured gRPC | -| `--root-namespaces=` | Restrict contour to searching these namespaces for root ingress routes | -| `--watch-namespaces=` | Restrict contour to searching these namespaces for all resources | -| `--ingress-class-name=` | Contour IngressClass name (comma-separated list allowed) | -| `--ingress-status-address=
` | Address to set in Ingress object status | -| `--envoy-http-access-log=` | Envoy HTTP access log | -| `--envoy-https-access-log=` | Envoy HTTPS access log | -| `--envoy-service-http-address=` | Kubernetes Service address for HTTP requests | -| `--envoy-service-https-address=` | Kubernetes Service address for HTTPS requests | -| `--envoy-service-http-port=` | Kubernetes Service port for HTTP requests | -| `--envoy-service-https-port=` | Kubernetes Service port for HTTPS requests | -| `--envoy-service-name=` | Name of the Envoy service to inspect for Ingress status details. | -| `--envoy-service-namespace=` | Envoy Service Namespace | -| `--use-proxy-protocol` | Use PROXY protocol for all listeners | -| `--accesslog-format=` | Format for Envoy access logs | -| `--disable-leader-election` | Disable leader election mechanism | -| `--disable-feature=` | Do not start an informer for the specified resources. Flag can be given multiple times. | -| `--leader-election-lease-duration` | The duration of the leadership lease. | -| `--leader-election-renew-deadline` | The duration leader will retry refreshing leadership before giving up. | -| `--leader-election-retry-period` | The interval which Contour will attempt to acquire leadership lease. | -| `--leader-election-resource-name` | The name of the resource (Lease) leader election will lease. | -| `--leader-election-resource-namespace` | The namespace of the resource (Lease) leader election will lease. | -| `-d, --debug` | Enable debug logging | -| `--kubernetes-debug=` | Enable Kubernetes client debug logging | -| `--log-format=` | Log output format for Contour. Either text (default) or json. | -| `--kubernetes-client-qps=` | QPS allowed for the Kubernetes client. | -| `--kubernetes-client-burst=` | Burst allowed for the Kubernetes client. | +| Flag Name | Description | +|----------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| +| `--config-path` | Path to base configuration | +| `--contour-config-name` | Name of the ContourConfiguration resource to use | +| `--incluster` | Use in cluster configuration | +| `--kubeconfig=` | Path to kubeconfig (if not in running inside a cluster) | +| `--xds-address=` | xDS gRPC API address | +| `--xds-port=` | xDS gRPC API port | +| `--stats-address=` | Envoy /stats interface address | +| `--stats-port=` | Envoy /stats interface port | +| `--debug-http-address=
` | Address the debug http endpoint will bind to. | +| `--debug-http-port=` | Port the debug http endpoint will bind to | +| `--http-address=` | Address the metrics HTTP endpoint will bind to | +| `--http-port=` | Port the metrics HTTP endpoint will bind to. | +| `--health-address=` | Address the health HTTP endpoint will bind to | +| `--health-port=` | Port the health HTTP endpoint will bind to | +| `--contour-cafile=` | CA bundle file name for serving gRPC with TLS | +| `--contour-cert-file=` | Contour certificate file name for serving gRPC over TLS | +| `--contour-key-file=` | Contour key file name for serving gRPC over TLS | +| `--insecure` | Allow serving without TLS secured gRPC | +| `--root-namespaces=` | Restrict contour to searching these namespaces for root ingress routes | +| `--watch-namespaces=` | Restrict contour to searching these namespaces for all resources | +| `--ingress-class-name=` | Contour IngressClass name (comma-separated list allowed) | +| `--ingress-status-address=
` | Address to set in Ingress object status | +| `--envoy-http-access-log=` | Envoy HTTP access log | +| `--envoy-https-access-log=` | Envoy HTTPS access log | +| `--envoy-service-http-address=` | Kubernetes Service address for HTTP requests | +| `--envoy-service-https-address=` | Kubernetes Service address for HTTPS requests | +| `--envoy-service-http-port=` | Kubernetes Service port for HTTP requests | +| `--envoy-service-https-port=` | Kubernetes Service port for HTTPS requests | +| `--envoy-service-name=` | Name of the Envoy service to inspect for Ingress status details. | +| `--envoy-service-namespace=` | Envoy Service Namespace | +| `--use-proxy-protocol` | Use PROXY protocol for all listeners | +| `--accesslog-format=` | Format for Envoy access logs | +| `--disable-leader-election` | Disable leader election mechanism | +| `--disable-feature=` | Do not start an informer for the specified resources. Flag can be given multiple times. | +| `--leader-election-lease-duration` | The duration of the leadership lease. | +| `--leader-election-renew-deadline` | The duration leader will retry refreshing leadership before giving up. | +| `--leader-election-retry-period` | The interval which Contour will attempt to acquire leadership lease. | +| `--leader-election-resource-name` | The name of the resource (Lease) leader election will lease. | +| `--leader-election-resource-namespace` | The namespace of the resource (Lease) leader election will lease. | +| `--load-balancer-status= | Address to set (kind=hostname) or the source to inspect for ingress status (kind=service or ingress). | +| `-d, --debug` | Enable debug logging | +| `--kubernetes-debug=` | Enable Kubernetes client debug logging | +| `--log-format=` | Log output format for Contour. Either text (default) or json. | +| `--kubernetes-client-qps=` | QPS allowed for the Kubernetes client. | +| `--kubernetes-client-burst=` | Burst allowed for the Kubernetes client. | ## Configuration File From 3c4b45340217bc9f6ec4f30e033611f792d4df03 Mon Sep 17 00:00:00 2001 From: Tero Saarni Date: Fri, 12 Jun 2026 18:17:41 +0300 Subject: [PATCH 3/3] Address remaining review feedback and add e2e test for --load-balancer-status Signed-off-by: Tero Saarni --- apis/projectcontour/v1alpha1/contourconfig.go | 31 +++- .../v1alpha1/zz_generated.deepcopy.go | 35 +++++ changelogs/unreleased/7595-tsaarni-minor.md | 7 + cmd/contour/ingressstatus.go | 26 +++- cmd/contour/serve.go | 135 +++--------------- cmd/contour/serve_test.go | 123 +++++----------- cmd/contour/servecontext.go | 69 ++++++++- examples/contour/01-crds.yaml | 20 +-- examples/render/contour-deployment.yaml | 20 +-- .../render/contour-gateway-provisioner.yaml | 20 +-- examples/render/contour-gateway.yaml | 20 +-- examples/render/contour.yaml | 20 +-- internal/k8s/statusaddress.go | 7 +- internal/k8s/statusaddress_test.go | 70 +++++++++ internal/provisioner/model/names.go | 5 - .../objects/deployment/deployment.go | 1 - pkg/config/parameters.go | 17 ++- .../docs/main/config/api-reference.html | 10 +- site/content/docs/main/configuration.md | 3 +- test/e2e/httpproxy/httpproxy_test.go | 17 +++ .../httpproxy/load_balancer_status_test.go | 118 +++++++++++++++ 21 files changed, 508 insertions(+), 266 deletions(-) create mode 100644 changelogs/unreleased/7595-tsaarni-minor.md create mode 100644 test/e2e/httpproxy/load_balancer_status_test.go diff --git a/apis/projectcontour/v1alpha1/contourconfig.go b/apis/projectcontour/v1alpha1/contourconfig.go index 103b31781fc..cae60302dbd 100644 --- a/apis/projectcontour/v1alpha1/contourconfig.go +++ b/apis/projectcontour/v1alpha1/contourconfig.go @@ -239,15 +239,15 @@ type EnvoyConfig struct { // +optional Service *NamespacedName `json:"service,omitempty"` - // LoadBalancer specifies how Contour should set the ingress status address. - // If provided, the value can be in one of the formats: - // - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. - // - service:/: Contour will use the address of the designated service. - // - ingress:/: Contour will use the address of the designated ingress. + // LoadBalancerStatus specifies the source for load balancer status addresses + // that Contour will set on HTTPProxy, Ingress, and Gateway resources. // - // Contour's default is an empty string. + // Exactly one of the fields must be set. + // + // If spec.ingress.statusAddress is set, it takes precedence over this field. + // If this field is empty, spec.envoy.service is used as default. // +optional - LoadBalancer string `json:"loadBalancer,omitempty"` + LoadBalancerStatus *LoadBalancerStatusConfig `json:"loadBalancerStatus,omitempty"` // Defines the HTTP Listener for Envoy. // @@ -320,6 +320,23 @@ type EnvoyConfig struct { OMEnforcedHealth *HealthConfig `json:"omEnforcedHealth,omitempty"` } +// LoadBalancerStatusConfig defines the source for load balancer status addresses. +// Exactly one of the fields must be set. +// +kubebuilder:validation:XValidation:message="exactly one of service, ingress, or addresses must be set",rule="[has(self.service), has(self.ingress), has(self.addresses)].exists_one(x, x)" +type LoadBalancerStatusConfig struct { + // Service watches the named Service's status.loadBalancer for addresses. + // +optional + Service *NamespacedName `json:"service,omitempty"` + + // Ingress watches the named Ingress's status.loadBalancer for addresses. + // +optional + Ingress *NamespacedName `json:"ingress,omitempty"` + + // Addresses specifies static address(es) to use directly (IP or FQDN). + // +optional + Addresses []string `json:"addresses,omitempty"` +} + // DebugConfig contains Contour specific troubleshooting options. type DebugConfig struct { // Defines the Contour debug address interface. diff --git a/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go b/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go index 88946e87c9a..831a9142533 100644 --- a/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go +++ b/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go @@ -522,6 +522,11 @@ func (in *EnvoyConfig) DeepCopyInto(out *EnvoyConfig) { *out = new(NamespacedName) **out = **in } + if in.LoadBalancerStatus != nil { + in, out := &in.LoadBalancerStatus, &out.LoadBalancerStatus + *out = new(LoadBalancerStatusConfig) + (*in).DeepCopyInto(*out) + } if in.HTTPListener != nil { in, out := &in.HTTPListener, &out.HTTPListener *out = new(EnvoyListener) @@ -1060,6 +1065,36 @@ func (in *IngressConfig) DeepCopy() *IngressConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerStatusConfig) DeepCopyInto(out *LoadBalancerStatusConfig) { + *out = *in + if in.Service != nil { + in, out := &in.Service, &out.Service + *out = new(NamespacedName) + **out = **in + } + if in.Ingress != nil { + in, out := &in.Ingress, &out.Ingress + *out = new(NamespacedName) + **out = **in + } + if in.Addresses != nil { + in, out := &in.Addresses, &out.Addresses + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerStatusConfig. +func (in *LoadBalancerStatusConfig) DeepCopy() *LoadBalancerStatusConfig { + if in == nil { + return nil + } + out := new(LoadBalancerStatusConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MetricsConfig) DeepCopyInto(out *MetricsConfig) { *out = *in diff --git a/changelogs/unreleased/7595-tsaarni-minor.md b/changelogs/unreleased/7595-tsaarni-minor.md new file mode 100644 index 00000000000..4d9aabcf3ce --- /dev/null +++ b/changelogs/unreleased/7595-tsaarni-minor.md @@ -0,0 +1,7 @@ +## Load balancer status source from Ingress objects + +Contour can now watch an Ingress object's LoadBalancer status as the source for address propagation to `Ingress` and `HTTPProxy` status. +A new `--load-balancer-status` flag unifies address source configuration with support for `address:`, `service:`, and `ingress:` source kinds. +The same setting is available as `spec.envoy.loadBalancer` in the ContourConfiguration CRD or `load-balancer-status` in the config file. + +(@hligit, @kahirokunn and @tsaarni) diff --git a/cmd/contour/ingressstatus.go b/cmd/contour/ingressstatus.go index 0fd002a934b..eb5ae89dadb 100644 --- a/cmd/contour/ingressstatus.go +++ b/cmd/contour/ingressstatus.go @@ -59,6 +59,8 @@ type loadBalancerStatusWriter struct { statusAddress string serviceName string serviceNamespace string + ingressName string + ingressNamespace string } func (isw *loadBalancerStatusWriter) NeedLeaderElection() bool { @@ -66,13 +68,13 @@ func (isw *loadBalancerStatusWriter) NeedLeaderElection() bool { } func (isw *loadBalancerStatusWriter) Start(ctx context.Context) error { - // Register an informer to watch envoy's service if we haven't been given static details. + // Register an informer to watch envoy's service or ingress if we haven't been given static details. // The informer is registered only after leader election to prevent events from being sent before the status writer // is ready to process them. if lbAddress := isw.statusAddress; len(lbAddress) > 0 { isw.log.WithField("loadbalancer-address", lbAddress).Info("Using supplied information for Ingress status") isw.lbStatus <- parseStatusFlag(lbAddress) - } else { + } else if isw.serviceName != "" { // Register Service informer to watch for status updates. var serviceHandler client_go_cache.ResourceEventHandler = &k8s.ServiceStatusLoadBalancerWatcher{ ServiceName: isw.serviceName, @@ -92,6 +94,26 @@ func (isw *loadBalancerStatusWriter) Start(ctx context.Context) error { isw.log.WithError(err).Fatal("failed to add Service event handler") return err } + } else if isw.ingressName != "" { + // Register Ingress informer to watch for status updates. + var ingressHandler client_go_cache.ResourceEventHandler = &k8s.IngressStatusLoadBalancerWatcher{ + IngressName: isw.ingressName, + LBStatus: isw.lbStatus, + Log: isw.log.WithField("context", "ingressStatusLoadBalancerWatcher"), + } + if isw.ingressNamespace != "" { + ingressHandler = k8s.NewNamespaceFilter([]string{isw.ingressNamespace}, ingressHandler) + } + inf, err := isw.cache.GetInformer(ctx, &networking_v1.Ingress{}) + if err != nil { + isw.log.WithError(err).Fatal("failed to get Ingress informer") + return err + } + _, err = inf.AddEventHandler(ingressHandler) + if err != nil { + isw.log.WithError(err).Fatal("failed to add Ingress event handler") + return err + } } u := &k8s.StatusAddressUpdater{ diff --git a/cmd/contour/serve.go b/cmd/contour/serve.go index 2c83082916c..1bcfc280ae1 100644 --- a/cmd/contour/serve.go +++ b/cmd/contour/serve.go @@ -170,7 +170,7 @@ func registerServe(app *kingpin.Application) (*kingpin.CmdClause, *serveContext) serve.Flag("leader-election-resource-namespace", "The namespace of the resource (Lease) leader election will lease.").Default(config.GetenvOr("CONTOUR_NAMESPACE", "projectcontour")).StringVar(&ctx.LeaderElection.Namespace) serve.Flag("leader-election-retry-period", "The interval which Contour will attempt to acquire leadership lease.").Default("2s").DurationVar(&ctx.LeaderElection.RetryPeriod) - serve.Flag("load-balancer-status", "Address to set or the source to inspect for ingress status.").PlaceHolder("").StringVar(&ctx.Config.LoadBalancerStatus) + serve.Flag("load-balancer-status", "Address to set or the source to inspect for load balancer status.").PlaceHolder("").StringVar(&ctx.loadBalancerStatus) serve.Flag("root-namespaces", "Restrict contour to searching these namespaces for root ingress routes.").PlaceHolder("").StringVar(&ctx.rootNamespaces) @@ -678,8 +678,8 @@ func (s *Server) doServe() error { return err } - // Set up ingress load balancer status writer. - if err := s.setupIngressLoadBalancerStatusWriter(contourConfiguration, ingressClassNames, gatewayRef, sh.Writer()); err != nil { + // Set up load balancer status writer. + if err := s.setupLoadBalancerStatusWriter(contourConfiguration, ingressClassNames, gatewayRef, sh.Writer()); err != nil { return err } @@ -706,7 +706,7 @@ func (s *Server) doServe() error { return s.mgr.Start(signals.SetupSignalHandler()) } -func (s *Server) setupIngressLoadBalancerStatusWriter( +func (s *Server) setupLoadBalancerStatusWriter( contourConfiguration contour_v1alpha1.ContourConfigurationSpec, ingressClassNames []string, gatewayRef *types.NamespacedName, @@ -719,121 +719,32 @@ func (s *Server) setupIngressLoadBalancerStatusWriter( ingressClassNames: ingressClassNames, gatewayRef: gatewayRef, statusUpdater: statusUpdater, - statusAddress: contourConfiguration.Ingress.StatusAddress, - serviceName: contourConfiguration.Envoy.Service.Name, - serviceNamespace: contourConfiguration.Envoy.Service.Namespace, - } - if err := s.mgr.Add(lbsw); err != nil { - return err } + // Resolve the load balancer status source from configuration. + // Priority: + // 1. spec.ingress.statusAddress + // 2. spec.envoy.loadBalancerStatus + // 3. spec.envoy.service - elbs := &envoyLoadBalancerStatus{} if lbAddress := contourConfiguration.Ingress.StatusAddress; len(lbAddress) > 0 { - elbs.Kind = "hostname" - elbs.FQDNs = lbAddress - } else if contourConfiguration.Envoy.LoadBalancer != "" { - status, err := parseEnvoyLoadBalancerStatus(contourConfiguration.Envoy.LoadBalancer) - if err != nil { - return err + lbsw.statusAddress = lbAddress + } else if lb := contourConfiguration.Envoy.LoadBalancerStatus; lb != nil { + switch { + case lb.Service != nil: + lbsw.serviceName = lb.Service.Name + lbsw.serviceNamespace = lb.Service.Namespace + case lb.Ingress != nil: + lbsw.ingressName = lb.Ingress.Name + lbsw.ingressNamespace = lb.Ingress.Namespace + case len(lb.Addresses) > 0: + lbsw.statusAddress = strings.Join(lb.Addresses, ",") } - elbs = status } else { - elbs.Kind = "service" - elbs.Namespace = contourConfiguration.Envoy.Service.Namespace - elbs.Name = contourConfiguration.Envoy.Service.Name - } - switch strings.ToLower(elbs.Kind) { - case "hostname": - s.log.WithField("loadbalancer-fqdns", lbAddress).Info("Using supplied hostname for Ingress status") - lbsw.lbStatus <- parseStatusFlag(elbs.FQDNs) - case "service": - // Register an informer to watch supplied service - serviceHandler := &k8s.ServiceStatusLoadBalancerWatcher{ - ServiceName: elbs.Name, - LBStatus: lbsw.lbStatus, - Log: s.log.WithField("context", "serviceStatusLoadBalancerWatcher"), - } - - var handler cache.ResourceEventHandler = serviceHandler - if elbs.Namespace != "" { - handler = k8s.NewNamespaceFilter([]string{elbs.Namespace}, handler) - } - - if err := s.informOnResource(&core_v1.Service{}, handler); err != nil { - s.log.WithError(err).WithField("resource", "services").Fatal("failed to create services informer") - } - s.log.Infof("Watching %s for Ingress status", elbs) - case "ingress": - // Register an informer to watch supplied ingress - ingressHandler := &k8s.IngressStatusLoadBalancerWatcher{ - IngressName: elbs.Name, - LBStatus: lbsw.lbStatus, - Log: s.log.WithField("context", "ingressStatusLoadBalancerWatcher"), - } - - var handler cache.ResourceEventHandler = ingressHandler - if elbs.Namespace != "" { - handler = k8s.NewNamespaceFilter([]string{elbs.Namespace}, handler) - } - - if err := s.informOnResource(&networking_v1.Ingress{}, handler); err != nil { - s.log.WithError(err).WithField("resource", "ingresses").Fatal("failed to create ingresses informer") - } - s.log.Infof("Watching %s for Ingress status", elbs) - default: - return fmt.Errorf("unsupported ingress kind: %s", elbs.Kind) - } - - return nil -} - -type envoyLoadBalancerStatus struct { - Kind string - FQDNs string - config.NamespacedName -} - -func (elbs *envoyLoadBalancerStatus) String() string { - if elbs.Kind == "hostname" { - return fmt.Sprintf("%s:%s", elbs.Kind, elbs.FQDNs) - } - return fmt.Sprintf("%s:%s/%s", elbs.Kind, elbs.Namespace, elbs.Name) -} - -func parseEnvoyLoadBalancerStatus(s string) (*envoyLoadBalancerStatus, error) { - parts := strings.SplitN(s, ":", 2) - if len(parts) != 2 { - return nil, fmt.Errorf("invalid load-balancer-status: %s", s) - } - - if parts[1] == "" { - return nil, fmt.Errorf("invalid load-balancer-status: empty object reference") - } - - elbs := envoyLoadBalancerStatus{} - - elbs.Kind = strings.ToLower(parts[0]) - switch elbs.Kind { - case "ingress", "service": - parts = strings.Split(parts[1], "/") - if len(parts) != 2 { - return nil, fmt.Errorf("invalid load-balancer-status: %s is not in the format of /", s) - } - - if parts[0] == "" || parts[1] == "" { - return nil, fmt.Errorf("invalid load-balancer-status: or is empty") - } - elbs.Namespace = parts[0] - elbs.Name = parts[1] - case "hostname": - elbs.FQDNs = parts[1] - case "": - return nil, fmt.Errorf("invalid load-balancer-status: kind is empty") - default: - return nil, fmt.Errorf("invalid load-balancer-status: unsupported kind: %s", elbs.Kind) + lbsw.serviceName = contourConfiguration.Envoy.Service.Name + lbsw.serviceNamespace = contourConfiguration.Envoy.Service.Namespace } - return &elbs, nil + return s.mgr.Add(lbsw) } func (s *Server) getExtensionSvcConfig(name, namespace string) (xdscache_v3.ExtensionServiceConfig, error) { diff --git a/cmd/contour/serve_test.go b/cmd/contour/serve_test.go index ac00bbdf1be..70b5cc00327 100644 --- a/cmd/contour/serve_test.go +++ b/cmd/contour/serve_test.go @@ -24,7 +24,6 @@ import ( contour_v1alpha1 "github.com/projectcontour/contour/apis/projectcontour/v1alpha1" "github.com/projectcontour/contour/internal/dag" - "github.com/projectcontour/contour/pkg/config" ) func TestGetDAGBuilder(t *testing.T) { @@ -309,107 +308,63 @@ func mustGetIngressProcessor(t *testing.T, builder *dag.Builder) *dag.IngressPro return nil } -func TestParseEnvoyLoadBalancerStatus(t *testing.T) { +func TestParseLoadBalancerStatusSource(t *testing.T) { tests := []struct { - name string - status string - want envoyLoadBalancerStatus + name string + input string + wantKind string + wantAddress string + wantNN types.NamespacedName }{ { - name: "Service", - status: "service:namespace-1/name-1", - want: envoyLoadBalancerStatus{ - Kind: "service", - NamespacedName: config.NamespacedName{ - Name: "name-1", - Namespace: "namespace-1", - }, - }, + name: "service", + input: "service:namespace-1/name-1", + wantKind: "service", + wantNN: types.NamespacedName{Namespace: "namespace-1", Name: "name-1"}, }, { - name: "Ingress", - status: "ingress:namespace-1/name-1", - want: envoyLoadBalancerStatus{ - Kind: "ingress", - NamespacedName: config.NamespacedName{ - Name: "name-1", - Namespace: "namespace-1", - }, - }, + name: "ingress", + input: "ingress:namespace-1/name-1", + wantKind: "ingress", + wantNN: types.NamespacedName{Namespace: "namespace-1", Name: "name-1"}, }, { - name: "hostname", - status: "hostname:example.com", - want: envoyLoadBalancerStatus{ - Kind: "hostname", - FQDNs: "example.com", - }, + name: "address with hostname", + input: "address:example.com", + wantKind: "address", + wantAddress: "example.com", + }, + { + name: "address with IP", + input: "address:1.2.3.4", + wantKind: "address", + wantAddress: "1.2.3.4", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - r, err := parseEnvoyLoadBalancerStatus(tt.status) + kind, address, nn, err := parseLoadBalancerStatusSource(tt.input) require.NoError(t, err) - assert.Equal(t, tt.want, *r) + assert.Equal(t, tt.wantKind, kind) + assert.Equal(t, tt.wantAddress, address) + assert.Equal(t, tt.wantNN, nn) }) } - tests2 := []struct { - name string - status string - error string + errors := []struct { + name string + input string }{ - { - name: "Empty", - status: "", - error: "invalid", - }, - { - name: "No kind", - status: ":n", - error: "kind is empty", - }, - { - name: "Invalid kind", - status: "test:n", - error: "unsupported kind", - }, - { - name: "No reference", - status: "service:", - error: "empty object reference", - }, - { - name: "No colon", - status: "service", - error: "invalid", - }, - { - name: "No slash", - status: "service:name-1", - error: "not in the format", - }, - { - name: "starts with slash", - status: "service:/name-1", - error: "is empty", - }, - { - name: "ends with slash", - status: "service:name-1/", - error: "is empty", - }, - { - name: "two many slashes", - status: "service:name/x/y", - error: "not in the format", - }, + {name: "empty string", input: ""}, + {name: "empty kind", input: ":value"}, + {name: "empty value", input: "service:"}, + {name: "no colon", input: "service"}, + {name: "unsupported kind", input: "unknown:ns/name"}, } - for _, tt := range tests2 { + for _, tt := range errors { t.Run(tt.name, func(t *testing.T) { - _, err := parseEnvoyLoadBalancerStatus(tt.status) - require.Error(t, err) - assert.Contains(t, err.Error(), tt.error) + _, _, _, err := parseLoadBalancerStatusSource(tt.input) + assert.Error(t, err) }) } } diff --git a/cmd/contour/servecontext.go b/cmd/contour/servecontext.go index 91bf90ed7b8..4f75f38b5e7 100644 --- a/cmd/contour/servecontext.go +++ b/cmd/contour/servecontext.go @@ -90,6 +90,9 @@ type serveContext struct { // PermitInsecureGRPC disables TLS on Contour's gRPC listener. PermitInsecureGRPC bool + // loadBalancerStatus is the CLI flag value for --load-balancer-status, before parsing into a structured format in loadBalancerStatusConfig. + loadBalancerStatus string + // Leader election configuration. LeaderElection LeaderElection @@ -629,8 +632,8 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_v1alpha1.Co EnvoyAdminPort: &ctx.Config.Network.EnvoyAdminPort, EnvoyStripTrailingHostDot: &ctx.Config.Network.EnvoyStripTrailingHostDot, }, - OMEnforcedHealth: envoyOMEnforcedHealthListenerConfig, - LoadBalancer: ctx.Config.LoadBalancerStatus, + OMEnforcedHealth: envoyOMEnforcedHealthListenerConfig, + LoadBalancerStatus: ctx.loadBalancerStatusConfig(), }, Gateway: gatewayConfig, HTTPProxy: &contour_v1alpha1.HTTPProxyConfig{ @@ -678,3 +681,65 @@ func setMetricsFromConfig(src config.MetricsServerParameters, dst *contour_v1alp } } } + +// loadBalancerStatusConfig returns the LoadBalancerStatusConfig, preferring +// the CLI flag over the config file struct. +func (ctx *serveContext) loadBalancerStatusConfig() *contour_v1alpha1.LoadBalancerStatusConfig { + // CLI flag takes precedence over config file. + if ctx.loadBalancerStatus != "" { + parts := strings.SplitN(ctx.loadBalancerStatus, ":", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return nil + } + + switch strings.ToLower(parts[0]) { + case "service": + nn := k8s.NamespacedNameFrom(parts[1]) + return &contour_v1alpha1.LoadBalancerStatusConfig{ + Service: &contour_v1alpha1.NamespacedName{ + Namespace: nn.Namespace, + Name: nn.Name, + }, + } + case "ingress": + nn := k8s.NamespacedNameFrom(parts[1]) + return &contour_v1alpha1.LoadBalancerStatusConfig{ + Ingress: &contour_v1alpha1.NamespacedName{ + Namespace: nn.Namespace, + Name: nn.Name, + }, + } + case "address": + return &contour_v1alpha1.LoadBalancerStatusConfig{ + Addresses: strings.Split(parts[1], ","), + } + default: + return nil + } + } + + // Fallback to config file. + src := ctx.Config.LoadBalancerStatus + switch { + case src.Service != nil: + return &contour_v1alpha1.LoadBalancerStatusConfig{ + Service: &contour_v1alpha1.NamespacedName{ + Namespace: src.Service.Namespace, + Name: src.Service.Name, + }, + } + case src.Ingress != nil: + return &contour_v1alpha1.LoadBalancerStatusConfig{ + Ingress: &contour_v1alpha1.NamespacedName{ + Namespace: src.Ingress.Namespace, + Name: src.Ingress.Name, + }, + } + case len(src.Addresses) > 0: + return &contour_v1alpha1.LoadBalancerStatusConfig{ + Addresses: src.Addresses, + } + default: + return nil + } +} diff --git a/examples/contour/01-crds.yaml b/examples/contour/01-crds.yaml index 3aff8eaf35c..3b8c0e062af 100644 --- a/examples/contour/01-crds.yaml +++ b/examples/contour/01-crds.yaml @@ -477,11 +477,13 @@ spec: type: object loadBalancer: description: |- - LoadBalancer specifies how Contour should set the ingress status address. + LoadBalancer specifies the source for ingress status address. If provided, the value can be in one of the formats: - - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. - - service:/: Contour will use the address of the designated service. - - ingress:/: Contour will use the address of the designated ingress. + - address:: use the provided addresses directly (FQDN or IP). + - service:/: watch the named Service for its LoadBalancer status. + - ingress:/: watch the named Ingress for its LoadBalancer status. + If spec.ingress.statusAddress is set, it takes precedence over this field. + If this field is empty, spec.envoy.service is used as default. Contour's default is an empty string. type: string logging: @@ -4572,11 +4574,13 @@ spec: type: object loadBalancer: description: |- - LoadBalancer specifies how Contour should set the ingress status address. + LoadBalancer specifies the source for ingress status address. If provided, the value can be in one of the formats: - - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. - - service:/: Contour will use the address of the designated service. - - ingress:/: Contour will use the address of the designated ingress. + - address:: use the provided addresses directly (FQDN or IP). + - service:/: watch the named Service for its LoadBalancer status. + - ingress:/: watch the named Ingress for its LoadBalancer status. + If spec.ingress.statusAddress is set, it takes precedence over this field. + If this field is empty, spec.envoy.service is used as default. Contour's default is an empty string. type: string logging: diff --git a/examples/render/contour-deployment.yaml b/examples/render/contour-deployment.yaml index 1d168425f63..13a15a31acd 100644 --- a/examples/render/contour-deployment.yaml +++ b/examples/render/contour-deployment.yaml @@ -696,11 +696,13 @@ spec: type: object loadBalancer: description: |- - LoadBalancer specifies how Contour should set the ingress status address. + LoadBalancer specifies the source for ingress status address. If provided, the value can be in one of the formats: - - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. - - service:/: Contour will use the address of the designated service. - - ingress:/: Contour will use the address of the designated ingress. + - address:: use the provided addresses directly (FQDN or IP). + - service:/: watch the named Service for its LoadBalancer status. + - ingress:/: watch the named Ingress for its LoadBalancer status. + If spec.ingress.statusAddress is set, it takes precedence over this field. + If this field is empty, spec.envoy.service is used as default. Contour's default is an empty string. type: string logging: @@ -4791,11 +4793,13 @@ spec: type: object loadBalancer: description: |- - LoadBalancer specifies how Contour should set the ingress status address. + LoadBalancer specifies the source for ingress status address. If provided, the value can be in one of the formats: - - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. - - service:/: Contour will use the address of the designated service. - - ingress:/: Contour will use the address of the designated ingress. + - address:: use the provided addresses directly (FQDN or IP). + - service:/: watch the named Service for its LoadBalancer status. + - ingress:/: watch the named Ingress for its LoadBalancer status. + If spec.ingress.statusAddress is set, it takes precedence over this field. + If this field is empty, spec.envoy.service is used as default. Contour's default is an empty string. type: string logging: diff --git a/examples/render/contour-gateway-provisioner.yaml b/examples/render/contour-gateway-provisioner.yaml index 751164811c7..338526e75f6 100644 --- a/examples/render/contour-gateway-provisioner.yaml +++ b/examples/render/contour-gateway-provisioner.yaml @@ -488,11 +488,13 @@ spec: type: object loadBalancer: description: |- - LoadBalancer specifies how Contour should set the ingress status address. + LoadBalancer specifies the source for ingress status address. If provided, the value can be in one of the formats: - - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. - - service:/: Contour will use the address of the designated service. - - ingress:/: Contour will use the address of the designated ingress. + - address:: use the provided addresses directly (FQDN or IP). + - service:/: watch the named Service for its LoadBalancer status. + - ingress:/: watch the named Ingress for its LoadBalancer status. + If spec.ingress.statusAddress is set, it takes precedence over this field. + If this field is empty, spec.envoy.service is used as default. Contour's default is an empty string. type: string logging: @@ -4583,11 +4585,13 @@ spec: type: object loadBalancer: description: |- - LoadBalancer specifies how Contour should set the ingress status address. + LoadBalancer specifies the source for ingress status address. If provided, the value can be in one of the formats: - - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. - - service:/: Contour will use the address of the designated service. - - ingress:/: Contour will use the address of the designated ingress. + - address:: use the provided addresses directly (FQDN or IP). + - service:/: watch the named Service for its LoadBalancer status. + - ingress:/: watch the named Ingress for its LoadBalancer status. + If spec.ingress.statusAddress is set, it takes precedence over this field. + If this field is empty, spec.envoy.service is used as default. Contour's default is an empty string. type: string logging: diff --git a/examples/render/contour-gateway.yaml b/examples/render/contour-gateway.yaml index 604e3cd5696..98a1b35b35e 100644 --- a/examples/render/contour-gateway.yaml +++ b/examples/render/contour-gateway.yaml @@ -513,11 +513,13 @@ spec: type: object loadBalancer: description: |- - LoadBalancer specifies how Contour should set the ingress status address. + LoadBalancer specifies the source for ingress status address. If provided, the value can be in one of the formats: - - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. - - service:/: Contour will use the address of the designated service. - - ingress:/: Contour will use the address of the designated ingress. + - address:: use the provided addresses directly (FQDN or IP). + - service:/: watch the named Service for its LoadBalancer status. + - ingress:/: watch the named Ingress for its LoadBalancer status. + If spec.ingress.statusAddress is set, it takes precedence over this field. + If this field is empty, spec.envoy.service is used as default. Contour's default is an empty string. type: string logging: @@ -4608,11 +4610,13 @@ spec: type: object loadBalancer: description: |- - LoadBalancer specifies how Contour should set the ingress status address. + LoadBalancer specifies the source for ingress status address. If provided, the value can be in one of the formats: - - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. - - service:/: Contour will use the address of the designated service. - - ingress:/: Contour will use the address of the designated ingress. + - address:: use the provided addresses directly (FQDN or IP). + - service:/: watch the named Service for its LoadBalancer status. + - ingress:/: watch the named Ingress for its LoadBalancer status. + If spec.ingress.statusAddress is set, it takes precedence over this field. + If this field is empty, spec.envoy.service is used as default. Contour's default is an empty string. type: string logging: diff --git a/examples/render/contour.yaml b/examples/render/contour.yaml index 8fcb311041f..be5d2711c26 100644 --- a/examples/render/contour.yaml +++ b/examples/render/contour.yaml @@ -696,11 +696,13 @@ spec: type: object loadBalancer: description: |- - LoadBalancer specifies how Contour should set the ingress status address. + LoadBalancer specifies the source for ingress status address. If provided, the value can be in one of the formats: - - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. - - service:/: Contour will use the address of the designated service. - - ingress:/: Contour will use the address of the designated ingress. + - address:: use the provided addresses directly (FQDN or IP). + - service:/: watch the named Service for its LoadBalancer status. + - ingress:/: watch the named Ingress for its LoadBalancer status. + If spec.ingress.statusAddress is set, it takes precedence over this field. + If this field is empty, spec.envoy.service is used as default. Contour's default is an empty string. type: string logging: @@ -4791,11 +4793,13 @@ spec: type: object loadBalancer: description: |- - LoadBalancer specifies how Contour should set the ingress status address. + LoadBalancer specifies the source for ingress status address. If provided, the value can be in one of the formats: - - address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. - - service:/: Contour will use the address of the designated service. - - ingress:/: Contour will use the address of the designated ingress. + - address:: use the provided addresses directly (FQDN or IP). + - service:/: watch the named Service for its LoadBalancer status. + - ingress:/: watch the named Ingress for its LoadBalancer status. + If spec.ingress.statusAddress is set, it takes precedence over this field. + If this field is empty, spec.envoy.service is used as default. Contour's default is an empty string. type: string logging: diff --git a/internal/k8s/statusaddress.go b/internal/k8s/statusaddress.go index fed0f6eaf9b..136a77542fe 100644 --- a/internal/k8s/statusaddress.go +++ b/internal/k8s/statusaddress.go @@ -316,7 +316,6 @@ type IngressStatusLoadBalancerWatcher struct { func (s *IngressStatusLoadBalancerWatcher) OnAdd(obj any, _ bool) { ingress, ok := obj.(*networking_v1.Ingress) if !ok { - // not a service return } if ingress.Name != s.IngressName { @@ -324,7 +323,7 @@ func (s *IngressStatusLoadBalancerWatcher) OnAdd(obj any, _ bool) { } s.Log.WithField("name", ingress.Name). WithField("namespace", ingress.Namespace). - Debug("received new service address") + Debug("received new ingress address") s.notify(ingress.Status.LoadBalancer) } @@ -332,7 +331,6 @@ func (s *IngressStatusLoadBalancerWatcher) OnAdd(obj any, _ bool) { func (s *IngressStatusLoadBalancerWatcher) OnUpdate(_, newObj any) { ingress, ok := newObj.(*networking_v1.Ingress) if !ok { - // not a service return } if ingress.Name != s.IngressName { @@ -340,7 +338,7 @@ func (s *IngressStatusLoadBalancerWatcher) OnUpdate(_, newObj any) { } s.Log.WithField("name", ingress.Name). WithField("namespace", ingress.Namespace). - Debug("received new service address") + Debug("received new ingress address") s.notify(ingress.Status.LoadBalancer) } @@ -348,7 +346,6 @@ func (s *IngressStatusLoadBalancerWatcher) OnUpdate(_, newObj any) { func (s *IngressStatusLoadBalancerWatcher) OnDelete(obj any) { ingress, ok := obj.(*networking_v1.Ingress) if !ok { - // not a service return } if ingress.Name != s.IngressName { diff --git a/internal/k8s/statusaddress_test.go b/internal/k8s/statusaddress_test.go index e8bbf88cf18..139fbcccccc 100644 --- a/internal/k8s/statusaddress_test.go +++ b/internal/k8s/statusaddress_test.go @@ -607,3 +607,73 @@ func simpleProxyGenerator(name, ingressClass string, lbstatus core_v1.LoadBalanc }, } } + +func TestIngressStatusLoadBalancerWatcher(t *testing.T) { + lbstatus := make(chan core_v1.LoadBalancerStatus, 1) + sw := IngressStatusLoadBalancerWatcher{ + IngressName: "envoy", + LBStatus: lbstatus, + Log: fixture.NewTestLogger(t), + } + + recv := func() (core_v1.LoadBalancerStatus, bool) { + select { + case lbs := <-sw.LBStatus: + return lbs, true + default: + return core_v1.LoadBalancerStatus{}, false + } + } + + ingress := func(name string, status ...networking_v1.IngressLoadBalancerIngress) *networking_v1.Ingress { + ing := &networking_v1.Ingress{} + ing.Name = name + ing.Status.LoadBalancer.Ingress = status + return ing + } + + t.Run("OnAdd wrong name", func(t *testing.T) { + sw.OnAdd(ingress("potato"), false) + _, ok := recv() + assert.False(t, ok, "expected no notification") + }) + + t.Run("OnAdd matching name", func(t *testing.T) { + sw.OnAdd(ingress("envoy", networking_v1.IngressLoadBalancerIngress{Hostname: "projectcontour.io"}), false) + got, ok := recv() + assert.True(t, ok, "expected notification") + assert.Equal(t, core_v1.LoadBalancerStatus{ + Ingress: []core_v1.LoadBalancerIngress{{Hostname: "projectcontour.io", Ports: []core_v1.PortStatus{}}}, + }, got) + }) + + t.Run("OnUpdate wrong name", func(t *testing.T) { + sw.OnUpdate(ingress("potato"), ingress("elephant")) + _, ok := recv() + assert.False(t, ok, "expected no notification") + }) + + t.Run("OnUpdate matching name", func(t *testing.T) { + sw.OnUpdate(ingress("potato"), ingress("envoy", networking_v1.IngressLoadBalancerIngress{IP: "1.2.3.4"})) + got, ok := recv() + assert.True(t, ok, "expected notification") + assert.Equal(t, core_v1.LoadBalancerStatus{ + Ingress: []core_v1.LoadBalancerIngress{{IP: "1.2.3.4", Ports: []core_v1.PortStatus{}}}, + }, got) + }) + + t.Run("OnDelete wrong name", func(t *testing.T) { + sw.OnDelete(ingress("potato")) + _, ok := recv() + assert.False(t, ok, "expected no notification") + }) + + t.Run("OnDelete matching name", func(t *testing.T) { + sw.OnDelete(ingress("envoy", networking_v1.IngressLoadBalancerIngress{Hostname: "projectcontour.io"})) + got, ok := recv() + assert.True(t, ok, "expected notification") + assert.Equal(t, core_v1.LoadBalancerStatus{ + Ingress: []core_v1.LoadBalancerIngress{}, + }, got) + }) +} diff --git a/internal/provisioner/model/names.go b/internal/provisioner/model/names.go index 58706d83931..081c25c0205 100644 --- a/internal/provisioner/model/names.go +++ b/internal/provisioner/model/names.go @@ -33,11 +33,6 @@ func (c *Contour) EnvoyServiceName() string { return "envoy-" + c.Name } -// EnvoyIngressName returns the name of the Envoy Ingress resource. -func (c *Contour) EnvoyIngressName() string { - return "envoy-" + c.Name -} - // ContourDeploymentName returns the name of the Contour Deployment resource. func (c *Contour) ContourDeploymentName() string { return "contour-" + c.Name diff --git a/internal/provisioner/objects/deployment/deployment.go b/internal/provisioner/objects/deployment/deployment.go index 956b7e5b8a9..21c23531283 100644 --- a/internal/provisioner/objects/deployment/deployment.go +++ b/internal/provisioner/objects/deployment/deployment.go @@ -96,7 +96,6 @@ func DesiredDeployment(contour *model.Contour, image string) *apps_v1.Deployment fmt.Sprintf("--contour-config-name=%s", contour.ContourConfigurationName()), fmt.Sprintf("--leader-election-resource-name=%s", contour.LeaderElectionLeaseName()), fmt.Sprintf("--envoy-service-name=%s", contour.EnvoyServiceName()), - fmt.Sprintf("--envoy-ingress-name=%s", contour.EnvoyIngressName()), fmt.Sprintf("--kubernetes-debug=%d", contour.Spec.KubernetesLogLevel), } diff --git a/pkg/config/parameters.go b/pkg/config/parameters.go index bef3d0db550..932cad1bd1e 100644 --- a/pkg/config/parameters.go +++ b/pkg/config/parameters.go @@ -165,6 +165,18 @@ func (n NamespacedName) Validate() error { return nil } +// LoadBalancerStatusSource defines the source for load balancer status addresses in the config file. +type LoadBalancerStatusSource struct { + // Service watches the named Service's status.loadBalancer for addresses. + Service *NamespacedName `yaml:"service,omitempty"` + + // Ingress watches the named Ingress's status.loadBalancer for addresses. + Ingress *NamespacedName `yaml:"ingress,omitempty"` + + // Addresses specifies static address(es) to use directly (IP or FQDN). + Addresses []string `yaml:"addresses,omitempty"` +} + // TLSParameters holds configuration file TLS configuration details. type TLSParameters struct { ProtocolParameters `yaml:",inline"` @@ -682,9 +694,8 @@ type Parameters struct { // Name of the envoy service to inspect for Ingress status details. EnvoyServiceName string `yaml:"envoy-service-name,omitempty"` - // Identifier of ingress object for load balancer address in the format of - // (ingress|service):/, or hostname:fqdn1[,fqdn2]. - LoadBalancerStatus string `yaml:"load-balancer-status,omitempty"` + // LoadBalancerStatus specifies the source for load balancer status addresses. + LoadBalancerStatus LoadBalancerStatusSource `yaml:"load-balancer-status,omitempty"` // DefaultHTTPVersions defines the default set of HTTPS // versions the proxy should accept. HTTP versions are diff --git a/site/content/docs/main/config/api-reference.html b/site/content/docs/main/config/api-reference.html index 2e73a4a33ef..e857215e06b 100644 --- a/site/content/docs/main/config/api-reference.html +++ b/site/content/docs/main/config/api-reference.html @@ -6986,11 +6986,13 @@

EnvoyConfig (Optional) -

LoadBalancer specifies how Contour should set the ingress status address. +

LoadBalancer specifies the source for ingress status address. If provided, the value can be in one of the formats: -- address:: Contour will use the provided comma separated list of addresses directly. The address can be a fully qualified domain name or an IP address. -- service:/: Contour will use the address of the designated service. -- ingress:/: Contour will use the address of the designated ingress.

+- address:: use the provided addresses directly (FQDN or IP). +- service:/: watch the named Service for its LoadBalancer status. +- ingress:/: watch the named Ingress for its LoadBalancer status.

+

If spec.ingress.statusAddress is set, it takes precedence over this field. +If this field is empty, spec.envoy.service is used as default.

Contour’s default is an empty string.

diff --git a/site/content/docs/main/configuration.md b/site/content/docs/main/configuration.md index 111d3c9e1fe..54e0afe8b43 100644 --- a/site/content/docs/main/configuration.md +++ b/site/content/docs/main/configuration.md @@ -57,7 +57,7 @@ Many of these flags are mirrored in the [Contour Configuration File](#configurat | `--leader-election-retry-period` | The interval which Contour will attempt to acquire leadership lease. | | `--leader-election-resource-name` | The name of the resource (Lease) leader election will lease. | | `--leader-election-resource-namespace` | The namespace of the resource (Lease) leader election will lease. | -| `--load-balancer-status= | Address to set (kind=hostname) or the source to inspect for ingress status (kind=service or ingress). | +| `--load-balancer-status=` | Source for ingress status address: `address:`, `service:/`, or `ingress:/`. If `--ingress-status-address` is also set, it takes precedence. If empty, `--envoy-service-name`/`--envoy-service-namespace` is used. | | `-d, --debug` | Enable debug logging | | `--kubernetes-debug=` | Enable Kubernetes client debug logging | | `--log-format=` | Log output format for Contour. Either text (default) or json. | @@ -88,6 +88,7 @@ Where Contour settings can also be specified with command-line flags, the comman | envoy-service-name | string | `envoy` | This sets the service name that will be inspected for address details to be applied to Ingress objects. | | envoy-service-namespace | string | `projectcontour` | This sets the namespace of the service that will be inspected for address details to be applied to Ingress objects. If the `CONTOUR_NAMESPACE` environment variable is present, Contour will populate this field with its value. | | ingress-status-address | string | None | If present, this specifies the address that will be copied into the Ingress status for each Ingress that Contour manages. It is exclusive with `envoy-service-name` and `envoy-service-namespace`. | +| load-balancer-status | string | None | Source for ingress status address: `address:`, `service:/`, or `ingress:/`. If `ingress-status-address` is also set, it takes precedence. If empty, `envoy-service-name`/`envoy-service-namespace` is used. | | incluster | boolean | `false` | This field specifies that Contour is running in a Kubernetes cluster and should use the in-cluster client access configuration. | | json-fields | string array | [fields][5] | This is the list the field names to include in the JSON [access log format][2]. This field only has effect if `accesslog-format` is `json`. | | kubeconfig | string | `$HOME/.kube/config` | Path to a Kubernetes [kubeconfig file][3] for when Contour is executed outside a cluster. | diff --git a/test/e2e/httpproxy/httpproxy_test.go b/test/e2e/httpproxy/httpproxy_test.go index 050113eca3f..b5aff0f21fa 100644 --- a/test/e2e/httpproxy/httpproxy_test.go +++ b/test/e2e/httpproxy/httpproxy_test.go @@ -381,6 +381,23 @@ var _ = Describe("HTTPProxy", func() { }) }) + f.NamespacedTest("httpproxy-load-balancer-status-from-ingress", func(namespace string) { + Context("with load-balancer-status sourced from Ingress", func() { + BeforeEach(func() { + additionalContourArgs = []string{ + "--load-balancer-status=ingress:projectcontour/lb-status-source", + } + contourConfiguration.Spec.Envoy.LoadBalancerStatus = &contour_v1alpha1.LoadBalancerStatusConfig{ + Ingress: &contour_v1alpha1.NamespacedName{ + Namespace: "projectcontour", + Name: "lb-status-source", + }, + } + }) + testLoadBalancerStatusFromIngress(namespace) + }) + }) + f.NamespacedTest("httpproxy-external-name-service-insecure", func(namespace string) { Context("with ExternalName Services enabled", func() { BeforeEach(func() { diff --git a/test/e2e/httpproxy/load_balancer_status_test.go b/test/e2e/httpproxy/load_balancer_status_test.go new file mode 100644 index 00000000000..29df05069a7 --- /dev/null +++ b/test/e2e/httpproxy/load_balancer_status_test.go @@ -0,0 +1,118 @@ +// Copyright Project Contour Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build e2e + +package httpproxy + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + networking_v1 "k8s.io/api/networking/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + contour_v1 "github.com/projectcontour/contour/apis/projectcontour/v1" + "github.com/projectcontour/contour/test/e2e" +) + +func testLoadBalancerStatusFromIngress(namespace string) { + Specify("status address is propagated from source Ingress to HTTPProxy", func() { + // Create Ingress that Contour watches for LB status and set its status. + sourceIngress := &networking_v1.Ingress{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "lb-status-source", + Namespace: "projectcontour", + }, + Spec: networking_v1.IngressSpec{ + IngressClassName: ptr.To("not-controlled-by-contour"), + DefaultBackend: &networking_v1.IngressBackend{ + Service: &networking_v1.IngressServiceBackend{ + Name: "placeholder", + Port: networking_v1.ServiceBackendPort{Number: 80}, + }, + }, + }, + } + require.NoError(f.T(), f.Client.Create(context.TODO(), sourceIngress)) + DeferCleanup(func() { + _ = f.Client.Delete(context.TODO(), sourceIngress) + }) + + // setSourceStatus updates the Ingress LB status. + setSourceStatus := func(ingresses []networking_v1.IngressLoadBalancerIngress) { + require.NoError(f.T(), f.Client.Get(context.TODO(), client.ObjectKeyFromObject(sourceIngress), sourceIngress)) + sourceIngress.Status.LoadBalancer = networking_v1.IngressLoadBalancerStatus{ + Ingress: ingresses, + } + require.NoError(f.T(), f.Client.Status().Update(context.TODO(), sourceIngress)) + } + + // Create HTTPProxy and wait for Contour to propagate the LB status. + f.Fixtures.Echo.Deploy(namespace, "echo") + p := &contour_v1.HTTPProxy{ + ObjectMeta: meta_v1.ObjectMeta{ + Namespace: namespace, + Name: "lb-status-test", + }, + Spec: contour_v1.HTTPProxySpec{ + VirtualHost: &contour_v1.VirtualHost{ + Fqdn: "lb-status-test.projectcontour.io", + }, + Routes: []contour_v1.Route{{ + Services: []contour_v1.Service{{Name: "echo", Port: 80}}, + }}, + }, + } + + proxyHasLBStatus := func(expected []networking_v1.IngressLoadBalancerIngress) bool { + if err := f.Client.Get(context.TODO(), client.ObjectKeyFromObject(p), p); err != nil { + return false + } + if len(p.Status.LoadBalancer.Ingress) != len(expected) { + return false + } + for i, exp := range expected { + if p.Status.LoadBalancer.Ingress[i].IP != exp.IP || p.Status.LoadBalancer.Ingress[i].Hostname != exp.Hostname { + return false + } + } + return true + } + + // Set initial status and verify it propagates after HTTPProxy is created. + setSourceStatus([]networking_v1.IngressLoadBalancerIngress{{IP: "1.2.3.4"}}) + require.True(f.T(), f.CreateHTTPProxyAndWaitFor(p, e2e.HTTPProxyValid)) + assert.Eventually(f.T(), func() bool { + return proxyHasLBStatus([]networking_v1.IngressLoadBalancerIngress{{IP: "1.2.3.4"}}) + }, f.RetryTimeout, f.RetryInterval) + + // Verify updates propagate and that multiple entries including both IP and hostname are handled correctly. + setSourceStatus([]networking_v1.IngressLoadBalancerIngress{ + {IP: "10.0.0.1"}, + {IP: "10.0.0.2"}, + {Hostname: "lb.example.com"}, + }) + assert.Eventually(f.T(), func() bool { + return proxyHasLBStatus([]networking_v1.IngressLoadBalancerIngress{ + {IP: "10.0.0.1"}, + {IP: "10.0.0.2"}, + {Hostname: "lb.example.com"}, + }) + }, f.RetryTimeout, f.RetryInterval) + }) +}