diff --git a/api/v1alpha1/envoyproxy_helpers.go b/api/v1alpha1/envoyproxy_helpers.go
index 33111e1229..585092da18 100644
--- a/api/v1alpha1/envoyproxy_helpers.go
+++ b/api/v1alpha1/envoyproxy_helpers.go
@@ -9,12 +9,17 @@ import (
"fmt"
"sort"
"strings"
+ "time"
autoscalingv2 "k8s.io/api/autoscaling/v2"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
)
+// DefaultDrainTimeout is the default drain timeout for the graceful drain
+// sequence, used when ShutdownConfig.DrainTimeout is not specified.
+const DefaultDrainTimeout = 60 * time.Second
+
// DefaultEnvoyProxyProvider returns a new EnvoyProxyProvider with default settings.
func DefaultEnvoyProxyProvider() *EnvoyProxyProvider {
return &EnvoyProxyProvider{
diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go
index a4ef2ee57a..27e20b93e5 100644
--- a/api/v1alpha1/envoyproxy_types.go
+++ b/api/v1alpha1/envoyproxy_types.go
@@ -489,6 +489,11 @@ type EnvoyProxyProvider struct {
// ShutdownConfig defines configuration for graceful envoy shutdown process.
type ShutdownConfig struct {
+ // DrainDelay defines the delay before starting the drain process.
+ // If unspecified, defaults to 0 seconds.
+ //
+ // +optional
+ DrainDelay *gwapiv1.Duration `json:"drainDelay,omitempty"`
// DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.
// If unspecified, defaults to 60 seconds.
//
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index f235298d88..805ea67c1e 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -7965,6 +7965,11 @@ func (in *SessionResumption) DeepCopy() *SessionResumption {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ShutdownConfig) DeepCopyInto(out *ShutdownConfig) {
*out = *in
+ if in.DrainDelay != nil {
+ in, out := &in.DrainDelay, &out.DrainDelay
+ *out = new(v1.Duration)
+ **out = **in
+ }
if in.DrainTimeout != nil {
in, out := &in.DrainTimeout, &out.DrainTimeout
*out = new(v1.Duration)
diff --git a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml
index 93db35107b..323e15b246 100644
--- a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml
+++ b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml
@@ -11283,6 +11283,12 @@ spec:
description: Shutdown defines configuration for graceful envoy shutdown
process.
properties:
+ drainDelay:
+ description: |-
+ DrainDelay defines the delay before starting the drain process.
+ If unspecified, defaults to 0 seconds.
+ pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+ type: string
drainTimeout:
description: |-
DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.
diff --git a/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml
index eafaa9486a..c0aa103339 100644
--- a/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml
+++ b/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml
@@ -11282,6 +11282,12 @@ spec:
description: Shutdown defines configuration for graceful envoy shutdown
process.
properties:
+ drainDelay:
+ description: |-
+ DrainDelay defines the delay before starting the drain process.
+ If unspecified, defaults to 0 seconds.
+ pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+ type: string
drainTimeout:
description: |-
DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.
diff --git a/internal/cmd/envoy.go b/internal/cmd/envoy.go
index 8c2d0f46ac..53602313a3 100644
--- a/internal/cmd/envoy.go
+++ b/internal/cmd/envoy.go
@@ -10,6 +10,7 @@ import (
"github.com/spf13/cobra"
+ egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
"github.com/envoyproxy/gateway/internal/cmd/envoy"
)
@@ -28,6 +29,7 @@ func GetEnvoyCommand() *cobra.Command {
// getShutdownCommand returns the shutdown cobra command to be executed.
func getShutdownCommand() *cobra.Command {
+ var drainDelay time.Duration
var drainTimeout time.Duration
var minDrainDuration time.Duration
var exitAtConnections int
@@ -36,11 +38,14 @@ func getShutdownCommand() *cobra.Command {
Use: "shutdown",
Short: "Gracefully drain open connections prior to pod shutdown.",
RunE: func(_ *cobra.Command, _ []string) error {
- return envoy.Shutdown(drainTimeout, minDrainDuration, exitAtConnections)
+ return envoy.Shutdown(drainDelay, drainTimeout, minDrainDuration, exitAtConnections)
},
}
- cmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 60*time.Second,
+ cmd.PersistentFlags().DurationVar(&drainDelay, "drain-delay", 0*time.Second,
+ "Delay before starting the drain process.")
+
+ cmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", egv1a1.DefaultDrainTimeout,
"Graceful shutdown timeout. This should be less than the pod's terminationGracePeriodSeconds.")
cmd.PersistentFlags().DurationVar(&minDrainDuration, "min-drain-duration", 10*time.Second,
diff --git a/internal/cmd/envoy/shutdown_manager.go b/internal/cmd/envoy/shutdown_manager.go
index cd56853392..e50533f530 100644
--- a/internal/cmd/envoy/shutdown_manager.go
+++ b/internal/cmd/envoy/shutdown_manager.go
@@ -118,8 +118,7 @@ func shutdownReadyHandler(w http.ResponseWriter, readyTimeout time.Duration, rea
// Shutdown is called from a preStop hook on the shutdown-manager container where
// it will initiate a drain sequence on the Envoy proxy and block until
// connections are drained or a timeout is exceeded.
-func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error {
- startTime := time.Now()
+func Shutdown(drainDelay, drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error {
allowedToExit := false
// Reconfigure logger to write to stdout of main process if running in Kubernetes
@@ -127,6 +126,13 @@ func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections in
logger = logging.FileLogger("/proc/1/fd/1", "shutdown-manager", egv1a1.LogLevelInfo)
}
+ if drainDelay > 0 {
+ logger.Info(fmt.Sprintf("waiting %.0f seconds before starting drain", drainDelay.Seconds()))
+ time.Sleep(drainDelay)
+ }
+
+ startTime := time.Now()
+
logger.Info(fmt.Sprintf("initiating drain with %.0f second minimum drain period and %.0f second timeout",
minDrainDuration.Seconds(), drainTimeout.Seconds()))
diff --git a/internal/infrastructure/common/proxy_args.go b/internal/infrastructure/common/proxy_args.go
index 6c3ec688ca..5e9a11a1d8 100644
--- a/internal/infrastructure/common/proxy_args.go
+++ b/internal/infrastructure/common/proxy_args.go
@@ -84,7 +84,7 @@ func BuildProxyArgs(
}
// Default drain timeout.
- drainTimeout := 60.0
+ drainTimeout := egv1a1.DefaultDrainTimeout.Seconds()
if shutdownConfig != nil && shutdownConfig.DrainTimeout != nil {
d, err := time.ParseDuration(string(*shutdownConfig.DrainTimeout))
if err != nil {
diff --git a/internal/infrastructure/kubernetes/proxy/resource.go b/internal/infrastructure/kubernetes/proxy/resource.go
index 40a39d4e91..b6940059b7 100644
--- a/internal/infrastructure/kubernetes/proxy/resource.go
+++ b/internal/infrastructure/kubernetes/proxy/resource.go
@@ -266,15 +266,32 @@ func expectedShutdownManagerImage(shutdownManager *egv1a1.ShutdownManager) strin
}
func expectedShutdownManagerArgs(cfg *egv1a1.ShutdownConfig) []string {
- args := []string{"envoy", "shutdown-manager"}
- if cfg != nil && cfg.DrainTimeout != nil {
+ args := make([]string, 0, 3)
+ args = append(args, "envoy", "shutdown-manager")
+
+ // Use default --ready-timeout when neither drain field is configured.
+ if cfg == nil || (cfg.DrainTimeout == nil && cfg.DrainDelay == nil) {
+ return args
+ }
+
+ readyTimeout := egv1a1.DefaultDrainTimeout
+ if cfg.DrainTimeout != nil {
d, err := time.ParseDuration(string(*cfg.DrainTimeout))
if err != nil {
return nil
}
- args = append(args, fmt.Sprintf("--ready-timeout=%.0fs", d.Seconds()+10))
+ readyTimeout = d
}
- return args
+
+ if cfg.DrainDelay != nil {
+ delay, err := time.ParseDuration(string(*cfg.DrainDelay))
+ if err != nil {
+ return nil
+ }
+ readyTimeout += delay
+ }
+
+ return append(args, fmt.Sprintf("--ready-timeout=%.0fs", readyTimeout.Seconds()+10))
}
func expectedShutdownPreStopCommand(cfg *egv1a1.ShutdownConfig) []string {
@@ -284,6 +301,14 @@ func expectedShutdownPreStopCommand(cfg *egv1a1.ShutdownConfig) []string {
return command
}
+ if cfg.DrainDelay != nil {
+ d, err := time.ParseDuration(string(*cfg.DrainDelay))
+ if err != nil {
+ return nil
+ }
+ command = append(command, fmt.Sprintf("--drain-delay=%.0fs", d.Seconds()))
+ }
+
if cfg.DrainTimeout != nil {
d, err := time.ParseDuration(string(*cfg.DrainTimeout))
if err != nil {
diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider.go b/internal/infrastructure/kubernetes/proxy/resource_provider.go
index b8635d89b3..2a5aa85ea7 100644
--- a/internal/infrastructure/kubernetes/proxy/resource_provider.go
+++ b/internal/infrastructure/kubernetes/proxy/resource_provider.go
@@ -620,6 +620,15 @@ func expectedTerminationGracePeriodSeconds(cfg *egv1a1.ShutdownConfig) *int64 {
}
s = int(d.Seconds() + 300) // 5 minutes longer than drain timeout
}
+
+ if cfg != nil && cfg.DrainDelay != nil {
+ d, err := time.ParseDuration(string(*cfg.DrainDelay))
+ if err != nil {
+ return nil
+ }
+ s += int(d.Seconds())
+ }
+
return new(int64(s))
}
diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go
index 9b8275bcda..f1e248f7b6 100644
--- a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go
+++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go
@@ -292,6 +292,7 @@ func TestDeployment(t *testing.T) {
},
},
shutdown: &egv1a1.ShutdownConfig{
+ DrainDelay: new(gwapiv1.Duration("15s")),
DrainTimeout: new(gwapiv1.Duration("30s")),
MinDrainDuration: new(gwapiv1.Duration("15s")),
},
@@ -2120,3 +2121,52 @@ func writeTestDataToFile(filename string, resources []any) error {
return os.WriteFile(filename, combinedYAML, 0o600)
}
+
+func TestExpectedTerminationGracePeriodSeconds(t *testing.T) {
+ tests := []struct {
+ name string
+ cfg *egv1a1.ShutdownConfig
+ expected int64
+ }{
+ {
+ name: "nil config",
+ cfg: nil,
+ expected: 360,
+ },
+ {
+ name: "empty config",
+ cfg: &egv1a1.ShutdownConfig{},
+ expected: 360,
+ },
+ {
+ name: "only drainTimeout",
+ cfg: &egv1a1.ShutdownConfig{
+ DrainTimeout: new(gwapiv1.Duration("30s")),
+ },
+ expected: 330,
+ },
+ {
+ name: "only drainDelay",
+ cfg: &egv1a1.ShutdownConfig{
+ DrainDelay: new(gwapiv1.Duration("15s")),
+ },
+ expected: 375,
+ },
+ {
+ name: "drainDelay and drainTimeout",
+ cfg: &egv1a1.ShutdownConfig{
+ DrainDelay: new(gwapiv1.Duration("15s")),
+ DrainTimeout: new(gwapiv1.Duration("30s")),
+ },
+ expected: 345,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := expectedTerminationGracePeriodSeconds(tt.cfg)
+ require.NotNil(t, got)
+ require.Equal(t, tt.expected, *got)
+ })
+ }
+}
diff --git a/internal/infrastructure/kubernetes/proxy/resource_test.go b/internal/infrastructure/kubernetes/proxy/resource_test.go
index 2428311495..2742ce272b 100644
--- a/internal/infrastructure/kubernetes/proxy/resource_test.go
+++ b/internal/infrastructure/kubernetes/proxy/resource_test.go
@@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
+ gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
"github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource"
@@ -182,3 +183,50 @@ func TestGetImageTag(t *testing.T) {
})
}
}
+
+func TestExpectedShutdownManagerArgs(t *testing.T) {
+ tests := []struct {
+ name string
+ cfg *egv1a1.ShutdownConfig
+ expected []string
+ }{
+ {
+ name: "nil config",
+ cfg: nil,
+ expected: []string{"envoy", "shutdown-manager"},
+ },
+ {
+ name: "empty config",
+ cfg: &egv1a1.ShutdownConfig{},
+ expected: []string{"envoy", "shutdown-manager"},
+ },
+ {
+ name: "only drainTimeout",
+ cfg: &egv1a1.ShutdownConfig{
+ DrainTimeout: new(gwapiv1.Duration("30s")),
+ },
+ expected: []string{"envoy", "shutdown-manager", "--ready-timeout=40s"},
+ },
+ {
+ name: "only drainDelay",
+ cfg: &egv1a1.ShutdownConfig{
+ DrainDelay: new(gwapiv1.Duration("15s")),
+ },
+ expected: []string{"envoy", "shutdown-manager", "--ready-timeout=85s"},
+ },
+ {
+ name: "drainDelay and drainTimeout",
+ cfg: &egv1a1.ShutdownConfig{
+ DrainDelay: new(gwapiv1.Duration("15s")),
+ DrainTimeout: new(gwapiv1.Duration("30s")),
+ },
+ expected: []string{"envoy", "shutdown-manager", "--ready-timeout=55s"},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ require.Equal(t, tt.expected, expectedShutdownManagerArgs(tt.cfg))
+ })
+ }
+}
diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml
index be75b9f274..89d928a885 100644
--- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml
+++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml
@@ -309,7 +309,7 @@ spec:
- args:
- envoy
- shutdown-manager
- - --ready-timeout=40s
+ - --ready-timeout=55s
command:
- envoy-gateway
env:
@@ -341,6 +341,7 @@ spec:
- envoy-gateway
- envoy
- shutdown
+ - --drain-delay=15s
- --drain-timeout=30s
- --min-drain-duration=15s
livenessProbe:
@@ -395,7 +396,7 @@ spec:
restartPolicy: Always
schedulerName: default-scheduler
serviceAccountName: envoy-default-37a8eec1
- terminationGracePeriodSeconds: 330
+ terminationGracePeriodSeconds: 345
volumes:
- name: certs
secret:
diff --git a/release-notes/current.yaml b/release-notes/current.yaml
index 1f810452ce..1f954ab9ac 100644
--- a/release-notes/current.yaml
+++ b/release-notes/current.yaml
@@ -17,6 +17,7 @@ new features: |
Added a `requestBody` field to the HTTP active health checker in `BackendTrafficPolicy`, allowing a request body payload to be sent during HTTP health checking. The field requires the health check `method` to be `POST` or `PUT`.
Added a `xdsNACKTotal` metric to track the number of NACKs received from Envoy, labeled by node ID and resource type URL. A NACK is a DiscoveryRequest carrying an ErrorDetail, indicating that Envoy rejected the last config update. This metric can be used to alert on config issues causing xDS rejections.
Added the `autoSNIFromEndpointHostname` TLS setting to Backends, allowing the SNI value sent to the backend to be automatically derived from the backend endpoint hostname instead of using a fixed SNI value.
+ Added a `drainDelay` field to `ShutdownConfig`, delaying the start of the Envoy proxy drain sequence on pod termination. During the delay the proxy continues to accept and serve new connections, allowing external load balancers that rely on health-check based de-registration (e.g. L4 passthrough load balancers) to stop sending traffic to the terminating pod before it stops serving.
Added support for `filterContext` field in Lua EnvoyExtensionPolicy, allowing shared Lua scripts to be parameterized per route via `request_handle:filterContext()`.
Added support for `ListenerSet` as a `targetRef` kind in `ClientTrafficPolicy`, allowing client traffic settings to be applied to a named group of listeners without a gateway-wide policy.
Added a `fromMetadata` field to global rate limit `limit` in `BackendTrafficPolicy`, allowing the limit value to be sourced from per-request dynamic metadata (e.g. set by an upstream ext_proc), falling back to the static `requests`/`unit` when the metadata is absent.
diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md
index 767d8a3af4..8b4ebf0d1e 100644
--- a/site/content/en/latest/api/extension_types.md
+++ b/site/content/en/latest/api/extension_types.md
@@ -5867,6 +5867,7 @@ _Appears in:_
| Field | Type | Required | Default | Description |
| --- | --- | --- | --- | --- |
+| `drainDelay` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | DrainDelay defines the delay before starting the drain process.
If unspecified, defaults to 0 seconds. |
| `drainTimeout` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.
If unspecified, defaults to 60 seconds. |
| `minDrainDuration` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.
If unspecified, defaults to 10 seconds. |
diff --git a/test/helm/gateway-crds-helm/all.out.yaml b/test/helm/gateway-crds-helm/all.out.yaml
index 357bc518cc..58caabfc80 100644
--- a/test/helm/gateway-crds-helm/all.out.yaml
+++ b/test/helm/gateway-crds-helm/all.out.yaml
@@ -42862,6 +42862,12 @@ spec:
description: Shutdown defines configuration for graceful envoy shutdown
process.
properties:
+ drainDelay:
+ description: |-
+ DrainDelay defines the delay before starting the drain process.
+ If unspecified, defaults to 0 seconds.
+ pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+ type: string
drainTimeout:
description: |-
DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.
diff --git a/test/helm/gateway-crds-helm/e2e.out.yaml b/test/helm/gateway-crds-helm/e2e.out.yaml
index 413f4840c4..85dfc57ab4 100644
--- a/test/helm/gateway-crds-helm/e2e.out.yaml
+++ b/test/helm/gateway-crds-helm/e2e.out.yaml
@@ -20835,6 +20835,12 @@ spec:
description: Shutdown defines configuration for graceful envoy shutdown
process.
properties:
+ drainDelay:
+ description: |-
+ DrainDelay defines the delay before starting the drain process.
+ If unspecified, defaults to 0 seconds.
+ pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+ type: string
drainTimeout:
description: |-
DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.
diff --git a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml
index a173638a1e..fce39eb73d 100644
--- a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml
+++ b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml
@@ -20835,6 +20835,12 @@ spec:
description: Shutdown defines configuration for graceful envoy shutdown
process.
properties:
+ drainDelay:
+ description: |-
+ DrainDelay defines the delay before starting the drain process.
+ If unspecified, defaults to 0 seconds.
+ pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+ type: string
drainTimeout:
description: |-
DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.