feat: add drain delay to graceful shutdown process

api/v1alpha1/envoyproxy_helpers.go

@@ -9,12 +9,17 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+	"time"
 
 	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 )
 
+// DefaultDrainTimeout is the default drain timeout for the graceful drain
+// sequence, used when ShutdownConfig.DrainTimeout is not specified.
+const DefaultDrainTimeout = 60 * time.Second
+
 // DefaultEnvoyProxyProvider returns a new EnvoyProxyProvider with default settings.
 func DefaultEnvoyProxyProvider() *EnvoyProxyProvider {
 	return &EnvoyProxyProvider{

api/v1alpha1/envoyproxy_types.go

@@ -489,6 +489,11 @@ type EnvoyProxyProvider struct {
 
 // ShutdownConfig defines configuration for graceful envoy shutdown process.
 type ShutdownConfig struct {
+	// DrainDelay defines the delay before starting the drain process.
+	// If unspecified, defaults to 0 seconds.
+	//
+	// +optional
+	DrainDelay *gwapiv1.Duration `json:"drainDelay,omitempty"`
 	// DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.
 	// If unspecified, defaults to 60 seconds.
 	//

charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml

@@ -11283,6 +11283,12 @@ spec:
                 description: Shutdown defines configuration for graceful envoy shutdown
                   process.
                 properties:
+                  drainDelay:
+                    description: |-
+                      DrainDelay defines the delay before starting the drain process.
+                      If unspecified, defaults to 0 seconds.
+                    pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+                    type: string
                   drainTimeout:
                     description: |-
                       DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.

charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml

@@ -11282,6 +11282,12 @@ spec:
                 description: Shutdown defines configuration for graceful envoy shutdown
                   process.
                 properties:
+                  drainDelay:
+                    description: |-
+                      DrainDelay defines the delay before starting the drain process.
+                      If unspecified, defaults to 0 seconds.
+                    pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+                    type: string
                   drainTimeout:
                     description: |-
                       DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.

internal/cmd/envoy.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
 	"github.com/envoyproxy/gateway/internal/cmd/envoy"
 )
 
@@ -28,6 +29,7 @@ func GetEnvoyCommand() *cobra.Command {
 
 // getShutdownCommand returns the shutdown cobra command to be executed.
 func getShutdownCommand() *cobra.Command {
+	var drainDelay time.Duration
 	var drainTimeout time.Duration
 	var minDrainDuration time.Duration
 	var exitAtConnections int
@@ -36,11 +38,14 @@ func getShutdownCommand() *cobra.Command {
 		Use:   "shutdown",
 		Short: "Gracefully drain open connections prior to pod shutdown.",
 		RunE: func(_ *cobra.Command, _ []string) error {
-			return envoy.Shutdown(drainTimeout, minDrainDuration, exitAtConnections)
+			return envoy.Shutdown(drainDelay, drainTimeout, minDrainDuration, exitAtConnections)
 		},
 	}
 
-	cmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 60*time.Second,
+	cmd.PersistentFlags().DurationVar(&drainDelay, "drain-delay", 0*time.Second,
+		"Delay before starting the drain process.")
+
+	cmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", egv1a1.DefaultDrainTimeout,
 		"Graceful shutdown timeout. This should be less than the pod's terminationGracePeriodSeconds.")
 
 	cmd.PersistentFlags().DurationVar(&minDrainDuration, "min-drain-duration", 10*time.Second,

internal/cmd/envoy/shutdown_manager.go

@@ -118,15 +118,21 @@ func shutdownReadyHandler(w http.ResponseWriter, readyTimeout time.Duration, rea
 // Shutdown is called from a preStop hook on the shutdown-manager container where
 // it will initiate a drain sequence on the Envoy proxy and block until
 // connections are drained or a timeout is exceeded.
-func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error {
-	startTime := time.Now()
+func Shutdown(drainDelay, drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error {
 	allowedToExit := false
 
 	// Reconfigure logger to write to stdout of main process if running in Kubernetes
 	if _, k8s := os.LookupEnv("KUBERNETES_SERVICE_HOST"); k8s && os.Getpid() != 1 {
 		logger = logging.FileLogger("/proc/1/fd/1", "shutdown-manager", egv1a1.LogLevelInfo)
 	}
 
+	if drainDelay > 0 {
+		logger.Info(fmt.Sprintf("waiting %.0f seconds before starting drain", drainDelay.Seconds()))
+		time.Sleep(drainDelay)
+	}
+
+	startTime := time.Now()
+
 	logger.Info(fmt.Sprintf("initiating drain with %.0f second minimum drain period and %.0f second timeout",
 		minDrainDuration.Seconds(), drainTimeout.Seconds()))
 

internal/infrastructure/common/proxy_args.go

@@ -84,7 +84,7 @@ func BuildProxyArgs(
 	}
 
 	// Default drain timeout.
-	drainTimeout := 60.0
+	drainTimeout := egv1a1.DefaultDrainTimeout.Seconds()
 	if shutdownConfig != nil && shutdownConfig.DrainTimeout != nil {
 		d, err := time.ParseDuration(string(*shutdownConfig.DrainTimeout))
 		if err != nil {

internal/infrastructure/kubernetes/proxy/resource.go

@@ -266,15 +266,32 @@ func expectedShutdownManagerImage(shutdownManager *egv1a1.ShutdownManager) strin
 }
 
 func expectedShutdownManagerArgs(cfg *egv1a1.ShutdownConfig) []string {
-	args := []string{"envoy", "shutdown-manager"}
-	if cfg != nil && cfg.DrainTimeout != nil {
+	args := make([]string, 0, 3)
+	args = append(args, "envoy", "shutdown-manager")
+
+	// Use default --ready-timeout when neither drain field is configured.
+	if cfg == nil || (cfg.DrainTimeout == nil && cfg.DrainDelay == nil) {
+		return args
+	}
+
+	readyTimeout := egv1a1.DefaultDrainTimeout
+	if cfg.DrainTimeout != nil {
 		d, err := time.ParseDuration(string(*cfg.DrainTimeout))
 		if err != nil {
 			return nil
 		}
-		args = append(args, fmt.Sprintf("--ready-timeout=%.0fs", d.Seconds()+10))
+		readyTimeout = d
 	}
-	return args
+
+	if cfg.DrainDelay != nil {
+		delay, err := time.ParseDuration(string(*cfg.DrainDelay))
+		if err != nil {
+			return nil
+		}
+		readyTimeout += delay
+	}
+
+	return append(args, fmt.Sprintf("--ready-timeout=%.0fs", readyTimeout.Seconds()+10))
 }
 
 func expectedShutdownPreStopCommand(cfg *egv1a1.ShutdownConfig) []string {
@@ -284,6 +301,14 @@ func expectedShutdownPreStopCommand(cfg *egv1a1.ShutdownConfig) []string {
 		return command
 	}
 
+	if cfg.DrainDelay != nil {
+		d, err := time.ParseDuration(string(*cfg.DrainDelay))
+		if err != nil {
+			return nil
+		}
+		command = append(command, fmt.Sprintf("--drain-delay=%.0fs", d.Seconds()))
+	}
+
 	if cfg.DrainTimeout != nil {
 		d, err := time.ParseDuration(string(*cfg.DrainTimeout))
 		if err != nil {

internal/infrastructure/kubernetes/proxy/resource_provider.go

@@ -620,6 +620,15 @@ func expectedTerminationGracePeriodSeconds(cfg *egv1a1.ShutdownConfig) *int64 {
 		}
 		s = int(d.Seconds() + 300) // 5 minutes longer than drain timeout
 	}
+
+	if cfg != nil && cfg.DrainDelay != nil {
+		d, err := time.ParseDuration(string(*cfg.DrainDelay))
+		if err != nil {
+			return nil
+		}
+		s += int(d.Seconds())
+	}
+
 	return new(int64(s))
 }
 

internal/infrastructure/kubernetes/proxy/resource_provider_test.go

@@ -292,6 +292,7 @@ func TestDeployment(t *testing.T) {
 				},
 			},
 			shutdown: &egv1a1.ShutdownConfig{
+				DrainDelay:       new(gwapiv1.Duration("15s")),
 				DrainTimeout:     new(gwapiv1.Duration("30s")),
 				MinDrainDuration: new(gwapiv1.Duration("15s")),
 			},
@@ -2120,3 +2121,52 @@ func writeTestDataToFile(filename string, resources []any) error {
 
 	return os.WriteFile(filename, combinedYAML, 0o600)
 }
+
+func TestExpectedTerminationGracePeriodSeconds(t *testing.T) {
+	tests := []struct {
+		name     string
+		cfg      *egv1a1.ShutdownConfig
+		expected int64
+	}{
+		{
+			name:     "nil config",
+			cfg:      nil,
+			expected: 360,
+		},
+		{
+			name:     "empty config",
+			cfg:      &egv1a1.ShutdownConfig{},
+			expected: 360,
+		},
+		{
+			name: "only drainTimeout",
+			cfg: &egv1a1.ShutdownConfig{
+				DrainTimeout: new(gwapiv1.Duration("30s")),
+			},
+			expected: 330,
+		},
+		{
+			name: "only drainDelay",
+			cfg: &egv1a1.ShutdownConfig{
+				DrainDelay: new(gwapiv1.Duration("15s")),
+			},
+			expected: 375,
+		},
+		{
+			name: "drainDelay and drainTimeout",
+			cfg: &egv1a1.ShutdownConfig{
+				DrainDelay:   new(gwapiv1.Duration("15s")),
+				DrainTimeout: new(gwapiv1.Duration("30s")),
+			},
+			expected: 345,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := expectedTerminationGracePeriodSeconds(tt.cfg)
+			require.NotNil(t, got)
+			require.Equal(t, tt.expected, *got)
+		})
+	}
+}

internal/infrastructure/kubernetes/proxy/resource_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
+	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
 	"github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource"
@@ -182,3 +183,50 @@ func TestGetImageTag(t *testing.T) {
 		})
 	}
 }
+
+func TestExpectedShutdownManagerArgs(t *testing.T) {
+	tests := []struct {
+		name     string
+		cfg      *egv1a1.ShutdownConfig
+		expected []string
+	}{
+		{
+			name:     "nil config",
+			cfg:      nil,
+			expected: []string{"envoy", "shutdown-manager"},
+		},
+		{
+			name:     "empty config",
+			cfg:      &egv1a1.ShutdownConfig{},
+			expected: []string{"envoy", "shutdown-manager"},
+		},
+		{
+			name: "only drainTimeout",
+			cfg: &egv1a1.ShutdownConfig{
+				DrainTimeout: new(gwapiv1.Duration("30s")),
+			},
+			expected: []string{"envoy", "shutdown-manager", "--ready-timeout=40s"},
+		},
+		{
+			name: "only drainDelay",
+			cfg: &egv1a1.ShutdownConfig{
+				DrainDelay: new(gwapiv1.Duration("15s")),
+			},
+			expected: []string{"envoy", "shutdown-manager", "--ready-timeout=85s"},
+		},
+		{
+			name: "drainDelay and drainTimeout",
+			cfg: &egv1a1.ShutdownConfig{
+				DrainDelay:   new(gwapiv1.Duration("15s")),
+				DrainTimeout: new(gwapiv1.Duration("30s")),
+			},
+			expected: []string{"envoy", "shutdown-manager", "--ready-timeout=55s"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.expected, expectedShutdownManagerArgs(tt.cfg))
+		})
+	}
+}

internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml

@@ -309,7 +309,7 @@ spec:
       - args:
         - envoy
         - shutdown-manager
-        - --ready-timeout=40s
+        - --ready-timeout=55s
         command:
         - envoy-gateway
         env:
@@ -341,6 +341,7 @@ spec:
               - envoy-gateway
               - envoy
               - shutdown
+              - --drain-delay=15s
               - --drain-timeout=30s
               - --min-drain-duration=15s
         livenessProbe:
@@ -395,7 +396,7 @@ spec:
       restartPolicy: Always
       schedulerName: default-scheduler
       serviceAccountName: envoy-default-37a8eec1
-      terminationGracePeriodSeconds: 330
+      terminationGracePeriodSeconds: 345
       volumes:
       - name: certs
         secret:

release-notes/current.yaml

@@ -15,6 +15,7 @@ new features: |
   Added a `requestBody` field to the HTTP active health checker in `BackendTrafficPolicy`, allowing a request body payload to be sent during HTTP health checking. The field requires the health check `method` to be `POST` or `PUT`.
   Added a `xdsNACKTotal` metric to track the number of NACKs received from Envoy, labeled by node ID and resource type URL. A NACK is a DiscoveryRequest carrying an ErrorDetail, indicating that Envoy rejected the last config update. This metric can be used to alert on config issues causing xDS rejections.
   Added the `autoSNIFromEndpointHostname` TLS setting to Backends, allowing the SNI value sent to the backend to be automatically derived from the backend endpoint hostname instead of using a fixed SNI value.
+  Added a `drainDelay` field to `ShutdownConfig`, delaying the start of the Envoy proxy drain sequence on pod termination. During the delay the proxy continues to accept and serve new connections, allowing external load balancers that rely on health-check based de-registration (e.g. L4 passthrough load balancers) to stop sending traffic to the terminating pod before it stops serving.
 
 bug fixes: |
   Fixed HTTPRoute, GRPCRoute, TLSRoute, TCPRoute, and UDPRoute Accepted condition being set to False when an attached listener is not programmed due to a missing TLS certificate ref; listener programmed state is now correctly separated from route acceptance.

site/content/en/latest/api/extension_types.md

@@ -5849,6 +5849,7 @@ _Appears in:_
 
 | Field | Type | Required | Default | Description |
 | ---   | ---  | ---      | ---     | ---         |
+| `drainDelay` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ |  false  |  | DrainDelay defines the delay before starting the drain process.<br />If unspecified, defaults to 0 seconds. |
 | `drainTimeout` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ |  false  |  | DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.<br />If unspecified, defaults to 60 seconds. |
 | `minDrainDuration` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ |  false  |  | MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.<br />If unspecified, defaults to 10 seconds. |
 

test/helm/gateway-crds-helm/all.out.yaml

@@ -42788,6 +42788,12 @@ spec:
                 description: Shutdown defines configuration for graceful envoy shutdown
                   process.
                 properties:
+                  drainDelay:
+                    description: |-
+                      DrainDelay defines the delay before starting the drain process.
+                      If unspecified, defaults to 0 seconds.
+                    pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+                    type: string
                   drainTimeout:
                     description: |-
                       DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.

test/helm/gateway-crds-helm/e2e.out.yaml

@@ -20761,6 +20761,12 @@ spec:
                 description: Shutdown defines configuration for graceful envoy shutdown
                   process.
                 properties:
+                  drainDelay:
+                    description: |-
+                      DrainDelay defines the delay before starting the drain process.
+                      If unspecified, defaults to 0 seconds.
+                    pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+                    type: string
                   drainTimeout:
                     description: |-
                       DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.