eks-auto-mode-gpu/deepseek_helm.tf at main · trackit/eks-auto-mode-gpu · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
resource "helm_release" "deepseek_gpu" {
  count            = var.enable_gpu && var.deploy_deepseek ? 1 : 0
  name             = "deepseek-gpu"
  chart            = "./vllm-chart"
  create_namespace = true
  wait             = false
  replace          = true
  namespace        = "deepseek"

  values = [
    <<-EOT
    nodeSelector:
      owner: "${"devops"}"
      instanceType: "gpu"
    tolerations:
      - key: "nvidia.com/gpu"
        operator: "Exists"
        effect: "NoSchedule"
    resources:
      limits:
        cpu: "32"
        memory: 100G
        nvidia.com/gpu: "1"
      requests:
        cpu: "16"
        memory: 30G
        nvidia.com/gpu: "1"
    command: "vllm serve ${var.model_name} --max_model_len ${var.max_model_len}"

    livenessProbe:
      httpGet:
        path: /health
        port: 8000
      initialDelaySeconds: 180
      periodSeconds: 10

    readinessProbe:
      httpGet:
        path: /health
        port: 8000
      initialDelaySeconds: 180
      periodSeconds: 5
    EOT
  ]
  depends_on = [module.eks, kubernetes_manifest.gpu_nodepool]
}

resource "helm_release" "deepseek_neuron" {
  count            = var.enable_neuron && var.deploy_deepseek ? 1 : 0
  name             = "deepseek-neuron"
  chart            = "./vllm-chart"
  create_namespace = true
  wait             = false
  replace          = true
  namespace        = "deepseek"

  values = [
    <<-EOT
    image:
      repository: ${aws_ecr_repository.neuron-ecr.repository_url}
      tag: 0.1
      pullPolicy: IfNotPresent

    nodeSelector:
      owner: "${"devops"}"
      instanceType: "neuron"
    tolerations:
      - key: "aws.amazon.com/neuron"
        operator: "Exists"
        effect: "NoSchedule"

    command: "vllm serve ${var.model_name} --device neuron --tensor-parallel-size 2 --max-num-seqs 4 --block-size 8 --use-v2-block-manager --max-model-len ${var.max_model_len}"

    env:
      - name: NEURON_RT_NUM_CORES
        value: "2"
      - name: NEURON_RT_VISIBLE_CORES
        value: "0,1"
      - name: VLLM_LOGGING_LEVEL
        value: "INFO"

    resources:
      limits:
        cpu: "30"
        memory: 64G
        aws.amazon.com/neuron: "1"
      requests:
        cpu: "30"
        memory: 64G
        aws.amazon.com/neuron: "1"

    livenessProbe:
      httpGet:
        path: /health
        port: 8000
      initialDelaySeconds: 180
      periodSeconds: 10

    readinessProbe:
      httpGet:
        path: /health
        port: 8000
      initialDelaySeconds: 180
      periodSeconds: 5
    EOT
  ]
  depends_on = [module.eks, kubernetes_manifest.neuron_nodepool]
}