Added llama3.1-70b Benchmarking recipe on A3-Mega nodes

inference/a3mega/llama3.1-70b/values.yaml

@@ -0,0 +1,68 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+targetPlatform: "gke"
+clusterName: ""
+queue: ""
+
+huggingface:
+  secretName: "hf-secret"
+  secretData:
+    token: "hf_api_token"
+
+workload:
+  framework: "trtllm"
+  gpus: 8
+  image: ""
+  model:
+    name: ""
+  configPath: "/workload/configs"
+  configFile: "serving-args.yaml"
+
+  benchmarks:
+    experiments:
+      - isl: 128
+        osl: 128
+        num_requests: 1000
+
+volumes:
+  gcsVolumes: true
+  ssdMountPath: "/ssd"
+  gcsMounts:
+    - bucketName: ""
+      mountPath: "/gcs"
+
+gpuPlatformSettings:
+  useHostPlugin: false
+  ncclPluginImage: "us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.15"
+  rxdmImage: "us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:v1.0.21"
+  ncclBuildType: "228"
+
+network:
+  hostNetwork: true
+  ncclSettings:
+    - name: NCCL_DEBUG
+      value: "WARN"
+  subnetworks: []
+
+trtllm:
+  replicaCount: 1
+  service:
+    type: ClusterIP
+    ports:
+      http: 8000
+  serverArgs:
+    max-model-len: 32768
+    max-num-seqs: 128
+    gpu-memory-utilization: 0.90

src/frameworks/a3mega/trtllm-configs/llama3.1-70b.yaml

@@ -0,0 +1,4 @@
+tp_size: 8
+pp_size: 1
+kv_cache_free_gpu_mem_fraction: 0.85
+backend: pytorch

src/helm-charts/a3mega/trtllm-inference/single-node/chart.yaml

@@ -0,0 +1,20 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v2
+name: trtllm-llama-3-1-70b-inference
+description: A Helm chart for running TensorRT-LLM inference on a single A3 Mega GKE node for llama-3-1-70b.
+type: application
+version: 0.1.0
+appVersion: "1.0.0"