From 21ba9f64431dd6d94aabd7b8ed8e21a3c3ed6f3a Mon Sep 17 00:00:00 2001 From: krishnakanthankam-qt Date: Fri, 5 Jun 2026 16:07:04 +0530 Subject: [PATCH 1/9] Added new recipe for llama3.1-70b on A3-mega nodes --- inference/a3mega/llama3.1-70b/README.md | 415 ++++++++++++++++ inference/a3mega/llama3.1-70b/values.yaml | 76 +++ .../a3mega/trtllm-configs/llama3.1-70b.yaml | 8 + .../trtllm-inference/single-node/chart.yaml | 20 + .../templates/model-serve-launcher.yaml | 448 ++++++++++++++++++ .../templates/model-serve-svc.yaml | 26 + src/launchers/trtllm-launcher.sh | 156 ++++-- 7 files changed, 1113 insertions(+), 36 deletions(-) create mode 100644 inference/a3mega/llama3.1-70b/README.md create mode 100644 inference/a3mega/llama3.1-70b/values.yaml create mode 100644 src/frameworks/a3mega/trtllm-configs/llama3.1-70b.yaml create mode 100644 src/helm-charts/a3mega/trtllm-inference/single-node/chart.yaml create mode 100644 src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml create mode 100644 src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-svc.yaml diff --git a/inference/a3mega/llama3.1-70b/README.md b/inference/a3mega/llama3.1-70b/README.md new file mode 100644 index 00000000..13a76f9c --- /dev/null +++ b/inference/a3mega/llama3.1-70b/README.md @@ -0,0 +1,415 @@ +# Single Host Model Serving with NVIDIA TensorRT-LLM (TRT-LLM) on A3mega GKE Node Pool + +This document outlines the steps to serve and benchmark various Large Language Models (LLMs) using the [NVIDIA TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) framework on a single [A3-Mega GKE Node pool](https://cloud.google.com/kubernetes-engine). + +This guide walks you through setting up the necessary cloud infrastructure, configuring your environment, and deploying a high-performance LLM for inference. + + +## Table of Contents + +* [1. Test Environment](#test-environment) +* [2. High-Level Architecture](#architecture) +* [3. Environment Setup (One-Time)](#environment-setup) + * [3.1. Clone the Repository](#clone-repo) + * [3.2. Configure Environment Variables](#configure-vars) + * [3.3. Connect to your GKE Cluster](#connect-cluster) + * [3.4. Get Hugging Face Token](#get-hf-token) + * [3.5. Create Hugging Face Kubernetes Secret](#setup-hf-secret) +* [4. Run the Recipe](#run-the-recipe) + * [4.1. Supported Models](#supported-models) + * [4.2. Deploy and Benchmark a Model](#deploy-model) +* [5. Monitoring and Troubleshooting](#monitoring) + * [5.1. Check Deployment Status](#check-status) + * [5.2. View Logs](#view-logs) +* [6. Cleanup](#cleanup) + + +## 1. Test Environment + +[Back to Top](#table-of-contents) + +The recipe uses the following setup: + +* **Orchestration**: [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) +* **Deployment Configuration**: A [Helm chart](https://helm.sh/) is used to configure and deploy a [Kubernetes Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/). This deployment encapsulates the inference of the target LLM using the TensorRT-LLM framework. + +This recipe has been optimized for and tested with the following configuration: + +* **GKE Cluster**: + * A [regional standard cluster](https://cloud.google.com/kubernetes-engine/docs/concepts/configuration-overview) version: `1.33.4-gke.1036000` or later. + * A GPU node pool with 1 [a3-megagpu-8g](https://docs.cloud.google.com/compute/docs/accelerator-optimized-machines#a3-mega-vms) machine. + * [Workload Identity Federation for GKE](https://cloud.google.com/kubernetes-engine/docs/concepts/workload-identity) enabled. + * [Cloud Storage FUSE CSI driver for GKE](https://cloud.google.com/kubernetes-engine/docs/concepts/cloud-storage-fuse-csi-driver) enabled. + * [DCGM metrics](https://cloud.google.com/kubernetes-engine/docs/how-to/dcgm-metrics) enabled. + * [Kueue](https://kueue.sigs.k8s.io/docs/reference/kueue.v1beta1/) and [JobSet](https://jobset.sigs.k8s.io/docs/overview/) APIs installed. + * Kueue configured to support [Topology Aware Scheduling](https://kueue.sigs.k8s.io/docs/concepts/topology_aware_scheduling/). +* A regional Google Cloud Storage (GCS) bucket to store logs generated by the recipe runs. + +> [!IMPORTANT] +> To prepare the required environment, see the [GKE environment setup guide](../../../../docs/configuring-environment-gke-a3-mega.md). +> Provisioning a new GKE cluster is a long-running operation and can take **20-30 minutes**. + + +## 2. High-Level Flow + +[Back to Top](#table-of-contents) + +Here is a simplified diagram of the flow that we follow in this recipe: + +```mermaid +--- +config: + layout: dagre +--- +flowchart TD + subgraph workstation["Client Workstation"] + T["Cluster Toolkit"] + B("Kubernetes API") + A["helm install"] + end + subgraph huggingface["Hugging Face Hub"] + I["Model Weights"] + end + subgraph gke["GKE Cluster (A3-Mega)"] + C["Deployment"] + D["Pod"] + E["TensorRT-LLM container"] + F["Service"] + end + subgraph storage["Cloud Storage"] + J["Bucket"] + end + + %% Logical/actual flow + T -- Create Cluster --> gke + A --> B + B --> C & F + C --> D + D --> E + F --> C + E -- Downloads at runtime --> I + E -- Write logs --> J + + + %% Layout control + gke +``` + +* **helm:** A package manager for Kubernetes to define, install, and upgrade applications. It's used here to configure and deploy the Kubernetes Deployment. +* **Deployment:** Manages the lifecycle of your model server pod, ensuring it stays running. +* **Service:** Provides a stable network endpoint (a DNS name and IP address) to access your model server. +* **Pod:** The smallest deployable unit in Kubernetes. The Triton server container with TensorRT-LLM runs inside this pod on a GPU-enabled node. +* **Cloud Storage:** A Cloud Storage bucket to store benchmark logs and other artifacts. + + +## 3. Environment Setup (One-Time) + +[Back to Top](#table-of-contents) + +First, you'll configure your local environment. These steps are required once before you can deploy any models. + + +### 3.1. Clone the Repository + +```bash +git clone https://github.com/ai-hypercomputer/gpu-recipes.git +cd gpu-recipes +export REPO_ROOT=$(pwd) +export RECIPE_ROOT=$REPO_ROOT/inference/a3mega/llama3.1-70b/trtllm-gke +``` + + +### 3.2. Configure Environment Variables + +This is the most critical step. These variables are used in subsequent commands to target the correct resources. + +```bash +export PROJECT_ID= +export CLUSTER_REGION= +export CLUSTER_NAME= +export KUEUE_NAME= +export GCS_BUCKET= +export TRTLLM_VERSION=1.3.0rc3 + +# Set the project for gcloud commands +gcloud config set project $PROJECT_ID +``` + +Replace the following values: + +| Variable | Description | Example | +| --------------------- | ------------------------------------------------------------------------------------------------------- | ------------------------------------------------------- | +| `PROJECT_ID` | Your Google Cloud Project ID. | `gcp-project-12345` | +| `CLUSTER_REGION` | The GCP region where your GKE cluster is located. | `us-central1` | +| `CLUSTER_NAME` | The name of your GKE cluster. | `a3-mega` | +| `KUEUE_NAME` | The name of the Kueue local queue. The default queue created by the cluster toolkit is `a3mega`. Verify the name in your cluster. | `a3mega` | +| `ARTIFACT_REGISTRY` | Full path to your Artifact Registry repository. | `us-central1-docker.pkg.dev/gcp-project-12345/my-repo` | +| `GCS_BUCKET` | Name of your GCS bucket (do not include `gs://`). | `my-benchmark-logs-bucket` | +| `TRTLLM_VERSION` | The tag/version for the Docker image. Other verions can be found at https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tensorrt-llm/containers/release | `1.3.0rc3` | + + + +### 3.3. Connect to your GKE Cluster + +Fetch credentials for `kubectl` to communicate with your cluster. + +```bash +gcloud container clusters get-credentials $CLUSTER_NAME --region $CLUSTER_REGION +``` + + +### 3.4. Get Hugging Face token + +To access models through Hugging Face, you'll need a Hugging Face token. + 1. Create a [Hugging Face account](https://huggingface.co/) if you don't have one. + 2. For **gated models** like Llama 4, ensure you have requested and been granted access on Hugging Face before proceeding. + 3. Generate an Access Token: Go to **Your Profile > Settings > Access Tokens**. + 4. Select **New Token**. + 5. Specify a Name and a Role of at least `Read`. + 6. Select **Generate a token**. + 7. Copy the generated token to your clipboard. You'll use this later. + + + +### 3.5. Create Hugging Face Kubernetes Secret + +Create a Kubernetes Secret with your Hugging Face token to enable the pod to download model checkpoints from Hugging Face. + +```bash +# Paste your Hugging Face token here +export HF_TOKEN= + +kubectl create secret generic hf-secret \ +--from-literal=hf_api_token=${HF_TOKEN} \ +--dry-run=client -o yaml | kubectl apply -f - +``` + + +## 4. Run the recipe + +[Back to Top](#table-of-contents) + +> [!NOTE] +> After running the recipe with `helm install`, it can take **up to 30 minutes** for the deployment to become fully available. This is because the GKE node must first pull the Docker image and then download the model weights from Hugging Face. + +> [!TIP] +> You can use the [NVIDIA Model Optimizer](https://github.com/NVIDIA/Model-Optimizer/tree/main/examples/llm_ptq) to quantize these models to FP8 for improved performance. + + +### 4.1. Supported Models + +[Back to Top](#table-of-contents) + +This recipe supports the deployment of the following models: + +| Model Name | Hugging Face ID | Configuration File | Release Name Suffix | +| :--- | :--- | :--- | :--- | +| **Llama 3.1 70B** | `meta-llama/Llama-3.1-70B-Instruct` | `llama-3.1-70b.yaml` | `llama-3-1-70b` | + + +### 4.2. Deploy and Benchmark a Model + +[Back to Top](#table-of-contents) + +The recipe uses [`trtllm-bench`](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/legacy/performance/perf-benchmarking.md), a command-line tool from NVIDIA to benchmark the performance of TensorRT-LLM engine. + +1. **Configure model-specific variables.** Choose a model from the [table above](#supported-models) and set the variables: + + ```bash + # Example for Llama 3.1 70B + export HF_MODEL_ID="meta-llama/Llama-3.1-70B-Instruct" + export CONFIG_FILE="llama-3.1-70b.yaml" + export RELEASE_NAME="$USER-serving-llama-3-1-70b" + ``` + +2. **Install the helm chart.** You can run a single benchmark configuration or a sequence of multiple experiments by indexing the `experiments` list. + + **Option A: Single Benchmark Configuration** + ```bash + cd $RECIPE_ROOT + helm install -f values.yaml \ + --set workload.benchmarks.experiments[0].isl=128 \ + --set workload.benchmarks.experiments[0].osl=128 \ + --set workload.benchmarks.experiments[0].num_requests=1000 \ + --set-file workload_launcher=$REPO_ROOT/src/launchers/trtllm-launcher.sh \ + --set-file serving_config=$REPO_ROOT/src/frameworks/a3mega/trtllm-configs/${CONFIG_FILE} \ + --set queue=${KUEUE_NAME} \ + --set "volumes.gcsMounts[0].bucketName=${GCS_BUCKET}" \ + --set workload.model.name=${HF_MODEL_ID} \ + --set workload.image=nvcr.io/nvidia/tensorrt-llm/release:${TRTLLM_VERSION} \ + --set workload.framework=trtllm \ + --set gpuPlatformSettings.useHostPlugin=true \ + ${RELEASE_NAME} \ + $REPO_ROOT/src/helm-charts/a3mega/trtllm-inference/single-node + ``` + + **Option B: Multiple Benchmark Experiments (Sequence)** + The launcher supports running multiple configurations sequentially by providing comma-separated lists. + + > [!IMPORTANT] + > When using `--set` in Helm, commas in list strings must be escaped with a backslash (`\,`), otherwise Helm will fail to parse the command. + + ```bash + cd $RECIPE_ROOT + helm install -f values.yaml \ + --set workload.benchmarks.experiments[0].isl="2048\,2048" \ + --set workload.benchmarks.experiments[0].osl="128\,2048" \ + --set workload.benchmarks.experiments[0].num_requests=1000 \ + --set-file workload_launcher=$REPO_ROOT/src/launchers/trtllm-launcher.sh \ + --set-file serving_config=$REPO_ROOT/src/frameworks/a3mega/trtllm-configs/${CONFIG_FILE} \ + --set queue=${KUEUE_NAME} \ + --set "volumes.gcsMounts[0].bucketName=${GCS_BUCKET}" \ + --set workload.model.name=${HF_MODEL_ID} \ + --set workload.image=nvcr.io/nvidia/tensorrt-llm/release:${TRTLLM_VERSION} \ + --set workload.framework=trtllm \ + --set gpuPlatformSettings.useHostPlugin=true \ + ${RELEASE_NAME} \ + $REPO_ROOT/src/helm-charts/a3mega/trtllm-inference/single-node + ``` + +3. **Check the deployment status:** + + ```bash + kubectl get deployment/${RELEASE_NAME} + ``` + + Wait until the `READY` column shows `1/1`. See the [Monitoring and Troubleshooting](#monitoring) section to view the deployment logs. + + +## 5. Monitoring and Troubleshooting + +[Back to Top](#table-of-contents) + +After the model is deployed via Helm as described in the sections [above](#run-the-recipe), use the following steps to monitor the deployment and interact with the model. Replace `` and `` with the appropriate names from the model-specific deployment instructions (e.g., `$USER-serving-llama3.1-70b` and `$USER-serving-llama3.1-70b-svc`). + + +### 5.1. Check Deployment Status + +Check the status of your deployment. Replace the name if you deployed a different model. + +```bash +# Example for Llama 3.1 70B +kubectl get deployment/$USER-serving-llama3.1-70b +``` + +Wait until the `READY` column shows `1/1`. If it shows `0/1`, the pod is still starting up. + +> [!NOTE] +> In the GKE UI on Cloud Console, you might see a status of "Does not have minimum availability" during startup. This is normal and will resolve once the pod is ready. + + +### 5.2. View Logs + +To see the logs from the TRTLLM server (useful for debugging), use the `-f` flag to follow the log stream: + +```bash +kubectl logs -f deployment/$USER-serving-llama3.1-70b +``` + +You should see logs indicating preparing the model, and then running the throughput benchmark test, similar to this: + +```bash +Running benchmark for nvidia/Llama3.1-70b with ISL=128, OSL=128, TP=8, EP=1, PP=1 + +=========================================================== += PYTORCH BACKEND +=========================================================== +Model: nvidia/Llama3.1-70b +Model Path: /ssd/nvidia/Llama3.1-70b +TensorRT LLM Version: 1.2 +Dtype: bfloat16 +KV Cache Dtype: FP8 +Quantization: FP8 + +=========================================================== += REQUEST DETAILS +=========================================================== +Number of requests: 1000 +Number of concurrent requests: 985.9849 +Average Input Length (tokens): 128.0000 +Average Output Length (tokens): 128.0000 +=========================================================== += WORLD + RUNTIME INFORMATION +=========================================================== +TP Size: 8 +PP Size: 1 +EP Size: 1 +Max Runtime Batch Size: 2304 +Max Runtime Tokens: 4608 +Scheduling Policy: GUARANTEED_NO_EVICT +KV Memory Percentage: 85.00% +Issue Rate (req/sec): 8.3913E+13 + +=========================================================== += PERFORMANCE OVERVIEW +=========================================================== +Request Throughput (req/sec): X.XX +Total Output Throughput (tokens/sec): X.XX +Total Token Throughput (tokens/sec): X.XX +Total Latency (ms): X.XX +Average request latency (ms): X.XX +Per User Output Throughput [w/ ctx] (tps/user): X.XX +Per GPU Output Throughput (tps/gpu): X.XX + +-- Request Latency Breakdown (ms) ----------------------- + +[Latency] P50 : X.XX +[Latency] P90 : X.XX +[Latency] P95 : X.XX +[Latency] P99 : X.XX +[Latency] MINIMUM: X.XX +[Latency] MAXIMUM: X.XX +[Latency] AVERAGE: X.XX + +=========================================================== += DATASET DETAILS +=========================================================== +Dataset Path: /ssd/token-norm-dist_llama3.1-70b_128_128_tp4.json +Number of Sequences: 1000 + +-- Percentiles statistics --------------------------------- + + Input Output Seq. Length +----------------------------------------------------------- +MIN: 128.0000 128.0000 256.0000 +MAX: 128.0000 128.0000 256.0000 +AVG: 128.0000 128.0000 256.0000 +P50: 128.0000 128.0000 256.0000 +P90: 128.0000 128.0000 256.0000 +P95: 128.0000 128.0000 256.0000 +P99: 128.0000 128.0000 256.0000 +=========================================================== +``` + + +## 6. Cleanup + +To avoid incurring further charges, clean up the resources you created. + +1. **Uninstall the Helm Release:** + + First, list your releases to get the deployed models: + + ```bash + # list deployed models + helm list --filter $USER-serving- + ``` + + Then, uninstall the desired release: + + ```bash + # uninstall the deployed model + helm uninstall + ``` + Replace `` with the helm release names listed. + +2. **Delete the Kubernetes Secret:** + + ```bash + kubectl delete secret hf-secret --ignore-not-found=true + ``` + +3. (Optional) Delete the built Docker image from Artifact Registry if no longer needed. +4. (Optional) Delete Cloud Build logs. +5. (Optional) Clean up files in your GCS bucket if benchmarking was performed. +6. (Optional) Delete the [test environment](#test-environment) provisioned including GKE cluster. \ No newline at end of file diff --git a/inference/a3mega/llama3.1-70b/values.yaml b/inference/a3mega/llama3.1-70b/values.yaml new file mode 100644 index 00000000..0a9f31eb --- /dev/null +++ b/inference/a3mega/llama3.1-70b/values.yaml @@ -0,0 +1,76 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +targetPlatform: "gke" +clusterName: "" +queue: "" + +huggingface: + secretName: "hf-secret" + secretData: + token: "hf_api_token" + +workload: + framework: "trtllm" + gpus: 8 + image: "" + model: + # NOTE: tp_size, pp_size, quantization, kv_cache_free_gpu_mem_fraction, and backend + # will be OVERWRITTEN if a model config file (e.g., llama3.1-70b.yaml) + # is passed via --set-file serving_config during helm install. + name: "" + tp_size: 8 + pp_size: 1 + quantization: "FP8" + kv_cache_free_gpu_mem_fraction: 0.70 + backend: "tensorrt" + configPath: "/workload/configs" + configFile: "serving-args.yaml" + + benchmarks: + experiments: + - isl: 128 + osl: 128 + num_requests: 1000 + +volumes: + gcsVolumes: true + ssdMountPath: "/ssd" + gcsMounts: + - bucketName: "" + mountPath: "/gcs" + +gpuPlatformSettings: + useHostPlugin: false + ncclPluginImage: "us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.15" + rxdmImage: "us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:v1.0.21" + ncclBuildType: "228" + +network: + hostNetwork: true + ncclSettings: + - name: NCCL_DEBUG + value: "TRACE" + subnetworks: [] + +trtllm: + replicaCount: 1 + service: + type: ClusterIP + ports: + http: 8000 + serverArgs: + max-model-len: 32768 + max-num-seqs: 128 + gpu-memory-utilization: 0.90 \ No newline at end of file diff --git a/src/frameworks/a3mega/trtllm-configs/llama3.1-70b.yaml b/src/frameworks/a3mega/trtllm-configs/llama3.1-70b.yaml new file mode 100644 index 00000000..c3dbdb30 --- /dev/null +++ b/src/frameworks/a3mega/trtllm-configs/llama3.1-70b.yaml @@ -0,0 +1,8 @@ +# Model Profile: Llama-3.1-70B +# Note: Values defined here override the defaults in values.yaml and the launcher script. +tp_size: 8 +pp_size: 1 +quantization: FP8 +kv_cache_free_gpu_mem_fraction: 0.70 +# Supported backends: "tensorrt" (builds/uses optimized engine), "pytorch" (uses high-level LLM API) +backend: tensorrt \ No newline at end of file diff --git a/src/helm-charts/a3mega/trtllm-inference/single-node/chart.yaml b/src/helm-charts/a3mega/trtllm-inference/single-node/chart.yaml new file mode 100644 index 00000000..e5835b43 --- /dev/null +++ b/src/helm-charts/a3mega/trtllm-inference/single-node/chart.yaml @@ -0,0 +1,20 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: trtllm-llama-3-1-70b-inference +description: A Helm chart for running TensorRT-LLM inference on a single A3 Mega GKE node for llama-3-1-70b. +type: application +version: 0.1.0 +appVersion: "1.0.0" \ No newline at end of file diff --git a/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml b/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml new file mode 100644 index 00000000..09b343bc --- /dev/null +++ b/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml @@ -0,0 +1,448 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# --- CONFIGMAPS --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: "{{ .Release.Name }}-config" +data: + serving-configuration: |- +{{- if .Values.serving_config }} +{{ .Values.serving_config | nindent 4 }} +{{- else }} + tp_size: {{ .Values.workload.model.tp_size | default 8 }} + pp_size: {{ .Values.workload.model.pp_size | default 1 }} + quantization: {{ .Values.workload.quantization | default "FP8" }} + kv_cache_free_gpu_mem_fraction: {{ .Values.workload.kv_cache_free_gpu_mem_fraction | default 0.70 }} + backend: {{ .Values.workload.backend | default "tensorrt" }} +{{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: "{{ .Release.Name }}-launcher" +data: + launch-workload.sh: |- +{{- if .Values.workload_launcher }} +{{ .Values.workload_launcher | nindent 4 }} +{{- else }} + #!/bin/bash + echo "No workload launcher specified" + exit 1 +{{- end }} + +--- + +{{ $nodes := div .Values.workload.gpus 8 | max 1 }} +{{ $gpusPerNode := min .Values.workload.gpus 8 }} + +{{- $root := . -}} + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-serving + labels: + app: {{ .Release.Name }}-serving +spec: + replicas: {{ $nodes }} + selector: + matchLabels: + app: {{ .Release.Name }}-serving + template: + metadata: + labels: + app: {{ .Release.Name }}-serving + annotations: + checksum/serving-config: {{ .Values.serving_config | default "" | sha256sum }} + checksum/launcher-config: {{ .Values.workload_launcher | default "" | sha256sum }} + kubectl.kubernetes.io/default-container: serving + gke-gcsfuse/volumes: "true" + gke-gcsfuse/cpu-limit: "0" + gke-gcsfuse/memory-limit: "0" + gke-gcsfuse/ephemeral-storage-limit: "0" + devices.gke.io/container.tcpxo-daemon: |+ + - path: /dev/nvidia0 + - path: /dev/nvidia1 + - path: /dev/nvidia2 + - path: /dev/nvidia3 + - path: /dev/nvidia4 + - path: /dev/nvidia5 + - path: /dev/nvidia6 + - path: /dev/nvidia7 + - path: /dev/nvidiactl + - path: /dev/nvidia-uvm + - path: /dev/dmabuf_import_helper + networking.gke.io/default-interface: "eth0" + spec: + restartPolicy: Always + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + subdomain: "{{.Release.Name}}" + {{ if $root.Values.targetNodes }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + {{- range $hostname := $root.Values.targetNodes }} + - {{ $hostname }} + {{- end }} + {{ end }} + + tolerations: + - key: nvidia.com/gpu + operator: Exists + - key: cloud.google.com/impending-node-termination + operator: Exists + volumes: + - name: nvidia-dir-host + hostPath: + path: /home/kubernetes/bin/nvidia + + # INFRASTRUCTURE CORE: Dynamic configuration maps for cluster workers + - name: serving-configuration + configMap: + name: "{{ .Release.Name }}-config" + items: + - key: "serving-configuration" + path: "serving-args.yaml" + - name: serving-launcher + configMap: + name: "{{ .Release.Name }}-launcher" + defaultMode: 0700 + + {{- if not $root.Values.gpuPlatformSettings.useHostPlugin }} + - name: nccl-plugin-volume + emptyDir: {} + {{- end }} + - name: sys + hostPath: + path: /sys + - name: proc-sys + hostPath: + path: /proc/sys + - name: aperture-devices + hostPath: + path: /dev/aperture_devices + - name: workload-terminated-volume + emptyDir: {} + - name: local-ssd + hostPath: + path: /mnt/stateful_partition/kube-ephemeral-ssd + - name: shared-memory + emptyDir: + medium: "Memory" + sizeLimit: 250Gi + {{- if and $root.Values.volumes (hasKey $root.Values.volumes "gcsMounts") }} + {{- range $gcs := $root.Values.volumes.gcsMounts }} + - name: "{{ $gcs.bucketName }}" + csi: + driver: gcsfuse.csi.storage.gke.io + volumeAttributes: + bucketName: "{{ $gcs.bucketName }}" + {{- end}} + {{- end}} + + initContainers: + {{- if not $root.Values.gpuPlatformSettings.useHostPlugin }} + - name: nccl-plugin-installer + image: "{{ $root.Values.gpuPlatformSettings.ncclPluginImage }}" + imagePullPolicy: Always + volumeMounts: + - name: nccl-plugin-volume + mountPath: /usr/local/tcpxo + env: + - name: BUILD_TYPE + value: "{{ $root.Values.gpuPlatformSettings.ncclBuildType }}" + command: + - bash + - -c + - | + set -ex + chmod 755 /scripts/container_entry.sh + /scripts/container_entry.sh install --install-nccl --nccl-buildtype ${BUILD_TYPE} + cp -r /var/lib/tcpxo/* /usr/local/tcpxo/ + {{- end }} + + - name: tcpxo-daemon + image: {{ $root.Values.gpuPlatformSettings.rxdmImage }} + imagePullPolicy: Always + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_BIND_SERVICE + restartPolicy: Always + volumeMounts: + - name: nvidia-dir-host + mountPath: /usr/local/nvidia + - name: sys + mountPath: /hostsysfs + - name: proc-sys + mountPath: /hostprocsysfs + env: + - name: LD_LIBRARY_PATH + value: /usr/local/nvidia/lib64 + command: + - bash + - -c + - | + cleanup() { + echo "Received SIGTERM, exiting RxDM" + if [ -n "$child_pid" ]; then + echo "Sending SIGTERM to child process" + kill -TERM "$child_pid" + fi + exit 0 + } + trap cleanup SIGTERM + + chmod 755 /fts/entrypoint_rxdm_container.sh + /fts/entrypoint_rxdm_container.sh --num_hops=2 --num_nics=8 --uid= --alsologtostderr & child_pid=$! + + wait "$child_pid" + + containers: + - name: serving + {{- if typeIs "string" $root.Values.workload.image }} + image: "{{ $root.Values.workload.image }}" + {{- else }} + image: "{{ $root.Values.workload.image.repository }}:{{ $root.Values.workload.image.tag }}" + {{- end }} + securityContext: + privileged: true + resources: + requests: + nvidia.com/gpu: {{ $gpusPerNode }} + limits: + nvidia.com/gpu: {{ $gpusPerNode }} + env: + - name: JOB_ORCHESTRATOR + value: "gke" + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: "{{ $root.Values.huggingface.secretName }}" + key: "{{ $root.Values.huggingface.secretData.token }}" + - name: HF_HUB_ENABLE_HF_TRANSFER + value: "0" + - name: HF_HOME + value: "/ssd" + - name: MODEL_DOWNLOAD_DIR + value: "/ssd" + - name: MODEL_NAME + value: "{{ $root.Values.workload.model.name }}" + - name: LAUNCHER_SCRIPT + value: "/workload/launcher/launch-workload.sh" + - name: SERVER_ARGS_FILE + value: "/workload/configs/serving-args.yaml" + - name: TRTLLM_DIR + value: "/app/tensorrt_llm" + - name: TORCH_DISTRIBUTED_DEBUG + value: "INFO" + - name: GLOO_SOCKET_IFNAME + value: "eth0" + + # RUNTIME MAPPING MATRIX (Includes fallback OS search parameters) + {{- if $root.Values.gpuPlatformSettings.useHostPlugin }} + - name: LD_LIBRARY_PATH + value: "/usr/local/nvidia/lib64" + - name: NCCL_LIB_DIR + value: "/usr/local/nvidia/lib64" + {{- else }} + - name: LD_LIBRARY_PATH + value: "/usr/local/tcpxo/lib64:/usr/local/nvidia/lib64" + - name: NCCL_LIB_DIR + value: "/usr/local/tcpxo/lib64" + {{- end }} + + - name: NCCL_FASTRAK_LLCM_DEVICE_DIRECTORY + value: /dev/aperture_devices + - name: NCCL_FASTRAK_CTRL_DEV + value: "eth0" + - name: NCCL_SOCKET_IFNAME + value: "eth0" + - name: NCCL_CROSS_NIC + value: "0" + - name: NCCL_ALGO + value: "Ring,Tree" + - name: NCCL_PROTO + value: "Simple,LL128" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: NCCL_DYNAMIC_CHUNK_SIZE + value: "524288" + - name: NCCL_P2P_NET_CHUNKSIZE + value: "524288" + - name: NCCL_P2P_PCI_CHUNKSIZE + value: "524288" + - name: NCCL_P2P_NVL_CHUNKSIZE + value: "1048576" + - name: NCCL_FASTRAK_NUM_FLOWS + value: "2" + - name: NCCL_BUFFSIZE + value: "8388608" + - name: NCCL_NET_GDR_LEVEL + value: "PIX" + - name: NCCL_DEBUG_SUBSYS + value: "INIT,GRAPH,ENV,TUNING,NET" + - name: NCCL_FASTRAK_ENABLE_HOTPATH_LOGGING + value: "0" + - name: NCCL_FASTRAK_USE_SNAP + value: "1" + - name: NCCL_FASTRAK_ENABLE_CONTROL_CHANNEL + value: "0" + - name: NCCL_FASTRAK_USE_LLCM + value: "1" + - name: NCCL_TUNER_PLUGIN + value: "libnccl-tuner.so" + - name: NCCL_TUNER_CONFIG_PATH + value: "/usr/local/tcpxo/lib64/a3plus_tuner_config.textproto" + - name: NCCL_SHIMNET_GUEST_CONFIG_CHECKER_CONFIG_FILE + value: "/usr/local/tcpxo/lib64/a3plus_guest_config.textproto" + - name: NCCL_FASTRAK_PLUGIN_ACCEPT_TIMEOUT_MS + value: "600000" + - name: CUDA_VISIBLE_DEVICES + value: "0,1,2,3,4,5,6,7" + - name: NCCL_FASTRAK_IFNAME + value: "eth1,eth2,eth3,eth4,eth5,eth6,eth7,eth8" + - name: NCCL_NVLSTREE_MAX_CHUNKSIZE + value: "131072" + - name: NVTE_FWD_LAYERNORM_SM_MARGIN + value: "8" + - name: NVTE_BWD_LAYERNORM_SM_MARGIN + value: "8" + - name: NCCL_P2P_PXN_LEVEL + value: "0" + - name: NCCL_NET_PLUGIN_TELEMETRY_MODE + value: "1" + - name: NCCL_GPUVIZ_ENABLE_MILLISECOND_BANDWIDTH_OUTPUT + value: "1" + - name: NCCL_GPUVIZ_FILE_ROTATION_INTERVAL_IN_SECONDS + value: "300" + + {{- range $environment_variable := $root.Values.network.ncclSettings }} + - name: {{ $environment_variable.name }} + value: "{{ $environment_variable.value }}" + {{- end }} + + command: + - bash + - -c + - | + # Point 14: Environment and NCCL Fixes for A3 Mega + export HF_HOME=/ssd + export LD_LIBRARY_PATH=/usr/local/cuda/compat/lib:/usr/local/lib/python3.12/dist-packages/torch/lib:/usr/local/lib/python3.12/dist-packages/torch_tensorrt/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/tensorrt/lib:/usr/local/tcpxo/lib64 + export NCCL_DEBUG=WARN + export NCCL_NET_PLUGIN=none + export NCCL_TUNER_PLUGIN=none + export NCCL_P2P_LEVEL=NVL + + # Force load the container's internal NCCL to resolve the 'ncclCommWindowDeregister' symbol mismatch + for loc in "/usr/local/lib/python3.12/dist-packages/torch/lib/libnccl.so.2" "/usr/local/lib/python3.12/dist-packages/nvidia/nccl/lib/libnccl.so.2" "/usr/local/cuda/lib64/libnccl.so.2" "/usr/lib/x86_64-linux-gnu/libnccl.so.2"; do + if [ -f "$loc" ]; then + if nm -D "$loc" 2>/dev/null | grep -q "ncclCommWindowDeregister"; then + export LD_PRELOAD="$loc" + break + fi + fi + done + + echo "--- START DEBUG INFO ---" + echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" + echo "LD_PRELOAD: ${LD_PRELOAD:-NOT_SET}" + echo "Searching for all libnccl.so.2 instances:" + find /usr -name "libnccl.so.2" 2>/dev/null | xargs ls -lh || true + echo "Checking for 'ncclCommWindowDeregister' symbol in found libraries:" + for lib in $(find /usr -name "libnccl.so.2" 2>/dev/null); do + if nm -D "$lib" 2>/dev/null | grep -q "ncclCommWindowDeregister"; then + echo " [OK] Symbol FOUND in: $lib" + else + echo " [!!] Symbol MISSING in: $lib" + fi + done + echo "Checking what libtorch_cuda.so is linked against:" + ldd /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so | grep nccl || true + echo "GPU Info:" + nvidia-smi --query-gpu=name,driver_version --format=csv || true + echo "--- END DEBUG INFO ---" + + trtllm-bench() { + unset NCCL_P2P_LEVEL + command trtllm-bench "$@" + } + export -f trtllm-bench + + ldconfig 2>/dev/null + chmod +x "$LAUNCHER_SCRIPT" + + ARGS=() + while IFS=': ' read -r key value || [ -n "$key" ]; do + [[ -z "$key" || "$key" == \#* ]] && continue + key=$(echo "$key" | xargs); value=$(echo "$value" | xargs) + if [[ "$value" == "true" ]]; then ARGS+=("--$key"); elif [[ "$value" == "false" ]]; then ARGS+=("--$key" "false"); elif [ -n "$value" ]; then ARGS+=("--$key" "$value"); else ARGS+=("--$key"); fi + done < "/workload/configs/serving-args.yaml" + + echo "Starting automated benchmark sequence..." + {{- $isl_list := list -}} + {{- $osl_list := list -}} + {{- $num_requests := 1024 -}} + {{- range .Values.workload.benchmarks.experiments -}} + {{- $isl_list = append $isl_list (toString .isl) -}} + {{- $osl_list = append $osl_list (toString .osl) -}} + {{- $num_requests = .num_requests -}} + {{- end -}} + {{- if $isl_list }} + "$LAUNCHER_SCRIPT" --model_name "{{ $.Values.workload.model.name }}" --isl "{{ join "," $isl_list }}" --osl "{{ join "," $osl_list }}" --num_requests {{ $num_requests }} -- "${ARGS[@]}" + {{- else }} + "$LAUNCHER_SCRIPT" --model_name "{{ $.Values.workload.model.name }}" -- "${ARGS[@]}" + {{- end }} + echo "Benchmarks complete. Job exiting." + ports: + - containerPort: {{ $root.Values.trtllm.service.ports.http }} + + volumeMounts: + - name: nvidia-dir-host + mountPath: /usr/local/nvidia + - name: aperture-devices + mountPath: /dev/aperture_devices + {{- if not $root.Values.gpuPlatformSettings.useHostPlugin }} + - name: nccl-plugin-volume + mountPath: /usr/local/tcpxo + {{- end }} + - name: sys + mountPath: /hostsysfs + - name: proc-sys + mountPath: /hostprocsysfs + - name: workload-terminated-volume + mountPath: /semaphore + - name: shared-memory + mountPath: /dev/shm + - name: local-ssd + mountPath: "{{ $root.Values.volumes.ssdMountPath }}" + - name: serving-configuration + mountPath: {{ $root.Values.workload.configPath | default "/workload/configs" }} + - name: serving-launcher + mountPath: /workload/launcher + {{- if and $root.Values.volumes (hasKey $root.Values.volumes "gcsMounts") }} + {{- range $gcs := $root.Values.volumes.gcsMounts }} + - name: "{{ $gcs.bucketName }}" + mountPath: "{{ $gcs.mountPath }}" + {{- end }} + {{- end }} \ No newline at end of file diff --git a/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-svc.yaml b/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-svc.yaml new file mode 100644 index 00000000..0daf7ab9 --- /dev/null +++ b/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-svc.yaml @@ -0,0 +1,26 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-svc +spec: + selector: + app: {{ .Release.Name }}-serving + ports: + - name: http + port: {{ .Values.trtllm.service.ports.http }} + targetPort: {{ .Values.trtllm.service.ports.http }} + type: {{ .Values.trtllm.service.type }} \ No newline at end of file diff --git a/src/launchers/trtllm-launcher.sh b/src/launchers/trtllm-launcher.sh index dc3f828b..55431a35 100644 --- a/src/launchers/trtllm-launcher.sh +++ b/src/launchers/trtllm-launcher.sh @@ -21,6 +21,9 @@ echo "TensorRT-LLM benchmark arguments received:" echo " $@" echo "" +# Define model download directory if not set +MODEL_DOWNLOAD_DIR=${MODEL_DOWNLOAD_DIR:-/ssd} + # Function to validate model name validate_model_name() { if [ -z "$MODEL_NAME" ]; then @@ -33,9 +36,9 @@ validate_model_name() { # Function to parse arguments parse_arguments() { model_name=$MODEL_NAME - isl=128 - osl=128 - num_requests=30000 + isl_list="128" + osl_list="128" + num_requests=1024 # Parse known arguments and check for unknown option or missing argument PARSED_OPTIONS=$(getopt -o "" -l model_name:,isl:,osl:,num_requests: -- "$@") @@ -54,11 +57,11 @@ parse_arguments() { shift 2 ;; --isl) - isl="$2" + isl_list="$2" shift 2 ;; --osl) - osl="$2" + osl_list="$2" shift 2 ;; --num_requests) @@ -119,12 +122,13 @@ parse_serving_config() { pp_size=${SERVING_CONFIG_DICT["pp_size"]:=1} ep_size=${SERVING_CONFIG_DICT["ep_size"]:=1} backend=${SERVING_CONFIG_DICT["backend"]:="tensorrt"} - kv_cache_free_gpu_mem_fraction=${SERVING_CONFIG_DICT["kv_cache_free_gpu_mem_fraction"]:=0.95} + kv_cache_free_gpu_mem_fraction=${SERVING_CONFIG_DICT["kv_cache_free_gpu_mem_fraction"]:=0.70} modality=${SERVING_CONFIG_DICT["modality"]:=""} streaming=${SERVING_CONFIG_DICT["streaming"]:="false"} max_input_len=${SERVING_CONFIG_DICT["max_input_len"]:=""} max_batch_size=${SERVING_CONFIG_DICT["max_batch_size"]:=""} custom_dataset=${SERVING_CONFIG_DICT["dataset"]:=""} + quantization=${SERVING_CONFIG_DICT["quantization"]:="FP8"} } print_configuration() { @@ -134,8 +138,8 @@ print_configuration() { echo "--------------------------------" echo "--- Parsed Arguments Summary ---" echo "model name: $model_name" - echo "input seq length: $isl" - echo "output seq length: $osl" + echo "input seq lengths: $isl_list" + echo "output seq lengths: $osl_list" echo "number of requests: $num_requests" echo "tensor parallel size: $tp_size" echo "pipeline parallel size: $pp_size" @@ -144,6 +148,7 @@ print_configuration() { echo "modality: $modality" echo "streaming: $streaming" echo "max input length: $max_input_len" + echo "quantization: $quantization" echo "max batch size: $max_batch_size" echo "kv_cache_free_gpu_mem_fraction: $kv_cache_free_gpu_mem_fraction" echo "--------------------------------" @@ -152,7 +157,8 @@ print_configuration() { download_model() { echo "Downloading model from HuggingFace... This may take a while when downloading for the first time." echo "NOTE: by default, huggingface-cli response can be verbose." - huggingface-cli download $model_name --exclude="*original*" --local-dir ${MODEL_DOWNLOAD_DIR} --local-dir-use-symlinks False + # Ensure we download into a model-specific subdirectory + huggingface-cli download $model_name --exclude="*original*" --local-dir ${MODEL_DOWNLOAD_DIR}/${model_name##*/} --local-dir-use-symlinks False } # Function to run benchmarks @@ -167,6 +173,24 @@ run_benchmark() { local backend=$8 local kv_cache_free_gpu_mem_fraction=$9 + local engine_dir="" + local generated_dataset="" + local output_file="${MODEL_DOWNLOAD_DIR}/output_${model_name##*/}_isl${isl}_osl${osl}_tp${tp_size}.txt" + local benchmark_rc=0 + + export TOKENIZERS_PARALLELISM=false + + # Emergency cleanup on crash + cleanup_on_exit() { + local exit_code=$? + if [[ $exit_code -ne 0 ]]; then + echo "Benchmark interrupted (Exit code: $exit_code). Cleaning up temporary files..." + [[ -n "${engine_dir:-}" && -d "${engine_dir:-}" ]] && rm -rf "${engine_dir}" || true + [[ -n "${generated_dataset:-}" && -f "${generated_dataset:-}" ]] && rm -f "${generated_dataset}" || true + fi + } + trap cleanup_on_exit EXIT SIGINT SIGTERM + echo "Running benchmark for $model_name with ISL=$isl, OSL=$osl, TP=$tp_size, PP=$pp_size, EP=$ep_size, backend=$backend" vl_args="" @@ -176,12 +200,18 @@ run_benchmark() { if [ -n "$max_batch_size" ]; then vl_args="$vl_args --max_batch_size $max_batch_size"; fi dataset_file=$custom_dataset - # If custom_dataset is not set, generate a textual dataset with tokens sampled in normal distribution + local tokenizer_arg=$model_name + if [ -d "/serving-model" ]; then + tokenizer_arg="/serving-model" + fi + + # Point 3 & 4: Tokenizer logic if [ -z "$dataset_file" ]; then - dataset_file="/ssd/token-norm-dist_${model_name##*/}_${isl}_${osl}_tp${tp_size}.json" + dataset_file="${MODEL_DOWNLOAD_DIR}/token-norm-dist_${model_name##*/}_${isl}_${osl}_tp${tp_size}.json" + generated_dataset=$dataset_file echo "Preparing dataset" python3 $TRTLLM_DIR/benchmarks/cpp/prepare_dataset.py \ - --tokenizer=$model_name \ + --tokenizer=$tokenizer_arg \ --stdout token-norm-dist \ --num-requests=$num_requests \ --input-mean=$isl \ @@ -190,58 +220,83 @@ run_benchmark() { --output-stdev=0 >$dataset_file fi - output_file="/ssd/output_${model_name##*/}_isl${isl}_osl${osl}_tp${tp_size}.txt" + # Clean up any leftover extra args to prevent "invalid argument: enable_cuda_graph" + rm -f /tmp/extra_llm_api_args.yaml || true + extra_args_file="/tmp/extra_llm_api_args.yaml" extra_args="" if [ -f "$extra_args_file" ]; then extra_args="--extra_llm_api_options $extra_args_file" fi - export TOKENIZERS_PARALLELISM=false - echo "enable_cuda_graph: false" > /tmp/extra_llm_api_args.yaml + # Point 5: Determine model path + local model_path_arg="${MODEL_DOWNLOAD_DIR}/${model_name}" + if [ -d "/serving-model" ]; then + model_path_arg="/serving-model" + fi + # Point 6, 7, 8: Throughput call if [[ $backend == "pytorch" ]]; then echo "Running throughput benchmark" - export NCCL_P2P_LEVEL=PHB + set +e trtllm-bench \ --model $model_name \ - --model_path /ssd/${model_name} throughput \ + --model_path $model_path_arg throughput \ --dataset $dataset_file \ --num_requests $num_requests \ --tp $tp_size \ --pp $pp_size \ --ep $ep_size \ - --backend "pytorch" \ + --backend $backend \ --kv_cache_free_gpu_mem_fraction $kv_cache_free_gpu_mem_fraction \ - $extra_args $vl_args > $output_file + $extra_args $vl_args | tee "$output_file" + benchmark_rc=${PIPESTATUS[0]} + set -e else - echo "Building engine" + # Point 9: Build engine trtllm-bench \ --model $model_name \ - --model_path /ssd/${model_name} \ - --workspace /ssd build \ + --model_path $model_path_arg \ + --workspace ${MODEL_DOWNLOAD_DIR} build \ --tp_size $tp_size \ --pp_size $pp_size \ - --quantization FP8 \ + --quantization $quantization \ --dataset $dataset_file - engine_dir="/ssd/${model_name}/tp_${tp_size}_pp_${pp_size}" + # Point 10: Engine Dir path + engine_dir="${model_path_arg}/tp_${tp_size}_pp_${pp_size}" - # Save throughput output to a file + # Point 11 & 12: Run throughput with engine echo "Running throughput benchmark" + set +e trtllm-bench \ --model $model_name \ - --model_path /ssd/${model_name} throughput \ + --model_path $model_path_arg throughput \ --dataset $dataset_file \ --engine_dir $engine_dir \ - --kv_cache_free_gpu_mem_fraction $kv_cache_free_gpu_mem_fraction $extra_args >$output_file + --tp $tp_size \ + --pp $pp_size \ + --ep $ep_size \ + --backend $backend \ + --kv_cache_free_gpu_mem_fraction $kv_cache_free_gpu_mem_fraction \ + $extra_args | tee "$output_file" + benchmark_rc=${PIPESTATUS[0]} + set -e fi - cat $output_file - gcloud storage cp $output_file /gcs/benchmark_logs/trtllm/ + # Point 13: Immediate sync of results to GCS + mkdir -p /gcs/benchmark_logs/trtllm || true + cp "$output_file" /gcs/benchmark_logs/trtllm/ || true + + # Standard cleanup after successful iteration + [[ -n "${engine_dir:-}" && -d "${engine_dir:-}" ]] && rm -rf "${engine_dir}" || true + [[ -n "${dataset_file:-}" ]] && rm -f "${dataset_file}" || true + trap - EXIT SIGINT SIGTERM - rm -rf $engine_dir - rm -f $dataset_file + if [ $benchmark_rc -ne 0 ]; then + echo "Error: Benchmark command failed with exit code $benchmark_rc." + exit $benchmark_rc + fi } # Main function to run the benchmark @@ -250,20 +305,49 @@ main() { validate_model_name parse_arguments "$@" parse_serving_config - print_configuration "$@" # download model download_model + # Convert comma-separated lists to arrays + # Remove spaces and split by comma + IFS=',' read -ra ISL_ARRAY <<< "${isl_list// /}" + IFS=',' read -ra OSL_ARRAY <<< "${osl_list// /}" + + # Validate array lengths match + if [[ ${#ISL_ARRAY[@]} -ne ${#OSL_ARRAY[@]} ]]; then + echo "Error: The number of values in --isl and --osl must be the same." + echo "ISL count: ${#ISL_ARRAY[@]}, OSL count: ${#OSL_ARRAY[@]}" + exit 1 + fi + + print_configuration "$@" + # run benchmark mkdir -p /gcs/benchmark_logs/trtllm echo "Running benchmarks" - run_benchmark "$model_name" $isl $osl $num_requests $tp_size $pp_size $ep_size $backend $kv_cache_free_gpu_mem_fraction + for i in "${!ISL_ARRAY[@]}"; do + current_isl="${ISL_ARRAY[$i]}" + current_osl="${OSL_ARRAY[$i]}" + + echo "Starting iteration $((i+1))/${#ISL_ARRAY[@]}: ISL=$current_isl, OSL=$current_osl, REQ=$num_requests" + run_benchmark "$model_name" "$current_isl" "$current_osl" "$num_requests" $tp_size $pp_size $ep_size $backend $kv_cache_free_gpu_mem_fraction + done + + echo "-----------------------------------------------------------" + echo "Benchmarks complete. Keeping container alive for result inspection." + sleep infinity } -# Set environment variables -export HF_HOME=/ssd -export LD_LIBRARY_PATH=/usr/local/lib/python3.12/dist-packages/torch/lib:/usr/local/lib/python3.12/dist-packages/torch_tensorrt/lib:/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/tensorrt/lib +# Force load the container's internal NCCL to resolve symbol mismatches on GKE +for loc in "/usr/local/lib/python3.12/dist-packages/torch/lib/libnccl.so.2" "/usr/local/lib/python3.12/dist-packages/nvidia/nccl/lib/libnccl.so.2" "/usr/local/cuda/lib64/libnccl.so.2" "/usr/lib/x86_64-linux-gnu/libnccl.so.2"; do + if [ -f "$loc" ]; then + if nm -D "$loc" 2>/dev/null | grep -q "ncclCommWindowDeregister"; then + export LD_PRELOAD="$loc" + break + fi + fi +done # Run the main function main "$@" \ No newline at end of file From d4c2a9c14901a6a2c88189b528ec5ef29b80fe4e Mon Sep 17 00:00:00 2001 From: krishnakanthankam-qt Date: Fri, 5 Jun 2026 16:32:21 +0530 Subject: [PATCH 2/9] modified trtllm-launcher.sh for backward compatibility --- src/launchers/trtllm-launcher.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/launchers/trtllm-launcher.sh b/src/launchers/trtllm-launcher.sh index 55431a35..48cce187 100644 --- a/src/launchers/trtllm-launcher.sh +++ b/src/launchers/trtllm-launcher.sh @@ -36,8 +36,8 @@ validate_model_name() { # Function to parse arguments parse_arguments() { model_name=$MODEL_NAME - isl_list="128" - osl_list="128" + isl="128" + osl="128" num_requests=1024 # Parse known arguments and check for unknown option or missing argument @@ -57,11 +57,11 @@ parse_arguments() { shift 2 ;; --isl) - isl_list="$2" + isl="$2" shift 2 ;; --osl) - osl_list="$2" + osl="$2" shift 2 ;; --num_requests) @@ -138,8 +138,8 @@ print_configuration() { echo "--------------------------------" echo "--- Parsed Arguments Summary ---" echo "model name: $model_name" - echo "input seq lengths: $isl_list" - echo "output seq lengths: $osl_list" + echo "input seq length(s): $isl" + echo "output seq length(s): $osl" echo "number of requests: $num_requests" echo "tensor parallel size: $tp_size" echo "pipeline parallel size: $pp_size" @@ -311,12 +311,12 @@ main() { # Convert comma-separated lists to arrays # Remove spaces and split by comma - IFS=',' read -ra ISL_ARRAY <<< "${isl_list// /}" - IFS=',' read -ra OSL_ARRAY <<< "${osl_list// /}" + IFS=',' read -ra ISL_ARRAY <<< "${isl// /}" + IFS=',' read -ra OSL_ARRAY <<< "${osl// /}" # Validate array lengths match if [[ ${#ISL_ARRAY[@]} -ne ${#OSL_ARRAY[@]} ]]; then - echo "Error: The number of values in --isl and --osl must be the same." + echo "Error: The number of values in --isl and --osl list(s) must be the same." echo "ISL count: ${#ISL_ARRAY[@]}, OSL count: ${#OSL_ARRAY[@]}" exit 1 fi From 5feb2d38df11ae9512ec79244b13007c1fafb8c9 Mon Sep 17 00:00:00 2001 From: krishnakanthankam-qt Date: Thu, 11 Jun 2026 14:29:32 +0530 Subject: [PATCH 3/9] streamlined launcher script and modified helm deployment --- inference/a3mega/llama3.1-70b/README.md | 47 ++---- inference/a3mega/llama3.1-70b/values.yaml | 10 +- .../a3mega/trtllm-configs/llama3.1-70b.yaml | 8 +- .../templates/model-serve-launcher.yaml | 80 +++------- src/launchers/trtllm-launcher.sh | 149 ++++-------------- 5 files changed, 73 insertions(+), 221 deletions(-) diff --git a/inference/a3mega/llama3.1-70b/README.md b/inference/a3mega/llama3.1-70b/README.md index 13a76f9c..90c4e986 100644 --- a/inference/a3mega/llama3.1-70b/README.md +++ b/inference/a3mega/llama3.1-70b/README.md @@ -200,11 +200,18 @@ kubectl create secret generic hf-secret \ [Back to Top](#table-of-contents) -This recipe supports the deployment of the following models: +This recipe supports the following models. Running TRTLLM inference benchmarking on these models are only tested and validated on A3-Mega GKE nodes with certain combination of TP, PP, EP, number of GPU chips, input & output sequence length, precision, etc. + +Example model configuration YAML files included in this repo only show a certain combination of parallelism hyperparameters and configs for benchmarking purposes. Input and output length in `/home/akrishnakanth/gpu-recipes/inference/a3mega/llama3.1-70b/trtllm-gke/values.yaml` need to be adjusted according to the model and its configs. + +As the PyTorch backend requires pre-quantized models for optimal performance, we use the FP8 quantized version for Llama 3.1 70B. | Model Name | Hugging Face ID | Configuration File | Release Name Suffix | | :--- | :--- | :--- | :--- | -| **Llama 3.1 70B** | `meta-llama/Llama-3.1-70B-Instruct` | `llama-3.1-70b.yaml` | `llama-3-1-70b` | +| **Llama 3.1 70B (FP8)** | `nvidia/Llama-3.1-70B-Instruct-FP8` | `llama-3.1-70b.yaml` | `llama-3-1-70b` | + +> [!TIP] +> You can use the NVIDIA Model Optimizer to quantize these models to FP8 for improved performance. ### 4.2. Deploy and Benchmark a Model @@ -216,15 +223,14 @@ The recipe uses [`trtllm-bench`](https://github.com/NVIDIA/TensorRT-LLM/blob/mai 1. **Configure model-specific variables.** Choose a model from the [table above](#supported-models) and set the variables: ```bash - # Example for Llama 3.1 70B - export HF_MODEL_ID="meta-llama/Llama-3.1-70B-Instruct" + # Example for Llama 3.1 70B (FP8) + export HF_MODEL_ID="nvidia/Llama-3.1-70B-Instruct-FP8" export CONFIG_FILE="llama-3.1-70b.yaml" export RELEASE_NAME="$USER-serving-llama-3-1-70b" ``` 2. **Install the helm chart.** You can run a single benchmark configuration or a sequence of multiple experiments by indexing the `experiments` list. - **Option A: Single Benchmark Configuration** ```bash cd $RECIPE_ROOT helm install -f values.yaml \ @@ -238,35 +244,12 @@ The recipe uses [`trtllm-bench`](https://github.com/NVIDIA/TensorRT-LLM/blob/mai --set workload.model.name=${HF_MODEL_ID} \ --set workload.image=nvcr.io/nvidia/tensorrt-llm/release:${TRTLLM_VERSION} \ --set workload.framework=trtllm \ - --set gpuPlatformSettings.useHostPlugin=true \ ${RELEASE_NAME} \ $REPO_ROOT/src/helm-charts/a3mega/trtllm-inference/single-node ``` - - **Option B: Multiple Benchmark Experiments (Sequence)** - The launcher supports running multiple configurations sequentially by providing comma-separated lists. + > [!NOTE] + > You can modify the benchmark configuration at runtime by changing the values for `isl`, `osl`, and `num_requests` (number of prompts) in the Helm command to test different scenarios. - > [!IMPORTANT] - > When using `--set` in Helm, commas in list strings must be escaped with a backslash (`\,`), otherwise Helm will fail to parse the command. - - ```bash - cd $RECIPE_ROOT - helm install -f values.yaml \ - --set workload.benchmarks.experiments[0].isl="2048\,2048" \ - --set workload.benchmarks.experiments[0].osl="128\,2048" \ - --set workload.benchmarks.experiments[0].num_requests=1000 \ - --set-file workload_launcher=$REPO_ROOT/src/launchers/trtllm-launcher.sh \ - --set-file serving_config=$REPO_ROOT/src/frameworks/a3mega/trtllm-configs/${CONFIG_FILE} \ - --set queue=${KUEUE_NAME} \ - --set "volumes.gcsMounts[0].bucketName=${GCS_BUCKET}" \ - --set workload.model.name=${HF_MODEL_ID} \ - --set workload.image=nvcr.io/nvidia/tensorrt-llm/release:${TRTLLM_VERSION} \ - --set workload.framework=trtllm \ - --set gpuPlatformSettings.useHostPlugin=true \ - ${RELEASE_NAME} \ - $REPO_ROOT/src/helm-charts/a3mega/trtllm-inference/single-node - ``` - 3. **Check the deployment status:** ```bash @@ -288,8 +271,8 @@ After the model is deployed via Helm as described in the sections [above](#run-t Check the status of your deployment. Replace the name if you deployed a different model. ```bash -# Example for Llama 3.1 70B -kubectl get deployment/$USER-serving-llama3.1-70b +# Example for Llama 3.1 70B (FP8) +kubectl get deployment/$USER-serving-llama-3-1-70b ``` Wait until the `READY` column shows `1/1`. If it shows `0/1`, the pod is still starting up. diff --git a/inference/a3mega/llama3.1-70b/values.yaml b/inference/a3mega/llama3.1-70b/values.yaml index 0a9f31eb..8ffb3e73 100644 --- a/inference/a3mega/llama3.1-70b/values.yaml +++ b/inference/a3mega/llama3.1-70b/values.yaml @@ -26,15 +26,7 @@ workload: gpus: 8 image: "" model: - # NOTE: tp_size, pp_size, quantization, kv_cache_free_gpu_mem_fraction, and backend - # will be OVERWRITTEN if a model config file (e.g., llama3.1-70b.yaml) - # is passed via --set-file serving_config during helm install. name: "" - tp_size: 8 - pp_size: 1 - quantization: "FP8" - kv_cache_free_gpu_mem_fraction: 0.70 - backend: "tensorrt" configPath: "/workload/configs" configFile: "serving-args.yaml" @@ -61,7 +53,7 @@ network: hostNetwork: true ncclSettings: - name: NCCL_DEBUG - value: "TRACE" + value: "WARN" subnetworks: [] trtllm: diff --git a/src/frameworks/a3mega/trtllm-configs/llama3.1-70b.yaml b/src/frameworks/a3mega/trtllm-configs/llama3.1-70b.yaml index c3dbdb30..bd1d6a12 100644 --- a/src/frameworks/a3mega/trtllm-configs/llama3.1-70b.yaml +++ b/src/frameworks/a3mega/trtllm-configs/llama3.1-70b.yaml @@ -1,8 +1,4 @@ -# Model Profile: Llama-3.1-70B -# Note: Values defined here override the defaults in values.yaml and the launcher script. tp_size: 8 pp_size: 1 -quantization: FP8 -kv_cache_free_gpu_mem_fraction: 0.70 -# Supported backends: "tensorrt" (builds/uses optimized engine), "pytorch" (uses high-level LLM API) -backend: tensorrt \ No newline at end of file +kv_cache_free_gpu_mem_fraction: 0.85 +backend: pytorch \ No newline at end of file diff --git a/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml b/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml index 09b343bc..1e566a85 100644 --- a/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml +++ b/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml @@ -48,6 +48,12 @@ data: {{ $nodes := div .Values.workload.gpus 8 | max 1 }} {{ $gpusPerNode := min .Values.workload.gpus 8 }} +{{- /* Extract benchmark parameters from Helm values */ -}} +{{- $experiment := index .Values.workload.benchmarks.experiments 0 -}} +{{- $isl := $experiment.isl -}} +{{- $osl := $experiment.osl -}} +{{- $num_requests := $experiment.num_requests -}} + {{- $root := . -}} apiVersion: apps/v1 @@ -114,8 +120,7 @@ spec: - name: nvidia-dir-host hostPath: path: /home/kubernetes/bin/nvidia - - # INFRASTRUCTURE CORE: Dynamic configuration maps for cluster workers + - name: serving-configuration configMap: name: "{{ .Release.Name }}-config" @@ -245,7 +250,7 @@ spec: - name: HF_HOME value: "/ssd" - name: MODEL_DOWNLOAD_DIR - value: "/ssd" + value: "/ssd/{{ $root.Values.workload.model.name }}" - name: MODEL_NAME value: "{{ $root.Values.workload.model.name }}" - name: LAUNCHER_SCRIPT @@ -259,7 +264,6 @@ spec: - name: GLOO_SOCKET_IFNAME value: "eth0" - # RUNTIME MAPPING MATRIX (Includes fallback OS search parameters) {{- if $root.Values.gpuPlatformSettings.useHostPlugin }} - name: LD_LIBRARY_PATH value: "/usr/local/nvidia/lib64" @@ -267,11 +271,12 @@ spec: value: "/usr/local/nvidia/lib64" {{- else }} - name: LD_LIBRARY_PATH - value: "/usr/local/tcpxo/lib64:/usr/local/nvidia/lib64" - - name: NCCL_LIB_DIR - value: "/usr/local/tcpxo/lib64" + value: "/usr/local/tcpxo/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib64" {{- end }} - + - name: NCCL_NET_PLUGIN_PATH + value: "/usr/local/tcpxo/lib64" + - name: NCCL_P2P_LEVEL + value: "NVL" - name: NCCL_FASTRAK_LLCM_DEVICE_DIRECTORY value: /dev/aperture_devices - name: NCCL_FASTRAK_CTRL_DEV @@ -346,42 +351,13 @@ spec: - bash - -c - | - # Point 14: Environment and NCCL Fixes for A3 Mega - export HF_HOME=/ssd - export LD_LIBRARY_PATH=/usr/local/cuda/compat/lib:/usr/local/lib/python3.12/dist-packages/torch/lib:/usr/local/lib/python3.12/dist-packages/torch_tensorrt/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/tensorrt/lib:/usr/local/tcpxo/lib64 - export NCCL_DEBUG=WARN + # locate NCCL library + export LD_PRELOAD=$(find /usr/local/lib -name "libnccl.so.2" | head -n 1) + + # Performance Tuning and NCCL Fixes for A3 Mega + ulimit -l unlimited export NCCL_NET_PLUGIN=none export NCCL_TUNER_PLUGIN=none - export NCCL_P2P_LEVEL=NVL - - # Force load the container's internal NCCL to resolve the 'ncclCommWindowDeregister' symbol mismatch - for loc in "/usr/local/lib/python3.12/dist-packages/torch/lib/libnccl.so.2" "/usr/local/lib/python3.12/dist-packages/nvidia/nccl/lib/libnccl.so.2" "/usr/local/cuda/lib64/libnccl.so.2" "/usr/lib/x86_64-linux-gnu/libnccl.so.2"; do - if [ -f "$loc" ]; then - if nm -D "$loc" 2>/dev/null | grep -q "ncclCommWindowDeregister"; then - export LD_PRELOAD="$loc" - break - fi - fi - done - - echo "--- START DEBUG INFO ---" - echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" - echo "LD_PRELOAD: ${LD_PRELOAD:-NOT_SET}" - echo "Searching for all libnccl.so.2 instances:" - find /usr -name "libnccl.so.2" 2>/dev/null | xargs ls -lh || true - echo "Checking for 'ncclCommWindowDeregister' symbol in found libraries:" - for lib in $(find /usr -name "libnccl.so.2" 2>/dev/null); do - if nm -D "$lib" 2>/dev/null | grep -q "ncclCommWindowDeregister"; then - echo " [OK] Symbol FOUND in: $lib" - else - echo " [!!] Symbol MISSING in: $lib" - fi - done - echo "Checking what libtorch_cuda.so is linked against:" - ldd /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so | grep nccl || true - echo "GPU Info:" - nvidia-smi --query-gpu=name,driver_version --format=csv || true - echo "--- END DEBUG INFO ---" trtllm-bench() { unset NCCL_P2P_LEVEL @@ -399,21 +375,11 @@ spec: if [[ "$value" == "true" ]]; then ARGS+=("--$key"); elif [[ "$value" == "false" ]]; then ARGS+=("--$key" "false"); elif [ -n "$value" ]; then ARGS+=("--$key" "$value"); else ARGS+=("--$key"); fi done < "/workload/configs/serving-args.yaml" - echo "Starting automated benchmark sequence..." - {{- $isl_list := list -}} - {{- $osl_list := list -}} - {{- $num_requests := 1024 -}} - {{- range .Values.workload.benchmarks.experiments -}} - {{- $isl_list = append $isl_list (toString .isl) -}} - {{- $osl_list = append $osl_list (toString .osl) -}} - {{- $num_requests = .num_requests -}} - {{- end -}} - {{- if $isl_list }} - "$LAUNCHER_SCRIPT" --model_name "{{ $.Values.workload.model.name }}" --isl "{{ join "," $isl_list }}" --osl "{{ join "," $osl_list }}" --num_requests {{ $num_requests }} -- "${ARGS[@]}" - {{- else }} - "$LAUNCHER_SCRIPT" --model_name "{{ $.Values.workload.model.name }}" -- "${ARGS[@]}" - {{- end }} - echo "Benchmarks complete. Job exiting." + echo "Starting benchmark..." + "$LAUNCHER_SCRIPT" --model_name "{{ .Values.workload.model.name }}" --isl "{{ $isl }}" --osl "{{ $osl }}" --num_requests "{{ $num_requests }}" -- "${ARGS[@]}" + echo "-----------------------------------------------------------" + echo "Benchmarks complete. Keeping container alive for result inspection." + sleep infinity ports: - containerPort: {{ $root.Values.trtllm.service.ports.http }} diff --git a/src/launchers/trtllm-launcher.sh b/src/launchers/trtllm-launcher.sh index 48cce187..ffd2a2a6 100644 --- a/src/launchers/trtllm-launcher.sh +++ b/src/launchers/trtllm-launcher.sh @@ -21,9 +21,6 @@ echo "TensorRT-LLM benchmark arguments received:" echo " $@" echo "" -# Define model download directory if not set -MODEL_DOWNLOAD_DIR=${MODEL_DOWNLOAD_DIR:-/ssd} - # Function to validate model name validate_model_name() { if [ -z "$MODEL_NAME" ]; then @@ -36,9 +33,9 @@ validate_model_name() { # Function to parse arguments parse_arguments() { model_name=$MODEL_NAME - isl="128" - osl="128" - num_requests=1024 + isl=128 + osl=128 + num_requests=30000 # Parse known arguments and check for unknown option or missing argument PARSED_OPTIONS=$(getopt -o "" -l model_name:,isl:,osl:,num_requests: -- "$@") @@ -122,13 +119,12 @@ parse_serving_config() { pp_size=${SERVING_CONFIG_DICT["pp_size"]:=1} ep_size=${SERVING_CONFIG_DICT["ep_size"]:=1} backend=${SERVING_CONFIG_DICT["backend"]:="tensorrt"} - kv_cache_free_gpu_mem_fraction=${SERVING_CONFIG_DICT["kv_cache_free_gpu_mem_fraction"]:=0.70} + kv_cache_free_gpu_mem_fraction=${SERVING_CONFIG_DICT["kv_cache_free_gpu_mem_fraction"]:=0.95} modality=${SERVING_CONFIG_DICT["modality"]:=""} streaming=${SERVING_CONFIG_DICT["streaming"]:="false"} max_input_len=${SERVING_CONFIG_DICT["max_input_len"]:=""} max_batch_size=${SERVING_CONFIG_DICT["max_batch_size"]:=""} custom_dataset=${SERVING_CONFIG_DICT["dataset"]:=""} - quantization=${SERVING_CONFIG_DICT["quantization"]:="FP8"} } print_configuration() { @@ -138,8 +134,8 @@ print_configuration() { echo "--------------------------------" echo "--- Parsed Arguments Summary ---" echo "model name: $model_name" - echo "input seq length(s): $isl" - echo "output seq length(s): $osl" + echo "input seq length: $isl" + echo "output seq length: $osl" echo "number of requests: $num_requests" echo "tensor parallel size: $tp_size" echo "pipeline parallel size: $pp_size" @@ -148,7 +144,6 @@ print_configuration() { echo "modality: $modality" echo "streaming: $streaming" echo "max input length: $max_input_len" - echo "quantization: $quantization" echo "max batch size: $max_batch_size" echo "kv_cache_free_gpu_mem_fraction: $kv_cache_free_gpu_mem_fraction" echo "--------------------------------" @@ -157,8 +152,7 @@ print_configuration() { download_model() { echo "Downloading model from HuggingFace... This may take a while when downloading for the first time." echo "NOTE: by default, huggingface-cli response can be verbose." - # Ensure we download into a model-specific subdirectory - huggingface-cli download $model_name --exclude="*original*" --local-dir ${MODEL_DOWNLOAD_DIR}/${model_name##*/} --local-dir-use-symlinks False + huggingface-cli download $model_name --exclude="*original*" --local-dir ${MODEL_DOWNLOAD_DIR} --local-dir-use-symlinks False } # Function to run benchmarks @@ -173,24 +167,6 @@ run_benchmark() { local backend=$8 local kv_cache_free_gpu_mem_fraction=$9 - local engine_dir="" - local generated_dataset="" - local output_file="${MODEL_DOWNLOAD_DIR}/output_${model_name##*/}_isl${isl}_osl${osl}_tp${tp_size}.txt" - local benchmark_rc=0 - - export TOKENIZERS_PARALLELISM=false - - # Emergency cleanup on crash - cleanup_on_exit() { - local exit_code=$? - if [[ $exit_code -ne 0 ]]; then - echo "Benchmark interrupted (Exit code: $exit_code). Cleaning up temporary files..." - [[ -n "${engine_dir:-}" && -d "${engine_dir:-}" ]] && rm -rf "${engine_dir}" || true - [[ -n "${generated_dataset:-}" && -f "${generated_dataset:-}" ]] && rm -f "${generated_dataset}" || true - fi - } - trap cleanup_on_exit EXIT SIGINT SIGTERM - echo "Running benchmark for $model_name with ISL=$isl, OSL=$osl, TP=$tp_size, PP=$pp_size, EP=$ep_size, backend=$backend" vl_args="" @@ -200,18 +176,12 @@ run_benchmark() { if [ -n "$max_batch_size" ]; then vl_args="$vl_args --max_batch_size $max_batch_size"; fi dataset_file=$custom_dataset - local tokenizer_arg=$model_name - if [ -d "/serving-model" ]; then - tokenizer_arg="/serving-model" - fi - - # Point 3 & 4: Tokenizer logic + # If custom_dataset is not set, generate a textual dataset with tokens sampled in normal distribution if [ -z "$dataset_file" ]; then - dataset_file="${MODEL_DOWNLOAD_DIR}/token-norm-dist_${model_name##*/}_${isl}_${osl}_tp${tp_size}.json" - generated_dataset=$dataset_file + dataset_file="/ssd/token-norm-dist_${model_name##*/}_${isl}_${osl}_tp${tp_size}.json" echo "Preparing dataset" python3 $TRTLLM_DIR/benchmarks/cpp/prepare_dataset.py \ - --tokenizer=$tokenizer_arg \ + --tokenizer=$model_name \ --stdout token-norm-dist \ --num-requests=$num_requests \ --input-mean=$isl \ @@ -220,83 +190,57 @@ run_benchmark() { --output-stdev=0 >$dataset_file fi - # Clean up any leftover extra args to prevent "invalid argument: enable_cuda_graph" - rm -f /tmp/extra_llm_api_args.yaml || true - + output_file="/ssd/output_${model_name##*/}_isl${isl}_osl${osl}_tp${tp_size}.txt" extra_args_file="/tmp/extra_llm_api_args.yaml" extra_args="" if [ -f "$extra_args_file" ]; then extra_args="--extra_llm_api_options $extra_args_file" fi - # Point 5: Determine model path - local model_path_arg="${MODEL_DOWNLOAD_DIR}/${model_name}" - if [ -d "/serving-model" ]; then - model_path_arg="/serving-model" - fi + export TOKENIZERS_PARALLELISM=false + echo "enable_cuda_graph: false" > /tmp/extra_llm_api_args.yaml - # Point 6, 7, 8: Throughput call if [[ $backend == "pytorch" ]]; then echo "Running throughput benchmark" - set +e + export NCCL_P2P_LEVEL=PHB trtllm-bench \ --model $model_name \ - --model_path $model_path_arg throughput \ + --model_path /ssd/${model_name} throughput \ --dataset $dataset_file \ --num_requests $num_requests \ --tp $tp_size \ --pp $pp_size \ --ep $ep_size \ - --backend $backend \ + --backend "pytorch" \ --kv_cache_free_gpu_mem_fraction $kv_cache_free_gpu_mem_fraction \ $extra_args $vl_args | tee "$output_file" - benchmark_rc=${PIPESTATUS[0]} - set -e else - # Point 9: Build engine + echo "Building engine" trtllm-bench \ --model $model_name \ - --model_path $model_path_arg \ - --workspace ${MODEL_DOWNLOAD_DIR} build \ + --model_path /ssd/${model_name} \ + --workspace /ssd build \ --tp_size $tp_size \ --pp_size $pp_size \ - --quantization $quantization \ + --quantization FP8 \ --dataset $dataset_file - # Point 10: Engine Dir path - engine_dir="${model_path_arg}/tp_${tp_size}_pp_${pp_size}" + engine_dir="/ssd/${model_name}/tp_${tp_size}_pp_${pp_size}" - # Point 11 & 12: Run throughput with engine + # Save throughput output to a file echo "Running throughput benchmark" - set +e trtllm-bench \ --model $model_name \ - --model_path $model_path_arg throughput \ + --model_path /ssd/${model_name} throughput \ --dataset $dataset_file \ --engine_dir $engine_dir \ - --tp $tp_size \ - --pp $pp_size \ - --ep $ep_size \ - --backend $backend \ - --kv_cache_free_gpu_mem_fraction $kv_cache_free_gpu_mem_fraction \ - $extra_args | tee "$output_file" - benchmark_rc=${PIPESTATUS[0]} - set -e + --kv_cache_free_gpu_mem_fraction $kv_cache_free_gpu_mem_fraction $extra_args | tee $output_file fi - # Point 13: Immediate sync of results to GCS - mkdir -p /gcs/benchmark_logs/trtllm || true - cp "$output_file" /gcs/benchmark_logs/trtllm/ || true - - # Standard cleanup after successful iteration - [[ -n "${engine_dir:-}" && -d "${engine_dir:-}" ]] && rm -rf "${engine_dir}" || true - [[ -n "${dataset_file:-}" ]] && rm -f "${dataset_file}" || true - trap - EXIT SIGINT SIGTERM + gcloud storage cp "$output_file" /gcs/benchmark_logs/trtllm/ || true - if [ $benchmark_rc -ne 0 ]; then - echo "Error: Benchmark command failed with exit code $benchmark_rc." - exit $benchmark_rc - fi + rm -rf $engine_dir || true + rm -f $dataset_file || true } # Main function to run the benchmark @@ -305,49 +249,20 @@ main() { validate_model_name parse_arguments "$@" parse_serving_config + print_configuration "$@" # download model download_model - # Convert comma-separated lists to arrays - # Remove spaces and split by comma - IFS=',' read -ra ISL_ARRAY <<< "${isl// /}" - IFS=',' read -ra OSL_ARRAY <<< "${osl// /}" - - # Validate array lengths match - if [[ ${#ISL_ARRAY[@]} -ne ${#OSL_ARRAY[@]} ]]; then - echo "Error: The number of values in --isl and --osl list(s) must be the same." - echo "ISL count: ${#ISL_ARRAY[@]}, OSL count: ${#OSL_ARRAY[@]}" - exit 1 - fi - - print_configuration "$@" - # run benchmark mkdir -p /gcs/benchmark_logs/trtllm echo "Running benchmarks" - for i in "${!ISL_ARRAY[@]}"; do - current_isl="${ISL_ARRAY[$i]}" - current_osl="${OSL_ARRAY[$i]}" - - echo "Starting iteration $((i+1))/${#ISL_ARRAY[@]}: ISL=$current_isl, OSL=$current_osl, REQ=$num_requests" - run_benchmark "$model_name" "$current_isl" "$current_osl" "$num_requests" $tp_size $pp_size $ep_size $backend $kv_cache_free_gpu_mem_fraction - done - - echo "-----------------------------------------------------------" - echo "Benchmarks complete. Keeping container alive for result inspection." - sleep infinity + run_benchmark "$model_name" $isl $osl $num_requests $tp_size $pp_size $ep_size $backend $kv_cache_free_gpu_mem_fraction } -# Force load the container's internal NCCL to resolve symbol mismatches on GKE -for loc in "/usr/local/lib/python3.12/dist-packages/torch/lib/libnccl.so.2" "/usr/local/lib/python3.12/dist-packages/nvidia/nccl/lib/libnccl.so.2" "/usr/local/cuda/lib64/libnccl.so.2" "/usr/lib/x86_64-linux-gnu/libnccl.so.2"; do - if [ -f "$loc" ]; then - if nm -D "$loc" 2>/dev/null | grep -q "ncclCommWindowDeregister"; then - export LD_PRELOAD="$loc" - break - fi - fi -done +# Set environment variables +export HF_HOME=/ssd +export LD_LIBRARY_PATH=/usr/local/lib/python3.12/dist-packages/torch/lib:/usr/local/lib/python3.12/dist-packages/torch_tensorrt/lib:/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/tensorrt/lib # Run the main function main "$@" \ No newline at end of file From ebc6650d90d9164faed892b32f225efefd4d898a Mon Sep 17 00:00:00 2001 From: krishnakanthankam-qt Date: Thu, 11 Jun 2026 16:28:23 +0530 Subject: [PATCH 4/9] added ld_prelaod path as env to the container --- inference/a3mega/llama3.1-70b/README.md | 10 +++++----- .../single-node/templates/model-serve-launcher.yaml | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/inference/a3mega/llama3.1-70b/README.md b/inference/a3mega/llama3.1-70b/README.md index 90c4e986..4925ce8e 100644 --- a/inference/a3mega/llama3.1-70b/README.md +++ b/inference/a3mega/llama3.1-70b/README.md @@ -297,12 +297,12 @@ Running benchmark for nvidia/Llama3.1-70b with ISL=128, OSL=128, TP=8, EP=1, PP= =========================================================== = PYTORCH BACKEND =========================================================== -Model: nvidia/Llama3.1-70b -Model Path: /ssd/nvidia/Llama3.1-70b +Model: nvidia/Llama3.1-70b +Model Path: /ssd/nvidia/Llama3.1-70b TensorRT LLM Version: 1.2 -Dtype: bfloat16 -KV Cache Dtype: FP8 -Quantization: FP8 +Dtype: bfloat16 +KV Cache Dtype: FP8 +Quantization: FP8 =========================================================== = REQUEST DETAILS diff --git a/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml b/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml index 1e566a85..5f93682b 100644 --- a/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml +++ b/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml @@ -273,6 +273,8 @@ spec: - name: LD_LIBRARY_PATH value: "/usr/local/tcpxo/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib64" {{- end }} + - name: LD_PRELOAD + value: "/usr/local/lib/python3.12/dist-packages/nvidia/nccl/lib/libnccl.so.2" - name: NCCL_NET_PLUGIN_PATH value: "/usr/local/tcpxo/lib64" - name: NCCL_P2P_LEVEL @@ -351,8 +353,6 @@ spec: - bash - -c - | - # locate NCCL library - export LD_PRELOAD=$(find /usr/local/lib -name "libnccl.so.2" | head -n 1) # Performance Tuning and NCCL Fixes for A3 Mega ulimit -l unlimited From 8bc5552f6fddbe28cab9f61da34471e274fdee4e Mon Sep 17 00:00:00 2001 From: krishnakanthankam-qt Date: Fri, 12 Jun 2026 12:47:10 +0530 Subject: [PATCH 5/9] standardize readme --- inference/a3mega/llama3.1-70b/README.md | 32 ++++++++----------- inference/a3mega/llama3.1-70b/values.yaml | 2 +- .../templates/model-serve-svc.yaml | 2 +- src/launchers/trtllm-launcher.sh | 6 ++-- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/inference/a3mega/llama3.1-70b/README.md b/inference/a3mega/llama3.1-70b/README.md index 4925ce8e..3afefd3b 100644 --- a/inference/a3mega/llama3.1-70b/README.md +++ b/inference/a3mega/llama3.1-70b/README.md @@ -202,8 +202,6 @@ kubectl create secret generic hf-secret \ This recipe supports the following models. Running TRTLLM inference benchmarking on these models are only tested and validated on A3-Mega GKE nodes with certain combination of TP, PP, EP, number of GPU chips, input & output sequence length, precision, etc. -Example model configuration YAML files included in this repo only show a certain combination of parallelism hyperparameters and configs for benchmarking purposes. Input and output length in `/home/akrishnakanth/gpu-recipes/inference/a3mega/llama3.1-70b/trtllm-gke/values.yaml` need to be adjusted according to the model and its configs. - As the PyTorch backend requires pre-quantized models for optimal performance, we use the FP8 quantized version for Llama 3.1 70B. | Model Name | Hugging Face ID | Configuration File | Release Name Suffix | @@ -211,7 +209,7 @@ As the PyTorch backend requires pre-quantized models for optimal performance, we | **Llama 3.1 70B (FP8)** | `nvidia/Llama-3.1-70B-Instruct-FP8` | `llama-3.1-70b.yaml` | `llama-3-1-70b` | > [!TIP] -> You can use the NVIDIA Model Optimizer to quantize these models to FP8 for improved performance. +> You can use the NVIDIA Model Optimizer to quantize these models to FP8. ### 4.2. Deploy and Benchmark a Model @@ -247,8 +245,8 @@ The recipe uses [`trtllm-bench`](https://github.com/NVIDIA/TensorRT-LLM/blob/mai ${RELEASE_NAME} \ $REPO_ROOT/src/helm-charts/a3mega/trtllm-inference/single-node ``` - > [!NOTE] - > You can modify the benchmark configuration at runtime by changing the values for `isl`, `osl`, and `num_requests` (number of prompts) in the Helm command to test different scenarios. + > [!NOTE] + > You can modify the benchmark configuration at runtime by changing the values for `isl`, `osl`, and `num_requests` (number of prompts) in the Helm command to test different scenarios. 3. **Check the deployment status:** @@ -271,7 +269,6 @@ After the model is deployed via Helm as described in the sections [above](#run-t Check the status of your deployment. Replace the name if you deployed a different model. ```bash -# Example for Llama 3.1 70B (FP8) kubectl get deployment/$USER-serving-llama-3-1-70b ``` @@ -295,24 +292,24 @@ You should see logs indicating preparing the model, and then running the through Running benchmark for nvidia/Llama3.1-70b with ISL=128, OSL=128, TP=8, EP=1, PP=1 =========================================================== -= PYTORCH BACKEND +PYTORCH BACKEND =========================================================== -Model: nvidia/Llama3.1-70b -Model Path: /ssd/nvidia/Llama3.1-70b -TensorRT LLM Version: 1.2 -Dtype: bfloat16 -KV Cache Dtype: FP8 -Quantization: FP8 +Model: nvidia/Llama3.1-70b +Model Path: /ssd/nvidia/Llama3.1-70b +TensorRT LLM Version: 1.2 +Dtype: bfloat16 +KV Cache Dtype: FP8 +Quantization: FP8 =========================================================== -= REQUEST DETAILS +REQUEST DETAILS =========================================================== Number of requests: 1000 Number of concurrent requests: 985.9849 Average Input Length (tokens): 128.0000 Average Output Length (tokens): 128.0000 =========================================================== -= WORLD + RUNTIME INFORMATION +WORLD + RUNTIME INFORMATION =========================================================== TP Size: 8 PP Size: 1 @@ -324,7 +321,7 @@ KV Memory Percentage: 85.00% Issue Rate (req/sec): 8.3913E+13 =========================================================== -= PERFORMANCE OVERVIEW +PERFORMANCE OVERVIEW =========================================================== Request Throughput (req/sec): X.XX Total Output Throughput (tokens/sec): X.XX @@ -345,7 +342,7 @@ Per GPU Output Throughput (tps/gpu): X.XX [Latency] AVERAGE: X.XX =========================================================== -= DATASET DETAILS +DATASET DETAILS =========================================================== Dataset Path: /ssd/token-norm-dist_llama3.1-70b_128_128_tp4.json Number of Sequences: 1000 @@ -381,7 +378,6 @@ To avoid incurring further charges, clean up the resources you created. Then, uninstall the desired release: ```bash - # uninstall the deployed model helm uninstall ``` Replace `` with the helm release names listed. diff --git a/inference/a3mega/llama3.1-70b/values.yaml b/inference/a3mega/llama3.1-70b/values.yaml index 8ffb3e73..3e83c133 100644 --- a/inference/a3mega/llama3.1-70b/values.yaml +++ b/inference/a3mega/llama3.1-70b/values.yaml @@ -1,4 +1,4 @@ -# Copyright 2025 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-svc.yaml b/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-svc.yaml index 0daf7ab9..5566e370 100644 --- a/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-svc.yaml +++ b/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-svc.yaml @@ -1,4 +1,4 @@ -# Copyright 2025 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/launchers/trtllm-launcher.sh b/src/launchers/trtllm-launcher.sh index ffd2a2a6..bee444e4 100644 --- a/src/launchers/trtllm-launcher.sh +++ b/src/launchers/trtllm-launcher.sh @@ -237,10 +237,10 @@ run_benchmark() { --kv_cache_free_gpu_mem_fraction $kv_cache_free_gpu_mem_fraction $extra_args | tee $output_file fi - gcloud storage cp "$output_file" /gcs/benchmark_logs/trtllm/ || true + gcloud storage cp "$output_file" /gcs/benchmark_logs/trtllm/ - rm -rf $engine_dir || true - rm -f $dataset_file || true + rm -rf $engine_dir + rm -f $dataset_file } # Main function to run the benchmark From 8cdcfe12bb539d104613cbd24b95e2d41b66ebfb Mon Sep 17 00:00:00 2001 From: krishnakanthankam-qt Date: Fri, 12 Jun 2026 13:55:51 +0530 Subject: [PATCH 6/9] modified readme --- inference/a3mega/llama3.1-70b/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inference/a3mega/llama3.1-70b/README.md b/inference/a3mega/llama3.1-70b/README.md index 3afefd3b..222f2a67 100644 --- a/inference/a3mega/llama3.1-70b/README.md +++ b/inference/a3mega/llama3.1-70b/README.md @@ -145,7 +145,7 @@ Replace the following values: | `KUEUE_NAME` | The name of the Kueue local queue. The default queue created by the cluster toolkit is `a3mega`. Verify the name in your cluster. | `a3mega` | | `ARTIFACT_REGISTRY` | Full path to your Artifact Registry repository. | `us-central1-docker.pkg.dev/gcp-project-12345/my-repo` | | `GCS_BUCKET` | Name of your GCS bucket (do not include `gs://`). | `my-benchmark-logs-bucket` | -| `TRTLLM_VERSION` | The tag/version for the Docker image. Other verions can be found at https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tensorrt-llm/containers/release | `1.3.0rc3` | +| `TRTLLM_VERSION` | The tag/version for the Docker image. Other verions can be found at [NGC Catalog](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tensorrt-llm/containers/release) | `1.3.0rc3` | @@ -193,7 +193,7 @@ kubectl create secret generic hf-secret \ > After running the recipe with `helm install`, it can take **up to 30 minutes** for the deployment to become fully available. This is because the GKE node must first pull the Docker image and then download the model weights from Hugging Face. > [!TIP] -> You can use the [NVIDIA Model Optimizer](https://github.com/NVIDIA/Model-Optimizer/tree/main/examples/llm_ptq) to quantize these models to FP8 for improved performance. +> You can use the [NVIDIA Model Optimizer](https://github.com/NVIDIA/Model-Optimizer/tree/main/examples/llm_ptq) to quantize these models to FP8. ### 4.1. Supported Models From e1c28af906e4522ee0963de82d16f1e52f29c1c2 Mon Sep 17 00:00:00 2001 From: krishnakanthankam-qt Date: Fri, 12 Jun 2026 16:52:45 +0530 Subject: [PATCH 7/9] updated readme --- inference/a3mega/llama3.1-70b/{ => trtllm-gke}/README.md | 2 +- inference/a3mega/llama3.1-70b/{ => trtllm-gke}/values.yaml | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename inference/a3mega/llama3.1-70b/{ => trtllm-gke}/README.md (99%) rename inference/a3mega/llama3.1-70b/{ => trtllm-gke}/values.yaml (100%) diff --git a/inference/a3mega/llama3.1-70b/README.md b/inference/a3mega/llama3.1-70b/trtllm-gke/README.md similarity index 99% rename from inference/a3mega/llama3.1-70b/README.md rename to inference/a3mega/llama3.1-70b/trtllm-gke/README.md index 222f2a67..6e3c4da0 100644 --- a/inference/a3mega/llama3.1-70b/README.md +++ b/inference/a3mega/llama3.1-70b/trtllm-gke/README.md @@ -223,7 +223,7 @@ The recipe uses [`trtllm-bench`](https://github.com/NVIDIA/TensorRT-LLM/blob/mai ```bash # Example for Llama 3.1 70B (FP8) export HF_MODEL_ID="nvidia/Llama-3.1-70B-Instruct-FP8" - export CONFIG_FILE="llama-3.1-70b.yaml" + export CONFIG_FILE="llama3.1-70b.yaml" export RELEASE_NAME="$USER-serving-llama-3-1-70b" ``` diff --git a/inference/a3mega/llama3.1-70b/values.yaml b/inference/a3mega/llama3.1-70b/trtllm-gke/values.yaml similarity index 100% rename from inference/a3mega/llama3.1-70b/values.yaml rename to inference/a3mega/llama3.1-70b/trtllm-gke/values.yaml From caba8324478c5cdff8d5bdf9f6cc3e72ff401110 Mon Sep 17 00:00:00 2001 From: krishnakanthankam-qt Date: Fri, 12 Jun 2026 18:06:16 +0530 Subject: [PATCH 8/9] fixed readme.md --- inference/a3mega/llama3.1-70b/trtllm-gke/README.md | 12 ++++++------ .../single-node/{chart.yaml => Chart.yaml} | 0 2 files changed, 6 insertions(+), 6 deletions(-) rename src/helm-charts/a3mega/trtllm-inference/single-node/{chart.yaml => Chart.yaml} (100%) diff --git a/inference/a3mega/llama3.1-70b/trtllm-gke/README.md b/inference/a3mega/llama3.1-70b/trtllm-gke/README.md index 6e3c4da0..009b3751 100644 --- a/inference/a3mega/llama3.1-70b/trtllm-gke/README.md +++ b/inference/a3mega/llama3.1-70b/trtllm-gke/README.md @@ -224,7 +224,7 @@ The recipe uses [`trtllm-bench`](https://github.com/NVIDIA/TensorRT-LLM/blob/mai # Example for Llama 3.1 70B (FP8) export HF_MODEL_ID="nvidia/Llama-3.1-70B-Instruct-FP8" export CONFIG_FILE="llama3.1-70b.yaml" - export RELEASE_NAME="$USER-serving-llama-3-1-70b" + export RELEASE_NAME="$USER-llama-3-1-70b" ``` 2. **Install the helm chart.** You can run a single benchmark configuration or a sequence of multiple experiments by indexing the `experiments` list. @@ -251,7 +251,7 @@ The recipe uses [`trtllm-bench`](https://github.com/NVIDIA/TensorRT-LLM/blob/mai 3. **Check the deployment status:** ```bash - kubectl get deployment/${RELEASE_NAME} + kubectl get deployment/${RELEASE_NAME}-serving ``` Wait until the `READY` column shows `1/1`. See the [Monitoring and Troubleshooting](#monitoring) section to view the deployment logs. @@ -261,7 +261,7 @@ The recipe uses [`trtllm-bench`](https://github.com/NVIDIA/TensorRT-LLM/blob/mai [Back to Top](#table-of-contents) -After the model is deployed via Helm as described in the sections [above](#run-the-recipe), use the following steps to monitor the deployment and interact with the model. Replace `` and `` with the appropriate names from the model-specific deployment instructions (e.g., `$USER-serving-llama3.1-70b` and `$USER-serving-llama3.1-70b-svc`). +After the model is deployed via Helm as described in the sections [above](#run-the-recipe), use the following steps to monitor the deployment and interact with the model. Replace `` and `` with the appropriate names from the model-specific deployment instructions (e.g., `$USER-llama3.1-70b` and `$USER-llama3.1-70b-svc`). ### 5.1. Check Deployment Status @@ -269,7 +269,7 @@ After the model is deployed via Helm as described in the sections [above](#run-t Check the status of your deployment. Replace the name if you deployed a different model. ```bash -kubectl get deployment/$USER-serving-llama-3-1-70b +kubectl get deployment/$RELEASE_NAME-serving ``` Wait until the `READY` column shows `1/1`. If it shows `0/1`, the pod is still starting up. @@ -283,7 +283,7 @@ Wait until the `READY` column shows `1/1`. If it shows `0/1`, the pod is still s To see the logs from the TRTLLM server (useful for debugging), use the `-f` flag to follow the log stream: ```bash -kubectl logs -f deployment/$USER-serving-llama3.1-70b +kubectl logs -f deployment/$RELEASE_NAME-serving ``` You should see logs indicating preparing the model, and then running the throughput benchmark test, similar to this: @@ -372,7 +372,7 @@ To avoid incurring further charges, clean up the resources you created. ```bash # list deployed models - helm list --filter $USER-serving- + helm list --filter $USER ``` Then, uninstall the desired release: diff --git a/src/helm-charts/a3mega/trtllm-inference/single-node/chart.yaml b/src/helm-charts/a3mega/trtllm-inference/single-node/Chart.yaml similarity index 100% rename from src/helm-charts/a3mega/trtllm-inference/single-node/chart.yaml rename to src/helm-charts/a3mega/trtllm-inference/single-node/Chart.yaml From 58b87deb832e49f58804999b9a662a2814e8e364 Mon Sep 17 00:00:00 2001 From: krishnakanthankam-qt Date: Fri, 19 Jun 2026 11:58:11 +0530 Subject: [PATCH 9/9] updated llama3.1-70b recipe --- .../a3mega/llama3.1-70b/trtllm-gke/README.md | 8 +++---- .../llama3.1-70b/trtllm-gke/values.yaml | 8 +++---- .../trtllm-inference/single-node/Chart.yaml | 2 +- .../templates/model-serve-launcher.yaml | 23 +++++++++++-------- src/launchers/trtllm-launcher.sh | 2 +- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/inference/a3mega/llama3.1-70b/trtllm-gke/README.md b/inference/a3mega/llama3.1-70b/trtllm-gke/README.md index 009b3751..cda6e6a4 100644 --- a/inference/a3mega/llama3.1-70b/trtllm-gke/README.md +++ b/inference/a3mega/llama3.1-70b/trtllm-gke/README.md @@ -206,7 +206,7 @@ As the PyTorch backend requires pre-quantized models for optimal performance, we | Model Name | Hugging Face ID | Configuration File | Release Name Suffix | | :--- | :--- | :--- | :--- | -| **Llama 3.1 70B (FP8)** | `nvidia/Llama-3.1-70B-Instruct-FP8` | `llama-3.1-70b.yaml` | `llama-3-1-70b` | +| **Llama 3.1 70B (FP8)** | `nvidia/Llama-3.1-70B-Instruct-FP8` | `llama3.1-70b.yaml` | `llama-3-1-70b` | > [!TIP] > You can use the NVIDIA Model Optimizer to quantize these models to FP8. @@ -294,8 +294,8 @@ Running benchmark for nvidia/Llama3.1-70b with ISL=128, OSL=128, TP=8, EP=1, PP= =========================================================== PYTORCH BACKEND =========================================================== -Model: nvidia/Llama3.1-70b -Model Path: /ssd/nvidia/Llama3.1-70b +Model: nvidia/Llama-3.1-70B-Instruct-FP8 +Model Path: /ssd/nvidia/Llama-3.1-70B-Instruct-FP8 TensorRT LLM Version: 1.2 Dtype: bfloat16 KV Cache Dtype: FP8 @@ -344,7 +344,7 @@ Per GPU Output Throughput (tps/gpu): X.XX =========================================================== DATASET DETAILS =========================================================== -Dataset Path: /ssd/token-norm-dist_llama3.1-70b_128_128_tp4.json +Dataset Path: /ssd/token-norm-dist_llama3.1-70b_128_128_tp8.json Number of Sequences: 1000 -- Percentiles statistics --------------------------------- diff --git a/inference/a3mega/llama3.1-70b/trtllm-gke/values.yaml b/inference/a3mega/llama3.1-70b/trtllm-gke/values.yaml index 3e83c133..71635b3d 100644 --- a/inference/a3mega/llama3.1-70b/trtllm-gke/values.yaml +++ b/inference/a3mega/llama3.1-70b/trtllm-gke/values.yaml @@ -35,6 +35,8 @@ workload: - isl: 128 osl: 128 num_requests: 1000 + concurrency: 128 + max_seq_len: 32768 volumes: gcsVolumes: true @@ -61,8 +63,4 @@ trtllm: service: type: ClusterIP ports: - http: 8000 - serverArgs: - max-model-len: 32768 - max-num-seqs: 128 - gpu-memory-utilization: 0.90 \ No newline at end of file + http: 8000 \ No newline at end of file diff --git a/src/helm-charts/a3mega/trtllm-inference/single-node/Chart.yaml b/src/helm-charts/a3mega/trtllm-inference/single-node/Chart.yaml index e5835b43..4bebfbfc 100644 --- a/src/helm-charts/a3mega/trtllm-inference/single-node/Chart.yaml +++ b/src/helm-charts/a3mega/trtllm-inference/single-node/Chart.yaml @@ -1,4 +1,4 @@ -# Copyright 2025 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml b/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml index 5f93682b..cc117919 100644 --- a/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml +++ b/src/helm-charts/a3mega/trtllm-inference/single-node/templates/model-serve-launcher.yaml @@ -12,6 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- /* Extract benchmark parameters from Helm values */ -}} +{{- $experiment := index .Values.workload.benchmarks.experiments 0 -}} +{{- $isl := $experiment.isl -}} +{{- $osl := $experiment.osl -}} +{{- $concurrency := $experiment.concurrency | default "" -}} +{{- $max_seq_len := $experiment.max_seq_len | default "32768" -}} +{{- $num_requests := $experiment.num_requests -}} + # --- CONFIGMAPS --- apiVersion: v1 kind: ConfigMap @@ -48,12 +56,6 @@ data: {{ $nodes := div .Values.workload.gpus 8 | max 1 }} {{ $gpusPerNode := min .Values.workload.gpus 8 }} -{{- /* Extract benchmark parameters from Helm values */ -}} -{{- $experiment := index .Values.workload.benchmarks.experiments 0 -}} -{{- $isl := $experiment.isl -}} -{{- $osl := $experiment.osl -}} -{{- $num_requests := $experiment.num_requests -}} - {{- $root := . -}} apiVersion: apps/v1 @@ -343,6 +345,8 @@ spec: value: "1" - name: NCCL_GPUVIZ_FILE_ROTATION_INTERVAL_IN_SECONDS value: "300" + - name: TLLM_NUMA_AWARE_WORKER_AFFINITY + value: "1" {{- range $environment_variable := $root.Values.network.ncclSettings }} - name: {{ $environment_variable.name }} @@ -361,7 +365,7 @@ spec: trtllm-bench() { unset NCCL_P2P_LEVEL - command trtllm-bench "$@" + command trtllm-bench "$@" --concurrency {{ $concurrency }} --max_seq_len {{ $max_seq_len }} } export -f trtllm-bench @@ -376,10 +380,9 @@ spec: done < "/workload/configs/serving-args.yaml" echo "Starting benchmark..." - "$LAUNCHER_SCRIPT" --model_name "{{ .Values.workload.model.name }}" --isl "{{ $isl }}" --osl "{{ $osl }}" --num_requests "{{ $num_requests }}" -- "${ARGS[@]}" + exec "$LAUNCHER_SCRIPT" --model_name "{{ .Values.workload.model.name }}" --isl "{{ $isl }}" --osl "{{ $osl }}" --num_requests "{{ $num_requests }}" -- "${ARGS[@]}" echo "-----------------------------------------------------------" - echo "Benchmarks complete. Keeping container alive for result inspection." - sleep infinity + echo "Benchmarks complete." ports: - containerPort: {{ $root.Values.trtllm.service.ports.http }} diff --git a/src/launchers/trtllm-launcher.sh b/src/launchers/trtllm-launcher.sh index bee444e4..dbdd6cfc 100644 --- a/src/launchers/trtllm-launcher.sh +++ b/src/launchers/trtllm-launcher.sh @@ -237,7 +237,7 @@ run_benchmark() { --kv_cache_free_gpu_mem_fraction $kv_cache_free_gpu_mem_fraction $extra_args | tee $output_file fi - gcloud storage cp "$output_file" /gcs/benchmark_logs/trtllm/ + gcloud storage cp $output_file /gcs/benchmark_logs/trtllm/ rm -rf $engine_dir rm -f $dataset_file