llmaz/docs/examples/serverless/basic.yaml at 66faed7eb621aaf957e687a114534f45d265c45f · InftyAI/llmaz · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
apiVersion: llmaz.io/v1alpha1
kind: OpenModel
metadata:
  name: qwen2-0--5b
spec:
  familyName: qwen2
  source:
    modelHub:
      modelID: Qwen/Qwen2-0.5B-Instruct-GGUF
      filename: qwen2-0_5b-instruct-q5_k_m.gguf
---
apiVersion: inference.llmaz.io/v1alpha1
kind: Playground
metadata:
  name: qwen2-0--5b
spec:
  replicas: 0
  modelClaim:
    modelName: qwen2-0--5b
  backendRuntimeConfig:
    backendName: llamacpp
    configName: default
    args:
      - -fa # use flash attention
---
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
  name: default-envoy-ai-gateway
spec:
  controllerName: gateway.envoyproxy.io/gatewayclass-controller
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: default-envoy-ai-gateway
spec:
  gatewayClassName: default-envoy-ai-gateway
  listeners:
    - name: http
      protocol: HTTP
      port: 80
---
apiVersion: aigateway.envoyproxy.io/v1alpha1
kind: AIGatewayRoute
metadata:
  name: default-envoy-ai-gateway
spec:
  schema:
    name: OpenAI
  targetRefs:
    - name: default-envoy-ai-gateway
      kind: Gateway
      group: gateway.networking.k8s.io
  rules:
    - matches:
        - headers:
            - type: Exact
              name: x-ai-eg-model
              value: qwen2-0--5b
      backendRefs:
        - name: qwen2-0--5b
---
apiVersion: aigateway.envoyproxy.io/v1alpha1
kind: AIServiceBackend
metadata:
  name: qwen2-0--5b
spec:
  timeouts:
    request: 3m
  schema:
    name: OpenAI
  backendRef:
    name: qwen2-0--5b-lb
    kind: Service
    port: 8080