Merge pull request #66 from GPS-Solutions/main

syncing aica to css

Author: Robert Koehlmoos

components/llm_service/src/config/models.json

@@ -153,7 +153,7 @@
         "temperature": 0.2,
         "top_p": 0.95,
         "top_k": 40,
-        "max_length": 2048
+        "max_tokens": 2048
       }
     },
     "VertexAI-ModelGarden-LLAMA2-Chat": {

experimental/vllm_gemma/README.md

@@ -1,7 +1,38 @@
 # Deploying Gemma 2B
+Reference: https://cloud.google.com/kubernetes-engine/docs/tutorials/serve-gemma-gpu-vllm
 
 ## Pre-Requisites
 Kubernetes cluster with L4 GPUs nodepool
+```shell
+export CLUSTER_NAME="main-cluster"
+export REGION="us-central1"
+gcloud container node-pools create gpu-node-pool \
+  --accelerator type=nvidia-l4,count=2,gpu-driver-version=latest \
+  --project=${PROJECT_ID} \
+  --location=${REGION} \
+  --node-locations=${REGION}-a \
+  --cluster=${CLUSTER_NAME} \
+  --service-account gke-sa@${PROJECT_ID}.iam.gserviceaccount.com \
+  --machine-type=g2-standard-24 \
+  --disk-type pd-balanced \
+  --disk-size 100 \
+  --num-nodes=1
+
+gcloud container node-pools list --region=${REGION} --cluster=${CLUSTER_NAME}
+```
+
+
+## HuggingFace API Token
+```shell
+export HF_TOKEN=...
+```
+Create secret:
+```shell
+kubectl create secret generic hf-secret \
+  --from-literal=hf_api_token=$HF_TOKEN \
+  --dry-run=client -o yaml | kubectl apply -f -
+kubectl describe secret hf-secret
+```
 
 ## Deployment
 Deploy Gemma 2B LLM using `kubectl`

experimental/vllm_gemma/vllm-gemma-2b-it.yaml

@@ -11,13 +11,13 @@ spec:
     metadata:
       labels:
         app: gemma-server
-        ai.gke.io/model: gemma-2b-it
+        ai.gke.io/model: gemma-1.1-2b-it
         ai.gke.io/inference-server: vllm
         examples.ai.gke.io/source: user-guide
     spec:
       containers:
       - name: inference-server
-        image: us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20240220_0936_RC01
+        image: us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20240527_0916_RC00
         resources:
           requests:
             cpu: "2"
@@ -61,5 +61,5 @@ spec:
   type: ClusterIP
   ports:
     - protocol: TCP
-      port: 8000
+      port: 80
       targetPort: 8000