diff --git a/Dockerfile.lmes-job b/Dockerfile.lmes-job index 50f36063..0cee7068 100644 --- a/Dockerfile.lmes-job +++ b/Dockerfile.lmes-job @@ -8,6 +8,7 @@ WORKDIR /opt/app-root/src RUN mkdir /opt/app-root/src/hf_home && chmod g+rwx /opt/app-root/src/hf_home RUN mkdir /opt/app-root/src/output && chmod g+rwx /opt/app-root/src/output RUN mkdir /opt/app-root/src/my_tasks && chmod g+rwx /opt/app-root/src/my_tasks +RUN mkdir -p /opt/app-root/src/my_catalogs/cards && chmod -R g+rwx /opt/app-root/src/my_catalogs RUN mkdir /opt/app-root/src/.cache ENV PATH="/opt/app-root/bin:/opt/app-root/src/.local/bin/:/opt/app-root/src/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" @@ -23,6 +24,7 @@ RUN python -c 'from lm_eval.tasks.unitxt import task; import os.path; print("cla ENV PYTHONPATH=/opt/app-root/src/.local/lib/python3.11/site-packages:/opt/app-root/src/lm-evaluation-harness:/opt/app-root/src:/opt/app-root/src/server ENV HF_HOME=/opt/app-root/src/hf_home +ENV UNITXT_ARTIFACTORIES=/opt/app-root/src/my_catalogs CMD ["/opt/app-root/bin/python"] diff --git a/api/lmes/v1alpha1/lmevaljob_types.go b/api/lmes/v1alpha1/lmevaljob_types.go index 55ec8f04..e1c26291 100644 --- a/api/lmes/v1alpha1/lmevaljob_types.go +++ b/api/lmes/v1alpha1/lmevaljob_types.go @@ -63,22 +63,15 @@ type Arg struct { Value string `json:"value,omitempty"` } -type EnvSecret struct { - // Environment's name - Env string `json:"env"` - // The secret is from a secret object +type Card struct { + // Unitxt card's ID // +optional - SecretRef *corev1.SecretKeySelector `json:"secretRef,omitempty"` - // The secret is from a plain text + Name string `json:"name,omitempty"` + // A JSON string for a custom unitxt card which contains the custom dataset. + // Use the documentation here: https://www.unitxt.ai/en/latest/docs/adding_dataset.html#adding-to-the-catalog + // to compose a custom card, store it as a JSON file, and use the JSON content as the value here. // +optional - Secret *string `json:"secret,omitempty"` -} - -type FileSecret struct { - // The secret object - SecretRef corev1.SecretVolumeSource `json:"secretRef,omitempty"` - // The path to mount the secret - MountPath string `json:"mountPath"` + Custom string `json:"custom,omitempty"` } // Use a task recipe to form a custom task. It maps to the Unitxt Recipe @@ -86,7 +79,7 @@ type FileSecret struct { // https://www.unitxt.ai/en/latest/unitxt.standard.html#unitxt.standard.StandardRecipe type TaskRecipe struct { // The Unitxt dataset card - Card string `json:"card"` + Card Card `json:"card"` // The Unitxt template Template string `json:"template"` // The Unitxt Task @@ -118,7 +111,7 @@ type TaskList struct { func (t *TaskRecipe) String() string { var b strings.Builder - b.WriteString(fmt.Sprintf("card=%s,template=%s", t.Card, t.Template)) + b.WriteString(fmt.Sprintf("card=%s,template=%s", t.Card.Name, t.Template)) if t.Task != nil { b.WriteString(fmt.Sprintf(",task=%s", *t.Task)) } @@ -140,6 +133,76 @@ func (t *TaskRecipe) String() string { return b.String() } +type LMEvalContainer struct { + // Define Env information for the main container + // +optional + Env []corev1.EnvVar `json:"env,omitempty"` + // Define the volume mount information + // +optional + VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"` + // Compute Resources required by this container. + // More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + // +optional + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` +} + +// The following Getter-ish functions avoid nil pointer panic +func (c *LMEvalContainer) GetEnv() []corev1.EnvVar { + if c == nil { + return nil + } + return c.Env +} + +func (c *LMEvalContainer) GetVolumMounts() []corev1.VolumeMount { + if c == nil { + return nil + } + return c.VolumeMounts +} + +func (c *LMEvalContainer) GetResources() *corev1.ResourceRequirements { + if c == nil { + return nil + } + return c.Resources +} + +type LMEvalPodSpec struct { + // Extra container data for the lm-eval container + // +optional + Container *LMEvalContainer `json:"container,omitempty"` + // Specify the volumes information for the lm-eval and sidecar containers + // +optional + Volumes []corev1.Volume `json:"volumes,omitempty"` + // Specify extra containers for the lm-eval job + // FIXME: aggregate the sidecar containers into the pod + // +optional + SideCars []corev1.Container `json:"sideCars,omitempty"` +} + +// The following Getter-ish functions avoid nil pointer panic +func (p *LMEvalPodSpec) GetContainer() *LMEvalContainer { + if p == nil { + return nil + } + return p.Container +} + +func (p *LMEvalPodSpec) GetVolumes() []corev1.Volume { + if p == nil { + return nil + } + return p.Volumes +} + +func (p *LMEvalPodSpec) GetSideCards() []corev1.Container { + if p == nil { + return nil + } + return p.SideCars +} + // LMEvalJobSpec defines the desired state of LMEvalJob type LMEvalJobSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster @@ -167,14 +230,12 @@ type LMEvalJobSpec struct { // model, will be saved at per-document granularity // +optional LogSamples *bool `json:"logSamples,omitempty"` - // Assign secrets to the environment variables - // +optional - EnvSecrets []EnvSecret `json:"envSecrets,omitempty"` - // Use secrets as files - FileSecrets []FileSecret `json:"fileSecrets,omitempty"` // Batch size for the evaluation. This is used by the models that run and are loaded // locally and not apply for the commercial APIs. BatchSize *int `json:"batchSize,omitempty"` + // Specify extra information for the lm-eval job's pod + // +optional + Pod *LMEvalPodSpec `json:"pod,omitempty"` } // LMEvalJobStatus defines the observed state of LMEvalJob diff --git a/api/lmes/v1alpha1/zz_generated.deepcopy.go b/api/lmes/v1alpha1/zz_generated.deepcopy.go index 52d5cd3b..0c9b1528 100644 --- a/api/lmes/v1alpha1/zz_generated.deepcopy.go +++ b/api/lmes/v1alpha1/zz_generated.deepcopy.go @@ -42,42 +42,50 @@ func (in *Arg) DeepCopy() *Arg { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EnvSecret) DeepCopyInto(out *EnvSecret) { +func (in *Card) DeepCopyInto(out *Card) { *out = *in - if in.SecretRef != nil { - in, out := &in.SecretRef, &out.SecretRef - *out = new(v1.SecretKeySelector) - (*in).DeepCopyInto(*out) - } - if in.Secret != nil { - in, out := &in.Secret, &out.Secret - *out = new(string) - **out = **in - } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvSecret. -func (in *EnvSecret) DeepCopy() *EnvSecret { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Card. +func (in *Card) DeepCopy() *Card { if in == nil { return nil } - out := new(EnvSecret) + out := new(Card) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *FileSecret) DeepCopyInto(out *FileSecret) { +func (in *LMEvalContainer) DeepCopyInto(out *LMEvalContainer) { *out = *in - in.SecretRef.DeepCopyInto(&out.SecretRef) + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.VolumeMounts != nil { + in, out := &in.VolumeMounts, &out.VolumeMounts + *out = make([]v1.VolumeMount, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileSecret. -func (in *FileSecret) DeepCopy() *FileSecret { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LMEvalContainer. +func (in *LMEvalContainer) DeepCopy() *LMEvalContainer { if in == nil { return nil } - out := new(FileSecret) + out := new(LMEvalContainer) in.DeepCopyInto(out) return out } @@ -165,25 +173,16 @@ func (in *LMEvalJobSpec) DeepCopyInto(out *LMEvalJobSpec) { *out = new(bool) **out = **in } - if in.EnvSecrets != nil { - in, out := &in.EnvSecrets, &out.EnvSecrets - *out = make([]EnvSecret, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.FileSecrets != nil { - in, out := &in.FileSecrets, &out.FileSecrets - *out = make([]FileSecret, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } if in.BatchSize != nil { in, out := &in.BatchSize, &out.BatchSize *out = new(int) **out = **in } + if in.Pod != nil { + in, out := &in.Pod, &out.Pod + *out = new(LMEvalPodSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LMEvalJobSpec. @@ -219,6 +218,40 @@ func (in *LMEvalJobStatus) DeepCopy() *LMEvalJobStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LMEvalPodSpec) DeepCopyInto(out *LMEvalPodSpec) { + *out = *in + if in.Container != nil { + in, out := &in.Container, &out.Container + *out = new(LMEvalContainer) + (*in).DeepCopyInto(*out) + } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]v1.Volume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.SideCars != nil { + in, out := &in.SideCars, &out.SideCars + *out = make([]v1.Container, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LMEvalPodSpec. +func (in *LMEvalPodSpec) DeepCopy() *LMEvalPodSpec { + if in == nil { + return nil + } + out := new(LMEvalPodSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TaskList) DeepCopyInto(out *TaskList) { *out = *in @@ -249,6 +282,7 @@ func (in *TaskList) DeepCopy() *TaskList { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TaskRecipe) DeepCopyInto(out *TaskRecipe) { *out = *in + out.Card = in.Card if in.Task != nil { in, out := &in.Task, &out.Task *out = new(string) diff --git a/cmd/lmes_driver/main.go b/cmd/lmes_driver/main.go index 78224b0b..e7ba3437 100644 --- a/cmd/lmes_driver/main.go +++ b/cmd/lmes_driver/main.go @@ -37,20 +37,21 @@ const ( OutputPath = "/opt/app-root/src/output" ) -type taskRecipeArg []string +type strArrayArg []string -func (t *taskRecipeArg) Set(value string) error { +func (t *strArrayArg) Set(value string) error { *t = append(*t, value) return nil } -func (t *taskRecipeArg) String() string { +func (t *strArrayArg) String() string { // supposedly, use ":" as the separator for task recipe should be safe return strings.Join(*t, ":") } var ( - taskRecipes taskRecipeArg + taskRecipes strArrayArg + customCards strArrayArg copy = flag.String("copy", "", "copy this binary to specified destination path") jobNameSpace = flag.String("job-namespace", "", "Job's namespace ") jobName = flag.String("job-name", "", "Job's name") @@ -64,6 +65,7 @@ var ( func init() { flag.Var(&taskRecipes, "task-recipe", "task recipe") + flag.Var(&customCards, "custom-card", "A JSON string represents a custom card") } func main() { @@ -105,6 +107,7 @@ func main() { DetectDevice: *detectDevice, Logger: driverLog, TaskRecipes: taskRecipes, + CustomCards: customCards, Args: args, ReportInterval: *reportInterval, } diff --git a/cmd/lmes_driver/main_test.go b/cmd/lmes_driver/main_test.go index 4b8347fc..723f3ca1 100644 --- a/cmd/lmes_driver/main_test.go +++ b/cmd/lmes_driver/main_test.go @@ -60,7 +60,7 @@ func Test_ArgParsing(t *testing.T) { assert.Equal(t, "/opt/app-root/src/output", *outputPath) assert.Equal(t, true, *detectDevice) assert.Equal(t, time.Second*10, *reportInterval) - assert.Equal(t, taskRecipeArg{ + assert.Equal(t, strArrayArg{ "card=unitxt.card1,template=unitxt.template,metrics=[unitxt.metric1,unitxt.metric2],format=unitxt.format,num_demos=5,demos_pool_size=10", "card=unitxt.card2,template=unitxt.template2,metrics=[unitxt.metric3,unitxt.metric4],format=unitxt.format,num_demos=5,demos_pool_size=10", }, taskRecipes) diff --git a/config/crd/bases/trustyai.opendatahub.io_lmevaljobs.yaml b/config/crd/bases/trustyai.opendatahub.io_lmevaljobs.yaml index a67eece8..84fcbb62 100644 --- a/config/crd/bases/trustyai.opendatahub.io_lmevaljobs.yaml +++ b/config/crd/bases/trustyai.opendatahub.io_lmevaljobs.yaml @@ -44,113 +44,6 @@ spec: that run and are loaded locally and not apply for the commercial APIs. type: integer - envSecrets: - description: Assign secrets to the environment variables - items: - properties: - env: - description: Environment's name - type: string - secret: - description: The secret is from a plain text - type: string - secretRef: - description: The secret is from a secret object - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must - be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - required: - - env - type: object - type: array - fileSecrets: - description: Use secrets as files - items: - properties: - mountPath: - description: The path to mount the secret - type: string - secretRef: - description: The secret object - properties: - defaultMode: - description: 'defaultMode is Optional: mode bits used to - set permissions on created files by default. Must be an - octal value between 0000 and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and decimal values, - JSON requires decimal values for mode bits. Defaults to - 0644. Directories within the path are not affected by - this setting. This might be in conflict with other options - that affect the file mode, like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - items: - description: items If unspecified, each key-value pair in - the Data field of the referenced Secret will be projected - into the volume as a file whose name is the key and content - is the value. If specified, the listed keys will be projected - into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in - the Secret, the volume setup will error unless it is marked - optional. Paths must be relative and may not contain the - '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to - set permissions on this file. Must be an octal value - between 0000 and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and decimal values, - JSON requires decimal values for mode bits. If not - specified, the volume defaultMode will be used. - This might be in conflict with other options that - affect the file mode, like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of the file - to map the key to. May not be an absolute path. - May not contain the path element '..'. May not start - with the string '..'. - type: string - required: - - key - - path - type: object - type: array - optional: - description: optional field specify whether the Secret or - its keys must be defined - type: boolean - secretName: - description: 'secretName is the name of the secret in the - pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - type: string - type: object - required: - - mountPath - type: object - type: array genArgs: description: Map to `--gen_kwargs` parameter for the underlying library. items: @@ -191,6 +84,3119 @@ spec: numFewShot: description: Sets the number of few-shot examples to place in context type: integer + pod: + description: Specify extra information for the lm-eval job's pod + properties: + container: + description: Extra container data for the lm-eval container + properties: + env: + description: Define Env information for the main container + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables + in the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. Double $$ are + reduced to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + resources: + description: 'Compute Resources required by this container. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. \n This field + is immutable. It can only be set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry + in pod.spec.resourceClaims of the Pod where this + field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of + compute resources required. If Requests is omitted for + a container, it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + volumeMounts: + description: Define the volume mount information + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: Path within the container at which the + volume should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts + are propagated from the host to container and the + other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's + root). + type: string + subPathExpr: + description: Expanded path within the volume from which + the container's volume should be mounted. Behaves + similarly to SubPath but environment variable references + $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). SubPathExpr and SubPath + are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + type: object + sideCars: + description: 'Specify extra containers for the lm-eval job FIXME: + aggregate the sidecar containers into the pod' + items: + description: A single application container that you want to + run within a pod. + properties: + args: + description: 'Arguments to the entrypoint. The container + image''s CMD is used if this is not provided. Variable + references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the reference + in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) + syntax: i.e. "$$(VAR_NAME)" will produce the string literal + "$(VAR_NAME)". Escaped references will never be expanded, + regardless of whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within a shell. + The container image''s ENTRYPOINT is used if this is not + provided. Variable references $(VAR_NAME) are expanded + using the container''s environment. If a variable cannot + be resolved, the reference in the input string will be + unchanged. Double $$ are reduced to a single $, which + allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". Escaped + references will never be expanded, regardless of whether + the variable exists or not. Cannot be updated. More info: + https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in the + container. Cannot be updated. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are + expanded using the previously defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, the + reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". Escaped + references will never be expanded, regardless of + whether the variable exists or not. Defaults to + "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment variables + in the container. The keys defined within a source must + be a C_IDENTIFIER. All invalid keys will be reported as + an event when the container is starting. When a key exists + in multiple sources, the value associated with the last + source will take precedence. Values defined by an Env + with a duplicate key will take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source of a + set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + description: 'Container image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management + to default or override container images in workload controllers + like Deployments and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent + otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system should take + in response to container lifecycle events. Cannot be updated. + properties: + postStart: + description: 'PostStart is called immediately after + a container is created. If the handler fails, the + container is terminated and restarted according to + its restart policy. Other management of the container + blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') in + the container's filesystem. The command is + simply exec'd, it is not run inside a shell, + so traditional shell instructions ('|', etc) + won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is + treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to + the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward + compatibility. There are no validation of this + field and lifecycle hooks will fail in runtime + when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before a + container is terminated due to an API request or management + event such as liveness/startup probe failure, preemption, + resource contention, etc. The handler is not called + if the container crashes or exits. The Pod''s termination + grace period countdown begins before the PreStop hook + is executed. Regardless of the outcome of the handler, + the container will eventually terminate within the + Pod''s termination grace period (unless delayed by + finalizers). Other management of the container blocks + until the hook completes or until the termination + grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') in + the container's filesystem. The command is + simply exec'd, it is not run inside a shell, + so traditional shell instructions ('|', etc) + won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is + treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to + the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward + compatibility. There are no validation of this + field and lifecycle hooks will fail in runtime + when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. Container + will be restarted if the probe fails. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's + filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you need + to explicitly call out to that shell. Exit status + of 0 is treated as live/healthy and non-zero is + unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe + to be considered failed after having succeeded. Defaults + to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC + port. This is a beta field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service + to place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in + httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the + host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe + to be considered successful after having failed. Defaults + to 1. Must be 1 for liveness and startup. Minimum + value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and + the time when the processes are forcibly halted with + a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, + the pod's terminationGracePeriodSeconds will be used. + Otherwise, this value overrides the value provided + by the pod spec. Value must be non-negative integer. + The value zero indicates stop immediately via the + kill signal (no opportunity to shut down). This is + a beta field and requires enabling ProbeTerminationGracePeriod + feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: List of ports to expose from the container. + Not specifying a port here DOES NOT prevent that port + from being exposed. Any port which is listening on the + default "0.0.0.0" address inside a container will be accessible + from the network. Modifying this array with strategic + merge patch may corrupt the data. For more information + See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in + a single container. + properties: + containerPort: + description: Number of port to expose on the pod's + IP address. This must be a valid port number, 0 + < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port + to. + type: string + hostPort: + description: Number of port to expose on the host. + If specified, this must be a valid port number, + 0 < x < 65536. If HostNetwork is specified, this + must match ContainerPort. Most containers do not + need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in a + pod must have a unique name. Name for the port that + can be referred to by services. + type: string + protocol: + default: TCP + description: Protocol for port. Must be UDP, TCP, + or SCTP. Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: 'Periodic probe of container service readiness. + Container will be removed from service endpoints if the + probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's + filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you need + to explicitly call out to that shell. Exit status + of 0 is treated as live/healthy and non-zero is + unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe + to be considered failed after having succeeded. Defaults + to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC + port. This is a beta field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service + to place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in + httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the + host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe + to be considered successful after having failed. Defaults + to 1. Must be 1 for liveness and startup. Minimum + value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and + the time when the processes are forcibly halted with + a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, + the pod's terminationGracePeriodSeconds will be used. + Otherwise, this value overrides the value provided + by the pod spec. Value must be non-negative integer. + The value zero indicates stop immediately via the + kill signal (no opportunity to shut down). This is + a beta field and requires enabling ProbeTerminationGracePeriod + feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resources: + description: 'Compute Resources required by this container. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. \n This field + is immutable. It can only be set for containers." + items: + description: ResourceClaim references one entry in + PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry + in pod.spec.resourceClaims of the Pod where + this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of + compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + securityContext: + description: 'SecurityContext defines the security options + the container should be run with. If set, the fields of + SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether + a process can gain more privileges than its parent + process. This bool directly controls if the no_new_privs + flag will be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as Privileged + 2) has CAP_SYS_ADMIN Note that this field cannot be + set when spec.os.name is windows.' + type: boolean + capabilities: + description: The capabilities to add/drop when running + containers. Defaults to the default set of capabilities + granted by the container runtime. Note that this field + cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes + in privileged containers are essentially equivalent + to root on the host. Defaults to false. Note that + this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: procMount denotes the type of proc mount + to use for the containers. The default is DefaultProcMount + which uses the container runtime defaults for readonly + paths and masked paths. This requires the ProcMountType + feature flag to be enabled. Note that this field cannot + be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only + root filesystem. Default is false. Note that this + field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be + set in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. Note that this field cannot be set + when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as + a non-root user. If true, the Kubelet will validate + the image at runtime to ensure that it does not run + as UID 0 (root) and fail to start the container if + it does. If unset or false, no such validation will + be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata + if unspecified. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name + is windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the + container. If unspecified, the container runtime will + allocate a random SELinux context for each container. May + also be set in PodSecurityContext. If set in both + SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name is + windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by this container. + If seccomp options are provided at both the pod & + container level, the container options override the + pod options. Note that this field cannot be set when + spec.os.name is windows. + properties: + localhostProfile: + description: localhostProfile indicates a profile + defined in a file on the node should be used. + The profile must be preconfigured on the node + to work. Must be a descending path, relative to + the kubelet's configured seccomp profile location. + Must only be set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp + profile will be applied. Valid options are: \n + Localhost - a profile defined in a file on the + node should be used. RuntimeDefault - the container + runtime default profile should be used. Unconfined + - no profile should be applied." + type: string + required: + - type + type: object + windowsOptions: + description: The Windows specific settings applied to + all containers. If unspecified, the options from the + PodSecurityContext will be used. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. Note that this field cannot be set + when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA + admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec + named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container + should be run as a 'Host Process' container. This + field is alpha-level and will only be honored + by components that enable the WindowsHostProcessContainers + feature flag. Setting this field without the feature + flag will result in errors when validating the + Pod. All of a Pod's containers must have the same + effective HostProcess value (it is not allowed + to have a mix of HostProcess containers and non-HostProcess + containers). In addition, if HostProcess is true + then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the + entrypoint of the container process. Defaults + to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set + in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: 'StartupProbe indicates that the Pod has successfully + initialized. If specified, no other probes are executed + until this completes successfully. If this probe fails, + the Pod will be restarted, just as if the livenessProbe + failed. This can be used to provide different probe parameters + at the beginning of a Pod''s lifecycle, when it might + take a long time to load data or warm a cache, than during + steady-state operation. This cannot be updated. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's + filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you need + to explicitly call out to that shell. Exit status + of 0 is treated as live/healthy and non-zero is + unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe + to be considered failed after having succeeded. Defaults + to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC + port. This is a beta field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service + to place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in + httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the + host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe + to be considered successful after having failed. Defaults + to 1. Must be 1 for liveness and startup. Minimum + value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and + the time when the processes are forcibly halted with + a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, + the pod's terminationGracePeriodSeconds will be used. + Otherwise, this value overrides the value provided + by the pod spec. Value must be non-negative integer. + The value zero indicates stop immediately via the + kill signal (no opportunity to shut down). This is + a beta field and requires enabling ProbeTerminationGracePeriod + feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate a buffer + for stdin in the container runtime. If this is not set, + reads from stdin in the container will always result in + EOF. Default is false. + type: boolean + stdinOnce: + description: Whether the container runtime should close + the stdin channel after it has been opened by a single + attach. When stdin is true the stdin stream will remain + open across multiple attach sessions. If stdinOnce is + set to true, stdin is opened on container start, is empty + until the first client attaches to stdin, and then remains + open and accepts data until the client disconnects, at + which time stdin is closed and remains closed until the + container is restarted. If this flag is false, a container + processes that reads from stdin will never receive an + EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file to which + the container''s termination message will be written is + mounted into the container''s filesystem. Message written + is intended to be brief final status, such as an assertion + failure message. Will be truncated by the node if greater + than 4096 bytes. The total message length across all containers + will be limited to 12kb. Defaults to /dev/termination-log. + Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should + be populated. File will use the contents of terminationMessagePath + to populate the container status message on both success + and failure. FallbackToLogsOnError will use the last chunk + of container log output if the termination message file + is empty and the container exited with an error. The log + output is limited to 2048 bytes or 80 lines, whichever + is smaller. Defaults to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate a TTY + for itself, also requires 'stdin' to be true. Default + is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices + to be used by the container. + items: + description: volumeDevice describes a mapping of a raw + block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the + container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: Path within the container at which the + volume should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts + are propagated from the host to container and the + other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write + otherwise (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the + container's volume should be mounted. Defaults to + "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from + which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable + references $(VAR_NAME) are expanded using the container's + environment. Defaults to "" (volume's root). SubPathExpr + and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which might + be configured in the container image. Cannot be updated. + type: string + required: + - name + type: object + type: array + volumes: + description: Specify the volumes information for the lm-eval and + sidecar containers + items: + description: Volume represents a named volume in a pod that + may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: 'awsElasticBlockStore represents an AWS Disk + resource that is attached to a kubelet''s host machine + and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'partition is the partition in the volume + that you want to mount. If omitted, the default is + to mount by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the volume + partition for /dev/sda is "0" (or you can leave the + property empty).' + format: int32 + type: integer + readOnly: + description: 'readOnly value true will force the readOnly + setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'volumeID is unique ID of the persistent + disk resource in AWS (Amazon EBS volume). More info: + https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data Disk mount + on the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: + None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in + the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the + blob storage + type: string + fsType: + description: fsType is Filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple + blob disks per storage account Dedicated: single + blob disk per storage account Managed: azure managed + data disk (only in managed availability set). defaults + to shared' + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service + mount on the host and bind mount to the pod. + properties: + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that + contains Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on the host + that shares a pod's lifetime + properties: + monitors: + description: 'monitors is Required: Monitors is a collection + of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'path is Optional: Used as the mounted + root, rather than the full Ceph tree, default is /' + type: string + readOnly: + description: 'readOnly is Optional: Defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'secretFile is Optional: SecretFile is + the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'secretRef is Optional: SecretRef is reference + to the authentication secret for User, default is + empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: 'user is optional: User is the rados user + name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'cinder represents a cinder volume attached + and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'secretRef is optional: points to a secret + object containing parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: 'volumeID used to identify the volume in + cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should + populate this volume + properties: + defaultMode: + description: 'defaultMode is optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in + conflict with other options that affect the file mode, + like fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + items: + description: items if unspecified, each key-value pair + in the Data field of the referenced ConfigMap will + be projected into the volume as a file whose name + is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. If a + key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. + Paths must be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits used + to set permissions on this file. Must be an + octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal + and decimal values, JSON requires decimal values + for mode bits. If not specified, the volume + defaultMode will be used. This might be in conflict + with other options that affect the file mode, + like fsGroup, and the result can be other mode + bits set.' + format: int32 + type: integer + path: + description: path is the relative path of the + file to map the key to. May not be an absolute + path. May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents + ephemeral storage that is handled by certain external + CSI drivers (Beta feature). + properties: + driver: + description: driver is the name of the CSI driver that + handles this volume. Consult with your admin for the + correct name as registered in the cluster. + type: string + fsType: + description: fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the + associated CSI driver which will determine the default + filesystem to apply. + type: string + nodePublishSecretRef: + description: nodePublishSecretRef is a reference to + the secret object containing sensitive information + to pass to the CSI driver to complete the CSI NodePublishVolume + and NodeUnpublishVolume calls. This field is optional, + and may be empty if no secret is required. If the + secret object contains more than one secret, all secret + references are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: readOnly specifies a read-only configuration + for the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: volumeAttributes stores driver-specific + properties that are passed to the CSI driver. Consult + your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the + pod that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a Optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in + conflict with other options that affect the file mode, + like fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume + file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the + pod: only annotations, labels, name and namespace + are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: 'Optional: mode bits used to set + permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict with + other options that affect the file mode, like + fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must not + be absolute or contain the ''..'' path. Must + be utf-8 encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'emptyDir represents a temporary directory + that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'medium represents what type of storage + medium should back this directory. The default is + "" which means to use the node''s default medium. + Must be an empty string (default) or Memory. More + info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'sizeLimit is the total amount of local + storage required for this EmptyDir volume. The size + limit is also applicable for memory medium. The maximum + usage on memory medium EmptyDir would be the minimum + value between the SizeLimit specified here and the + sum of memory limits of all containers in a pod. The + default is nil which means that the limit is undefined. + More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: "ephemeral represents a volume that is handled + by a cluster storage driver. The volume's lifecycle is + tied to the pod that defines it - it will be created before + the pod starts, and deleted when the pod is removed. \n + Use this if: a) the volume is only needed while the pod + runs, b) features of normal volumes like restoring from + snapshot or capacity tracking are needed, c) the storage + driver is specified through a storage class, and d) the + storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for + more information on the connection between this volume + type and PersistentVolumeClaim). \n Use PersistentVolumeClaim + or one of the vendor-specific APIs for volumes that persist + for longer than the lifecycle of an individual pod. \n + Use CSI for light-weight local ephemeral volumes if the + CSI driver is meant to be used that way - see the documentation + of the driver for more information. \n A pod can use both + types of ephemeral volumes and persistent volumes at the + same time." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC + to provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the + PVC will be deleted together with the pod. The name + of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` + array entry. Pod validation will reject the pod if + the concatenated name is not valid for a PVC (for + example, too long). \n An existing PVC with that name + that is not owned by the pod will *not* be used for + the pod to avoid using an unrelated volume by mistake. + Starting the pod is then blocked until the unrelated + PVC is removed. If such a pre-created PVC is meant + to be used by the pod, the PVC has to updated with + an owner reference to the pod once the pod exists. + Normally this should not be necessary, but it may + be useful when manually reconstructing a broken cluster. + \n This field is read-only and no changes will be + made by Kubernetes to the PVC after it has been created. + \n Required, must not be nil." + properties: + metadata: + description: May contain labels and annotations + that will be copied into the PVC when creating + it. No other fields are allowed and will be rejected + during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the + PVC that gets created from this template. The + same fields as in a PersistentVolumeClaim are + also valid here. + properties: + accessModes: + description: 'accessModes contains the desired + access modes the volume should have. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to + specify either: * An existing VolumeSnapshot + object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller + can support the specified data source, it + will create a new volume based on the contents + of the specified data source. When the AnyVolumeDataSource + feature gate is enabled, dataSource contents + will be copied to dataSourceRef, and dataSourceRef + contents will be copied to dataSource when + dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef + will not be copied to dataSource.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup + is not specified, the specified Kind must + be in the core API group. For any other + third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'dataSourceRef specifies the object + from which to populate the volume with data, + if a non-empty volume is desired. This may + be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding + will only succeed if the type of the specified + object matches some installed volume populator + or dynamic provisioner. This field will replace + the functionality of the dataSource field + and as such if both fields are non-empty, + they must have the same value. For backwards + compatibility, when namespace isn''t specified + in dataSourceRef, both fields (dataSource + and dataSourceRef) will be set to the same + value automatically if one of them is empty + and the other is non-empty. When namespace + is specified in dataSourceRef, dataSource + isn''t set to the same value and must be empty. + There are three important differences between + dataSource and dataSourceRef: * While dataSource + only allows two specific types of objects, + dataSourceRef allows any non-core object, + as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values + (dropping them), dataSourceRef preserves all + values, and generates an error if a disallowed + value is specified. * While dataSource only + allows local objects, dataSourceRef allows + objects in any namespaces. (Beta) Using this + field requires the AnyVolumeDataSource feature + gate to be enabled. (Alpha) Using the namespace + field of dataSourceRef requires the CrossNamespaceVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup + is not specified, the specified Kind must + be in the core API group. For any other + third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + namespace: + description: Namespace is the namespace + of resource being referenced Note that + when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant + object is required in the referent namespace + to allow that namespace's owner to accept + the reference. See the ReferenceGrant + documentation for details. (Alpha) This + field requires the CrossNamespaceVolumeDataSource + feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: 'resources represents the minimum + resources the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than + previous value but must still be higher than + capacity recorded in the status field of the + claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + claims: + description: "Claims lists the names of + resources, defined in spec.resourceClaims, + that are used by this container. \n This + is an alpha field and requires enabling + the DynamicResourceAllocation feature + gate. \n This field is immutable. It can + only be set for containers." + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name + of one entry in pod.spec.resourceClaims + of the Pod where this field is used. + It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. + If Requests is omitted for a container, + it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over + volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'storageClassName is the name of + the StorageClass required by the claim. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of + volume is required by the claim. Value of + Filesystem is implied when not included in + claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that + is attached to a kubelet's host machine and then exposed + to the pod. + properties: + fsType: + description: 'fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. TODO: how do we prevent + errors in the filesystem from compromising the machine' + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'readOnly is Optional: Defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide + names (WWNs)' + items: + type: string + type: array + wwids: + description: 'wwids Optional: FC volume world wide identifiers + (wwids) Either wwids or combination of targetWWNs + and lun must be set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: flexVolume represents a generic volume resource + that is provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver to use + for this volume. + type: string + fsType: + description: fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". The default filesystem + depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds + extra command options if any.' + type: object + readOnly: + description: 'readOnly is Optional: defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts.' + type: boolean + secretRef: + description: 'secretRef is Optional: secretRef is reference + to the secret object containing sensitive information + to pass to the plugin scripts. This may be empty if + no secret object is specified. If the secret object + contains more than one secret, all secrets are passed + to the plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached + to a kubelet's host machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: datasetName is Name of the dataset stored + as metadata -> name on the dataset for Flocker should + be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. + This is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'gcePersistentDisk represents a GCE Disk resource + that is attached to a kubelet''s host machine and then + exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'fsType is filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'partition is the partition in the volume + that you want to mount. If omitted, the default is + to mount by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the volume + partition for /dev/sda is "0" (or you can leave the + property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'pdName is unique name of the PD resource + in GCE. Used to identify the disk in GCE. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'gitRepo represents a git repository at a particular + revision. DEPRECATED: GitRepo is deprecated. To provision + a container with a git repo, mount an EmptyDir into an + InitContainer that clones the repo using git, then mount + the EmptyDir into the Pod''s container.' + properties: + directory: + description: directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, + the volume directory will be the git repository. Otherwise, + if specified, the volume will contain the git repository + in the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified + revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'glusterfs represents a Glusterfs mount on + the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'endpoints is the endpoint name that details + Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'path is the Glusterfs volume path. More + info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'readOnly here will force the Glusterfs + volume to be mounted with read-only permissions. Defaults + to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'hostPath represents a pre-existing file or + directory on the host machine that is directly exposed + to the container. This is generally used for system agents + or other privileged things that are allowed to see the + host machine. Most containers will NOT need this. More + info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host + directory mounts and who can/can not mount host directories + as read/write.' + properties: + path: + description: 'path of the directory on the host. If + the path is a symlink, it will follow the link to + the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'type for HostPath Volume Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'iscsi represents an ISCSI Disk resource that + is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support + iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support + iSCSI Session CHAP authentication + type: boolean + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + initiatorName: + description: initiatorName is the custom iSCSI Initiator + Name. If initiatorName is specified with iscsiInterface + simultaneously, new iSCSI interface : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iscsiInterface is the interface Name that + uses an iSCSI transport. Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: portals is the iSCSI Target Portal List. + The portal is either an IP or ip_addr:port if the + port is other than default (typically TCP ports 860 + and 3260). + items: + type: string + type: array + readOnly: + description: readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI + target and initiator authentication + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: targetPortal is iSCSI Target Portal. The + Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and + 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'name of the volume. Must be a DNS_LABEL and + unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'nfs represents an NFS mount on the host that + shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'readOnly here will force the NFS export + to be mounted with read-only permissions. Defaults + to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'server is the hostname or IP address of + the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'persistentVolumeClaimVolumeSource represents + a reference to a PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'claimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: readOnly Will force the ReadOnly setting + in VolumeMounts. Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host + machine + properties: + fsType: + description: fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume + attached and mounted on kubelets host machine + properties: + fsType: + description: fSType represents the filesystem type to + mount Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx + volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, + configmaps, and downward API + properties: + defaultMode: + description: defaultMode are the mode bits used to set + permissions on created files by default. Must be an + octal value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this + setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set. + format: int32 + type: integer + sources: + description: sources is the list of volume projections + items: + description: Projection that may be projected along + with other supported volume types + properties: + configMap: + description: configMap information about the configMap + data to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced + ConfigMap will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed keys + will be projected into the specified paths, + and unlisted keys will not be present. If + a key is specified which is not present + in the ConfigMap, the volume setup will + error unless it is marked optional. Paths + must be relative and may not contain the + '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode + bits used to set permissions on this + file. Must be an octal value between + 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal + and decimal values, JSON requires + decimal values for mode bits. If not + specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the + file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path + of the file to map the key to. May + not be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: optional specify whether the + ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about the + downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: 'Optional: mode bits used + to set permissions on this file, must + be an octal value between 0000 and + 0777 or a decimal value between 0 + and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute or + contain the ''..'' path. Must be utf-8 + encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + description: secret information about the secret + data to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced + Secret will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed keys + will be projected into the specified paths, + and unlisted keys will not be present. If + a key is specified which is not present + in the Secret, the volume setup will error + unless it is marked optional. Paths must + be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode + bits used to set permissions on this + file. Must be an octal value between + 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal + and decimal values, JSON requires + decimal values for mode bits. If not + specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the + file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path + of the file to map the key to. May + not be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: optional field specify whether + the Secret or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information + about the serviceAccountToken data to project + properties: + audience: + description: audience is the intended audience + of the token. A recipient of a token must + identify itself with an identifier specified + in the audience of the token, and otherwise + should reject the token. The audience defaults + to the identifier of the apiserver. + type: string + expirationSeconds: + description: expirationSeconds is the requested + duration of validity of the service account + token. As the token approaches expiration, + the kubelet volume plugin will proactively + rotate the service account token. The kubelet + will start trying to rotate the token if + the token is older than 80 percent of its + time to live or if the token is older than + 24 hours.Defaults to 1 hour and must be + at least 10 minutes. + format: int64 + type: integer + path: + description: path is the path relative to + the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + description: quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: group to map volume access to Default is + no group + type: string + readOnly: + description: readOnly here will force the Quobyte volume + to be mounted with read-only permissions. Defaults + to false. + type: boolean + registry: + description: registry represents a single or multiple + Quobyte Registry services specified as a string as + host:port pair (multiple entries are separated with + commas) which acts as the central registry for volumes + type: string + tenant: + description: tenant owning the given Quobyte volume + in the Backend Used with dynamically provisioned Quobyte + volumes, value is set by the plugin + type: string + user: + description: user to map volume access to Defaults to + serivceaccount user + type: string + volume: + description: volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'rbd represents a Rados Block Device mount + on the host that shares a pod''s lifetime. More info: + https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + image: + description: 'image is the rados image name. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'pool is the rados pool name. Default is + rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'secretRef is name of the authentication + secret for RBDUser. If provided overrides keyring. + Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: 'user is the rados user name. Default is + admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO + API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO + Protection Domain for the configured storage. + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef references to the secret for + ScaleIO user and other sensitive information. If this + is not provided, Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: storageMode indicates whether the storage + for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool + associated with the protection domain. + type: string + system: + description: system is the name of the storage system + as configured in ScaleIO. + type: string + volumeName: + description: volumeName is the name of a volume already + created in the ScaleIO system that is associated with + this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'defaultMode is Optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in + conflict with other options that affect the file mode, + like fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + items: + description: items If unspecified, each key-value pair + in the Data field of the referenced Secret will be + projected into the volume as a file whose name is + the key and content is the value. If specified, the + listed keys will be projected into the specified paths, + and unlisted keys will not be present. If a key is + specified which is not present in the Secret, the + volume setup will error unless it is marked optional. + Paths must be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits used + to set permissions on this file. Must be an + octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal + and decimal values, JSON requires decimal values + for mode bits. If not specified, the volume + defaultMode will be used. This might be in conflict + with other options that affect the file mode, + like fsGroup, and the result can be other mode + bits set.' + format: int32 + type: integer + path: + description: path is the relative path of the + file to map the key to. May not be an absolute + path. May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: optional field specify whether the Secret + or its keys must be defined + type: boolean + secretName: + description: 'secretName is the name of the secret in + the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef specifies the secret to use for + obtaining the StorageOS API credentials. If not specified, + default values will be attempted. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: volumeName is the human-readable name of + the StorageOS volume. Volume names are only unique + within a namespace. + type: string + volumeNamespace: + description: volumeNamespace specifies the scope of + the volume within StorageOS. If no namespace is specified + then the Pod's namespace will be used. This allows + the Kubernetes name scoping to be mirrored within + StorageOS for tighter integration. Set VolumeName + to any name to override the default behaviour. Set + to "default" if you are not using namespaces within + StorageOS. Namespaces that do not pre-exist within + StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: fsType is filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based + Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy + Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object taskList: description: Evaluation task list properties: @@ -208,7 +3214,18 @@ spec: properties: card: description: The Unitxt dataset card - type: string + properties: + custom: + description: 'A JSON string for a custom unitxt card + which contains the custom dataset. Use the documentation + here: https://www.unitxt.ai/en/latest/docs/adding_dataset.html#adding-to-the-catalog + to compose a custom card, store it as a JSON file, + and use the JSON content as the value here.' + type: string + name: + description: Unitxt card's ID + type: string + type: object demosPoolSize: description: The pool size for the fewshot type: integer diff --git a/controllers/lmes/driver/driver.go b/controllers/lmes/driver/driver.go index c50dc24f..fb65c74e 100644 --- a/controllers/lmes/driver/driver.go +++ b/controllers/lmes/driver/driver.go @@ -56,7 +56,9 @@ const ( GrpcServerCaEnv = "GRPC_SERVER_CA" DefaultDriverReportInterval = time.Second * 10 DefaultTaskRecipesPath = "/opt/app-root/src/my_tasks" - TaskRecipe_Prefix = "tr" + DefaultCatalogPath = "/opt/app-root/src/my_catalogs" + TaskRecipePrefix = "tr" + CustomCardPrefix = "custom" ) func init() { @@ -74,6 +76,8 @@ type DriverOption struct { DetectDevice bool TaskRecipesPath string TaskRecipes []string + CatalogPath string + CustomCards []string Logger logr.Logger Args []string ReportInterval time.Duration @@ -111,6 +115,11 @@ func NewDriver(opt *DriverOption) (Driver, error) { if opt.TaskRecipesPath == "" { opt.TaskRecipesPath = DefaultTaskRecipesPath } + + if opt.CatalogPath == "" { + opt.CatalogPath = DefaultCatalogPath + } + conn, err := getGRPCClientConn(opt) if err != nil { return nil, err @@ -259,6 +268,10 @@ func (d *driverImpl) exec() error { return fmt.Errorf("failed to create task recipes: %v", err) } + if err := d.createCustomCards(); err != nil { + return fmt.Errorf("failed to create custom cards: %v", err) + } + // Detect available devices if needed if err := d.detectDevice(); err != nil { return err @@ -312,7 +325,7 @@ func (d *driverImpl) exec() error { // temporally fix the trust_remote_code issue io.WriteString(stdin, "y\n") - if err = executor.Start(); err != nil { + if err := executor.Start(); err != nil { freeRes() return err } @@ -322,17 +335,17 @@ func (d *driverImpl) exec() error { go func() { for scanner.Scan() { msg := scanner.Text() - if err = d.reportProgress(msg); err != nil { + if err := d.reportProgress(msg); err != nil { d.Option.Logger.Error(err, "report progress failed") } } wg.Done() }() - err = executor.Wait() + finalError := executor.Wait() freeRes() wg.Wait() - return err + return finalError } func (d *driverImpl) updateStatus(state lmesv1alpha1.JobState, msg string) error { @@ -445,10 +458,10 @@ func (d *driverImpl) reportProgress(msg string) error { func (d *driverImpl) createTaskRecipes() error { for i, taskRecipe := range d.Option.TaskRecipes { err := os.WriteFile( - filepath.Join(d.Option.TaskRecipesPath, fmt.Sprintf("%s_%d.yaml", TaskRecipe_Prefix, i)), + filepath.Join(d.Option.TaskRecipesPath, fmt.Sprintf("%s_%d.yaml", TaskRecipePrefix, i)), []byte(fmt.Sprintf( "task: %s\ninclude: unitxt\nrecipe: %s", - fmt.Sprintf("%s_%d", TaskRecipe_Prefix, i), + fmt.Sprintf("%s_%d", TaskRecipePrefix, i), taskRecipe, )), 0666, @@ -459,3 +472,17 @@ func (d *driverImpl) createTaskRecipes() error { } return nil } + +func (d *driverImpl) createCustomCards() error { + for i, customCard := range d.Option.CustomCards { + err := os.WriteFile( + filepath.Join(d.Option.CatalogPath, "cards", fmt.Sprintf("%s_%d.json", CustomCardPrefix, i)), + []byte(customCard), + 0666, + ) + if err != nil { + return err + } + } + return nil +} diff --git a/controllers/lmes/driver/driver_test.go b/controllers/lmes/driver/driver_test.go index 66fe208e..25d567e8 100644 --- a/controllers/lmes/driver/driver_test.go +++ b/controllers/lmes/driver/driver_test.go @@ -116,7 +116,7 @@ func Test_ProgressUpdate(t *testing.T) { GrpcPort: 8082, OutputPath: ".", Logger: driverLog, - Args: []string{"sh", "-ec", "echo 'testing progress: 100%|' >&2; sleep 10"}, + Args: []string{"sh", "-ec", "echo 'testing progress: 100%|' >&2; sleep 6"}, ReportInterval: time.Second * 5, }) assert.Nil(t, err) @@ -227,7 +227,7 @@ func Test_TaskRecipes(t *testing.T) { "card=unitxt.card1,template=unitxt.template,metrics=[unitxt.metric1,unitxt.metric2],format=unitxt.format,num_demos=5,demos_pool_size=10", "card=unitxt.card2,template=unitxt.template2,metrics=[unitxt.metric3,unitxt.metric4],format=unitxt.format,num_demos=5,demos_pool_size=10", }, - Args: []string{"sh", "-ec", "echo 'testing progress: 100%|' >&2; sleep 10"}, + Args: []string{"sh", "-ec", "echo 'testing progress: 100%|' >&2; sleep 6"}, ReportInterval: time.Second * 5, }) assert.Nil(t, err) @@ -259,6 +259,65 @@ func Test_TaskRecipes(t *testing.T) { assert.Nil(t, os.Remove("./tr_1.yaml")) } +func Test_CustomCards(t *testing.T) { + server := grpc.NewServer() + progresssServer := ProgressUpdateServer{} + v1beta1.RegisterLMEvalJobUpdateServiceServer(server, &progresssServer) + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 8082)) + assert.Nil(t, err) + go server.Serve(lis) + + driver, err := NewDriver(&DriverOption{ + Context: context.Background(), + JobNamespace: "fms-lm-eval-service-system", + JobName: "evaljob-sample", + GrpcService: "localhost", + GrpcPort: 8082, + OutputPath: ".", + Logger: driverLog, + TaskRecipesPath: "./", + CatalogPath: "./", + TaskRecipes: []string{ + "card=cards.custom_0,template=unitxt.template,metrics=[unitxt.metric1,unitxt.metric2],format=unitxt.format,num_demos=5,demos_pool_size=10", + }, + CustomCards: []string{ + `{ "__type__": "task_card", "loader": { "__type__": "load_hf", "path": "wmt16", "name": "de-en" }, "preprocess_steps": [ { "__type__": "copy", "field": "translation/en", "to_field": "text" }, { "__type__": "copy", "field": "translation/de", "to_field": "translation" }, { "__type__": "set", "fields": { "source_language": "english", "target_language": "deutch" } } ], "task": "tasks.translation.directed", "templates": "templates.translation.directed.all" }`, + }, + Args: []string{"sh", "-ec", "echo 'testing progress: 100%|' >&2; sleep 6"}, + ReportInterval: time.Second * 5, + }) + assert.Nil(t, err) + + os.Mkdir("cards", 0750) + + assert.Nil(t, driver.Run()) + assert.Equal(t, []string{ + "update status from the driver: running", + "testing progress: 100%", + "update status from the driver: completed", + }, progresssServer.progressMsgs) + + server.Stop() + + tr0, err := os.ReadFile("./tr_0.yaml") + assert.Nil(t, err) + assert.Equal(t, + "task: tr_0\ninclude: unitxt\nrecipe: card=cards.custom_0,template=unitxt.template,metrics=[unitxt.metric1,unitxt.metric2],format=unitxt.format,num_demos=5,demos_pool_size=10", + string(tr0), + ) + custom0, err := os.ReadFile("./cards/custom_0.json") + assert.Nil(t, err) + assert.Equal(t, + `{ "__type__": "task_card", "loader": { "__type__": "load_hf", "path": "wmt16", "name": "de-en" }, "preprocess_steps": [ { "__type__": "copy", "field": "translation/en", "to_field": "text" }, { "__type__": "copy", "field": "translation/de", "to_field": "translation" }, { "__type__": "set", "fields": { "source_language": "english", "target_language": "deutch" } } ], "task": "tasks.translation.directed", "templates": "templates.translation.directed.all" }`, + string(custom0), + ) + assert.Nil(t, os.Remove("./stderr.log")) + assert.Nil(t, os.Remove("./stdout.log")) + assert.Nil(t, os.Remove("./tr_0.yaml")) + assert.Nil(t, os.Remove("./cards/custom_0.json")) + assert.Nil(t, os.Remove("./cards")) +} + func Test_ProgramError(t *testing.T) { server := grpc.NewServer() progresssServer := ProgressUpdateServer{} diff --git a/controllers/lmes/lmevaljob_controller.go b/controllers/lmes/lmevaljob_controller.go index 94485664..2b77909e 100644 --- a/controllers/lmes/lmevaljob_controller.go +++ b/controllers/lmes/lmevaljob_controller.go @@ -19,6 +19,7 @@ package lmes import ( "context" "fmt" + "maps" "reflect" "slices" "strconv" @@ -29,6 +30,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/json" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -70,6 +72,9 @@ var ( "MaxBatchSize": MaxBatchSizeKey, "DetectDevice": DetectDeviceKey, } + + labelFilterPrefixes = []string{} + annotationFilterPrefixes = []string{} ) type TLSMode int @@ -411,6 +416,20 @@ func (r *LMEvalJobReconciler) handleNewCR(ctx context.Context, log logr.Logger, return ctrl.Result{}, nil } + // Validate the custom card if exists + // FIXME: Move the validation to the webhook once we enable it. + if err := r.validateCustomCard(job, log); err != nil { + // custom card validation failed + job.Status.State = lmesv1alpha1.CompleteJobState + job.Status.Reason = lmesv1alpha1.FailedReason + job.Status.Message = err.Error() + if err := r.Status().Update(ctx, job); err != nil { + log.Error(err, "unable to update LMEvalJob status for custom card validation error") + } + log.Error(err, "Contain invalid custom card in the LMEvalJob", "name", job.Name) + return ctrl.Result{}, err + } + // construct a new pod and create a pod for the job currentTime := v1.Now() pod := r.createPod(job, log) @@ -578,13 +597,37 @@ func (r *LMEvalJobReconciler) handleCancel(ctx context.Context, log logr.Logger, return ctrl.Result{}, err } +func (r *LMEvalJobReconciler) validateCustomCard(job *lmesv1alpha1.LMEvalJob, log logr.Logger) error { + if job.Spec.TaskList.TaskRecipes == nil { + return nil + } + + for _, taskRecipe := range job.Spec.TaskList.TaskRecipes { + if taskRecipe.Card.Custom != "" { + var card map[string]interface{} + if err := json.Unmarshal([]byte(taskRecipe.Card.Custom), &card); err != nil { + log.Error(err, "failed to parse the custom card") + return fmt.Errorf("custom card is not a valid JSON string, %s", err.Error()) + } + // at least the custom card shall define its loader + if _, ok := card["loader"]; !ok { + missKeyError := fmt.Errorf("no loader definition in the custom card") + log.Error(missKeyError, "failed to parse the custom card") + return missKeyError + } + } + } + + return nil +} + func (r *LMEvalJobReconciler) createPod(job *lmesv1alpha1.LMEvalJob, log logr.Logger) *corev1.Pod { var allowPrivilegeEscalation = false var runAsNonRootUser = true var ownerRefController = true var runAsUser int64 = 1001030000 - var envVars = generateEnvs(job.Spec.EnvSecrets) + var envVars = job.Spec.Pod.GetContainer().GetEnv() var volumeMounts = []corev1.VolumeMount{ { @@ -602,7 +645,11 @@ func (r *LMEvalJobReconciler) createPod(job *lmesv1alpha1.LMEvalJob, log logr.Lo } envVars, volumes, volumeMounts = r.patch4GrpcTLS(envVars, volumes, volumeMounts, r.options.grpcTLSMode) - volumes, volumeMounts = patch4FileSecrets(volumes, volumeMounts, job.Spec.FileSecrets) + volumes = append(volumes, job.Spec.Pod.GetVolumes()...) + volumeMounts = append(volumeMounts, job.Spec.Pod.GetContainer().GetVolumMounts()...) + labels := getPodLabels(job.Labels, log) + annotations := getAnnotations(job.Annotations, log) + resources := getResources(job.Spec.Pod.GetContainer().GetResources()) // Then compose the Pod CR pod := corev1.Pod{ @@ -622,9 +669,8 @@ func (r *LMEvalJobReconciler) createPod(job *lmesv1alpha1.LMEvalJob, log logr.Lo UID: job.UID, }, }, - Labels: map[string]string{ - "app.kubernetes.io/name": "lm-eval-service", - }, + Labels: labels, + Annotations: annotations, }, Spec: corev1.PodSpec{ InitContainers: []corev1.Container{ @@ -668,6 +714,7 @@ func (r *LMEvalJobReconciler) createPod(job *lmesv1alpha1.LMEvalJob, log logr.Lo }, }, VolumeMounts: volumeMounts, + Resources: *resources, }, }, SecurityContext: &corev1.PodSecurityContext{ @@ -683,6 +730,50 @@ func (r *LMEvalJobReconciler) createPod(job *lmesv1alpha1.LMEvalJob, log logr.Lo return &pod } +func getPodLabels(src map[string]string, log logr.Logger) map[string]string { + labels := map[string]string{ + "app.kubernetes.io/name": "ta-lmes", + } + mergeMapWithFilters(labels, src, labelFilterPrefixes, log) + return labels +} + +func getAnnotations(annotations map[string]string, log logr.Logger) map[string]string { + if len(annotations) == 0 { + return nil + } + dest := map[string]string{} + mergeMapWithFilters(dest, annotations, annotationFilterPrefixes, log) + return dest +} + +func getResources(resources *corev1.ResourceRequirements) *corev1.ResourceRequirements { + if resources == nil { + return &corev1.ResourceRequirements{} + } + return resources +} + +// Merge the map based on the filters. If the names in the `src` map contains any prefixes +// in the prefixFilters list, those KV will be discarded, otherwise, KV will be merge into +// `dest` map. +func mergeMapWithFilters(dest, src map[string]string, prefixFitlers []string, log logr.Logger) { + if len(prefixFitlers) == 0 { + // Fast path if the labelFilterPrefix is empty. + maps.Copy(dest, src) + } else { + for k, v := range src { + if slices.ContainsFunc(prefixFitlers, func(prefix string) bool { + return strings.HasPrefix(k, prefix) + }) { + log.Info("the label is not propagated to the pod", k, v) + } else { + dest[k] = v + } + } + } +} + func (r *LMEvalJobReconciler) generateArgs(job *lmesv1alpha1.LMEvalJob, log logr.Logger) []string { if job == nil { return nil @@ -738,7 +829,7 @@ func concatTasks(tasks lmesv1alpha1.TaskList) []string { recipesName := make([]string, len(tasks.TaskRecipes)) for i := range tasks.TaskRecipes { // assign internal userd task name - recipesName[i] = fmt.Sprintf("%s_%d", driver.TaskRecipe_Prefix, i) + recipesName[i] = fmt.Sprintf("%s_%d", driver.TaskRecipePrefix, i) } return append(tasks.TaskNames, recipesName...) } @@ -761,8 +852,20 @@ func (r *LMEvalJobReconciler) generateCmd(job *lmesv1alpha1.LMEvalJob) []string cmds = append(cmds, "--detect-device") } + cr_idx := 0 for _, recipe := range job.Spec.TaskList.TaskRecipes { - cmds = append(cmds, "--task-recipe", recipe.String()) + if recipe.Card.Name != "" { + // built-in card, regular recipe + cmds = append(cmds, "--task-recipe", recipe.String()) + } else if recipe.Card.Custom != "" { + // custom card, need to inject --custom-card arg as well + dupRecipe := recipe.DeepCopy() + // the format of a custom card's name: custom_ + dupRecipe.Card.Name = fmt.Sprintf("cards.%s_%d", driver.CustomCardPrefix, cr_idx) + cmds = append(cmds, "--task-recipe", dupRecipe.String()) + cmds = append(cmds, "--custom-card", dupRecipe.Card.Custom) + cr_idx++ + } } cmds = append(cmds, "--") @@ -780,27 +883,6 @@ func argsToString(args []lmesv1alpha1.Arg) string { return strings.Join(equalForms, ",") } -func generateEnvs(secrets []lmesv1alpha1.EnvSecret) []corev1.EnvVar { - var envs = []corev1.EnvVar{} - for _, secret := range secrets { - if secret.Secret != nil { - envs = append(envs, corev1.EnvVar{ - Name: secret.Env, - Value: *secret.Secret, - }) - } else if secret.SecretRef != nil { - envs = append(envs, corev1.EnvVar{ - Name: secret.Env, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: secret.SecretRef, - }, - }) - } - } - - return envs -} - func (r *LMEvalJobReconciler) patch4GrpcTLS( envVars []corev1.EnvVar, volumes []corev1.Volume, @@ -894,26 +976,3 @@ func (r *LMEvalJobReconciler) patch4GrpcTLS( return envVars, volumes, volumeMounts } - -func patch4FileSecrets( - volumes []corev1.Volume, - volumeMounts []corev1.VolumeMount, - secrets []lmesv1alpha1.FileSecret) ([]corev1.Volume, []corev1.VolumeMount) { - - var counter = 1 - for _, secret := range secrets { - volumes = append(volumes, corev1.Volume{ - Name: fmt.Sprintf("secVol%d", counter), - VolumeSource: corev1.VolumeSource{ - Secret: &secret.SecretRef, - }, - }) - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: fmt.Sprintf("secVol%d", counter), - MountPath: secret.MountPath, - ReadOnly: true, - }) - counter++ - } - return volumes, volumeMounts -} diff --git a/controllers/lmes/lmevaljob_controller_test.go b/controllers/lmes/lmevaljob_controller_test.go index e6d12bfd..81390325 100644 --- a/controllers/lmes/lmevaljob_controller_test.go +++ b/controllers/lmes/lmevaljob_controller_test.go @@ -24,6 +24,7 @@ import ( lmesv1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/lmes/v1alpha1" "github.com/trustyai-explainability/trustyai-service-operator/controllers/lmes/driver" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -74,7 +75,7 @@ func Test_SimplePod(t *testing.T) { Name: "test", Namespace: "default", Labels: map[string]string{ - "app.kubernetes.io/name": "lm-eval-service", + "app.kubernetes.io/name": "ta-lmes", }, OwnerReferences: []metav1.OwnerReference{ { @@ -119,7 +120,6 @@ func Test_SimplePod(t *testing.T) { Name: "main", Image: lmevalRec.options.PodImage, ImagePullPolicy: lmevalRec.options.ImagePullPolicy, - Env: []corev1.EnvVar{}, Command: lmevalRec.generateCmd(job), Args: lmevalRec.generateArgs(job, log), SecurityContext: &corev1.SecurityContext{ @@ -161,6 +161,202 @@ func Test_SimplePod(t *testing.T) { assert.Equal(t, expect, newPod) } +func Test_WithLabelsAnnotationsResourcesVolumes(t *testing.T) { + log := log.FromContext(context.Background()) + lmevalRec := LMEvalJobReconciler{ + Namespace: "test", + options: &ServiceOptions{ + PodImage: "podimage:latest", + DriverImage: "driver:latest", + ImagePullPolicy: corev1.PullAlways, + GrpcPort: 8088, + GrpcService: "grpc-service", + }, + } + var job = &lmesv1alpha1.LMEvalJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + UID: "for-testing", + Labels: map[string]string{ + "custom/label1": "value1", + "custom/label2": "value2", + }, + Annotations: map[string]string{ + "custom/annotation1": "annotation1", + "custom/annotation2": "annotation2", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: lmesv1alpha1.KindName, + APIVersion: lmesv1alpha1.Version, + }, + Spec: lmesv1alpha1.LMEvalJobSpec{ + Model: "test", + ModelArgs: []lmesv1alpha1.Arg{ + {Name: "arg1", Value: "value1"}, + }, + TaskList: lmesv1alpha1.TaskList{ + TaskNames: []string{"task1", "task2"}, + }, + Pod: &lmesv1alpha1.LMEvalPodSpec{ + Container: &lmesv1alpha1.LMEvalContainer{ + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "additionalVolume", + MountPath: "/test", + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "addtionalVolume", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "mypvc", + ReadOnly: true, + }, + }, + }, + }, + }, + }, + } + + expect := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/name": "ta-lmes", + "custom/label1": "value1", + "custom/label2": "value2", + }, + Annotations: map[string]string{ + "custom/annotation1": "annotation1", + "custom/annotation2": "annotation2", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: lmesv1alpha1.Version, + Kind: lmesv1alpha1.KindName, + Name: "test", + Controller: &isController, + UID: "for-testing", + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: "driver", + Image: lmevalRec.options.DriverImage, + ImagePullPolicy: lmevalRec.options.ImagePullPolicy, + Command: []string{DriverPath, "--copy", DestDriverPath}, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: &allowPrivilegeEscalation, + RunAsUser: &runAsUser, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{ + "ALL", + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "shared", + MountPath: "/opt/app-root/src/bin", + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "main", + Image: lmevalRec.options.PodImage, + ImagePullPolicy: lmevalRec.options.ImagePullPolicy, + Command: lmevalRec.generateCmd(job), + Args: lmevalRec.generateArgs(job, log), + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: &allowPrivilegeEscalation, + RunAsUser: &runAsUser, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{ + "ALL", + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "shared", + MountPath: "/opt/app-root/src/bin", + }, + { + Name: "additionalVolume", + MountPath: "/test", + }, + }, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + }, + }, + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: &runAsNonRootUser, + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "shared", VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "addtionalVolume", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "mypvc", + ReadOnly: true, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + } + + newPod := lmevalRec.createPod(job, log) + + assert.Equal(t, expect, newPod) + + // with filter + labelFilterPrefixes = append(labelFilterPrefixes, "custom/label1") + annotationFilterPrefixes = append(annotationFilterPrefixes, "custom/annotation2") + expect.Labels = map[string]string{ + "app.kubernetes.io/name": "ta-lmes", + "custom/label2": "value2", + } + expect.Annotations = map[string]string{ + "custom/annotation1": "annotation1", + } + + newPod = lmevalRec.createPod(job, log) + assert.Equal(t, expect, newPod) +} + func Test_GrpcMTlsPod(t *testing.T) { log := log.FromContext(context.Background()) lmevalRec := LMEvalJobReconciler{ @@ -202,7 +398,7 @@ func Test_GrpcMTlsPod(t *testing.T) { Name: "test", Namespace: "default", Labels: map[string]string{ - "app.kubernetes.io/name": "lm-eval-service", + "app.kubernetes.io/name": "ta-lmes", }, OwnerReferences: []metav1.OwnerReference{ { @@ -366,13 +562,19 @@ func Test_EnvSecretsPod(t *testing.T) { TaskList: lmesv1alpha1.TaskList{ TaskNames: []string{"task1", "task2"}, }, - EnvSecrets: []lmesv1alpha1.EnvSecret{ - { - Env: "my_env", - SecretRef: &corev1.SecretKeySelector{ - Key: "my-key", - LocalObjectReference: corev1.LocalObjectReference{ - Name: "my-secret", + Pod: &lmesv1alpha1.LMEvalPodSpec{ + Container: &lmesv1alpha1.LMEvalContainer{ + Env: []corev1.EnvVar{ + { + Name: "my_env", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "my-key", + LocalObjectReference: corev1.LocalObjectReference{ + Name: "my-secret", + }, + }, + }, }, }, }, @@ -385,7 +587,7 @@ func Test_EnvSecretsPod(t *testing.T) { Name: "test", Namespace: "default", Labels: map[string]string{ - "app.kubernetes.io/name": "lm-eval-service", + "app.kubernetes.io/name": "ta-lmes", }, OwnerReferences: []metav1.OwnerReference{ { @@ -560,15 +762,28 @@ func Test_FileSecretsPod(t *testing.T) { TaskList: lmesv1alpha1.TaskList{ TaskNames: []string{"task1", "task2"}, }, - FileSecrets: []lmesv1alpha1.FileSecret{ - { - MountPath: "the_path", - SecretRef: corev1.SecretVolumeSource{ - SecretName: "my-secret", - Items: []corev1.KeyToPath{ - { - Key: "key1", - Path: "path1", + Pod: &lmesv1alpha1.LMEvalPodSpec{ + Container: &lmesv1alpha1.LMEvalContainer{ + VolumeMounts: []corev1.VolumeMount{ + { + Name: "secVol1", + MountPath: "the_path", + ReadOnly: true, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "secVol1", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "my-secret", + Items: []corev1.KeyToPath{ + { + Key: "key1", + Path: "path1", + }, + }, }, }, }, @@ -582,7 +797,7 @@ func Test_FileSecretsPod(t *testing.T) { Name: "test", Namespace: "default", Labels: map[string]string{ - "app.kubernetes.io/name": "lm-eval-service", + "app.kubernetes.io/name": "ta-lmes", }, OwnerReferences: []metav1.OwnerReference{ { @@ -828,7 +1043,7 @@ func Test_GenerateArgCmdTaskRecipes(t *testing.T) { TaskNames: []string{"task1", "task2"}, TaskRecipes: []lmesv1alpha1.TaskRecipe{ { - Card: "unitxt.card1", + Card: lmesv1alpha1.Card{Name: "unitxt.card1"}, Template: "unitxt.template", Format: &format, Metrics: []string{"unitxt.metric1", "unitxt.metric2"}, @@ -860,7 +1075,7 @@ func Test_GenerateArgCmdTaskRecipes(t *testing.T) { job.Spec.TaskList.TaskRecipes = append(job.Spec.TaskList.TaskRecipes, lmesv1alpha1.TaskRecipe{ - Card: "unitxt.card2", + Card: lmesv1alpha1.Card{Name: "unitxt.card2"}, Template: "unitxt.template2", Format: &format, Metrics: []string{"unitxt.metric3", "unitxt.metric4"}, @@ -890,13 +1105,187 @@ func Test_GenerateArgCmdTaskRecipes(t *testing.T) { }, lmevalRec.generateCmd(job)) } +func Test_GenerateArgCmdCustomCard(t *testing.T) { + log := log.FromContext(context.Background()) + lmevalRec := LMEvalJobReconciler{ + Namespace: "test", + options: &ServiceOptions{ + PodImage: "podimage:latest", + DriverImage: "driver:latest", + ImagePullPolicy: corev1.PullAlways, + GrpcPort: 8088, + GrpcService: "grpc-service", + grpcTLSMode: TLSMode_None, + DefaultBatchSize: DefaultBatchSize, + MaxBatchSize: DefaultMaxBatchSize, + }, + } + var format = "unitxt.format" + var numDemos = 5 + var demosPoolSize = 10 + var job = &lmesv1alpha1.LMEvalJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + UID: "for-testing", + }, + TypeMeta: metav1.TypeMeta{ + Kind: lmesv1alpha1.KindName, + APIVersion: lmesv1alpha1.Version, + }, + Spec: lmesv1alpha1.LMEvalJobSpec{ + Model: "test", + ModelArgs: []lmesv1alpha1.Arg{ + {Name: "arg1", Value: "value1"}, + }, + TaskList: lmesv1alpha1.TaskList{ + TaskNames: []string{"task1", "task2"}, + TaskRecipes: []lmesv1alpha1.TaskRecipe{ + { + Card: lmesv1alpha1.Card{ + Custom: `{ "__type__": "task_card", "loader": { "__type__": "load_hf", "path": "wmt16", "name": "de-en" }, "preprocess_steps": [ { "__type__": "copy", "field": "translation/en", "to_field": "text" }, { "__type__": "copy", "field": "translation/de", "to_field": "translation" }, { "__type__": "set", "fields": { "source_language": "english", "target_language": "deutch" } } ], "task": "tasks.translation.directed", "templates": "templates.translation.directed.all" }`, + }, + Template: "unitxt.template", + Format: &format, + Metrics: []string{"unitxt.metric1", "unitxt.metric2"}, + NumDemos: &numDemos, + DemosPoolSize: &demosPoolSize, + }, + }, + }, + }, + } + + assert.Equal(t, []string{ + "sh", "-ec", + "python -m lm_eval --output_path /opt/app-root/src/output --model test --model_args arg1=value1 --tasks task1,task2,tr_0 --include_path /opt/app-root/src/my_tasks --batch_size 8", + }, lmevalRec.generateArgs(job, log)) + + assert.Equal(t, []string{ + "/opt/app-root/src/bin/driver", + "--job-namespace", "default", + "--job-name", "test", + "--grpc-service", "grpc-service.test.svc", + "--grpc-port", "8088", + "--output-path", "/opt/app-root/src/output", + "--report-interval", "0s", + "--task-recipe", "card=cards.custom_0,template=unitxt.template,metrics=[unitxt.metric1,unitxt.metric2],format=unitxt.format,num_demos=5,demos_pool_size=10", + "--custom-card", `{ "__type__": "task_card", "loader": { "__type__": "load_hf", "path": "wmt16", "name": "de-en" }, "preprocess_steps": [ { "__type__": "copy", "field": "translation/en", "to_field": "text" }, { "__type__": "copy", "field": "translation/de", "to_field": "translation" }, { "__type__": "set", "fields": { "source_language": "english", "target_language": "deutch" } } ], "task": "tasks.translation.directed", "templates": "templates.translation.directed.all" }`, + "--", + }, lmevalRec.generateCmd(job)) +} + +func Test_CustomCardValidation(t *testing.T) { + log := log.FromContext(context.Background()) + lmevalRec := LMEvalJobReconciler{ + Namespace: "test", + options: &ServiceOptions{ + PodImage: "podimage:latest", + DriverImage: "driver:latest", + ImagePullPolicy: corev1.PullAlways, + GrpcPort: 8088, + GrpcService: "grpc-service", + }, + } + var job = &lmesv1alpha1.LMEvalJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + UID: "for-testing", + }, + TypeMeta: metav1.TypeMeta{ + Kind: lmesv1alpha1.KindName, + APIVersion: lmesv1alpha1.Version, + }, + Spec: lmesv1alpha1.LMEvalJobSpec{ + Model: "test", + ModelArgs: []lmesv1alpha1.Arg{ + {Name: "arg1", Value: "value1"}, + }, + TaskList: lmesv1alpha1.TaskList{ + TaskRecipes: []lmesv1alpha1.TaskRecipe{ + { + Card: lmesv1alpha1.Card{ + Custom: "invalid JSON", + }, + }, + }, + }, + }, + } + + assert.ErrorContains(t, lmevalRec.validateCustomCard(job, log), "custom card is not a valid JSON string") + + // no loader + job.Spec.TaskList.TaskRecipes[0].Card.Custom = ` + { + "__type__": "task_card", + "preprocess_steps": [ + { + "__type__": "copy", + "field": "translation/en", + "to_field": "text" + }, + { + "__type__": "copy", + "field": "translation/de", + "to_field": "translation" + }, + { + "__type__": "set", + "fields": { + "source_language": "english", + "target_language": "deutch" + } + } + ], + "task": "tasks.translation.directed", + "templates": "templates.translation.directed.all" + }` + assert.ErrorContains(t, lmevalRec.validateCustomCard(job, log), "no loader definition in the custom card") + + // ok + job.Spec.TaskList.TaskRecipes[0].Card.Custom = ` + { + "__type__": "task_card", + "loader": { + "__type__": "load_hf", + "path": "wmt16", + "name": "de-en" + }, + "preprocess_steps": [ + { + "__type__": "copy", + "field": "translation/en", + "to_field": "text" + }, + { + "__type__": "copy", + "field": "translation/de", + "to_field": "translation" + }, + { + "__type__": "set", + "fields": { + "source_language": "english", + "target_language": "deutch" + } + } + ], + "task": "tasks.translation.directed", + "templates": "templates.translation.directed.all" + }` + + assert.Nil(t, lmevalRec.validateCustomCard(job, log)) +} + func Test_ConcatTasks(t *testing.T) { tasks := concatTasks(lmesv1alpha1.TaskList{ TaskNames: []string{"task1", "task2"}, TaskRecipes: []lmesv1alpha1.TaskRecipe{ - {Template: "template3", Card: "format3"}, + {Template: "template3", Card: lmesv1alpha1.Card{Name: "format3"}}, }, }) - assert.Equal(t, []string{"task1", "task2", driver.TaskRecipe_Prefix + "_0"}, tasks) + assert.Equal(t, []string{"task1", "task2", driver.TaskRecipePrefix + "_0"}, tasks) }