feat: rely solely on metadata name

.vscode/launch.json

@@ -78,6 +78,16 @@
       "console": "integratedTerminal",
       "cwd": "${workspaceRoot}"
     },
+    {
+      "name": "Bootstrap-test-fixtures",
+      "request": "launch",
+      "runtimeArgs": ["run", "bootstrap-tests-fixtures"],
+      "runtimeExecutable": "npm",
+      "type": "node",
+      "envFile": ".env",
+      "console": "integratedTerminal",
+      "cwd": "${workspaceRoot}"
+    },
     {
       "name": "Migrate values",
       "request": "launch",

bin/dyff.sh

@@ -22,6 +22,6 @@ echo "$diff_output" | while read -r line; do
     # Use dyff to compare the files
     dyff between "$second_path" "$first_path" --omit-header \
       --exclude "data.tls.key" --exclude "/data/ca.crt" --exclude "/data/tls.crt" --exclude "/data/tls.key" \
-      --exclude-regexp "/checksum" --exclude-regexp "/webhooks.*"
+      --exclude-regexp "/checksum" --exclude-regexp "/webhooks.*" --ignore-order-changes
   fi
 done

package.json

@@ -163,7 +163,8 @@
     "validate-templates:all": "set -e; i=29; while [ $i -le 32 ]; do NODE_ENV=test binzx/otomi validate-templates -k 1.$i; i=$(($i+1)); done",
     "validate-values": "ENV_DIR=$PWD/tests/fixtures NODE_ENV=test binzx/otomi validate-values",
     "bootstrap-dev": "rm -rf /tmp/otomi-bootstrap-dev; CI=1 VALUES_INPUT=$PWD/tests/bootstrap/input-local-dev.yaml ENV_DIR=/tmp/otomi-bootstrap-dev binzx/otomi bootstrap",
-    "bootstrap-dev-with-repo": "CI=1 ENV_DIR=/tmp/otomi-bootstrap-dev binzx/otomi bootstrap"
+    "bootstrap-dev-with-repo": "CI=1 ENV_DIR=/tmp/otomi-bootstrap-dev binzx/otomi bootstrap",
+    "bootstrap-tests-fixtures": "CI=1 ENV_DIR=$PWD/tests/fixtures binzx/otomi bootstrap"
   },
   "standard-version": {
     "skip": {

src/cmd/bootstrap.ts

@@ -223,8 +223,8 @@
     deps.addPlatformAdmin(users, domainSuffix)
   }
   deps.addInitialPasswords(users)
   users.forEach((user) => {
-    set(user, 'id', user.id || randomUUID())
+    set(user, 'name', user.name || randomUUID())
   })
   return users
 }

src/cmd/migrate.ts

@@ -5,6 +5,7 @@
 import { diff } from 'deep-diff'
 import { copy, createFileSync, move, pathExists, renameSync, rm } from 'fs-extra'
 import { mkdir, readFile, writeFile } from 'fs/promises'
+import { glob } from 'glob'
 import { cloneDeep, each, get, isObject, mapKeys, mapValues, omit, pick, pull, set, unset } from 'lodash'
 import { basename, dirname, join } from 'path'
 import { prepareEnvironment } from 'src/common/cli'
@@ -20,7 +21,6 @@
 import { parse } from 'yaml'
 import { Argv } from 'yargs'
 import { $, cd } from 'zx'
-import { glob } from 'glob'
 const cmdName = getFilename(__filename)
 
 interface Arguments extends BasicArguments {
@@ -494,8 +494,8 @@
     oldTeams[teamName] = newTeam
   })
   const users = get(oldValues, 'users', [])
   users.forEach((user) => {
-    set(user, 'id', user.id || randomUUID())
+    set(user, 'name', user.id || randomUUID())
   })
   oldValues.versions = { specVersion: 1 }
   const teamNames = await getTeamNames(env.ENV_DIR)

src/common/crypt.ts

@@ -58,29 +58,31 @@
 
 const processFileChunk = async (crypt: CR, files: string[]): Promise<(ProcessOutput | undefined)[]> => {
   const d = terminal(`common:crypt:processFileChunk`)
   const commands = files.map(async (file) => {
     if (!crypt.condition || (await crypt.condition(file))) {
       d.debug(`${crypt.cmd} ${file}`)
       try {
         d.info(`${crypt.cmd} ${file}`)
         const result = await $`${[...crypt.cmd.split(' '), file]}`.quiet()
 
         if (crypt.cmd === CryptType.DECRYPT) {
           const outputFile = `${file}.dec`
           await writeFile(outputFile, result.stdout)
         }
 
         if (crypt.post) {
           await crypt.post(file)
         }
 
         return result
       } catch (error) {
         if (error.message.includes('Already encrypted') && (await pathExists(`${file}.dec`))) {
           const res = await $`helm secrets encrypt ${file}.dec`
           await writeFile(file, res.stdout)
           if (crypt.post) await crypt.post(file)
           return res
+        } else {
+          d.error(error.message)
         }
       }
     }

src/common/repo.ts

@@ -400,29 +400,33 @@
 
 export function renderManifest(fileMap: FileMap, jsonPath: jsonpath.PathComponent[], data: Record<string, any>) {
   //TODO remove this custom workaround for workloadValues
-  const manifest =
-    fileMap.kind === 'AplTeamWorkloadValues'
-      ? omit(data, ['id', 'name', 'teamId'])
-      : {
-          kind: fileMap.kind,
-          metadata: {
-            name: getResourceName(fileMap, jsonPath, data),
-            labels: {},
-          },
-          spec: data,
-        }
+  let spec = data
+  if (fileMap.resourceGroup === 'team') {
+    spec = omit(data, ['id', 'name', 'teamId'])
+  }
+  const manifest = {
+    kind: fileMap.kind,
+    metadata: {
+      name: getResourceName(fileMap, jsonPath, data),
+      labels: {},
+    },
+    spec,
+  }
   if (fileMap.resourceGroup === 'team' && fileMap.kind !== 'AplTeamWorkloadValues') {
     manifest.metadata.labels['apl.io/teamId'] = getTeamNameFromJsonPath(jsonPath)
   }
 
   return manifest
 }
 
-export function renderManifestForSecrets(fileMap: FileMap, data: Record<string, any>) {
+export function renderManifestForSecrets(fileMap: FileMap, resourceName: string, data: Record<string, any>) {
   return {
     kind: fileMap.kind,
+    metadata: {
+      name: resourceName,
+    },
     spec: data,
   }
 }
 
 export async function saveResourceGroupToFiles(
@@ -455,13 +459,14 @@
     }),
   )
 
   await Promise.all(
     jsonPathsvaluesSecrets.map(async (node) => {
       const nodePath = node.path
       const nodeValue = node.value
       try {
         const filePath = getFilePath(fileMap, nodePath, nodeValue, 'secrets.')
-        const manifest = renderManifestForSecrets(fileMap, nodeValue)
+        const resourceName = getResourceName(fileMap, nodePath, nodeValue)
+        const manifest = renderManifestForSecrets(fileMap, resourceName, nodeValue)
         await deps.writeValuesToFile(filePath, manifest)
       } catch (e) {
         console.log(nodePath)
@@ -554,7 +559,6 @@
     if (hasCorrespondingDecryptedFile(filePath, files)) return
     promises.push(deps.loadFileToSpec(filePath, fileMap, spec))
   })
-
   await Promise.all(promises)
 }
 
@@ -565,18 +569,32 @@
   deps = { loadYaml },
 ): Promise<void> {
   const jsonPath = getJsonPath(fileMap, filePath)
-  const data = await deps.loadYaml(filePath)
-  if (fileMap.processAs === 'arrayItem') {
-    const ref: Record<string, any>[] = get(spec, jsonPath)
-    ref.push(data?.spec)
-  } else {
-    const ref: Record<string, any> = get(spec, jsonPath)
-    // Decrypted secrets may need to be merged with plain text specs
-    const newRef = merge(cloneDeep(ref), data?.spec)
-    set(spec, jsonPath, newRef)
+  const data = (await deps.loadYaml(filePath)) || {}
+
+  try {
+    if (!filePath.includes('secrets.')) {
+      if (fileMap.resourceGroup === 'team' && fileMap.processAs === 'arrayItem') {
+        data.spec.name = data.metadata.name
+      }
+    }
+    if (fileMap.resourceGroup === 'users') {
+      data.spec.name = data.metadata.name
+    }
+    if (fileMap.processAs === 'arrayItem') {
+      const ref: Record<string, any>[] = get(spec, jsonPath)
+      ref.push(data?.spec)
+    } else {
+      const ref: Record<string, any> = get(spec, jsonPath)
+      // Decrypted secrets may need to be merged with plain text specs
+      const newRef = merge(cloneDeep(ref), data?.spec)
+      set(spec, jsonPath, newRef)
+    }
+  } catch (e) {
+    console.log(filePath)
+    console.log(fileMap)
+    throw e
   }
 }
-
 export async function getKmsSettings(envDir: string, deps = { loadToSpec }): Promise<Record<string, any>> {
   const kmsFiles = getFileMap('AplKms', envDir)
   const spec = {}

tests/fixtures/.gitignore

@@ -11,4 +11,5 @@ core.yaml
 *.sample
 .env
 env/status.yaml
-values-repo.yaml
+env/bootstrap.yaml
+values-repo.yaml

tests/fixtures/env/apps/alertmanager.yaml

@@ -4,3 +4,12 @@ metadata:
     labels: {}
 spec:
     enabled: true
+    _rawValues: {}
+    resources:
+        alertmanager:
+            limits:
+                cpu: 200m
+                memory: 256Mi
+            requests:
+                cpu: 10m
+                memory: 64Mi

tests/fixtures/env/apps/apl-gitea-operator.yaml

@@ -0,0 +1,14 @@
+kind: AplApp
+metadata:
+    name: apl-gitea-operator
+    labels: {}
+spec:
+    _rawValues: {}
+    resources:
+        operator:
+            limits:
+                cpu: '1'
+                memory: 1Gi
+            requests:
+                cpu: 50m
+                memory: 128Mi

tests/fixtures/env/apps/apl-harbor-operator.yaml

@@ -0,0 +1,14 @@
+kind: AplApp
+metadata:
+    name: apl-harbor-operator
+    labels: {}
+spec:
+    _rawValues: {}
+    resources:
+        operator:
+            limits:
+                cpu: '1'
+                memory: 1Gi
+            requests:
+                cpu: 50m
+                memory: 128Mi

tests/fixtures/env/apps/apl-keycloak-operator.yaml

@@ -0,0 +1,14 @@
+kind: AplApp
+metadata:
+    name: apl-keycloak-operator
+    labels: {}
+spec:
+    _rawValues: {}
+    resources:
+        operator:
+            limits:
+                cpu: '1'
+                memory: 1Gi
+            requests:
+                cpu: 50m
+                memory: 128Mi

tests/fixtures/env/apps/argocd.yaml

@@ -8,10 +8,14 @@ spec:
             enabled: true
             maxReplicas: 5
             minReplicas: 1
+            targetCPUUtilizationPercentage: 80
+            targetMemoryUtilizationPercentage: 80
         server:
             enabled: true
             maxReplicas: 5
             minReplicas: 1
+            targetCPUUtilizationPercentage: 80
+            targetMemoryUtilizationPercentage: 80
     resources:
         applicationSet:
             limits:
@@ -62,3 +66,10 @@ spec:
             requests:
                 cpu: 50m
                 memory: 256M
+    _rawValues: {}
+    applicationSet:
+        replicas: 1
+    controller:
+        replicas: 1
+    controllerOperationProcessors: 10
+    controllerStatusProcessors: 20

tests/fixtures/env/apps/cert-manager.yaml

@@ -5,3 +5,33 @@ metadata:
 spec:
     externallyManagedTlsSecretName: mysecret
     issuer: externally-managed-tls-secret
+    _rawValues: {}
+    resources:
+        cainjector:
+            limits:
+                cpu: 200m
+                memory: 1Gi
+            requests:
+                cpu: 50m
+                memory: 384Mi
+        certManager:
+            limits:
+                cpu: '1'
+                memory: 512Mi
+            requests:
+                cpu: 50m
+                memory: 128Mi
+        startupapicheck:
+            limits:
+                cpu: 200m
+                memory: 384Mi
+            requests:
+                cpu: 50m
+                memory: 64Mi
+        webhook:
+            limits:
+                cpu: 100m
+                memory: 256Mi
+            requests:
+                cpu: 50m
+                memory: 64Mi

tests/fixtures/env/apps/cnpg.yaml

@@ -10,3 +10,4 @@ spec:
         requests:
             cpu: 100m
             memory: 200Mi
+    _rawValues: {}

tests/fixtures/env/apps/drone.yaml

@@ -13,3 +13,5 @@ spec:
             server: https://gitea.demo.eks.otomi.cloud
         github: {}
         provider: gitea
+        username: otomi-admin
+    trace: false

tests/fixtures/env/apps/external-dns.yaml

@@ -0,0 +1,13 @@
+kind: AplApp
+metadata:
+    name: external-dns
+    labels: {}
+spec:
+    logLevel: info
+    resources:
+        limits:
+            cpu: 100m
+            memory: 128Mi
+        requests:
+            cpu: 10m
+            memory: 64Mi