Skip to content

Commit 0f89b2b

Browse files
j-zimnowodamerll
j-zimnowoda
and
merll
authored
feat: rely solely on metadata name (#2040)
Co-authored-by: Matthias Erll <[email protected]>
1 parent 7bee88b commit 0f89b2b

File tree

117 files changed

+1190
-77
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

117 files changed

+1190
-77
lines changed

‎.vscode/launch.json

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,16 @@
7878
"console": "integratedTerminal",
7979
"cwd": "${workspaceRoot}"
8080
},
81+
{
82+
"name": "Bootstrap-test-fixtures",
83+
"request": "launch",
84+
"runtimeArgs": ["run", "bootstrap-tests-fixtures"],
85+
"runtimeExecutable": "npm",
86+
"type": "node",
87+
"envFile": ".env",
88+
"console": "integratedTerminal",
89+
"cwd": "${workspaceRoot}"
90+
},
8191
{
8292
"name": "Migrate values",
8393
"request": "launch",

‎bin/dyff.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,6 @@ echo "$diff_output" | while read -r line; do
2222
# Use dyff to compare the files
2323
dyff between "$second_path" "$first_path" --omit-header \
2424
--exclude "data.tls.key" --exclude "/data/ca.crt" --exclude "/data/tls.crt" --exclude "/data/tls.key" \
25-
--exclude-regexp "/checksum" --exclude-regexp "/webhooks.*"
25+
--exclude-regexp "/checksum" --exclude-regexp "/webhooks.*" --ignore-order-changes
2626
fi
2727
done

‎charts/team-ns/templates/argocd/argocd-applicationset.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ spec:
8989
jqPathExpressions:
9090
- ".spec.volumeClaimTemplates[].apiVersion"
9191
- ".spec.volumeClaimTemplates[].kind"
92+
- ".spec.volumeClaimTemplates[].spec"
9293
{{- end }}
9394
destination:
9495
server: 'https://kubernetes.default.svc'

‎package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,8 @@
163163
"validate-templates:all": "set -e; i=29; while [ $i -le 32 ]; do NODE_ENV=test binzx/otomi validate-templates -k 1.$i; i=$(($i+1)); done",
164164
"validate-values": "ENV_DIR=$PWD/tests/fixtures NODE_ENV=test binzx/otomi validate-values",
165165
"bootstrap-dev": "rm -rf /tmp/otomi-bootstrap-dev; CI=1 VALUES_INPUT=$PWD/tests/bootstrap/input-local-dev.yaml ENV_DIR=/tmp/otomi-bootstrap-dev binzx/otomi bootstrap",
166-
"bootstrap-dev-with-repo": "CI=1 ENV_DIR=/tmp/otomi-bootstrap-dev binzx/otomi bootstrap"
166+
"bootstrap-dev-with-repo": "CI=1 ENV_DIR=/tmp/otomi-bootstrap-dev binzx/otomi bootstrap",
167+
"bootstrap-tests-fixtures": "CI=1 ENV_DIR=$PWD/tests/fixtures binzx/otomi bootstrap"
167168
},
168169
"standard-version": {
169170
"skip": {

‎src/cmd/bootstrap.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ export const getUsers = (originalInput: any, deps = { generatePassword, addIniti
224224
}
225225
deps.addInitialPasswords(users)
226226
users.forEach((user) => {
227-
set(user, 'id', user.id || randomUUID())
227+
set(user, 'name', user.name || randomUUID())
228228
})
229229
return users
230230
}

‎src/cmd/migrate.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { randomUUID } from 'crypto'
55
import { diff } from 'deep-diff'
66
import { copy, createFileSync, move, pathExists, renameSync, rm } from 'fs-extra'
77
import { mkdir, readFile, writeFile } from 'fs/promises'
8+
import { glob } from 'glob'
89
import { cloneDeep, each, get, isObject, isUndefined, mapKeys, mapValues, omit, pick, pull, set, unset } from 'lodash'
910
import { basename, dirname, join } from 'path'
1011
import { prepareEnvironment } from 'src/common/cli'
@@ -19,7 +20,7 @@ import { BasicArguments, getParsedArgs, setParsedArgs } from 'src/common/yargs'
1920
import { v4 as uuidv4 } from 'uuid'
2021
import { parse } from 'yaml'
2122
import { Argv } from 'yargs'
22-
import { $, cd, glob } from 'zx'
23+
import { $, cd } from 'zx'
2324
const cmdName = getFilename(__filename)
2425

2526
interface Arguments extends BasicArguments {
@@ -544,7 +545,7 @@ export const migrateLegacyValues = async (envDir: string, deps = { writeFile }):
544545
})
545546
const users = get(oldValues, 'users', [])
546547
users.forEach((user) => {
547-
set(user, 'id', user.id || randomUUID())
548+
set(user, 'name', user.id || randomUUID())
548549
})
549550
oldValues.versions = { specVersion: 1 }
550551
const teamNames = await getTeamNames(env.ENV_DIR)

‎src/common/crypt.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ const processFileChunk = async (crypt: CR, files: string[]): Promise<(ProcessOut
8181
await writeFile(file, res.stdout)
8282
if (crypt.post) await crypt.post(file)
8383
return res
84+
} else {
85+
d.error(error.message)
8486
}
8587
}
8688
}

‎src/common/repo.test.ts

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,26 @@ import {
77
getResourceFileName,
88
getResourceName,
99
getTeamNameFromJsonPath,
10+
getUniqueIdentifierFromFilePath,
1011
hasCorrespondingDecryptedFile,
1112
} from 'src/common/repo'
1213
import stubs from 'src/test-stubs'
1314

1415
const { terminal } = stubs
1516

17+
describe('getUniqueIdentifierFromFilePath', () => {
18+
it('should get user name from .dec file', () => {
19+
expect(getUniqueIdentifierFromFilePath('secrets.7f5d1670-ea3d-48b5-aa48-0f9d62f80fdb.yaml.dec')).toEqual(
20+
'7f5d1670-ea3d-48b5-aa48-0f9d62f80fdb',
21+
)
22+
})
23+
it('should get user name', () => {
24+
expect(getUniqueIdentifierFromFilePath('secrets.7f5d1670-ea3d-48b5-aa48-0f9d62f80fdb.yaml')).toEqual(
25+
'7f5d1670-ea3d-48b5-aa48-0f9d62f80fdb',
26+
)
27+
})
28+
})
29+
1630
describe('getFilePath', () => {
1731
it('should get path for apps', () => {
1832
const fileMap: FileMap = {

‎src/common/repo.ts

Lines changed: 56 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -400,28 +400,36 @@ export async function saveValues(
400400

401401
export function renderManifest(fileMap: FileMap, jsonPath: jsonpath.PathComponent[], data: Record<string, any>) {
402402
//TODO remove this custom workaround for workloadValues
403-
const manifest =
404-
fileMap.kind === 'AplTeamWorkloadValues'
405-
? omit(data, ['id', 'name', 'teamId'])
406-
: {
407-
kind: fileMap.kind,
408-
metadata: {
409-
name: getResourceName(fileMap, jsonPath, data),
410-
labels: {},
411-
},
412-
spec: data,
413-
}
414-
if (fileMap.resourceGroup === 'team' && fileMap.kind !== 'AplTeamWorkloadValues') {
403+
let spec = data
404+
if (fileMap.resourceGroup === 'team') {
405+
spec = omit(data, ['id', 'name', 'teamId'])
406+
}
407+
const manifest = {
408+
kind: fileMap.kind,
409+
metadata: {
410+
name: getResourceName(fileMap, jsonPath, data),
411+
labels: {},
412+
},
413+
spec,
414+
}
415+
if (fileMap.resourceGroup === 'team') {
415416
manifest.metadata.labels['apl.io/teamId'] = getTeamNameFromJsonPath(jsonPath)
416417
}
417418

418419
return manifest
419420
}
420421

421-
export function renderManifestForSecrets(fileMap: FileMap, data: Record<string, any>) {
422+
export function renderManifestForSecrets(fileMap: FileMap, resourceName: string, data: Record<string, any>) {
423+
let spec = data
424+
if (fileMap.resourceGroup === 'users') {
425+
spec = omit(data, ['id', 'name'])
426+
}
422427
return {
423428
kind: fileMap.kind,
424-
spec: data,
429+
metadata: {
430+
name: resourceName,
431+
},
432+
spec,
425433
}
426434
}
427435

@@ -461,7 +469,8 @@ export async function saveResourceGroupToFiles(
461469
const nodeValue = node.value
462470
try {
463471
const filePath = getFilePath(fileMap, nodePath, nodeValue, 'secrets.')
464-
const manifest = renderManifestForSecrets(fileMap, nodeValue)
472+
const resourceName = getResourceName(fileMap, nodePath, nodeValue)
473+
const manifest = renderManifestForSecrets(fileMap, resourceName, nodeValue)
465474
await deps.writeValuesToFile(filePath, manifest)
466475
} catch (e) {
467476
console.log(nodePath)
@@ -472,6 +481,14 @@ export async function saveResourceGroupToFiles(
472481
)
473482
}
474483

484+
export function getUniqueIdentifierFromFilePath(filePath: string): string {
485+
return path
486+
.basename(filePath)
487+
.replace(/^secrets\./, '')
488+
.replace(/\.yaml\.dec$/, '')
489+
.replace(/\.yaml$/, '')
490+
}
491+
475492
export async function setValuesFile(envDir: string, deps = { pathExists, loadValues, writeFile }): Promise<string> {
476493
const valuesPath = path.join(envDir, 'values-repo.yaml')
477494
// if (await deps.pathExists(valuesPath)) return valuesPath
@@ -554,7 +571,6 @@ export async function loadToSpec(
554571
if (hasCorrespondingDecryptedFile(filePath, files)) return
555572
promises.push(deps.loadFileToSpec(filePath, fileMap, spec))
556573
})
557-
558574
await Promise.all(promises)
559575
}
560576

@@ -565,18 +581,32 @@ export async function loadFileToSpec(
565581
deps = { loadYaml },
566582
): Promise<void> {
567583
const jsonPath = getJsonPath(fileMap, filePath)
568-
const data = await deps.loadYaml(filePath)
569-
if (fileMap.processAs === 'arrayItem') {
570-
const ref: Record<string, any>[] = get(spec, jsonPath)
571-
ref.push(data?.spec)
572-
} else {
573-
const ref: Record<string, any> = get(spec, jsonPath)
574-
// Decrypted secrets may need to be merged with plain text specs
575-
const newRef = merge(cloneDeep(ref), data?.spec)
576-
set(spec, jsonPath, newRef)
584+
try {
585+
const data = (await deps.loadYaml(filePath)) || {}
586+
587+
if (!filePath.includes('secrets.')) {
588+
if (fileMap.resourceGroup === 'team' && fileMap.processAs === 'arrayItem') {
589+
data.spec.name = data.metadata.name
590+
}
591+
}
592+
if (fileMap.resourceGroup === 'users') {
593+
data.spec.name = getUniqueIdentifierFromFilePath(filePath)
594+
}
595+
if (fileMap.processAs === 'arrayItem') {
596+
const ref: Record<string, any>[] = get(spec, jsonPath)
597+
ref.push(data?.spec)
598+
} else {
599+
const ref: Record<string, any> = get(spec, jsonPath)
600+
// Decrypted secrets may need to be merged with plain text specs
601+
const newRef = merge(cloneDeep(ref), data?.spec)
602+
set(spec, jsonPath, newRef)
603+
}
604+
} catch (e) {
605+
console.log(filePath)
606+
console.log(fileMap)
607+
throw e
577608
}
578609
}
579-
580610
export async function getKmsSettings(envDir: string, deps = { loadToSpec }): Promise<Record<string, any>> {
581611
const kmsFiles = getFileMap('AplKms', envDir)
582612
const spec = {}

‎tests/fixtures/.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,5 @@ core.yaml
1111
*.sample
1212
.env
1313
env/status.yaml
14-
values-repo.yaml
14+
env/bootstrap.yaml
15+
values-repo.yaml

‎tests/fixtures/env/apps/alertmanager.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,12 @@ metadata:
44
labels: {}
55
spec:
66
enabled: true
7+
_rawValues: {}
8+
resources:
9+
alertmanager:
10+
limits:
11+
cpu: 200m
12+
memory: 256Mi
13+
requests:
14+
cpu: 10m
15+
memory: 64Mi

‎tests/fixtures/env/apps/apl-gitea-operator.yaml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
kind: AplApp
2+
metadata:
3+
name: apl-gitea-operator
4+
labels: {}
5+
spec:
6+
_rawValues: {}
7+
resources:
8+
operator:
9+
limits:
10+
cpu: '1'
11+
memory: 1Gi
12+
requests:
13+
cpu: 50m
14+
memory: 128Mi

‎tests/fixtures/env/apps/apl-harbor-operator.yaml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
kind: AplApp
2+
metadata:
3+
name: apl-harbor-operator
4+
labels: {}
5+
spec:
6+
_rawValues: {}
7+
resources:
8+
operator:
9+
limits:
10+
cpu: '1'
11+
memory: 1Gi
12+
requests:
13+
cpu: 50m
14+
memory: 128Mi

‎tests/fixtures/env/apps/apl-keycloak-operator.yaml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
kind: AplApp
2+
metadata:
3+
name: apl-keycloak-operator
4+
labels: {}
5+
spec:
6+
_rawValues: {}
7+
resources:
8+
operator:
9+
limits:
10+
cpu: '1'
11+
memory: 1Gi
12+
requests:
13+
cpu: 50m
14+
memory: 128Mi

‎tests/fixtures/env/apps/argocd.yaml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,14 @@ spec:
88
enabled: true
99
maxReplicas: 5
1010
minReplicas: 1
11+
targetCPUUtilizationPercentage: 80
12+
targetMemoryUtilizationPercentage: 80
1113
server:
1214
enabled: true
1315
maxReplicas: 5
1416
minReplicas: 1
17+
targetCPUUtilizationPercentage: 80
18+
targetMemoryUtilizationPercentage: 80
1519
resources:
1620
applicationSet:
1721
limits:
@@ -62,3 +66,10 @@ spec:
6266
requests:
6367
cpu: 50m
6468
memory: 256M
69+
_rawValues: {}
70+
applicationSet:
71+
replicas: 1
72+
controller:
73+
replicas: 1
74+
controllerOperationProcessors: 10
75+
controllerStatusProcessors: 20

‎tests/fixtures/env/apps/cert-manager.yaml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,33 @@ metadata:
55
spec:
66
externallyManagedTlsSecretName: mysecret
77
issuer: externally-managed-tls-secret
8+
_rawValues: {}
9+
resources:
10+
cainjector:
11+
limits:
12+
cpu: 200m
13+
memory: 1Gi
14+
requests:
15+
cpu: 50m
16+
memory: 384Mi
17+
certManager:
18+
limits:
19+
cpu: '1'
20+
memory: 512Mi
21+
requests:
22+
cpu: 50m
23+
memory: 128Mi
24+
startupapicheck:
25+
limits:
26+
cpu: 200m
27+
memory: 384Mi
28+
requests:
29+
cpu: 50m
30+
memory: 64Mi
31+
webhook:
32+
limits:
33+
cpu: 100m
34+
memory: 256Mi
35+
requests:
36+
cpu: 50m
37+
memory: 64Mi

‎tests/fixtures/env/apps/cnpg.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,4 @@ spec:
1010
requests:
1111
cpu: 100m
1212
memory: 200Mi
13+
_rawValues: {}

‎tests/fixtures/env/apps/drone.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,5 @@ spec:
1313
server: https://gitea.demo.eks.otomi.cloud
1414
github: {}
1515
provider: gitea
16+
username: otomi-admin
17+
trace: false

‎tests/fixtures/env/apps/external-dns.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
kind: AplApp
2+
metadata:
3+
name: external-dns
4+
labels: {}
5+
spec:
6+
logLevel: info
7+
resources:
8+
limits:
9+
cpu: 100m
10+
memory: 128Mi
11+
requests:
12+
cpu: 10m
13+
memory: 64Mi

0 commit comments

Comments
 (0)