Merge branch 'main' into iain/web-3479-reduce-yarn-postinstall-steps-cache

Author: iainsproat

.circleci/config.yml

@@ -262,6 +262,11 @@ workflows:
           requires:
             - install-dependencies
 
+      - lint-and-prettier:
+          filters: *filters-allow-all
+          requires:
+            - install-dependencies
+
       - docker-build-server:
           context: &build-context
             - github-readonly-public-repos
@@ -336,6 +341,7 @@ workflows:
             - docker-build-server
             - get-version
             - pre-commit
+            - lint-and-prettier
             - publish-approval
             - test-frontend-2
             - test-viewer
@@ -352,6 +358,7 @@ workflows:
             - docker-build-frontend-2
             - get-version
             - pre-commit
+            - lint-and-prettier
             - publish-approval
             - test-frontend-2
             - test-viewer
@@ -377,6 +384,7 @@ workflows:
             - docker-build-webhooks
             - get-version
             - pre-commit
+            - lint-and-prettier
             - publish-approval
             - test-frontend-2
             - test-viewer
@@ -393,6 +401,7 @@ workflows:
             - docker-build-file-imports
             - get-version
             - pre-commit
+            - lint-and-prettier
             - publish-approval
             - test-frontend-2
             - test-viewer
@@ -409,6 +418,7 @@ workflows:
             - docker-build-previews
             - get-version
             - pre-commit
+            - lint-and-prettier
             - publish-approval
             - test-frontend-2
             - test-viewer
@@ -425,6 +435,7 @@ workflows:
             - docker-build-test-container
             - get-version
             - pre-commit
+            - lint-and-prettier
             - publish-approval
             - test-frontend-2
             - test-viewer
@@ -453,6 +464,7 @@ workflows:
             - docker-build-monitor-container
             - get-version
             - pre-commit
+            - lint-and-prettier
             - publish-approval
             - test-frontend-2
             - test-viewer
@@ -498,11 +510,6 @@ workflows:
             - get-version
             - publish-approval
 
-      - update-helm-documentation:
-          filters: *filters-publish
-          requires:
-            - publish-helm-chart
-
       - publish-npm:
           filters:
             tags:
@@ -568,44 +575,53 @@ jobs:
       - *pull-npm-global # workaround
       - *push-yarn-cache # to circleci cache storage
 
+  lint-and-prettier:
+    <<: *docker-node-image
+    resource_class: xlarge
+    steps:
+      - checkout
+      - *pull-yarn-cache # from circleci cache storage
+      - *pull-npm-global # workaround
+      - *yarn
+      - *restore-build-public
+      - run:
+          name: Lint everything
+          command: yarn eslint:projectwide
+      - run:
+          name: Run prettier check
+          command: yarn prettier:check
+      - run:
+          name: Check JSON Schema is up to date
+          command: |
+            yarn helm:jsonschema:generate
+            git diff --exit-code
+
   pre-commit:
     parameters:
       config_file:
         default: ./.pre-commit-config.yaml
         description: Optional, path to pre-commit config file.
         type: string
-      deployment_config_file:
-        default: ./.pre-commit-config.deployment.yaml
-        description: Optional, path to additional pre-commit config file.
-        type: string
-      cache_prefix:
-        default: ''
-        description: |
-          Optional cache prefix to be used on CircleCI. Can be used for cache busting or to ensure multiple jobs use different caches.
-        type: string
     docker:
       - image: speckle/pre-commit-runner:latest
     resource_class: xlarge
+    working_directory: *default-working-dir
     steps:
       - checkout
       - restore_cache:
-          name: Restore pre-commit & Yarn Package cache
+          name: Restore pre-commit
           keys:
-            - cache-pre-commit-<<parameters.cache_prefix>>-{{ checksum "<<parameters.config_file>>" }}
+            - cache-pre-commit-{{ checksum "<<parameters.config_file>>" }}
       - run:
           name: Install pre-commit hooks
           command: pre-commit install-hooks --config <<parameters.config_file>>
       - save_cache:
-          key: cache-pre-commit-<<parameters.cache_prefix>>-{{ checksum "<<parameters.config_file>>" }}
+          key: cache-pre-commit--{{ checksum "<<parameters.config_file>>" }}
           paths:
             - ~/.cache/pre-commit
-      - *restore-yarn-cache
-      - *pull-npm-global # workaround
-      - *yarn-from-cache
-      - *restore-build-public
       - run:
           name: Run pre-commit
-          command: ./.husky/pre-commit
+          command: pre-commit run --all-files --config <<parameters.config_file>>
       - run:
           command: git --no-pager diff
           name: git diff
@@ -1214,19 +1230,6 @@ jobs:
           name: Publish Helm Chart
           command: ./.circleci/publish_helm_chart.sh
 
-  update-helm-documentation:
-    <<: *docker-node-image
-    steps:
-      - checkout
-      - *restore_workspace
-      - *restore_env_vars
-      - add_ssh_keys:
-          fingerprints:
-            - '4d:68:70:66:49:97:ba:8b:8c:55:96:df:3d:be:6e:05'
-      - run:
-          name: Update Helm Documentation
-          command: ./.circleci/update_helm_documentation.sh
-
   publish-viewer-sandbox-cloudflare-pages:
     <<: *docker-node-image
     resource_class: xlarge

.circleci/deployment/Tiltfile.build

@@ -6,7 +6,7 @@ def speckle_image(package,original_package_name=None):
         original_package_name = package
 
     package_dir = 'packages/{}'.format(original_package_name)
-    if package == 'test-deployment' or package == 'monitor-deployment' or package == 'docker-compose-ingress':
+    if package == 'test-deployment' or package == 'docker-compose-ingress':
         package_dir = 'utils/{}'.format(package)
     docker_build('speckle/speckle-{}'.format(package),
                   context='../..',

.circleci/deployment/Tiltfile.helm

@@ -66,14 +66,14 @@ helm_resource('minio',
             deps=['./values/minio.values.yaml'],
             labels=['speckle-dependencies'])
 
-helm_resource('redis',
-            release_name='redis',
-            namespace='redis',
-            chart='oci://registry-1.docker.io/bitnamicharts/redis',
-            flags=['--version=18.7.1',
-                    '--values=./values/redis.values.yaml',
+helm_resource('valkey',
+            release_name='valkey',
+            namespace='valkey',
+            chart='oci://registry-1.docker.io/bitnamicharts/valkey',
+            flags=['--version=3.0.8',
+                    '--values=./values/valkey.values.yaml',
                     '--kube-context=kind-speckle-server'],
-            deps=['./values/redis.values.yaml'],
+            deps=['./values/valkey.values.yaml'],
             labels=['speckle-dependencies'])
 
 #FIXME this helm chart does not deploy any containers, so tilt incorrectly believes it never gets to a final state
@@ -98,11 +98,11 @@ helm_resource('ingress-nginx',
             release_name='ingress-nginx',
             namespace='ingress-nginx',
             chart='ingress-nginx-repo/ingress-nginx',
-            flags=['--version=^4.8.0',
+            flags=['--version=4.8.0',
                     '--values=./values/nginx.values.yaml',
                     '--kube-context=kind-speckle-server'],
             deps=['./values/nginx.values.yaml'],
-            resource_deps=['postgresql', 'minio', 'redis', 'ingress-nginx-repo'],
+            resource_deps=['postgresql', 'minio', 'valkey', 'ingress-nginx-repo'],
             labels=['speckle-dependencies'])
 
 # Uncomment the below, and comment out the other helm_resource('speckle-server'...) to use the speckle-server helm chart
@@ -113,7 +113,7 @@ helm_resource('ingress-nginx',
 #             namespace='speckle-server',
 #             chart='speckle-server-repo/speckle-server',
 #             deps=['./values/speckle-server.values.yaml'],
-#             resource_deps=['postgresql', 'minio', 'redis', 'ingress-nginx','speckle-server-repo'],
+#             resource_deps=['postgresql', 'minio', 'valkey', 'ingress-nginx','speckle-server-repo'],
 #             flags=['--values=./values/speckle-server.values.yaml', '--devel'],
 #             labels=['speckle-server'])
 
@@ -144,7 +144,7 @@ helm_resource('speckle-server',
                           'webhook_service.image',
                           ],
               deps=['../../utils/helm', './values/speckle-server.values.yaml'],
-              resource_deps=['postgresql', 'minio', 'redis', 'ingress-nginx'],
+              resource_deps=['postgresql', 'minio', 'valkey', 'ingress-nginx'],
               labels=['speckle-server'])
 
   # TODO this is not yet working as it is expecting the updated version of the test container which is not yet released as of 2.17.16

.circleci/deployment/manifests/speckle-server.secret.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 data:
   s3_secret_key: 'bWluaW9hZG1pbg=='
   session_secret: 'c3BvcmtsZXNzcHJlY2tsZXNzcGVrbGU='
-  redis_url: 'cmVkaXM6Ly86cmVkaXNAcmVkaXMtbWFzdGVyLnJlZGlzLnN2Yy5jbHVzdGVyLmxvY2FsOjYzNzk='
+  redis_url: 'cmVkaXM6Ly86dmFsa2V5QHZhbGtleS1wcmltYXJ5LnZhbGtleS5zdmMuY2x1c3Rlci5sb2NhbDo2Mzc5Cg=='
   postgres_url: 'cG9zdGdyZXNxbDovL3NwZWNrbGU6c3BlY2tsZUBwb3N0Z3Jlc3FsLnBvc3RncmVzLnN2Yy5jbHVzdGVyLmxvY2FsOjU0MzIvc3BlY2tsZQo='
 kind: Secret
 metadata:

.circleci/deployment/manifests/redis.namespace.yaml renamed to .circleci/deployment/manifests/valkey.namespace.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: 'redis'
+  name: 'valkey'

.circleci/deployment/values/nginx.values.yaml

@@ -3,7 +3,8 @@ controller:
   hostNetwork: true
   admissionWebhooks:
     enabled: false
+  # progressDeadlineSeconds: 600 #HACK helm chart was complaining that this was less than minReadySeconds https://github.com/kubernetes/ingress-nginx/blob/c72441585e1ab1a32df86e760613d36fa804315d/charts/ingress-nginx/templates/controller-deployment.yaml#L26
 tcp:
   5433: 'postgres/postgresql:5432'
-  6380: 'redis/redis-master:6379'
+  6380: 'valkey/valkey-primary:6379'
   9002: 'minio/minio:9000'

.circleci/deployment/values/redis.values.yaml renamed to .circleci/deployment/values/valkey.values.yaml

@@ -1,3 +1,3 @@
 architecture: standalone
 auth:
-  password: redis
+  password: valkey

.circleci/update_helm_documentation.sh

@@ -2,39 +2,27 @@
 # shellcheck disable=SC1091
 set -e
 
-if [ -n "$CI" ]
-then
-  echo "running eslint"
-  yarn eslint:projectwide
-  echo "...eslint done"
-  echo "running prettier"
-  yarn prettier:check
-  echo "...prettier done"
-else
 # shellcheck disable=SC1090
-  . "$(dirname "$0")/_/husky.sh"
-  yarn lint-staged
-fi
-
-echo "🔍 looking for additional linter dependencies"
+. "$(dirname "$0")/_/husky.sh"
+yarn lint-staged
 
 check_dependencies_available() {
+  echo "🔍 looking for additional linter dependencies"
   for i in "${@}"
   do
     if ! command -v "${i}"; then
       echo "No ${i} executable found skipping additional checks" >&2
       exit 0
     fi
   done
+  echo "✅ All additional dependencies found"
 }
 
-additional_dependencies=""
-if [ -z "${CI}" ]; then
-  additional_dependencies=" ggshield"
-fi
-
 # shellcheck disable=SC2086
-check_dependencies_available pre-commit hadolint helm shellcheck circleci${additional_dependencies}
+check_dependencies_available pre-commit hadolint helm shellcheck circleci ggshield
+
+echo "🍵 Updating JSON schema for Helm chart"
+yarn helm:jsonschema:generate
 
-echo "All systems functional, running additional pre-commit checks..."
+echo "▶️ Running pre-commit hooks"
 pre-commit run --all-files