diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a7806467b..fe529dc15d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,17 +10,135 @@ aliases: - image: cimg/base:2024.02 - &docker-node-image + working_directory: &default-working-dir /mnt/ramdisk/speckle docker: - image: cimg/node:22.6.0 - &docker-node-image-w-browsers + working_directory: *default-working-dir docker: - image: cimg/node:22.6.0-browsers + - &cache-key v4-yarn-packages-cachebust25-{{ checksum "yarn.lock" }} + + - &cache-key-global npm-global-cachebust3-{{ checksum "yarn.lock" }} + + # This is a workaround for the fact that some packages run postinstall scripts + # and do not respect the NPM_CONFIG_PREFIX setting. + # And we cannot store ~/.npm in the workspace, because it is not a sub-directory of our working directory + # So we need to store it in the CircleCI cache storage. + - &pull-npm-global + restore_cache: + name: Restore NPM global directory (workaround) + keys: + - *cache-key-global + + - &push-npm-global + save_cache: + name: Save NPM global directory (workaround) + key: *cache-key-global + paths: + - ~/.npm + - ~/.yarn/berry + + - &pull-yarn-cache + restore_cache: + name: Restore Yarn Package Cache + keys: + - *cache-key + + - &yarn-cache-paths + - .yarn/cache + - .yarn/global # generated from postinstall scripts + - .yarn/unplugged + - .husky/_ # generated from postinstall scripts + - .yarn/install-state.gz + # - node_modules + # - packages/ui-components-nuxt/node_modules + # - packages/monitor-deployment/node_modules + # - packages/webhook-service/node_modules + # - packages/fileimport-service/node_modules + # - packages/dui3/node_modules + # - packages/server/node_modules + # - packages/ui-components/node_modules + # - packages/shared/node_modules + # - packages/viewer-sandbox/node_modules + # - packages/viewer/node_modules + # - packages/preview-service/node_modules + # - packages/tailwind-theme/node_modules + # - packages/frontend-2/node_modules + # - packages/preview-frontend/node_modules + # - packages/objectloader/node_modules + # - packages/objectsender/node_modules + + - &push-yarn-cache + save_cache: + name: Save Yarn Package Cache + key: *cache-key + paths: *yarn-cache-paths + + - &save-yarn-cache + persist_to_workspace: + root: *default-working-dir + paths: *yarn-cache-paths + + - &restore-yarn-cache + attach_workspace: + at: *default-working-dir + + - &restore_workspace + attach_workspace: + at: /tmp/ci/workspace + + - &restore_env_vars + run: + name: Restore Env Vars + command: cat /tmp/ci/workspace/env-vars >> $BASH_ENV + + - &build-public-paths + - packages/frontend-2/.nuxt + - packages/objectloader/dist + - packages/objectloader2/.tshy + - packages/objectloader2/dist + - packages/objectsender/dist + - packages/shared/dist + - packages/tailwind-theme/dist + - packages/tailwind-theme/dist-cjs + - packages/ui-components/dist + - packages/viewer/dist + - node_modules/@speckle/ui-components/dist + + - &save-build-public + persist_to_workspace: + root: *default-working-dir + paths: *build-public-paths + + - &restore-build-public + attach_workspace: + at: *default-working-dir + - &yarn run: name: Install Dependencies - command: PUPPETEER_SKIP_DOWNLOAD=true PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn + # puppeteer and playwright attempt to download browsers, which we don't need + command: | + PUPPETEER_SKIP_DOWNLOAD=true \ + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 \ + yarn --immutable + + - &yarn-from-cache + run: + name: Install Dependencies + # puppeteer and playwright attempt to download browsers, which we don't need + command: | + PUPPETEER_SKIP_DOWNLOAD=true \ + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 \ + yarn --immutable --immutable-cache + + - &build-public + run: + name: Build public packages + command: yarn build:public workflows: test-build: @@ -33,6 +151,8 @@ workflows: only: - main - hotfix* + requires: &yarn-install + - install-dependencies - test-server: &test-server-job-definition context: @@ -45,28 +165,31 @@ workflows: only: /.*/ requires: - docker-build-postgres-container + - install-dependencies - - test-server-no-ff: - filters: *filters-allow-all - requires: - - docker-build-postgres-container + - test-server-no-ff: *test-server-job-definition - test-server-multiregion: *test-server-job-definition - test-frontend-2: filters: *filters-allow-all + requires: + - install-dependencies - test-viewer: filters: *filters-allow-all + requires: + - install-dependencies - test-objectsender: filters: *filters-allow-all + requires: + - install-dependencies - test-shared: filters: *filters-allow-all - - - test-preview-service: - filters: *filters-allow-all + requires: + - install-dependencies - test-ui-components: filters: *filters-allow-all @@ -75,6 +198,8 @@ workflows: context: - chromatic-ui-components filters: *filters-allow-all + requires: + - install-dependencies - deployment-testing-approval: type: approval @@ -127,9 +252,20 @@ workflows: - pre-commit: filters: *filters-allow-all + requires: *yarn-install + + - install-dependencies: + filters: *filters-allow-all + + - cache-dependencies: + filters: *filters-allow-all + requires: + - install-dependencies - lint-and-prettier: filters: *filters-allow-all + requires: + - install-dependencies - docker-build-server: context: &build-context @@ -214,7 +350,6 @@ workflows: - test-server - test-server-no-ff - test-server-multiregion - - test-preview-service - docker-publish-frontend-2: context: *docker-hub-context @@ -232,7 +367,6 @@ workflows: - test-server - test-server-no-ff - test-server-multiregion - - test-preview-service - docker-publish-frontend-2-sourcemaps: context: @@ -259,7 +393,6 @@ workflows: - test-server - test-server-no-ff - test-server-multiregion - - test-preview-service - docker-publish-file-imports: context: *docker-hub-context @@ -277,7 +410,6 @@ workflows: - test-server - test-server-no-ff - test-server-multiregion - - test-preview-service - docker-publish-previews: context: *docker-hub-context @@ -295,7 +427,6 @@ workflows: - test-server - test-server-no-ff - test-server-multiregion - - test-preview-service - docker-publish-test-container: context: *docker-hub-context @@ -313,7 +444,6 @@ workflows: - test-server - test-server-no-ff - test-server-multiregion - - test-preview-service - docker-publish-postgres-container: context: *docker-hub-context @@ -343,7 +473,6 @@ workflows: - test-server - test-server-no-ff - test-server-multiregion - - test-preview-service - docker-publish-docker-compose-ingress: context: *docker-hub-context @@ -360,7 +489,6 @@ workflows: - test-server - test-server-no-ff - test-server-multiregion - - test-preview-service - publish-helm-chart: filters: &filters-publish @@ -400,43 +528,62 @@ workflows: - test-viewer - test-objectsender - test-shared - - test-preview-service - publish-viewer-sandbox-cloudflare-pages: filters: *filters-publish context: - cloudflare-pages-edit requires: + - install-dependencies - test-viewer jobs: get-version: <<: *docker-base-image - working_directory: &work-dir /tmp/ci steps: - checkout - - run: mkdir -p workspace + - run: mkdir -p /tmp/ci/workspace - run: name: set version command: | - echo "export IMAGE_VERSION_TAG=$(.circleci/get_version.sh)" >> workspace/env-vars - - run: cat workspace/env-vars >> $BASH_ENV + echo "export IMAGE_VERSION_TAG=$(.circleci/get_version.sh)" >> /tmp/ci/workspace/env-vars + - *restore_env_vars - run: echo "IMAGE_VERSION_TAG=${IMAGE_VERSION_TAG}" - persist_to_workspace: - root: workspace + root: /tmp/ci/workspace paths: - env-vars + install-dependencies: + <<: *docker-node-image + resource_class: xlarge + steps: + - checkout + - *pull-yarn-cache # from circleci cache storage + - *yarn + - *save-yarn-cache # to workspace + - *push-npm-global # populate the workaround + - *build-public + - *save-build-public # to workspace + + cache-dependencies: + <<: *docker-node-image + resource_class: xlarge + steps: + - checkout + - *restore-yarn-cache # from workspace + - *pull-npm-global # workaround + - *push-yarn-cache # to circleci cache storage + lint-and-prettier: <<: *docker-node-image resource_class: xlarge - working_directory: *work-dir steps: - checkout + - *pull-yarn-cache # from circleci cache storage + - *pull-npm-global # workaround - *yarn - - run: - name: Build public packages - command: yarn build:public + - *restore-build-public - run: name: Lint everything command: yarn eslint:projectwide @@ -457,14 +604,14 @@ jobs: type: string docker: - image: speckle/pre-commit-runner:latest - resource_class: large - working_directory: *work-dir + resource_class: xlarge + working_directory: *default-working-dir steps: - checkout - restore_cache: - name: Restore pre-commit & Yarn Package cache + name: Restore pre-commit keys: - - cache-pre-commit--{{ checksum "<>" }} + - cache-pre-commit-{{ checksum "<>" }} - run: name: Install pre-commit hooks command: pre-commit install-hooks --config <> @@ -494,7 +641,7 @@ jobs: command: server /data --console-address ":9001" # environment: - resource_class: large + resource_class: xlarge environment: NODE_ENV: test DATABASE_URL: 'postgres://speckle:speckle@127.0.0.1:5432/speckle2_test' @@ -517,10 +664,10 @@ jobs: RATELIMITER_ENABLED: 'false' steps: - checkout - - *yarn - - run: - name: Build public packages - command: yarn build:public + - *restore-yarn-cache + - *pull-npm-global # workaround + - *yarn-from-cache + - *restore-build-public - run: name: Wait for dependencies to start command: 'dockerize -wait tcp://localhost:5432 -wait tcp://localhost:6379 -timeout 1m' @@ -651,10 +798,10 @@ jobs: resource_class: xlarge steps: - checkout - - *yarn - - run: - name: Build public packages - command: yarn build:public + - *restore-yarn-cache + - *pull-npm-global # workaround + - *yarn-from-cache + - *restore-build-public - run: name: Lint everything command: yarn lint:ci @@ -662,13 +809,13 @@ jobs: test-viewer: <<: *docker-node-image-w-browsers - resource_class: large + resource_class: xlarge steps: - checkout - - *yarn - - run: - name: Build public packages - command: yarn build:public + - *restore-yarn-cache + - *pull-npm-global # workaround + - *yarn-from-cache + - *restore-build-public - run: name: Lint viewer command: yarn lint:ci @@ -686,43 +833,14 @@ jobs: command: yarn build working_directory: 'packages/viewer-sandbox' - test-preview-service: - docker: - - image: cimg/node:22.6.0-browsers - - image: cimg/postgres:16.4@sha256:2e4f1a965bdd9ba77aa6a0a7b93968c07576ba2a8a7cf86d5eb7b31483db1378 - environment: - POSTGRES_DB: preview_service_test - POSTGRES_PASSWORD: preview_service_test - POSTGRES_USER: preview_service_test - resource_class: large - environment: {} - steps: - - checkout - - *yarn - - run: - name: Build public packages - command: yarn build:public - - run: - name: Lint everything - command: yarn lint:ci - working_directory: 'packages/preview-service' - - run: - name: Copy .env.example to .env - command: | - #!/usr/bin/env bash - cp packages/preview-service/.env.example packages/preview-service/.env - sed -i~ '/^PG_CONNECTION_STRING=/s/=.*/="postgres:\/\/preview_service_test:preview_service_test@127.0.0.1:5432\/preview_service_test"/' packages/preview-service/.env - - run: - name: Run tests - command: yarn test - working_directory: 'packages/preview-service' - test-shared: <<: *docker-node-image-w-browsers - resource_class: medium+ + resource_class: xlarge steps: - checkout - - *yarn + - *restore-yarn-cache + - *pull-npm-global # workaround + - *yarn-from-cache - run: name: Lint command: yarn lint:ci @@ -755,10 +873,10 @@ jobs: resource_class: large steps: - checkout - - *yarn - - run: - name: Build public packages - command: yarn build:public + - *restore-yarn-cache + - *pull-npm-global # workaround + - *yarn-from-cache + - *restore-build-public - run: name: Run tests command: yarn test:ci @@ -771,12 +889,12 @@ jobs: resource_class: xlarge steps: - checkout + - *restore-yarn-cache + - *pull-npm-global # workaround - run: - name: Install Dependencies - command: PUPPETEER_SKIP_DOWNLOAD=true yarn - - run: - name: Build public packages - command: yarn build:public + name: Install Dependencies and download Puppeteer browsers + command: PUPPETEER_SKIP_DOWNLOAD=true yarn --immutable + - *build-public - run: name: Lint tailwind theme command: yarn lint:ci @@ -796,13 +914,13 @@ jobs: ui-components-chromatic: <<: *docker-node-image - resource_class: medium+ + resource_class: xlarge steps: - checkout - - *yarn - - run: - name: Build shared packages - command: yarn build:public + - *restore-yarn-cache + - *pull-npm-global # workaround + - *yarn-from-cache + - *restore-build-public - run: name: Run chromatic command: yarn chromatic @@ -815,11 +933,12 @@ jobs: # because it requires node_modules # therefore this scanning has to be triggered via the cli <<: *docker-node-image - resource_class: large - working_directory: *work-dir + resource_class: xlarge steps: - checkout - - *yarn + - *restore-yarn-cache + - *pull-npm-global # workaround + - *yarn-from-cache - snyk/scan: additional-arguments: --yarn-workspaces --strict-out-of-sync=false fail-on-issues: false @@ -829,11 +948,9 @@ jobs: image: ubuntu-2204:2024.05.1 docker_layer_caching: true resource_class: large - working_directory: *work-dir steps: - checkout - - attach_workspace: - at: /tmp/ci/workspace + - *restore_workspace # create the nix folder with permissive write permissions - run: | sudo mkdir /nix @@ -855,7 +972,7 @@ jobs: key: nix-{{ checksum "./.circleci/deployment/docker-compose-shell.nix" }} paths: - /nix - - run: cat workspace/env-vars >> $BASH_ENV + - *restore_env_vars - run: nix-shell --run "LOAD_DOCKER='true' tilt ci --file ./.circleci/deployment/Tiltfile.dockercompose" ./.circleci/deployment/docker-compose-shell.nix deployment-test-helm-chart: @@ -863,11 +980,9 @@ jobs: image: ubuntu-2204:2024.01.1 docker_layer_caching: true resource_class: large - working_directory: *work-dir steps: - checkout - - attach_workspace: - at: /tmp/ci/workspace + - *restore_workspace # create the nix folder with permissive write permissions - run: | sudo mkdir /nix @@ -891,7 +1006,7 @@ jobs: key: nix-{{ checksum "./.circleci/deployment/helm-chart-shell.nix" }} paths: - /nix - - run: cat workspace/env-vars >> $BASH_ENV + - *restore_env_vars - run: echo "export KUBECONFIG=$(pwd)/.kube/config" >> "${BASH_ENV}" - run: echo "${KUBECONFIG}" - run: @@ -934,12 +1049,10 @@ jobs: docker-build: &build-job <<: *docker-base-image resource_class: medium - working_directory: *work-dir steps: - checkout - - attach_workspace: - at: /tmp/ci/workspace - - run: cat workspace/env-vars >> $BASH_ENV + - *restore_workspace + - *restore_env_vars - setup_remote_docker: version: default docker_layer_caching: true @@ -961,14 +1074,12 @@ jobs: docker-publish-frontend-2-sourcemaps: <<: *docker-node-image resource_class: xlarge - working_directory: *work-dir environment: SPECKLE_SERVER_PACKAGE: frontend-2 steps: - checkout - - attach_workspace: - at: /tmp/ci/workspace - - run: cat workspace/env-vars >> $BASH_ENV + - *restore_workspace + - *restore_env_vars - setup_remote_docker: version: default docker_layer_caching: true @@ -1017,12 +1128,10 @@ jobs: docker-publish: &publish-job <<: *docker-base-image resource_class: medium - working_directory: *work-dir steps: - checkout - - attach_workspace: - at: /tmp/ci/workspace - - run: cat workspace/env-vars >> $BASH_ENV + - *restore_workspace + - *restore_env_vars - setup_remote_docker: version: default docker_layer_caching: true @@ -1080,14 +1189,14 @@ jobs: publish-npm: <<: *docker-node-image - working_directory: *work-dir resource_class: large steps: - checkout - - attach_workspace: - at: /tmp/ci/workspace - - run: cat workspace/env-vars >> $BASH_ENV - - *yarn + - *restore_workspace + - *restore_env_vars + - *restore-yarn-cache + - *pull-npm-global # workaround + - *yarn-from-cache - run: name: auth to npm as Speckle command: | @@ -1110,12 +1219,10 @@ jobs: publish-helm-chart: docker: - image: cimg/python:3.12.1 - working_directory: *work-dir steps: - checkout - - attach_workspace: - at: /tmp/ci/workspace - - run: cat workspace/env-vars >> $BASH_ENV + - *restore_workspace + - *restore_env_vars - add_ssh_keys: fingerprints: - '4d:68:70:66:49:97:ba:8b:8c:55:96:df:3d:be:6e:05' @@ -1125,14 +1232,13 @@ jobs: publish-viewer-sandbox-cloudflare-pages: <<: *docker-node-image - working_directory: *work-dir - resource_class: large + resource_class: xlarge steps: - checkout - - *yarn - - run: - name: Build public packages - command: yarn build:public + - *restore-yarn-cache + - *pull-npm-global # workaround + - *yarn-from-cache + - *restore-build-public - run: name: Lint viewer-sandbox command: yarn lint:ci diff --git a/.yarnrc.yml b/.yarnrc.yml index 61defe11c2..df93231740 100644 --- a/.yarnrc.yml +++ b/.yarnrc.yml @@ -6,4 +6,11 @@ enableMirror: false nodeLinker: node-modules -yarnPath: .yarn/releases/yarn-4.5.0.cjs +yarnPath: ./.yarn/releases/yarn-4.5.0.cjs + +# Some items do not respect the enableGlobalCache: false setting +# and still use the global cache. This is a workaround to ensure they +# use the local directory instead. +globalFolder: ./.yarn/global + +cacheFolder: ./.yarn/cache diff --git a/packages/webhook-service/Dockerfile b/packages/webhook-service/Dockerfile index efcb272606..5779d8c1ef 100644 --- a/packages/webhook-service/Dockerfile +++ b/packages/webhook-service/Dockerfile @@ -41,7 +41,6 @@ RUN apt-get update -y \ && rm -rf /var/lib/apt/lists/* FROM node:22-bookworm-slim@sha256:221ee67425de7a3c11ce4e81e63e50caaec82ede3a7d34599ab20e59d29a0cb5 AS dependency-stage -# yarn install ARG NODE_ENV ENV NODE_ENV=${NODE_ENV}