🪟 🔧 Switch to pnpm for package managing (#22053)

* WIP

* WIP

* Adjust lockfile

* Fix issues

* Adjust github flows

* More fixes

* Adjust documentation

* Remove unnecessary tsconfig entry

* Cleanup .eslintrc

* Update dependencies

* Fix more issues

* Correct TS

* Make pnpm install run always via gradle

* Install @storybook/react explicitaly

* Update snapshots due to new libraries

* Add node-fetch types

* Add `url` package

* Freeze react router version (due to useBlocker)

* Fix typing issue

* Update airbyte-webapp/src/components/ui/BarChart/BarChart.tsx

Co-authored-by: Lake Mossman <[email protected]>

* Rewrite README

* Update e2e test library

* Remove retry

* Latest pnpm version

* Revert removal of retry logic

---------

Co-authored-by: Lake Mossman <[email protected]>

Author: Tim Roes

.github/actions/cache-build-artifacts/action.yml

@@ -31,6 +31,15 @@ runs:
         restore-keys: |
           ${{ inputs.cache-key }}-npm-${{ runner.os }}-
 
+    - name: pnpm Caching
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.pnpm-store
+        key: ${{ inputs.cache-key }}-pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+        restore-keys: |
+          ${{ inputs.cache-key }}-pnpm-${{ runner.os }}-
+
     # this intentionally does not use restore-keys so we don't mess with gradle caching
     - name: Gradle and Python Caching
       uses: actions/cache@v3

airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -126,6 +126,7 @@ definitions:
       - stream_slicers
     properties:
       type:
+        type: string
         enum: [CartesianProductStreamSlicer]
       stream_slicers:
         type: array
@@ -746,6 +747,7 @@ definitions:
       - type
     properties:
       type:
+        type: string
         enum: [JsonFileSchemaLoader, JsonSchema] # TODO As part of Beta, remove JsonSchema and update connectors to use JsonFileSchemaLoader
       file_path:
         type: string

airbyte-webapp-e2e-tests/README.md

@@ -12,8 +12,8 @@ Steps:
 4) When running the connector builder tests, start the dummy API server: `npm run createdummyapi`
 5) Start the OSS backend: `BASIC_AUTH_USERNAME="" BASIC_AUTH_PASSWORD="" VERSION=dev docker compose --file ../docker-compose.yaml up`. If you want, follow this with `docker compose stop webapp` to turn off the dockerized frontend build; interactive cypress sessions don't use it.
 6) The following two commands will start a separate long-running server, so open another terminal window. In it, `cd` into the `airbyte-webapp/` directory.
-7) If you have not already done so, run `npm install` to install the frontend app's dependencies.
-8) Start the frontend development server with `npm start`.
+7) If you have not already done so, run `pnpm install` to install the frontend app's dependencies.
+8) Start the frontend development server with `pnpm start`.
 9) Back in the `airbyte-webapp-e2e-tests/` directory, start the cypress test runner with `npm run cypress:open`.
 
 ## Reproducing CI test results with `npm run cypress:ci` or `npm run cypress:ci:record`

airbyte-webapp/.eslintrc.js

@@ -9,7 +9,7 @@ module.exports = {
     "plugin:jsx-a11y/recommended",
     "plugin:@airbyte/recommended",
   ],
-  plugins: ["react", "@typescript-eslint", "prettier", "unused-imports", "css-modules", "jsx-a11y", "@airbyte"],
+  plugins: ["@typescript-eslint", "prettier", "unused-imports", "css-modules", "jsx-a11y", "@airbyte"],
   parserOptions: {
     ecmaVersion: 2020,
     sourceType: "module",

airbyte-webapp/.npmrc

@@ -1 +1,2 @@
 engine-strict=true
+enable-pre-post-scripts=true

airbyte-webapp/README.md

@@ -1,31 +1,26 @@
 # airbyte-webapp
 
-This module contains the Airbyte Webapp. It is a React app written in TypeScript. It runs in a Docker container. A very lightweight nginx server runs in that Docker container and serves the webapp.
+This module contains the Airbyte Webapp. It is a React app written in TypeScript.
+The webapp compiles to static HTML, JavaScript and CSS, which is served (in OSS) via
+a nginx in the airbyte-webapp docker image. This nginx also serves as the reverse proxy
+for accessing the server APIs in other images.
 
-This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).
+## Building the webapp
 
-## Available Scripts
+You can build the webapp using Gradle in the root of the repository:
 
-In the project directory, you can run:
+```sh
+# Only compile and build the docker webapp image:
+SUB_BUILD=PLATFORM ./gradlew :airbyte-webapp:assemble
+# Build the webapp and additional artifacts and run tests:
+SUB_BUILD=PLATFORM ./gradlew :airbyte-webapp:build
+```
 
-### `npm start`
+## Developing the webapp
 
-Runs the app in the development mode.<br />
-Open [http://localhost:3000](http://localhost:3000) to view it in the browser.
+For an instruction how to develop on the webapp, please refer to our [documentation](https://docs.airbyte.com/contributing-to-airbyte/developing-locally/#develop-on-airbyte-webapp).
 
-### `npm test`
+### Entrypoints
 
-Launches the test runner in the interactive watch mode.<br />
-
-### `npm run build`
-
-Builds the app for production to the `build` folder.<br />
-
-### VERSION=yourtag ./gradlew :airbyte-webapp:assemble
-
-Builds the app and Docker image and tags the image with `yourtag`.
-Note: needs to be run from the root directory of the Airbyte project.
-
-## Entrypoints
 * `airbyte-webapp/src/App.tsx` is the entrypoint into the OSS version of the webapp.
 * `airbyte-webapp/src/packages/cloud/App.tsx` is the entrypoint into the Cloud version of the webapp.

airbyte-webapp/build.gradle

@@ -1,28 +1,49 @@
 plugins {
     id "base"
-    id "com.github.node-gradle.node" version "3.3.0"
+    id "com.github.node-gradle.node" version "3.4.0"
 }
 
 // Use the node version that's defined in the .nvmrc file
 def nodeVersion = new File("${projectDir}/.nvmrc").text.trim();
 
+// Read pnpm version to use from package.json engines.pnpm entry
+def pnpmVer = new groovy.json.JsonSlurper().parse(new File("${projectDir}/package.json")).engines.pnpm.trim();
+
 // This array should contain a path to all configs that are common to most build tasks and
 // might affect them (i.e. if any of those files change we want to rerun most tasks)
 def commonConfigs = [
     '.env',
     'package.json',
-    'package-lock.json',
+    'pnpm-lock.yaml',
     'tsconfig.json',
     '.prettierrc.js'
 ]
 
 node {
     download = true
     version = nodeVersion
-    npmInstallCommand = 'ci'
+    pnpmVersion = pnpmVer
+}
+
+task validateLockFiles {
+    description "Validate only a pnpm-lock.yaml lock file exists"
+    doLast {
+        assert file("pnpm-lock.yaml").exists()
+        assert !file("package-lock.json").exists()
+        assert !file("yarn.lock").exists()
+    }
 }
 
-npm_run_build {
+pnpmInstall.dependsOn validateLockFiles
+// Make sure `pnpmInstall` always runs, since it's better at determining
+// whether it's needs to do anything than Gradle can
+pnpmInstall.outputs.upToDateWhen { false }
+
+task pnpmBuild(type: PnpmTask) {
+    dependsOn pnpmInstall
+
+    args = ['build']
+
     inputs.files commonConfigs
     inputs.file '.eslintrc.js'
     inputs.dir 'public'
@@ -31,16 +52,16 @@ npm_run_build {
     outputs.dir 'build/app'
 }
 
-task test(type: NpmTask) {
+task test(type: PnpmTask) {
     dependsOn assemble
 
     args = ['run', 'test:ci']
     inputs.files commonConfigs
     inputs.dir 'src'
 }
 
-task licenseCheck(type: NpmTask) {
-    dependsOn npmInstall
+task licenseCheck(type: PnpmTask) {
+    dependsOn pnpmInstall
 
     args = ['run', 'license-check']
     inputs.files commonConfigs
@@ -51,8 +72,8 @@ task licenseCheck(type: NpmTask) {
     outputs.upToDateWhen { true }
 }
 
-task validateLinks(type: NpmTask) {
-    dependsOn npmInstall
+task validateLinks(type: PnpmTask) {
+    dependsOn pnpmInstall
 
     args = ['run', 'validate-links']
 
@@ -61,8 +82,8 @@ task validateLinks(type: NpmTask) {
     outputs.upToDateWhen { false }
 }
 
-task buildStorybook(type: NpmTask) {
-    dependsOn npmInstall
+task buildStorybook(type: PnpmTask) {
+    dependsOn pnpmInstall
     args = ['run', 'build:storybook']
 
     inputs.files commonConfigs
@@ -78,7 +99,7 @@ task buildStorybook(type: NpmTask) {
 }
 
 task copyBuildOutput(type: Copy) {
-    dependsOn copyDocker, npm_run_build
+    dependsOn copyDocker, pnpmBuild
 
     from "${project.projectDir}/build/app"
     into 'build/docker/bin/build'