feat(cli): add frontend dynamic plugins base build config (#747)

* feat(cli): build dynamic frontend plugins

chore(ci): bump test node version (#831)

feat(cli): add frontend dynamic plugins base build config

fix(cli): resolve post dynamic build stats errors

fix(cli): remove eager consumption settings from shared modules

Eager packages will always end up within the container entry JS files
which significantly increases the network bandwidth even if packages the
shared packages have been already initialized in the shared scope via
different container.

fix(cli): remove the requirement for root tsconfig.json

chore(dynamic-plugins): add sample dynamic plugin package

chore(app-next): add sample app with dynamic plugins

fix(cli): align versions of @backstage/backend-common package

feat(app-next): dynamically load entire backstage plugins

chore(app-next): add readme

chore(app-next): fix eslint errors

fix(cli): do not treat warnings as errors on CI

chore(dependencies): align dependency versions

* chore(cli): cleanup and enable quay dynamic build

Signed-off-by: Tomas Coufal <[email protected]>

---------

Signed-off-by: Tomas Coufal <[email protected]>
Co-authored-by: Tomas Coufal <[email protected]>

Author: Hyperkid123

.gitignore

@@ -31,6 +31,7 @@ node_modules/
 # Build output
 dist
 dist-types
+dist-scalprum
 
 # Temporary change files created by Vim
 *.swp
@@ -49,3 +50,6 @@ site
 
 # turbo
 .turbo
+
+# build cache
+.webpack-cache

package.json

@@ -9,10 +9,10 @@
     "preinstall": "npx only-allow yarn",
     "start": "turbo run start",
     "start:backstage": "turbo run start --filter=...{./packages/app} --filter=...{./packages/backend}",
-    "start:plugins": "turbo run start --filter=...{./plugins/*}",
+    "start:plugins": "turbo run start --filter=@janus-idp/*",
     "build": "turbo run build",
-    "build:backstage": "turbo run build --filter=...{./packages/*}",
-    "build:plugins": "turbo run build --filter=...{./plugins/*}",
+    "build:backstage": "turbo run build --filter=...{./packages/app} --filter=...{./packages/backend}",
+    "build:plugins": "turbo run build --filter=@janus-idp/*",
     "tsc": "turbo run tsc",
     "clean": "turbo run clean",
     "test": "turbo run test",

packages/cli/bin/janus-cli

@@ -24,6 +24,12 @@ const isLocal = require('fs').existsSync(path.resolve(__dirname, '../src'));
 if (!isLocal || process.env.BACKSTAGE_E2E_CLI_TEST) {
   require('..');
 } else {
+  /**
+   *  TODO: Figure out of we need to find `project` path leading to a local plugin
+   * tsconfig.json.
+   * This will become relevant once we start migration plugins to the Janus cli that have
+   * different tsconfig.json specifications.
+   *  */
   require('ts-node').register({
     transpileOnly: true,
     /* eslint-disable-next-line no-restricted-syntax */

packages/cli/package.json

@@ -41,6 +41,7 @@
     "@backstage/integration": "^1.7.1",
     "@backstage/release-manifests": "^0.0.10",
     "@backstage/types": "^1.1.1",
+    "@openshift/dynamic-plugin-sdk-webpack": "^3.0.0",
     "@esbuild-kit/cjs-loader": "^2.4.4",
     "@esbuild-kit/esm-loader": "^2.6.5",
     "@manypkg/get-packages": "^1.1.3",
@@ -119,7 +120,7 @@
     "react-refresh": "^0.14.0",
     "recursive-readdir": "^2.2.2",
     "replace-in-file": "^6.0.0",
-    "rollup": "^2.60.2",
+    "rollup": "^2.78.0",
     "rollup-plugin-dts": "^4.0.1",
     "rollup-plugin-esbuild": "^4.7.2",
     "rollup-plugin-postcss": "^4.0.0",
@@ -137,7 +138,7 @@
     "webpack-node-externals": "^3.0.0",
     "yaml": "^2.3.3",
     "yml-loader": "^2.1.0",
-    "yn": "^5.0.0",
+    "yn": "^4.0.0",
     "zod": "^3.22.4"
   },
   "devDependencies": {

packages/cli/src/commands/build/buildFrontend.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import { PluginBuildMetadata } from '@openshift/dynamic-plugin-sdk-webpack';
 import fs from 'fs-extra';
 
 import { resolve as resolvePath } from 'path';
@@ -26,16 +27,18 @@ interface BuildAppOptions {
   targetDir: string;
   writeStats: boolean;
   configPaths: string[];
+  pluginMetadata?: PluginBuildMetadata;
 }
 
 export async function buildFrontend(options: BuildAppOptions) {
-  const { targetDir, writeStats, configPaths } = options;
+  const { targetDir, writeStats, configPaths, pluginMetadata } = options;
   const { name } = await fs.readJson(resolvePath(targetDir, 'package.json'));
   await buildBundle({
     targetDir,
     entry: 'src/index',
     parallelism: getEnvironmentParallelism(),
     statsJsonEnabled: writeStats,
+    pluginMetadata,
     ...(await loadCliConfig({
       args: configPaths,
       fromPackage: name,

packages/cli/src/commands/export-dynamic-plugin/frontend.ts

@@ -17,10 +17,34 @@
 import { PackageRoleInfo } from '@backstage/cli-node';
 
 import { OptionValues } from 'commander';
+import fs from 'fs-extra';
+
+import { buildScalprumPlugin } from '../../lib/builder/buildScalprumPlugin';
+import { paths } from '../../lib/paths';
 
 export async function frontend(
   _: PackageRoleInfo,
   __: OptionValues,
 ): Promise<void> {
-  throw new Error('frontend not yet implemented');
+  const { name, version, scalprum } = await fs.readJson(
+    paths.resolveTarget('package.json'),
+  );
+  if (scalprum === undefined) {
+    throw new Error(
+      `Package doesn't seem to support dynamic loading. It should have a 'scalprum' key in 'package.json' containing the dynamic loading entrypoints.`,
+    );
+  }
+
+  await fs.remove(paths.resolveTarget('dist-scalprum'));
+
+  await buildScalprumPlugin({
+    writeStats: false,
+    configPaths: [],
+    targetDir: paths.targetDir,
+    pluginMetadata: {
+      ...scalprum,
+      version,
+    },
+    fromPackage: name,
+  });
 }

packages/cli/src/lib/builder/buildScalprumPlugin.ts

@@ -0,0 +1,27 @@
+import { PluginBuildMetadata } from '@openshift/dynamic-plugin-sdk-webpack';
+
+import { buildScalprumBundle } from '../bundler/bundlePlugin';
+import { loadCliConfig } from '../config';
+import { getEnvironmentParallelism } from '../parallel';
+
+interface BuildScalprumPluginOptions {
+  targetDir: string;
+  writeStats: boolean;
+  configPaths: string[];
+  pluginMetadata: PluginBuildMetadata;
+  fromPackage: string;
+}
+
+export async function buildScalprumPlugin(options: BuildScalprumPluginOptions) {
+  const { targetDir, pluginMetadata, fromPackage } = options;
+  await buildScalprumBundle({
+    targetDir,
+    entry: 'src/index',
+    parallelism: getEnvironmentParallelism(),
+    pluginMetadata,
+    ...(await loadCliConfig({
+      args: [],
+      fromPackage,
+    })),
+  });
+}

packages/cli/src/lib/bundler/bundlePlugin.ts

@@ -0,0 +1,123 @@
+import {
+  measureFileSizesBeforeBuild,
+  printFileSizesAfterBuild,
+} from 'react-dev-utils/FileSizeReporter';
+import formatWebpackMessages from 'react-dev-utils/formatWebpackMessages';
+
+import chalk from 'chalk';
+import fs from 'fs-extra';
+import webpack from 'webpack';
+import yn from 'yn';
+
+import { resolveBaseUrl } from './config';
+import { BundlingPathsOptions, resolveBundlingPaths } from './paths';
+import { createScalprumConfig } from './scalprumConfig';
+import { DynamicPluginOptions } from './types';
+
+// TODO(Rugvip): Limits from CRA, we might want to tweak these though.
+const WARN_AFTER_BUNDLE_GZIP_SIZE = 512 * 1024;
+const WARN_AFTER_CHUNK_GZIP_SIZE = 1024 * 1024;
+
+function applyContextToError(error: string, moduleName: string): string {
+  return `Failed to compile '${moduleName}':\n  ${error}`;
+}
+
+export async function buildScalprumBundle(
+  options: BundlingPathsOptions & DynamicPluginOptions,
+) {
+  const paths = resolveBundlingPaths(options);
+  const config = await createScalprumConfig(paths, {
+    ...options,
+    checksEnabled: false,
+    isDev: false,
+    baseUrl: resolveBaseUrl(options.frontendConfig),
+  });
+
+  const isCi = yn(process.env.CI, { default: false });
+
+  const previousFileSizes = await measureFileSizesBeforeBuild(
+    paths.targetScalprumDist,
+  );
+  await fs.emptyDir(paths.targetScalprumDist);
+
+  if (paths.targetPublic) {
+    await fs.copy(paths.targetPublic, paths.targetDist, {
+      dereference: true,
+      filter: file => file !== paths.targetHtml,
+    });
+  }
+
+  const { stats } = await build(config, isCi);
+
+  if (!stats) {
+    throw new Error('No stats returned');
+  }
+
+  printFileSizesAfterBuild(
+    stats,
+    previousFileSizes,
+    paths.targetScalprumDist,
+    WARN_AFTER_BUNDLE_GZIP_SIZE,
+    WARN_AFTER_CHUNK_GZIP_SIZE,
+  );
+}
+
+async function build(config: webpack.Configuration, isCi: boolean) {
+  const stats = await new Promise<webpack.Stats | undefined>(
+    (resolve, reject) => {
+      webpack(config, (err, buildStats) => {
+        if (err) {
+          if (err.message) {
+            const { errors } = formatWebpackMessages({
+              errors: [err.message],
+              warnings: new Array<string>(),
+              _showErrors: true,
+              _showWarnings: true,
+            });
+
+            throw new Error(errors[0]);
+          } else {
+            reject(err);
+          }
+        } else {
+          resolve(buildStats);
+        }
+      });
+    },
+  );
+
+  if (!stats) {
+    throw new Error('Failed to compile: No stats provided');
+  }
+
+  const serializedStats = stats.toJson({
+    all: false,
+    warnings: true,
+    errors: true,
+  });
+  const { errors, warnings } = formatWebpackMessages({
+    errors: serializedStats.errors,
+    warnings: serializedStats.warnings,
+  });
+
+  if (errors.length) {
+    // Only keep the first error. Others are often indicative
+    // of the same problem, but confuse the reader with noise.
+    const errorWithContext = applyContextToError(
+      errors[0],
+      serializedStats.errors?.[0]?.moduleName ?? '',
+    );
+    throw new Error(errorWithContext);
+  }
+  if (isCi && warnings.length) {
+    const warningsWithContext = warnings.map((warning, i) => {
+      return applyContextToError(
+        warning,
+        serializedStats.warnings?.[i]?.moduleName ?? '',
+      );
+    });
+    console.log(chalk.yellow(warningsWithContext.join('\n\n')));
+  }
+
+  return { stats };
+}

packages/cli/src/lib/bundler/config.ts

@@ -27,11 +27,15 @@ import fs from 'fs-extra';
 import HtmlWebpackPlugin from 'html-webpack-plugin';
 import pickBy from 'lodash/pickBy';
 import { RunScriptWebpackPlugin } from 'run-script-webpack-plugin';
-import webpack, { ProvidePlugin } from 'webpack';
+import webpack, { container, ProvidePlugin } from 'webpack';
 import nodeExternals from 'webpack-node-externals';
 import yn from 'yn';
 
-import { posix as posixPath, resolve as resolvePath } from 'path';
+import {
+  join as joinPath,
+  posix as posixPath,
+  resolve as resolvePath,
+} from 'path';
 
 import { paths as cliPaths } from '../../lib/paths';
 import { version } from '../../lib/version';
@@ -40,9 +44,18 @@ import { runPlain } from '../run';
 import { LinkedPackageResolvePlugin } from './LinkedPackageResolvePlugin';
 import { optimization } from './optimization';
 import { BundlingPaths } from './paths';
+import { sharedModules } from './scalprumConfig';
 import { transforms } from './transforms';
 import { BackendBundlingOptions, BundlingOptions } from './types';
 
+const { ModuleFederationPlugin } = container;
+
+const scalprumPlugin = new ModuleFederationPlugin({
+  name: 'backstageHost',
+  filename: 'backstageHost.[fullhash].js',
+  shared: [sharedModules],
+});
+
 const BUILD_CACHE_ENV_VAR = 'BACKSTAGE_CLI_EXPERIMENTAL_BUILD_CACHE';
 
 export function resolveBaseUrl(config: Config): URL {
@@ -142,6 +155,8 @@ export async function createConfig(
     }),
   );
 
+  plugins.push(scalprumPlugin);
+
   // These files are required by the transpiled code when using React Refresh.
   // They need to be excluded to the module scope plugin which ensures that files
   // that exist in the package are required.
@@ -156,6 +171,11 @@ export async function createConfig(
   const withCache = yn(process.env[BUILD_CACHE_ENV_VAR], { default: false });
 
   return {
+    cache: {
+      type: 'filesystem',
+      allowCollectingMemory: true,
+      cacheDirectory: joinPath(process.cwd(), '.webpack-cache'),
+    },
     mode: isDev ? 'development' : 'production',
     profile: false,
     optimization: optimization(options),
@@ -167,6 +187,16 @@ export async function createConfig(
     context: paths.targetPath,
     entry: [...(options.additionalEntryPoints ?? []), paths.targetEntry],
     resolve: {
+      alias: {
+        '@backstage/frontend-app-api/src': joinPath(
+          process.cwd(),
+          'src',
+          'overrides',
+          '@backstage',
+          'frontend-app-api',
+          'src',
+        ),
+      },
       extensions: ['.ts', '.tsx', '.mjs', '.js', '.jsx', '.json', '.wasm'],
       mainFields: ['browser', 'module', 'main'],
       fallback: {

packages/cli/src/lib/bundler/paths.ts

@@ -63,6 +63,7 @@ export function resolveBundlingPaths(options: BundlingPathsOptions) {
     targetPath: resolvePath(targetDir, '.'),
     targetRunFile: runFileExists ? targetRunFile : undefined,
     targetDist: resolvePath(targetDir, 'dist'),
+    targetScalprumDist: resolvePath(targetDir, 'dist-scalprum'),
     targetAssets: resolvePath(targetDir, 'assets'),
     targetSrc: resolvePath(targetDir, 'src'),
     targetDev: resolvePath(targetDir, 'dev'),