server(plugins): upate without downtime

Create a new folder each time a new plugin/theme is installed or
updated. The folder name is created based on the package.json content
hash.

closes Chocobozzz#4828

Author: kontrollanten

packages/server-commands/src/server/plugins-command.ts

@@ -253,6 +253,6 @@ export class PluginsCommand extends AbstractCommand {
   }
 
   private getPackageJSONPath (npmName: string) {
-    return this.server.servers.buildDirectory(join('plugins', 'node_modules', npmName, 'package.json'))
+    return this.server.servers.buildDirectory(join('plugins', 'latest', 'node_modules', npmName, 'package.json'))
   }
 }

packages/tests/src/api/server/plugins.ts

@@ -370,7 +370,7 @@ describe('Test plugins', function () {
     const query = `UPDATE "application" SET "nodeABIVersion" = 1`
     await sqlCommand.updateQuery(query)
 
-    const baseNativeModule = server.servers.buildDirectory(join('plugins', 'node_modules', 'a-native-example'))
+    const baseNativeModule = server.servers.buildDirectory(join('plugins', 'latest', 'node_modules', 'a-native-example'))
 
     await removeNativeModule()
     await server.kill()

server/core/helpers/core-utils.ts

@@ -7,7 +7,8 @@
 
 import { promisify1, promisify2, promisify3 } from '@peertube/peertube-core-utils'
 import { exec, ExecOptions } from 'child_process'
-import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto'
+import { createHash, ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto'
+import { createReadStream } from 'fs'
 import truncate from 'lodash-es/truncate.js'
 import { pipeline } from 'stream'
 import { URL } from 'url'
@@ -261,6 +262,17 @@ function generateED25519KeyPairPromise () {
   })
 }
 
+function getContentHash (filePath: string): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const hash = createHash('md5')
+    hash.update(Date.now().toString())
+    const stream = createReadStream(filePath)
+    stream.on('error', err => reject(err))
+    stream.on('data', chunk => hash.update(chunk))
+    stream.on('end', () => resolve(hash.digest('hex')))
+  })
+}
+
 // ---------------------------------------------------------------------------
 
 const randomBytesPromise = promisify1<number, Buffer>(randomBytes)
@@ -285,6 +297,8 @@ export {
 
   scryptPromise,
 
+  getContentHash,
+
   randomBytesPromise,
 
   generateRSAKeyPairPromise,

server/core/helpers/decache.ts

@@ -4,18 +4,6 @@
 import { Module } from 'module'
 import { extname } from 'path'
 
-function decachePlugin (require: NodeRequire, libraryPath: string) {
-  const moduleName = find(require, libraryPath)
-
-  if (!moduleName) return
-
-  searchCache(require, moduleName, function (mod) {
-    delete require.cache[mod.id]
-
-    removeCachedPath(mod.path)
-  })
-}
-
 function decacheModule (require: NodeRequire, name: string) {
   const moduleName = find(require, name)
 
@@ -31,8 +19,7 @@ function decacheModule (require: NodeRequire, name: string) {
 // ---------------------------------------------------------------------------
 
 export {
-  decacheModule,
-  decachePlugin
+  decacheModule
 }
 
 // ---------------------------------------------------------------------------

server/core/lib/plugins/plugin-manager.ts

@@ -1,6 +1,7 @@
 import express from 'express'
-import { createReadStream, createWriteStream } from 'fs'
-import { ensureDir, outputFile, readJSON } from 'fs-extra/esm'
+import { createReadStream, createWriteStream, statSync } from 'fs'
+import { mkdir, readlink } from 'fs/promises'
+import { ensureDir, outputFile, readJSON, ensureSymlink } from 'fs-extra/esm'
 import { Server } from 'http'
 import { createRequire } from 'module'
 import { basename, join } from 'path'
@@ -16,7 +17,6 @@ import {
   ServerHook,
   ServerHookName
 } from '@peertube/peertube-models'
-import { decachePlugin } from '@server/helpers/decache.js'
 import { ApplicationModel } from '@server/models/application/application.js'
 import { MOAuthTokenUser, MUser } from '@server/types/models/index.js'
 import { isLibraryCodeValid, isPackageJSONValid } from '../../helpers/custom-validators/plugins.js'
@@ -77,13 +77,23 @@ export class PluginManager implements ServerHook {
   private hooks: { [name: string]: HookInformationValue[] } = {}
   private translations: PluginLocalesTranslations = {}
 
+  private readonly latestDirectory = join(CONFIG.STORAGE.PLUGINS_DIR, 'latest')
+
   private server: Server
 
   private constructor () {
   }
 
-  init (server: Server) {
+  async init (server: Server) {
     this.server = server
+
+    try {
+      statSync(this.latestDirectory)
+    } catch (err) {
+      const workingDir = join(CONFIG.STORAGE.PLUGINS_DIR, Date.now().toString())
+      await mkdir(workingDir)
+      await ensureSymlink(workingDir, this.latestDirectory)
+    }
   }
 
   registerWebSocketRouter () {
@@ -374,6 +384,11 @@ export class PluginManager implements ServerHook {
       logger.info('Successful installation of plugin %s.', toInstall)
 
       if (register) {
+        // Unregister old hooks if it's an update
+        try {
+          await this.unregister(npmName)
+        } catch (err) {}
+
         await this.registerPluginOrTheme(plugin)
       }
     } catch (rootErr) {
@@ -394,6 +409,12 @@ export class PluginManager implements ServerHook {
       }
 
       throw rootErr
+    } finally {
+      // Update plugin paths
+      for (const npmName in this.registeredPlugins) {
+        const { name, type } = this.registeredPlugins[npmName]
+        this.registeredPlugins[npmName].path = await this.getPluginPath(name, type)
+      }
     }
 
     return plugin
@@ -411,9 +432,6 @@ export class PluginManager implements ServerHook {
       version = plugin.latestVersion
     }
 
-    // Unregister old hooks
-    await this.unregister(npmName)
-
     return this.install({ toInstall: toUpdate, version, fromDisk })
   }
 
@@ -463,7 +481,7 @@ export class PluginManager implements ServerHook {
     logger.info('Registering plugin or theme %s.', npmName)
 
     const packageJSON = await this.getPackageJSON(plugin.name, plugin.type)
-    const pluginPath = this.getPluginPath(plugin.name, plugin.type)
+    const pluginPath = await this.getPluginPath(plugin.name, plugin.type)
 
     this.sanitizeAndCheckPackageJSONOrThrow(packageJSON, plugin.type)
 
@@ -503,9 +521,7 @@ export class PluginManager implements ServerHook {
   private async registerPlugin (plugin: PluginModel, pluginPath: string, packageJSON: PluginPackageJSON) {
     const npmName = PluginModel.buildNpmName(plugin.name, plugin.type)
 
-    // Delete cache if needed
     const modulePath = join(pluginPath, packageJSON.library)
-    decachePlugin(require, modulePath)
     const library: PluginLibrary = require(modulePath)
 
     if (!isLibraryCodeValid(library)) {
@@ -530,7 +546,7 @@ export class PluginManager implements ServerHook {
   private async addTranslations (plugin: PluginModel, npmName: string, translationPaths: PluginTranslationPathsJSON) {
     for (const locale of Object.keys(translationPaths)) {
       const path = translationPaths[locale]
-      const json = await readJSON(join(this.getPluginPath(plugin.name, plugin.type), path))
+      const json = await readJSON(join(await this.getPluginPath(plugin.name, plugin.type), path))
 
       const completeLocale = getCompleteLocale(locale)
 
@@ -591,16 +607,17 @@ export class PluginManager implements ServerHook {
     }
   }
 
-  private getPackageJSON (pluginName: string, pluginType: PluginType_Type) {
-    const pluginPath = join(this.getPluginPath(pluginName, pluginType), 'package.json')
+  private async getPackageJSON (pluginName: string, pluginType: PluginType_Type) {
+    const pluginPath = join(await this.getPluginPath(pluginName, pluginType), 'package.json')
 
     return readJSON(pluginPath) as Promise<PluginPackageJSON>
   }
 
-  private getPluginPath (pluginName: string, pluginType: PluginType_Type) {
+  private async getPluginPath (pluginName: string, pluginType: PluginType_Type) {
     const npmName = PluginModel.buildNpmName(pluginName, pluginType)
+    const currentDirectory = await readlink(join(CONFIG.STORAGE.PLUGINS_DIR, 'latest'))
 
-    return join(CONFIG.STORAGE.PLUGINS_DIR, 'node_modules', npmName)
+    return join(currentDirectory, 'node_modules', npmName)
   }
 
   private getAuth (npmName: string, authName: string) {

server/core/lib/plugins/yarn.ts

@@ -1,6 +1,7 @@
-import { outputJSON, pathExists } from 'fs-extra/esm'
+import { copy, ensureSymlink, remove, outputJSON, pathExists, ensureDir } from 'fs-extra/esm'
+import { readlink } from 'fs/promises'
 import { join } from 'path'
-import { execShell } from '../../helpers/core-utils.js'
+import { execShell, getContentHash } from '../../helpers/core-utils.js'
 import { isNpmPluginNameValid, isPluginStableOrUnstableVersionValid } from '../../helpers/custom-validators/plugins.js'
 import { logger } from '../../helpers/logger.js'
 import { CONFIG } from '../../initializers/config.js'
@@ -16,7 +17,7 @@ async function installNpmPlugin (npmName: string, versionArg?: string) {
   let toInstall = npmName
   if (version) toInstall += `@${version}`
 
-  const { stdout } = await execYarn('add ' + toInstall)
+  const stdout = await execYarn('add ' + toInstall)
 
   logger.debug('Added a yarn package.', { yarnStdout: stdout })
 }
@@ -47,21 +48,55 @@ export {
 // ############################################################################
 
 async function execYarn (command: string) {
+  const latestDirectory = join(CONFIG.STORAGE.PLUGINS_DIR, 'latest')
+  const currentDirectory = await readlink(latestDirectory)
+  let workingDirectory: string
+  let stdout: string
+
   try {
-    const pluginDirectory = CONFIG.STORAGE.PLUGINS_DIR
-    const pluginPackageJSON = join(pluginDirectory, 'package.json')
+    const pluginPackageJSON = join(currentDirectory, 'package.json')
 
     // Create empty package.json file if needed
     if (!await pathExists(pluginPackageJSON)) {
       await outputJSON(pluginPackageJSON, {})
     }
 
-    return execShell(`yarn ${command}`, { cwd: pluginDirectory })
+    const hash = await getContentHash(pluginPackageJSON)
+
+    workingDirectory = join(CONFIG.STORAGE.PLUGINS_DIR, hash)
+    await ensureDir(workingDirectory)
+    await copy(join(currentDirectory, 'package.json'), join(workingDirectory, 'package.json'))
+
+    try {
+      await copy(join(currentDirectory, 'yarn.lock'), join(workingDirectory, 'yarn.lock'))
+    } catch (err) {
+      logger.debug('No yarn.lock file to copy, will continue without.')
+    }
+
+    const result = await execShell(`yarn ${command}`, { cwd: workingDirectory })
+    stdout = result.stdout
   } catch (result) {
-    logger.error('Cannot exec yarn.', { command, err: result.err, stderr: result.stderr })
+    logger.error('Cannot exec yarn.', { command, err: result, stderr: result.stderr })
+
+    await remove(workingDirectory)
 
     throw result.err
   }
+
+  try {
+    await remove(latestDirectory)
+    await ensureSymlink(workingDirectory, latestDirectory)
+  } catch (err) {
+    logger.error('Cannot create symlink for new plugin set. Trying to restore the old one.', { err })
+    await ensureSymlink(currentDirectory, latestDirectory)
+    logger.info('Succeeded to restore old plugin set.')
+
+    throw err
+  }
+
+  await remove(currentDirectory)
+
+  return stdout
 }
 
 function checkNpmPluginNameOrThrow (name: string) {

server/server.ts

@@ -335,7 +335,7 @@ async function startApplication () {
 
   OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer })
 
-  PluginManager.Instance.init(server)
+  await PluginManager.Instance.init(server)
   // Before PeerTubeSocket init
   PluginManager.Instance.registerWebSocketRouter()