feat: add external mod loading

A package called "useapi" is introduced to provide a dynamic import
system. This import system, rather than relying on the state of the
filesystem, is populated as modules are installed into Puter's kernel.

The "useapi" package is then used to add support for loading external
mod directories as Puter kernel modules, making it possible to mod puter
without any tooling.

Author: KernelDeimos

packages/backend/src/CoreModule.js

@@ -24,7 +24,8 @@ class CoreModule extends AdvancedBase {
     async install (context) {
         const services = context.get('services');
         const app = context.get('app');
-        await install({ services, app });
+        const useapi = context.get('useapi');
+        await install({ services, app, useapi });
     }
 
     // Some services were created before the BaseService
@@ -40,9 +41,16 @@ class CoreModule extends AdvancedBase {
 
 module.exports = CoreModule;
 
-const install = async ({ services, app }) => {
+const install = async ({ services, app, useapi }) => {
     const config = require('./config');
 
+    useapi.withuse(() => {
+        def('Service', require('./services/BaseService'));
+        def('Module', AdvancedBase);
+
+        def('puter.middlewares.auth', require('./middleware/auth2'));
+    });
+
     // /!\ IMPORTANT /!\
     // For new services, put the import immediate above the
     // call to services.registerService. We'll clean this up

packages/backend/src/Kernel.js

@@ -18,12 +18,20 @@
  */
 const { AdvancedBase } = require("@heyputer/puter-js-common");
 const { Context } = require('./util/context');
+const BaseService = require("./services/BaseService");
+const useapi = require('useapi');
 
 class Kernel extends AdvancedBase {
     constructor () {
         super();
 
         this.modules = [];
+        this.useapi = useapi();
+
+        this.useapi.withuse(() => {
+            def('Module', AdvancedBase);
+            def('Service', BaseService);
+        });
     }
 
     add_module (module) {
@@ -48,7 +56,8 @@ class Kernel extends AdvancedBase {
         const runtimeEnv = new RuntimeEnvironment({
             logger: bootLogger,
         });
-        runtimeEnv.init();
+        const environment = runtimeEnv.init();
+        this.environment = environment;
 
         // polyfills
         require('./polyfill/to-string-higher-radix');
@@ -89,6 +98,8 @@ class Kernel extends AdvancedBase {
         // app.set('services', services);
 
         const root_context = Context.create({
+            environment: this.environment,
+            useapi: this.useapi,
             services,
             config,
             logger: this.bootLogger,
@@ -108,10 +119,14 @@ class Kernel extends AdvancedBase {
     async _install_modules () {
         const { services } = this;
 
+        // Internal modules
         for ( const module of this.modules ) {
             await module.install(Context.get());
         }
 
+        // External modules
+        await this.install_extern_mods_();
+
         try {
             await services.init();
         } catch (e) {
@@ -173,6 +188,34 @@ class Kernel extends AdvancedBase {
         await services.emit('boot.activation');
         await services.emit('boot.ready');
     }
+
+    async install_extern_mods_ () {
+        const path_ = require('path');
+        const fs = require('fs');
+
+        const mod_paths = this.environment.mod_paths;
+        for ( const mods_dirpath of mod_paths ) {
+            const mod_dirnames = fs.readdirSync(mods_dirpath);
+            for ( const mod_dirname of mod_dirnames ) {
+                const mod_path = path_.join(mods_dirpath, mod_dirname);
+                if ( ! fs.lstatSync(mod_path).isDirectory() ) {
+                    continue;
+                }
+
+                const mod_class = this.useapi.withuse(() => require(mod_path));
+                const mod = new mod_class();
+                if ( ! mod ) {
+                    continue;
+                }
+
+                if ( mod.install ) {
+                    this.useapi.awithuse(async () => {
+                        await mod.install(Context.get());
+                    });
+                }
+            }
+        }
+    }
 }
 
 module.exports = { Kernel };

packages/backend/src/boot/RuntimeEnvironment.js

@@ -162,6 +162,30 @@ const runtime_paths = ({ path_checks }) => ({ path_ }) => [
     },
 ];
 
+// Suitable mod paths in order of precedence.
+const mod_paths = ({ path_checks }) => ({ path_ }) => [
+    {
+        label: '$MOD_PATH',
+        get path () { return process.env.MOD_PATH },
+        checks: [
+            path_checks.require_if_not_undefined,
+        ],
+    },
+    {
+        path: '/var/puter/mods',
+        checks: [
+            path_checks.skip_if_not_exists,
+            path_checks.env_not_set('NO_VAR_MODS'),
+        ],
+    },
+    {
+        get path () {
+            return path_.join(original_cwd, 'mods');
+        },
+        checks: [ path_checks.skip_if_not_exists ],
+    },
+];
+
 class RuntimeEnvironment extends AdvancedBase {
     static MODULES = {
         fs: require('node:fs'),
@@ -175,11 +199,12 @@ class RuntimeEnvironment extends AdvancedBase {
         this.path_checks = path_checks(this)(this.modules);
         this.config_paths = config_paths(this)(this.modules);
         this.runtime_paths = runtime_paths(this)(this.modules);
+        this.mod_paths = mod_paths(this)(this.modules);
     }
 
     init () {
         try {
-            this.init_();
+            return this.init_();
         } catch (e) {
             this.logger.error(e);
             print_error_help(e);
@@ -203,6 +228,12 @@ class RuntimeEnvironment extends AdvancedBase {
             [ this.path_checks.require_write_permission ]
         );
 
+        const mods_path_entry = this.get_first_suitable_path_(
+            { pathFor: 'mods', optional: true },
+            this.mod_paths,
+            [ this.path_checks.require_read_permission ],
+        );
+
         process.chdir(pwd_path_entry.path);
 
         // Check for a valid config file in the config path
@@ -266,6 +297,16 @@ class RuntimeEnvironment extends AdvancedBase {
         // console.log(config.services);
         // console.log(Object.keys(config.services));
         // console.log({ ...config.services });
+
+        const mod_paths = [];
+
+        if ( mods_path_entry ) {
+            mod_paths.push(mods_path_entry.path);
+        }
+
+        return {
+            mod_paths,
+        };
     }
 
     get_first_suitable_path_ (meta, paths, last_checks) {
@@ -295,6 +336,7 @@ class RuntimeEnvironment extends AdvancedBase {
             return entry;
         }
 
+        if ( meta.optional ) return;
         throw new TechnicalError(`No suitable path found for ${meta.pathFor}.`);
     }
 }

packages/backend/src/services/WebServerService.js

@@ -28,6 +28,7 @@ const fs = require('fs');
 const auth = require('../middleware/auth');
 const { osclink } = require('../util/strutil');
 const { surrounding_box, es_import_promise } = require('../fun/dev-console-ui-utils');
+const auth2 = require('../middleware/auth2.js');
 
 class WebServerService extends BaseService {
     static MODULES = {
@@ -393,6 +394,10 @@ class WebServerService extends BaseService {
         app.options('/*', (_, res) => {
             return res.sendStatus(200);
         });
+
+        this.router_user = express.Router();
+        this.router_user.use(auth2);
+        app.use(this.router_user);
     }
 
     _register_commands (commands) {

packages/backend/src/services/database/BaseDatabaseAccessService.js

@@ -18,8 +18,11 @@
  */
 const { AdvancedBase } = require("@heyputer/puter-js-common");
 const BaseService = require("../BaseService");
+const { DB_WRITE, DB_READ } = require("./consts");
 
 class BaseDatabaseAccessService extends BaseService {
+    static DB_WRITE = DB_WRITE;
+    static DB_READ = DB_READ;
     case ( choices ) {
         const engine_name = this.constructor.ENGINE_NAME;
         if ( choices.hasOwnProperty(engine_name) ) {

packages/useapi/main.js

@@ -0,0 +1,100 @@
+const globalwith = (vars, fn) => {
+    const original_values = {};
+    const keys = Object.keys(vars);
+
+    for ( const key of keys ) {
+        if ( key in globalThis ) {
+            original_values[key] = globalThis[key];
+        }
+        globalThis[key] = vars[key];
+    }
+
+    try {
+        return fn();
+    } finally {
+        for ( const key of keys ) {
+            if ( key in original_values ) {
+                globalThis[key] = original_values[key];
+            } else {
+                delete globalThis[key];
+            }
+        }
+    }
+};
+
+const aglobalwith = async (vars, fn) => {
+    const original_values = {};
+    const keys = Object.keys(vars);
+
+    for ( const key of keys ) {
+        if ( key in globalThis ) {
+            original_values[key] = globalThis[key];
+        }
+        globalThis[key] = vars[key];
+    }
+
+    try {
+        return await fn();
+    } finally {
+        for ( const key of keys ) {
+            if ( key in original_values ) {
+                globalThis[key] = original_values[key];
+            } else {
+                delete globalThis[key];
+            }
+        }
+    }
+};
+
+let default_fn = () => {
+    const use = name => {
+        const parts = name.split('.');
+        let obj = use;
+        for ( const part of parts ) {
+            if ( ! obj[part] ) {
+                obj[part] = {};
+            }
+            obj = obj[part];
+        }
+
+        return obj;
+    };
+    const library = {
+        use,
+        def: (name, value) => {
+            const parts = name.split('.');
+            let obj = use;
+            for ( const part of parts.slice(0, -1) ) {
+                if ( ! obj[part] ) {
+                    obj[part] = {};
+                }
+                obj = obj[part];
+            }
+
+            obj[parts[parts.length - 1]] = value;
+        },
+        withuse: fn => {
+            return globalwith({
+                use,
+                def: library.def,
+            }, fn);
+        },
+        awithuse: async fn => {
+            return await aglobalwith({
+                use,
+                def: library.def,
+            }, fn);
+        }
+    };
+
+    return library;
+};
+
+const useapi = function useapi () {
+    return default_fn();
+};
+
+// We export some things on the function itself
+useapi.globalwith = globalwith;
+
+module.exports = useapi;

packages/useapi/package.json

@@ -0,0 +1,8 @@
+{
+    "name": "useapi",
+    "version": "1.0.0",
+    "author": "Puter Technologies Inc.",
+    "license": "AGPL-3.0-only",
+    "description": "Dynamic import interface for Puter mods",
+    "main": "main.js"
+}