feat!: Store the typegraph on s3 (#620)

Store the typegraph on s3 for multiple instance support mode.

#### Motivation and context

Reduce Redis data.

#### Migration notes

Environment variables:
- `REDIS_URL` has been removed
- For multiple instance support, the following variables are required:
`SYNC_REDIS_URL`, `SYNC_S3_HOST`, `SYNC_S3_REGION`, `SYNC_S3_BUCKET`,
`SYNC_S3_ACCESS_KEY`, `SYNC_S3_SECRET_KEY`; and the following variables
are optional: `SYNC_REDIS_PASSWORD`, `SYNC_S3_PATH_STYLE`. Otherwise,
none of them can be set.


### Checklist

- [x] The change come with new or modified tests
- [ ] Hard-to-understand functions have explanatory comments
- [x] End-user documentation is updated to reflect the change

Author: Natoandro

.github/workflows/tests.yml

@@ -243,7 +243,7 @@ jobs:
         ports:
           - 6379:6379
         options: >-
-          --health-cmd "redis-cli ping"
+          --health-cmd "redis-cli -a password ping"
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5

dev/test.ts

@@ -100,7 +100,6 @@ const env: Record<string, string> = {
   "TG_SECRET":
     "a4lNi0PbEItlFZbus1oeH/+wyIxi9uH6TpL8AIqIaMBNvp7SESmuUBbfUwC0prxhGhZqHw8vMDYZAGMhSZ4fLw==",
   "TG_ADMIN_PASSWORD": "password",
-  "REDIS_URL": "redis://:password@localhost:6379/0",
   "DENO_TESTING": "true",
   "TMP_DIR": tmpDir,
   "TIMER_MAX_TIMEOUT_MS": "30000",

docs/workflows/typegraph_deployment.drawio.svg

@@ -1,6 +1,7 @@
 // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0.
 // SPDX-License-Identifier: MPL-2.0
 
+use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 
@@ -89,6 +90,26 @@ pub struct DeployOptions {
     /// Run in watch mode
     #[clap(long, default_value_t = false)]
     pub watch: bool,
+
+    /// secret overrides
+    #[clap(long = "secret")]
+    pub secrets: Vec<String>,
+}
+
+fn override_secrets(
+    secrets: HashMap<String, String>,
+    overrides: Vec<String>,
+) -> Result<HashMap<String, String>> {
+    let mut secrets = secrets;
+    for override_str in overrides {
+        let parts: Vec<&str> = override_str.splitn(2, '=').collect();
+        if parts.len() != 2 {
+            bail!("Invalid secret override: {}", override_str);
+        }
+        secrets.insert(parts[0].to_string(), parts[1].to_string());
+    }
+
+    Ok(secrets)
 }
 
 pub struct Deploy {
@@ -113,7 +134,7 @@ impl Deploy {
         let node = node_config
             .build(&dir)
             .await
-            .with_context(|| format!("building node from config: {node_config:#?}"))?;
+            .with_context(|| format!("error while building node from config: {node_config:#?}"))?;
 
         ServerStore::with(Some(Command::Deploy), Some(config.as_ref().to_owned()));
         ServerStore::set_migration_action_glob(MigrationAction {
@@ -203,8 +224,11 @@ mod default_mode {
     impl DefaultMode {
         pub async fn init(deploy: Deploy) -> Result<Self> {
             let console = ConsoleActor::new(Arc::clone(&deploy.config)).start();
-            let secrets =
-                lade_sdk::hydrate(deploy.node.env.clone(), deploy.base_dir.to_path_buf()).await?;
+            let secrets = lade_sdk::hydrate(
+                override_secrets(deploy.node.env.clone(), deploy.options.secrets.clone())?,
+                deploy.base_dir.to_path_buf(),
+            )
+            .await?;
 
             ServerStore::set_secrets(secrets);
 
@@ -323,8 +347,11 @@ mod watch_mode {
         .context("setting Ctrl-C handler")?;
 
         loop {
-            let secrets =
-                lade_sdk::hydrate(deploy.node.env.clone(), deploy.base_dir.to_path_buf()).await?;
+            let secrets = lade_sdk::hydrate(
+                override_secrets(deploy.node.env.clone(), deploy.options.secrets.clone())?,
+                deploy.base_dir.to_path_buf(),
+            )
+            .await?;
 
             ServerStore::set_secrets(secrets.clone());
 

meta-cli/src/cli/deploy.rs

@@ -25,6 +25,10 @@ pub struct Dev {
 
     #[clap(long)]
     max_parallel_loads: Option<usize>,
+
+    /// secrets overload
+    #[clap(long = "secret")]
+    secrets: Vec<String>,
 }
 
 #[async_trait]
@@ -38,6 +42,7 @@ impl Action for Dev {
             watch: true,
             no_migration: false,
             create_migration: true,
+            secrets: self.secrets.clone(),
         };
 
         let deploy = DeploySubcommand::new(

meta-cli/src/cli/dev.rs

@@ -14,7 +14,7 @@
     "zod": "https://deno.land/x/[email protected]/mod.ts",
     "monads": "https://deno.land/x/[email protected]/mod.ts",
     "jwt": "https://deno.land/x/[email protected]/mod.ts",
-    "redis": "https://deno.land/x/redis@v0.31.0/mod.ts",
+    "redis": "https://deno.land/x/redis@v0.32.1/mod.ts",
     "oauth2_client": "https://deno.land/x/[email protected]/mod.ts",
     "test/mock_fetch": "https://deno.land/x/[email protected]/mod.ts",
     "levenshtein": "https://deno.land/x/[email protected]/mod.ts",

typegate/deno.lock

@@ -6,7 +6,6 @@ import { mapKeys } from "std/collections/map_keys.ts";
 
 import * as base64 from "std/encoding/base64.ts";
 import { parse } from "std/flags/mod.ts";
-import { RedisConnectOptions } from "redis";
 import { join } from "std/path/mod.ts";
 // This import ensure log loads before config, important for the version hydration
 import { configOrExit, zBooleanString } from "./log.ts";
@@ -17,18 +16,6 @@ const schema = {
   // If false, auto reload system typegraphs on change. Default: to true.
   packaged: zBooleanString,
   hostname: z.string(),
-  redis_url: z
-    .string()
-    .transform((s: string) => {
-      if (s == "none") {
-        return new URL("redis://none");
-      }
-      const url = new URL(s);
-      if (url.password === "") {
-        url.password = Deno.env.get("REDIS_PASSWORD") ?? "";
-      }
-      return url;
-    }),
   tg_port: z.coerce.number().positive().max(65535),
   tg_secret: z.string().transform((s: string, ctx) => {
     const bytes = base64.decode(s);
@@ -94,12 +81,3 @@ const config = await configOrExit([
 ], schema);
 
 export default config;
-
-export const redisConfig: RedisConnectOptions = {
-  hostname: config.redis_url.hostname,
-  port: config.redis_url.port,
-  ...config.redis_url.password.length > 0
-    ? { password: config.redis_url.password }
-    : {},
-  db: parseInt(config.redis_url.pathname.substring(1), 10),
-};

typegate/import_map.json

@@ -1,19 +1,15 @@
 // Copyright Metatype OÜ, licensed under the Elastic License 2.0.
 // SPDX-License-Identifier: Elastic-2.0
 
-import { deferred } from "std/async/deferred.ts";
 import { init_native } from "native";
 
-import { Register, ReplicatedRegister } from "./typegate/register.ts";
-import config, { redisConfig } from "./config.ts";
+import config from "./config.ts";
 import { Typegate } from "./typegate/mod.ts";
-import { RateLimiter, RedisRateLimiter } from "./typegate/rate_limiter.ts";
 import { SystemTypegraph } from "./system_typegraphs.ts";
 import * as Sentry from "sentry";
 import { getLogger } from "./log.ts";
 import { init_runtimes } from "./runtimes/mod.ts";
-import { MemoryRegister } from "test-utils/memory_register.ts";
-import { NoLimiter } from "test-utils/no_limiter.ts";
+import { syncConfigFromEnv } from "./sync/config.ts";
 
 const logger = getLogger(import.meta);
 
@@ -50,32 +46,8 @@ try {
   // load all runtimes
   await init_runtimes();
 
-  const deferredTypegate = deferred<Typegate>();
-  let register: Register | undefined;
-  let limiter: RateLimiter | undefined;
-
-  if (redisConfig.hostname != "none") {
-    register = await ReplicatedRegister.init(deferredTypegate, redisConfig);
-    limiter = await RedisRateLimiter.init(redisConfig);
-  } else {
-    logger.warning("Entering Redis-less mode");
-    register = new MemoryRegister();
-    limiter = new NoLimiter();
-  }
-
-  const typegate = new Typegate(register!, limiter!);
-
-  deferredTypegate.resolve(typegate);
-
-  if (register instanceof ReplicatedRegister) {
-    const lastSync = await register.historySync().catch((err) => {
-      logger.error(err);
-      throw new Error(
-        `failed to load history at boot, aborting: ${err.message}`,
-      );
-    });
-    register.startSync(lastSync);
-  }
+  const syncConfig = await syncConfigFromEnv(["vars", "args"]);
+  const typegate = await Typegate.init(syncConfig);
 
   await SystemTypegraph.loadAll(typegate, !config.packaged);
 

typegate/src/config.ts

@@ -0,0 +1,130 @@
+// Copyright Metatype OÜ, licensed under the Elastic License 2.0.
+// SPDX-License-Identifier: Elastic-2.0
+
+import { z } from "zod";
+import { configOrExit, zBooleanString } from "../log.ts";
+import { RedisConnectOptions } from "redis";
+import { S3ClientConfig } from "aws-sdk/client-s3";
+import { mapKeys } from "std/collections/map_keys.ts";
+import { parse } from "std/flags/mod.ts";
+
+export const syncConfigSchemaNaked = {
+  sync_redis_url: z.string().optional().transform((s) => {
+    if (s == undefined) return undefined;
+    const url = new URL(s);
+    if (url.password === "") {
+      url.password = Deno.env.get("SYNC_REDIS_PASSWORD") ?? "";
+    }
+    return url;
+  }),
+  sync_s3_host: z.string().optional().transform((s) => {
+    if (s == undefined) return undefined;
+    return new URL(s);
+  }),
+  sync_s3_region: z.string().optional(),
+  sync_s3_bucket: z.string().optional(),
+  sync_s3_access_key: z.string().optional(),
+  sync_s3_secret_key: z.string().optional(),
+  sync_s3_path_style: zBooleanString.optional(),
+};
+
+export const syncConfigSchema = z.object(syncConfigSchemaNaked);
+
+export type SyncConfigRaw = Required<z.output<typeof syncConfigSchema>>;
+
+export function validateSyncConfig(
+  config: z.output<typeof syncConfigSchema>,
+): SyncConfigRaw | null {
+  const syncVars = new Set(
+    Object.keys(config).filter((key) => key.startsWith("sync_")),
+  );
+
+  if (syncVars.size === 0) {
+    return null;
+  }
+
+  const missingVars = Object.keys(syncConfigSchemaNaked).filter(
+    (key) => {
+      const value = config[key as keyof typeof config];
+      if (value != undefined) return false;
+
+      // not required - set to default
+      if (key === "sync_s3_path_style") {
+        config.sync_s3_path_style = false;
+        return false;
+      }
+      return true;
+    },
+  ).map((key) => key.toUpperCase());
+
+  if (missingVars.length > 0) {
+    const missingVarsStr = missingVars.join(", ");
+    const msg = `Environment variables required for sync: ${missingVarsStr}.`;
+    const suggestion =
+      "Make sure to set these variables or set SYNC_ENABLED=false.";
+    throw new Error(`${msg}\n${suggestion}`);
+  }
+
+  const finalConfig = config as SyncConfigRaw;
+  return finalConfig;
+}
+
+export type SyncConfig = {
+  redis: RedisConnectOptions;
+  s3: S3ClientConfig;
+  s3Bucket: string;
+};
+
+export function syncConfigFromRaw(
+  config: SyncConfigRaw | null,
+): SyncConfig | null {
+  if (config == null) return null;
+
+  const redisDbStr = config.sync_redis_url.pathname.substring(1);
+  const redisDb = parseInt(redisDbStr, 10);
+  if (isNaN(redisDb)) {
+    throw new Error(`Invalid redis db number: '${redisDbStr}'`);
+  }
+
+  return {
+    redis: {
+      hostname: config.sync_redis_url.hostname,
+      port: config.sync_redis_url.port,
+      ...config.sync_redis_url.password.length > 0
+        ? { password: config.sync_redis_url.password }
+        : {},
+      db: parseInt(config.sync_redis_url.pathname.substring(1), 10),
+    },
+    s3: {
+      endpoint: config.sync_s3_host.href,
+      region: config.sync_s3_region,
+      credentials: {
+        accessKeyId: config.sync_s3_access_key,
+        secretAccessKey: config.sync_s3_secret_key,
+      },
+      forcePathStyle: config.sync_s3_path_style,
+    },
+    s3Bucket: config.sync_s3_bucket,
+  };
+}
+
+export type ConfigSource = "vars" | "args";
+
+export async function syncConfigFromEnv(
+  sources: ConfigSource[],
+): Promise<SyncConfig | null> {
+  const rawObjects = sources.map((source) => {
+    switch (source) {
+      case "vars":
+        return mapKeys(
+          Deno.env.toObject(),
+          (k: string) => k.toLowerCase(),
+        );
+      case "args":
+        return parse(Deno.args) as Record<string, unknown>;
+    }
+  });
+  const syncConfigRaw = await configOrExit(rawObjects, syncConfigSchemaNaked);
+
+  return syncConfigFromRaw(validateSyncConfig(syncConfigRaw));
+}

typegate/src/main.ts

@@ -0,0 +1,4 @@
+// Copyright Metatype OÜ, licensed under the Elastic License 2.0.
+// SPDX-License-Identifier: Elastic-2.0
+
+//