feat!: Remove secret definitions through env vars (#666)

Remove the ability to define secrets  in the env vars of the typegate.

Secrets can now only be defined in the metatype config file and the
`--secret` CLI option.

#### Motivation and context

-
[MET-370](https://linear.app/metatypedev/issue/MET-370/easier-way-to-pass-secrets-in-metatypeyaml-config-file)
- Security
- Better DX

#### Migration notes

1. **Metatype config file**: On the node configuration, secrets are
defined at `secrets.<tg_name>.key`:

```yaml
# before
typegates:
  dev:
    env:
      TG_CONSOLE_POSTGRES_CONN: postgresql://postgres:password@localhost:5432/db?schema=console
      TG_CONSOLE_BASIC_ADMIN: password

#after
typegates:
  dev:
    secrets:
      console:
        POSTGRES_CONN: postgresql://postgres:password@localhost:5432/db?schema=console
        BASIC_ADMIN: password    
```

2. **Secret override option on meta/cli**

```sh
# before
meta deploy -f my-tg.py --secret TG_CONSOLE_POSTGRES_CONN=postgresql://postgres:password@localhost:5432/db?schema=console

# after
meta deploy -f my-tg.py --secret POSTGRES_CONN=postgresql://postgres:password@localhost:5432/db?schema=console
# or - with the typegraph name
meta deploy -f my-tg.py --secret console:POSTGRES_CONN=postgresql://postgres:password@localhost:5432/db?schema=console

```

### Checklist

- [x] The change come with new or modified tests
- [ ] Hard-to-understand functions have explanatory comments
- [x] End-user documentation is updated to reflect the change

Author: Natoandro

.ghjk/deno.lock

@@ -84,11 +84,6 @@
           "portRef": "[email protected]",
           "pluginRepo": "https://github.com/asdf-community/asdf-cmake"
         },
-        "468769528cbf87576285bfcd000271d6398bf3c2b20083d90105d083d61854ba": {
-          "version": "2.40.1",
-          "depConfigs": {},
-          "portRef": "[email protected]"
-        },
         "402fae54fb94ffdba9cd2617602c7b33c55002275f3c3bb588615b01ff0c3c26": {
           "version": "v1.6.4",
           "depConfigs": {},
@@ -99,7 +94,7 @@
           "depConfigs": {},
           "portRef": "[email protected]"
         },
-        "f6c92b4425a3c13dfde7fe516efd376287e66cfa74aefd7b6a7a7bb0f3547a47": {
+        "80635b3a3967246789cc5f09137a8334fcfae9e96d507b6f8143d385093f024f": {
           "version": "0.116.0",
           "depConfigs": {
             "cargo_binstall_ghrel": {
@@ -241,7 +236,7 @@
           "version": "1.0.0",
           "depConfigs": {
             "node_org": {
-              "version": "v21.7.1",
+              "version": "v21.7.2",
               "depConfigs": {
                 "tar_aa": {
                   "version": "1.35",
@@ -256,7 +251,7 @@
           "packageName": "@bytecodealliance/jco"
         },
         "77a0197285cae563176871bd72f2ea24504c195dc596f5c33d4c9cc95eda1fba": {
-          "version": "v21.7.1",
+          "version": "v21.7.2",
           "depConfigs": {
             "tar_aa": {
               "version": "1.35",
@@ -270,7 +265,7 @@
           "version": "10.0.1",
           "depConfigs": {
             "node_org": {
-              "version": "v21.7.1",
+              "version": "v21.7.2",
               "depConfigs": {
                 "tar_aa": {
                   "version": "1.35",
@@ -414,37 +409,6 @@
           "portRef": "[email protected]",
           "crateName": "whiz",
           "locked": true
-        },
-        "80635b3a3967246789cc5f09137a8334fcfae9e96d507b6f8143d385093f024f": {
-          "version": "0.116.0",
-          "depConfigs": {
-            "cargo_binstall_ghrel": {
-              "version": "v1.6.4",
-              "depConfigs": {},
-              "portRef": "[email protected]"
-            },
-            "rust_rustup": {
-              "version": "1.76.0",
-              "depConfigs": {
-                "rustup_rustlang": {
-                  "version": "1.27.0",
-                  "depConfigs": {
-                    "git_aa": {
-                      "version": "2.43.0",
-                      "depConfigs": {},
-                      "portRef": "[email protected]"
-                    }
-                  },
-                  "portRef": "[email protected]"
-                }
-              },
-              "portRef": "[email protected]",
-              "profile": "minimal"
-            }
-          },
-          "portRef": "[email protected]",
-          "crateName": "wasm-opt",
-          "locked": true
         }
       }
     },

.ghjk/lock.json

@@ -23,7 +23,6 @@ pub struct Node {
     pub base_url: Url,
     pub prefix: Option<String>,
     pub auth: Option<BasicAuth>,
-    pub env: HashMap<String, String>,
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -39,13 +38,11 @@ impl Node {
         url: U,
         prefix: Option<String>,
         auth: Option<BasicAuth>,
-        env: HashMap<String, String>,
     ) -> Result<Self> {
         Ok(Self {
             base_url: url.into_url()?,
             prefix,
             auth,
-            env,
         })
     }
 

libs/common/src/node.rs

@@ -113,7 +113,6 @@ async fn spin_up_typegate() -> anyhow::Result<(tokio::process::Child, common::no
             username: "admin".into(),
             password: tg_admin_password.into(),
         }),
-        Default::default(),
     )?;
     Ok((typegate, node))
 }

libs/metagen/src/tests/mod.rs

@@ -1,7 +1,6 @@
 // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0.
 // SPDX-License-Identifier: MPL-2.0
 
-use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 
@@ -16,12 +15,12 @@ use crate::deploy::actors::loader::{
 };
 use crate::deploy::actors::watcher::WatcherActor;
 use crate::deploy::push::pusher::PushResult;
+use crate::secrets::{RawSecrets, Secrets};
 use actix::prelude::*;
 use actix_web::dev::ServerHandle;
 use anyhow::{bail, Context, Result};
 use async_trait::async_trait;
 use clap::Parser;
-use common::node::Node;
 use log::warn;
 use normpath::PathExt;
 use tokio::sync::mpsc;
@@ -91,32 +90,16 @@ pub struct DeployOptions {
     #[clap(long, default_value_t = false)]
     pub watch: bool,
 
-    /// secret overrides
+    /// Overrides secrets in the format `[<typegraph-name>:]<secret-name>=<value>`
     #[clap(long = "secret")]
     pub secrets: Vec<String>,
 }
 
-fn override_secrets(
-    secrets: HashMap<String, String>,
-    overrides: Vec<String>,
-) -> Result<HashMap<String, String>> {
-    let mut secrets = secrets;
-    for override_str in overrides {
-        let parts: Vec<&str> = override_str.splitn(2, '=').collect();
-        if parts.len() != 2 {
-            bail!("Invalid secret override: {}", override_str);
-        }
-        secrets.insert(parts[0].to_string(), parts[1].to_string());
-    }
-
-    Ok(secrets)
-}
-
 pub struct Deploy {
     config: Arc<Config>,
     base_dir: Arc<Path>,
     options: DeployOptions,
-    node: Node,
+    secrets: RawSecrets,
     file: Option<Arc<Path>>,
     max_parallel_loads: Option<usize>,
 }
@@ -131,6 +114,7 @@ impl Deploy {
         let options = deploy.options.clone();
 
         let node_config = config.node(&deploy.node, &deploy.target);
+        let secrets = Secrets::load_from_node_config(&node_config);
         let node = node_config
             .build(&dir)
             .await
@@ -152,7 +136,7 @@ impl Deploy {
             config,
             base_dir: dir.into(),
             options,
-            node,
+            secrets,
             file: deploy
                 .file
                 .as_ref()
@@ -225,13 +209,11 @@ mod default_mode {
     impl DefaultMode {
         pub async fn init(deploy: Deploy) -> Result<Self> {
             let console = ConsoleActor::new(Arc::clone(&deploy.config)).start();
-            let secrets = lade_sdk::hydrate(
-                override_secrets(deploy.node.env.clone(), deploy.options.secrets.clone())?,
-                deploy.base_dir.to_path_buf(),
-            )
-            .await?;
 
-            ServerStore::set_secrets(secrets);
+            let mut secrets = deploy.secrets.clone();
+            secrets.apply_overrides(&deploy.options.secrets)?;
+
+            ServerStore::set_secrets(secrets.hydrate(deploy.base_dir.clone()).await?);
 
             let (loader_event_tx, loader_event_rx) = mpsc::unbounded_channel();
 
@@ -348,13 +330,10 @@ mod watch_mode {
         .context("setting Ctrl-C handler")?;
 
         loop {
-            let secrets = lade_sdk::hydrate(
-                override_secrets(deploy.node.env.clone(), deploy.options.secrets.clone())?,
-                deploy.base_dir.to_path_buf(),
-            )
-            .await?;
+            let mut secrets = deploy.secrets.clone();
+            secrets.apply_overrides(&deploy.options.secrets)?;
 
-            ServerStore::set_secrets(secrets.clone());
+            ServerStore::set_secrets(secrets.hydrate(deploy.base_dir.clone()).await?);
 
             let (loader_event_tx, loader_event_rx) = mpsc::unbounded_channel();
 

meta-cli/src/cli/deploy.rs

@@ -57,7 +57,7 @@ async fn config(req: HttpRequest) -> impl Responder {
     artefact_base_dir.pop(); // pop file.ext
 
     let endpoint = ServerStore::get_endpoint();
-    let secrets = ServerStore::get_secrets();
+    let secrets = ServerStore::get_secrets(&parsed.typegraph);
     let migration_action_glob = ServerStore::get_migration_action_glob();
     let disable_artifact_resolution = !ServerStore::get_artifact_resolution_flag();
     let codegen = ServerStore::get_codegen_flag();

meta-cli/src/com/server.rs

@@ -1,7 +1,7 @@
 // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0.
 // SPDX-License-Identifier: MPL-2.0
 
-use crate::config::Config;
+use crate::{config::Config, secrets::Secrets};
 use anyhow::{bail, Result};
 use common::node::BasicAuth;
 use lazy_static::lazy_static;
@@ -65,7 +65,7 @@ pub struct ServerStore {
     migration_action_glob: MigrationAction,
     /// 1 typegraph => n runtimes
     migration_action: HashMap<PathBuf, Arc<Vec<RuntimeMigrationAction>>>,
-    secrets: HashMap<String, String>,
+    secrets: Secrets,
     endpoint: Endpoint,
     prefix: Option<String>,
     sdk_responses: HashMap<PathBuf, Arc<HashMap<String, SDKResponse>>>,
@@ -94,12 +94,12 @@ impl ServerStore {
         with_store(|s| s.command.clone())
     }
 
-    pub fn set_secrets(secrets: HashMap<String, String>) {
+    pub fn set_secrets(secrets: Secrets) {
         with_store_mut(|s| s.secrets = secrets)
     }
 
-    pub fn get_secrets() -> HashMap<String, String> {
-        with_store(|s| s.secrets.clone())
+    pub fn get_secrets(tg_name: &str) -> HashMap<String, String> {
+        with_store(|s| s.secrets.get(tg_name))
     }
 
     pub fn set_endpoint(endpoint: Endpoint) {

meta-cli/src/com/store.rs

@@ -56,6 +56,9 @@ pub struct NodeConfig {
     password: Option<String>,
     #[serde(default)]
     pub env: HashMap<String, String>,
+    /// tg_name -> {key -> secret}
+    #[serde(default)]
+    pub secrets: HashMap<String, HashMap<String, String>>,
 }
 
 impl Default for NodeConfig {
@@ -66,6 +69,7 @@ impl Default for NodeConfig {
             username: None,
             password: None,
             env: HashMap::default(),
+            secrets: HashMap::default(),
         }
     }
 }
@@ -97,7 +101,6 @@ impl NodeConfig {
             self.url.clone(),
             self.prefix.clone(),
             Some(self.basic_auth(dir).await.context("basic auth")?.into()),
-            self.env.clone(),
         )
     }
 }

meta-cli/src/config.rs

@@ -9,6 +9,7 @@ pub mod deploy;
 mod fs;
 mod global_config;
 mod logger;
+mod secrets;
 
 #[cfg(test)]
 mod tests;