Merge branch 'main' into local-npm-registry

# Conflicts:
#	poetry.lock

Author: Natoandro

.github/workflows/tests.yml

@@ -340,7 +340,7 @@ jobs:
           cargo test --locked --package meta-cli
 
           # from old test-libs
-          cargo test --locked --exclude meta-cli --exclude typegate --exclude typegraph_engine --exclude typegraph_core --exclude metagen_mdk_rust_static  --workspace
+          cargo test --locked --exclude meta-cli --exclude typegate --exclude typegate_engine --exclude typegraph_core --exclude metagen_mdk_rust_static  --workspace
 
           deno run -A dev/update.ts --cache-only || deno run -A dev/update.ts --cache-only
           cargo build -p meta-cli

Cargo.lock

@@ -8,7 +8,7 @@ members = [
   "meta-cli",
   "typegraph/core", ]
 
-[workspace.package]
+[workspace.package] 
 version = "0.3.7-0"
 edition = "2021"
 
@@ -20,6 +20,7 @@ deno = { git = "https://github.com/metatypedev/deno", branch = "v1.41.0-embeddab
 mt_deno = { path = "libs/deno/" }
 common = { path = "libs/common/" }
 typescript = { path = "libs/typescript/" }
+metagen = { path = "libs/metagen/" }
 typegate_engine = { path = "typegate/engine" }
 
 # pinned to match version brought in

Cargo.toml

@@ -16,19 +16,34 @@
 //!             annotate_debug: true
 use crate::interlude::*;
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize, Debug, Clone)]
 pub struct Config {
     pub targets: HashMap<String, Target>,
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize, Debug, Clone)]
 pub struct Target(pub HashMap<String, serde_json::Value>);
 
+/// If both name and path are set, name is used to disambiguate
+/// from multiple typegrpahs loaded from file at path.
 #[derive(Serialize, Deserialize, Debug, garde::Validate)]
 pub struct MdkGeneratorConfigBase {
     #[garde(length(min = 1))]
     #[serde(rename = "typegraph")]
-    pub typegraph_name: String,
+    #[garde(custom(|_, _| either_typegraph_name_or_path(self)))]
+    pub typegraph_name: Option<String>,
+    #[garde(skip)]
+    pub typegraph_path: Option<PathBuf>,
     #[garde(skip)]
     pub path: PathBuf,
 }
+
+fn either_typegraph_name_or_path(config: &MdkGeneratorConfigBase) -> garde::Result {
+    if config.typegraph_name.is_none() && config.typegraph_path.is_none() {
+        Err(garde::Error::new(
+            "either typegraph or typegraph_path must be set",
+        ))
+    } else {
+        Ok(())
+    }
+}

libs/metagen/src/config.rs

@@ -28,14 +28,18 @@ mod utils;
 
 use crate::interlude::*;
 
+pub use config::*;
+
 #[derive(Debug)]
 pub enum GeneratorInputOrder {
-    TypegraphDesc { name: String },
+    TypegraphFromTypegate { name: String },
+    TypegraphFromPath { path: PathBuf, name: Option<String> },
 }
 
 #[derive(Debug)]
 pub enum GeneratorInputResolved {
-    TypegraphDesc { raw: Typegraph },
+    TypegraphFromTypegate { raw: Typegraph },
+    TypegraphFromPath { raw: Typegraph },
 }
 
 pub trait InputResolver {
@@ -48,20 +52,6 @@ pub trait InputResolver {
 #[derive(Debug)]
 pub struct GeneratorOutput(pub HashMap<PathBuf, String>);
 
-impl InputResolver for Ctx {
-    async fn resolve(&self, order: GeneratorInputOrder) -> anyhow::Result<GeneratorInputResolved> {
-        Ok(match order {
-            GeneratorInputOrder::TypegraphDesc { name } => GeneratorInputResolved::TypegraphDesc {
-                raw: self
-                    .typegate
-                    .typegraph(&name)
-                    .await?
-                    .with_context(|| format!("no typegraph found under \"{name}\""))?,
-            },
-        })
-    }
-}
-
 trait Plugin: Send + Sync {
     fn bill_of_inputs(&self) -> HashMap<String, GeneratorInputOrder>;
     fn generate(
@@ -70,11 +60,9 @@ trait Plugin: Send + Sync {
     ) -> anyhow::Result<GeneratorOutput>;
 }
 
-#[derive(Clone)]
-struct Ctx {
-    typegate: Arc<common::node::Node>,
-}
-
+/// This function makes use of a JoinSet to process
+/// items in parallel. This makes using actix workers in InputResolver
+/// is a no no.
 pub async fn generate_target(
     config: &config::Config,
     target_name: &str,

libs/metagen/src/lib.rs

@@ -38,8 +38,17 @@ impl crate::Plugin for Generator {
     fn bill_of_inputs(&self) -> HashMap<String, GeneratorInputOrder> {
         [(
             Self::INPUT_TG.to_string(),
-            GeneratorInputOrder::TypegraphDesc {
-                name: self.config.base.typegraph_name.clone(),
+            if let Some(tg_name) = &self.config.base.typegraph_name {
+                GeneratorInputOrder::TypegraphFromTypegate {
+                    name: tg_name.clone(),
+                }
+            } else if let Some(tg_path) = &self.config.base.typegraph_path {
+                GeneratorInputOrder::TypegraphFromPath {
+                    path: tg_path.clone(),
+                    name: self.config.base.typegraph_name.clone(),
+                }
+            } else {
+                unreachable!()
             },
         )]
         .into_iter()
@@ -51,17 +60,22 @@ impl crate::Plugin for Generator {
         inputs: HashMap<String, GeneratorInputResolved>,
     ) -> anyhow::Result<GeneratorOutput> {
         // return Ok(GeneratorOutput(Default::default()))
-        let GeneratorInputResolved::TypegraphDesc { raw: tg } = inputs
+        let tg = match inputs
             .get(Self::INPUT_TG)
-            .context("missing generator input")?;
+            .context("missing generator input")?
+        {
+            GeneratorInputResolved::TypegraphFromTypegate { raw } => raw,
+            GeneratorInputResolved::TypegraphFromPath { raw } => raw,
+        };
         let mut out = HashMap::new();
         out.insert(
             self.config.base.path.join("mod.rs"),
             gen_mod_rs(&self.config, tg)?,
         );
         if self.config.no_crate_manifest.unwrap_or(true) {
             use heck::ToSnekCase;
-            let crate_name = format!("{}_mdk", self.config.base.typegraph_name.to_snek_case());
+            let tg_name = tg.name().unwrap_or_else(|_| "generated".to_string());
+            let crate_name = format!("{}_mdk", tg_name.to_snek_case());
             out.insert(
                 self.config.base.path.join("Cargo.toml"),
                 gen_cargo_toml(Some(&crate_name)),
@@ -155,7 +169,8 @@ fn mdk_rs_e2e() -> anyhow::Result<()> {
                         stubbed_runtimes: None,
                         no_crate_manifest: None,
                         base: config::MdkGeneratorConfigBase {
-                            typegraph_name: tg_name.into(),
+                            typegraph_name: Some(tg_name.into()),
+                            typegraph_path: None,
                             // NOTE: root will map to the test's tempdir
                             path: "./gen/".into(),
                         },
@@ -181,11 +196,7 @@ fn mdk_rs_e2e() -> anyhow::Result<()> {
                 build_fn: |args| {
                     Box::pin(async move {
                         let status = tokio::process::Command::new("cargo")
-                            .args(
-                                "build --target wasm32-wasi --target-dir target/wasi"
-                                    .split(' ')
-                                    .collect::<Vec<_>>(),
-                            )
+                            .args("build --target wasm32-wasi".split(' ').collect::<Vec<_>>())
                             .current_dir(args.path)
                             .kill_on_drop(true)
                             .spawn()?

libs/metagen/src/mdk_rust/mod.rs

@@ -108,15 +108,16 @@ mod test {
         let generator = Generator::new(MdkRustGenConfig {
             base: crate::config::MdkGeneratorConfigBase {
                 path: "/tmp".into(),
-                typegraph_name: tg_name.clone(),
+                typegraph_name: Some(tg_name.clone()),
+                typegraph_path: None,
             },
             stubbed_runtimes: Some(vec!["wasm".into()]),
             no_crate_manifest: None,
         })?;
         let out = generator.generate(
             [(
                 Generator::INPUT_TG.to_owned(),
-                GeneratorInputResolved::TypegraphDesc { raw: tg },
+                GeneratorInputResolved::TypegraphFromTypegate { raw: tg },
             )]
             .into_iter()
             .collect(),

libs/metagen/src/mdk_rust/stubs.rs

@@ -7,8 +7,8 @@ use common::typegraph::*;
 pub async fn test_typegraph_1() -> anyhow::Result<Typegraph> {
     let out = tokio::process::Command::new("cargo")
         .args(
-            // "run -p meta-cli -- serialize -f tests/tg.ts"
-            "run -p meta-cli -- serialize -f ../../examples/typegraphs/reduce.py"
+            "run -p meta-cli -- serialize -f tests/tg.ts"
+                // "run -p meta-cli -- serialize -f ../../examples/typegraphs/reduce.py"
                 .split(' ')
                 .collect::<Vec<_>>(),
         )

libs/metagen/src/tests/fixtures.rs

@@ -14,11 +14,12 @@ struct TestCtx {
 impl InputResolver for TestCtx {
     async fn resolve(&self, order: GeneratorInputOrder) -> anyhow::Result<GeneratorInputResolved> {
         match order {
-            GeneratorInputOrder::TypegraphDesc { name } => {
-                Ok(GeneratorInputResolved::TypegraphDesc {
+            GeneratorInputOrder::TypegraphFromTypegate { name } => {
+                Ok(GeneratorInputResolved::TypegraphFromTypegate {
                     raw: self.typegraphs.get(&name).unwrap().clone(),
                 })
             }
+            GeneratorInputOrder::TypegraphFromPath { .. } => unimplemented!(),
         }
     }
 }
@@ -78,7 +79,7 @@ pub async fn e2e_test(cases: Vec<E2eTestCase>) -> anyhow::Result<()> {
         // TODO: query generated stub functions
 
         // cleanup
-        tokio::fs::remove_dir_all(tmp_dir).await?;
+        // tokio::fs::remove_dir_all(tmp_dir).await?;
         // node.try_undeploy(&typegraphs.keys().cloned().collect::<Vec<_>>()).await?;
     }
     Ok(())

libs/metagen/src/tests/mod.rs

@@ -58,8 +58,9 @@ dialoguer = "0.11.0"
 flate2 = "1.0.28"
 tar = "0.4.40"
 base64 = "0.21.5"
-common = { path = "../libs/common" }
-typescript = { path = "../libs/typescript" }
+common.workspace = true
+typescript.workspace = true
+metagen.workspace = true
 serde_yaml = "0.9.27"
 pathdiff = "0.2.1"
 openssl = { version = "0.10.59", features = ["vendored"] }
@@ -95,6 +96,7 @@ num_cpus = "1.16.0"
 assert_cmd = "2.0.12"
 insta = "1.34.0"
 project-root = "0.2.2"
+tempfile.workspace = true
 
 [build-dependencies]
 shadow-rs.workspace = true

meta-cli/Cargo.toml

@@ -13,7 +13,7 @@ use itertools::Itertools;
 
 use super::Action;
 use super::Args;
-use super::GenArgs;
+use super::ConfigArgs;
 
 #[derive(Parser, Debug)]
 pub struct Completion {
@@ -23,7 +23,7 @@ pub struct Completion {
 
 #[async_trait]
 impl Action for Completion {
-    async fn run(&self, _args: GenArgs, _: Option<ServerHandle>) -> Result<()> {
+    async fn run(&self, _args: ConfigArgs, _: Option<ServerHandle>) -> Result<()> {
         let mut cmd = Args::command();
         let name = cmd.get_name().to_string();
         match self.shell.or_else(Shell::from_env) {