refactor(cli/sdk): move all preps to sdk (wip)

michael-0acf4 · michael-0acf4 · commit 8156bf73a724 · 2024-02-14T17:46:03.000+03:00
diff --git a/meta-cli/src/cli/codegen.rs b/meta-cli/src/cli/codegen.rs
@@ -57,9 +57,7 @@ impl Action for Deno {
         );
 
         let loader_pool = LoaderPool::new(config, 1)
-            .with_postprocessor(postprocess::DenoModules::default().codegen(true))
-            .with_postprocessor(postprocess::PythonModules::default())
-            .with_postprocessor(postprocess::WasmdegeModules::default());
+            .with_postprocessor(postprocess::DenoModules::default().codegen(true));
 
         let loader = loader_pool.get_loader().await?;
 
diff --git a/meta-cli/src/deploy/actors/loader.rs b/meta-cli/src/deploy/actors/loader.rs
@@ -13,7 +13,7 @@ use tokio::sync::{mpsc, oneshot};
 
 use crate::config::Config;
 use crate::typegraph::loader::LoaderPool;
-use crate::typegraph::postprocess::{self, DenoModules, EmbedPrismaMigrations};
+use crate::typegraph::postprocess::{DenoModules, EmbedPrismaMigrations};
 use crate::utils::plural_suffix;
 
 use super::console::{Console, ConsoleActor};
@@ -120,10 +120,6 @@ impl LoaderActor {
             pool = pool.with_postprocessor(deno.clone());
         }
 
-        pool = pool
-            .with_postprocessor(postprocess::PythonModules::default())
-            .with_postprocessor(postprocess::WasmdegeModules::default());
-
         if let Some(prisma) = &postprocess_options.prisma {
             pool = pool.with_postprocessor(
                 prisma
diff --git a/meta-cli/src/typegraph/loader/mod.rs b/meta-cli/src/typegraph/loader/mod.rs
@@ -42,10 +42,7 @@ impl LoaderPool {
     pub fn new(config: Arc<Config>, max_parallel_loads: usize) -> Self {
         Self {
             config,
-            postprocessors: vec![
-                postprocess::Validator.into(),
-                postprocess::ReformatScripts.into(),
-            ],
+            postprocessors: vec![postprocess::Validator.into()],
             semaphore: Semaphore::new(max_parallel_loads),
         }
     }
diff --git a/meta-cli/src/typegraph/postprocess.rs b/meta-cli/src/typegraph/postprocess.rs
@@ -5,17 +5,11 @@ use std::sync::{Arc, RwLock};
 
 use crate::config::Config;
 
-use anyhow::Context;
 use anyhow::{bail, Result};
 use colored::Colorize;
-use common::archive::archive_entries;
-use common::typegraph::utils::{map_from_object, object_from_map};
 use common::typegraph::validator::validate_typegraph;
-use common::typegraph::{Materializer, Typegraph};
-use ignore::WalkBuilder;
+use common::typegraph::Typegraph;
 use log::error;
-use std::path::Path;
-use typescript::parser::transform_script;
 
 pub trait PostProcessor {
     fn postprocess(&self, tg: &mut Typegraph, config: &Config) -> Result<()>;
@@ -35,14 +29,6 @@ where
 #[derive(Clone)]
 pub struct PostProcessorWrapper(Arc<RwLock<Box<dyn PostProcessor + Sync + Send>>>);
 
-impl PostProcessorWrapper {
-    pub fn generic(
-        pp: impl Fn(&mut Typegraph, &Config) -> Result<()> + Sync + Send + 'static,
-    ) -> Self {
-        PostProcessorWrapper::from(GenericPostProcessor(pp))
-    }
-}
-
 impl<T> From<T> for PostProcessorWrapper
 where
     T: PostProcessor + Send + Sync + 'static,
@@ -63,60 +49,9 @@ pub fn apply_all<'a>(
     Ok(())
 }
 
-fn compress_and_encode(main_path: &Path, tg_path: &Path) -> Result<String> {
-    // Note: tg_path and main_path are all absolute
-    // tg_root/tg.py
-    // tg_root/* <= script location
-    if main_path.is_relative() {
-        bail!(
-            "script path {:?} is relative, absolute expected",
-            main_path.display()
-        );
-    }
-
-    if tg_path.is_relative() {
-        bail!(
-            "typegraph path {:?} is relative, absolute expected",
-            tg_path.display()
-        );
-    }
-
-    let tg_root = tg_path.parent().with_context(|| {
-        format!(
-            "invalid state: typegraph path {:?} does not have parent",
-            tg_path.display()
-        )
-    })?;
-
-    let dir_walker = WalkBuilder::new(tg_root)
-        .standard_filters(true)
-        // .add_custom_ignore_filename(".DStore")
-        .sort_by_file_path(|a, b| a.cmp(b))
-        .build();
-
-    let enc_content = match archive_entries(dir_walker, Some(tg_root)).unwrap() {
-        Some(b64) => b64,
-        None => "".to_string(),
-    };
-
-    let file = match main_path.strip_prefix(tg_root) {
-        Ok(ret) => ret,
-        Err(_) => bail!(
-            "{:?} does not contain script {:?}",
-            tg_root.display(),
-            main_path.display(),
-        ),
-    };
-
-    Ok(format!("file:{};base64:{}", file.display(), enc_content))
-}
-
 pub use deno_rt::DenoModules;
-pub use deno_rt::ReformatScripts;
 pub use prisma_rt::EmbedPrismaMigrations;
 pub use prisma_rt::EmbeddedPrismaMigrationOptionsPatch;
-pub use python_rt::PythonModules;
-pub use wasmedge_rt::WasmdegeModules;
 
 pub struct Validator;
 impl PostProcessor for Validator {
@@ -139,46 +74,8 @@ impl PostProcessor for Validator {
 }
 
 pub mod deno_rt {
-    use std::fs;
-
-    use common::typegraph::runtimes::deno::{FunctionMatData, ModuleMatData};
-    use common::typegraph::runtimes::{KnownRuntime, TGRuntime};
-
-    use common::typegraph::utils::{find_runtimes, get_materializers};
-
     use super::*;
 
-    pub struct ReformatScripts;
-
-    impl From<ReformatScripts> for PostProcessorWrapper {
-        fn from(_val: ReformatScripts) -> Self {
-            PostProcessorWrapper::generic(reformat_scripts)
-        }
-    }
-
-    fn reformat_materializer_script(mat: &mut Materializer) -> Result<()> {
-        if mat.name.as_str() == "function" {
-            let mut mat_data: FunctionMatData = object_from_map(std::mem::take(&mut mat.data))?;
-            // TODO check variable `_my_lambda` exists and is a function expression/lambda
-            mat_data.script = transform_script(mat_data.script)?;
-            mat.data = map_from_object(mat_data)?;
-        }
-        Ok(())
-    }
-
-    fn reformat_scripts(typegraph: &mut Typegraph, _c: &Config) -> Result<()> {
-        for rt_idx in find_runtimes(typegraph, |rt| {
-            matches!(rt, TGRuntime::Known(KnownRuntime::Deno(_)))
-        })
-        .into_iter()
-        {
-            for mat_idx in get_materializers(typegraph, rt_idx as u32) {
-                reformat_materializer_script(&mut typegraph.materializers[mat_idx])?;
-            }
-        }
-        Ok(())
-    }
-
     #[derive(Default, Debug, Clone)]
     pub struct DenoModules {
         codegen: bool,
@@ -196,82 +93,6 @@ pub mod deno_rt {
             if self.codegen {
                 crate::codegen::deno::codegen(tg, tg.path.as_ref().unwrap())?;
             }
-            for mat in tg.materializers.iter_mut().filter(|m| m.name == "module") {
-                let mut mat_data: ModuleMatData = object_from_map(std::mem::take(&mut mat.data))?;
-                log::debug!("processing module {:?}", mat_data.code);
-                let Some(path) = mat_data.code.strip_prefix("file:") else {
-                    continue;
-                };
-
-                // make sure tg_path is absolute
-                let tg_path = fs::canonicalize(tg.path.to_owned().unwrap()).unwrap();
-                let main_path = tg_path.parent().unwrap().join(path);
-                mat_data.code = compress_and_encode(&main_path, &tg_path)?;
-                log::debug!("compressed module {:?}", mat_data.code);
-
-                mat.data = map_from_object(mat_data)?;
-                tg.deps.push(main_path);
-            }
-            Ok(())
-        }
-    }
-}
-
-pub mod python_rt {
-    use super::*;
-    use common::typegraph::runtimes::python::ModuleMatData;
-    use std::fs;
-
-    #[derive(Default, Debug)]
-    pub struct PythonModules {}
-
-    impl PostProcessor for PythonModules {
-        fn postprocess(&self, tg: &mut Typegraph, _config: &Config) -> Result<()> {
-            for mat in tg.materializers.iter_mut().filter(|m| m.name == "pymodule") {
-                let mut mat_data: ModuleMatData = object_from_map(std::mem::take(&mut mat.data))?;
-                let path = mat_data
-                    .code
-                    .strip_prefix("file:")
-                    .context("\"file:\" prefix is not present")?;
-
-                // make sure tg_path is absolute
-                let tg_path = fs::canonicalize(tg.path.to_owned().unwrap()).unwrap();
-                let main_path = tg_path.parent().unwrap().join(path);
-                mat_data.code = compress_and_encode(&main_path, &tg_path)?;
-
-                mat.data = map_from_object(mat_data)?;
-                tg.deps.push(main_path);
-            }
-            Ok(())
-        }
-    }
-}
-
-pub mod wasmedge_rt {
-    use super::*;
-    use common::{archive::encode_to_base_64, typegraph::runtimes::wasmedge::WasiMatData};
-    use std::fs;
-
-    #[derive(Default, Debug)]
-    pub struct WasmdegeModules {}
-
-    impl PostProcessor for WasmdegeModules {
-        fn postprocess(&self, tg: &mut Typegraph, _config: &Config) -> Result<()> {
-            for mat in tg.materializers.iter_mut().filter(|m| m.name == "wasi") {
-                let mut mat_data: WasiMatData = object_from_map(std::mem::take(&mut mat.data))?;
-                let path = mat_data
-                    .wasm
-                    .strip_prefix("file:")
-                    .context("\"file:\" prefix is not present")?;
-
-                // make sure tg_path is absolute
-                let tg_path = fs::canonicalize(tg.path.to_owned().unwrap()).unwrap();
-                let wasi_path = tg_path.parent().unwrap().join(path);
-                mat_data.wasm = encode_to_base_64(&wasi_path)?;
-
-                mat.data = map_from_object(mat_data)?;
-                tg.deps.push(wasi_path);
-            }
             Ok(())
         }
     }
diff --git a/typegraph/node/sdk/src/typegraph.ts b/typegraph/node/sdk/src/typegraph.ts
@@ -169,7 +169,7 @@ export function typegraph(
   builder(g);
 
   const tgJson = core.finalizeTypegraph(
-    disableAutoSerialization ? "resolve-artifacts" : "simple",
+    /*disableAutoSerialization ?*/ "resolve-artifacts", /* : "simple"*/
   );
   if (!disableAutoSerialization) {
     console.log(tgJson);
diff --git a/typegraph/python/typegraph/graph/typegraph.py b/typegraph/python/typegraph/graph/typegraph.py
@@ -172,9 +172,9 @@ def decorator(builder: Callable[[Graph], None]) -> TypegraphOutput:
 
         tg_json = core.finalize_typegraph(
             store,
-            TypegraphFinalizeMode.RESOLVE_ARTIFACTS
-            if disable_auto_serialization
-            else TypegraphFinalizeMode.SIMPLE,
+            TypegraphFinalizeMode.RESOLVE_ARTIFACTS,
+            # if disable_auto_serialization
+            # else TypegraphFinalizeMode.SIMPLE,
         )
 
         if isinstance(tg_json, Err):

Original file line number	Diff line number	Diff line change
`@@ -42,10 +42,7 @@ impl LoaderPool {`
`42`	`42`	`pub fn new(config: Arc<Config>, max_parallel_loads: usize) -> Self {`
`43`	`43`	`Self {`
`44`	`44`	`config,`
`45`		`- postprocessors: vec![`
`46`		`- postprocess::Validator.into(),`
`47`		`- postprocess::ReformatScripts.into(),`
`48`		`- ],`
	`45`	`+ postprocessors: vec![postprocess::Validator.into()],`
`49`	`46`	`semaphore: Semaphore::new(max_parallel_loads),`
`50`	`47`	`}`
`51`	`48`	`}`