chore: resolve python artifacts during python_wasi init

Author: destifo

libs/common/src/typegraph/runtimes/python.rs

@@ -9,9 +9,9 @@ use serde_json::Value;
 
 #[cfg_attr(feature = "codegen", derive(JsonSchema))]
 #[derive(Serialize, Deserialize, Clone, Debug)]
-#[serde(rename_all = "lowercase")]
+#[serde(rename_all = "camelCase")]
 pub struct ModuleMatData {
-    pub python_artifact: String,
+    pub python_artifact: IndexMap<String, Value>,
     pub deps: Vec<String>,
     pub deps_meta: Option<Vec<IndexMap<String, Value>>>,
 }

typegate/src/runtimes/python_wasi/python_wasi.ts

@@ -14,6 +14,12 @@ import * as ast from "graphql/ast";
 
 const logger = getLogger(import.meta);
 
+interface Artifact {
+  path: string;
+  hash: string;
+  size: number;
+}
+
 function generateVmIdentifier(mat: Materializer, uuid: string) {
   const { mod } = mat.data ?? {};
   let identifier = "";
@@ -71,34 +77,40 @@ export class PythonWasiRuntime extends Runtime {
           const pyModMat = typegraph.materializers[m.data.mod as number];
 
           // resolve the python module artifacts/files
-          const { pythonArtifact, _depsMeta } = pyModMat.data;
+          const { pythonArtifact, depsMeta: depArtifacts } = pyModMat.data;
 
-          // const deserialized_deps_meta =
+          const deps = depArtifacts as Artifact[];
 
-          // const repr = await structureRepr(code);
           const vmId = generateVmIdentifier(m, uuid);
 
-          const modName = basename(pythonArtifact as string);
+          const artifact = pythonArtifact as Artifact;
+          const modName = basename(artifact.path);
 
           // TODO: move this logic to postprocess or python runtime
-          // m.data.name = `${modName}.${m.data.name as string}`;
-
+          m.data.name = `${modName}.${m.data.name as string}`;
+          console.log("*************", m.data.name);
           logger.info(`setup vm "${vmId}" for module ${modName}`);
           const vm = new PythonVirtualMachine();
 
           // for python modules, imports must be inside a folder above or same directory
-          const _typegraph = typegate.register.get(typegraphName)!;
-          const artifact =
-            _typegraph.tg.tg.meta.artifacts[pythonArtifact as string];
           const artifactMeta = {
             typegraphName: typegraphName,
             relativePath: artifact.path,
             hash: artifact.hash,
             sizeInBytes: artifact.size,
           };
+          const depMetas = deps.map((dep) => {
+            return {
+              typegraphName: typegraphName,
+              relativePath: dep.path,
+              hash: dep.hash,
+              sizeInBytes: dep.size,
+            };
+          });
 
           const entryPointFullPath = await typegate.artifactStore.getLocalPath(
             artifactMeta,
+            depMetas,
           );
           const sourceCode = Deno.readTextFileSync(entryPointFullPath);
 

typegate/src/typegate/artifacts/mod.ts

@@ -117,8 +117,12 @@ export abstract class ArtifactStore {
     tgName: string,
   ): Promise<[string, number]> {
     const expiresIn = 5 * 60;
-    const token = await signJWT({ expiresIn }, expiresIn);
-    const url = new URL(getUploadPath(tgName), origin);
+    const uuid = crypto.randomUUID();
+    const token = await signJWT({ uuid }, expiresIn);
+    const url = new URL(
+      `${getUploadPath(tgName)}`,
+      origin,
+    );
     url.searchParams.set("token", token);
     return [url.toString(), jwt.getNumericDate(expiresIn)];
   }

typegate/tests/runtimes/python_wasi/python_wasi_test.ts

@@ -342,12 +342,12 @@ Meta.test({
     await gql`
       query {
         identityMod(input: {
-           a: "hello",
-           b: [1, 2, "three"],
-         }) {
-           a
-           b
-         }
+          a: "hello",
+          b: [1, 2, "three"],
+        }) {
+          a
+          b
+        }
       }
     `
       .expectData({

typegraph/core/src/conversion/runtimes.rs

@@ -230,9 +230,11 @@ impl MaterializerConverter for PythonMaterializer {
             }
             Module(module) => {
                 let data = serde_json::from_value(json!({
-                    "python_artifact": module.file,
+                    "pythonArtifact":json!({
+                        "path": module.file
+                    }),
                     "deps": module.deps,
-                    "deps_meta": None::<serde_json::Value>,
+                    "depsMeta": None::<serde_json::Value>,
                 }))
                 .map_err(|e| e.to_string())?;
 

typegraph/core/src/utils/postprocess/python_rt.rs

@@ -19,7 +19,9 @@ impl PostProcessor for PythonProcessor {
             if mat.name.as_str() == "pymodule" {
                 let mut mat_data: ModuleMatData =
                     object_from_map(std::mem::take(&mut mat.data)).map_err(|e| e.to_string())?;
-                let path = PathBuf::from(&mat_data.python_artifact);
+                let path = mat_data.python_artifact.get("path").unwrap();
+                let path: PathBuf = path.as_str().unwrap().into();
+                // eprint(&format!("***************** {}", path_value));
 
                 if tg.meta.artifacts.contains_key(&path) {
                     continue;
@@ -35,7 +37,7 @@ impl PostProcessor for PythonProcessor {
                     Artifact {
                         hash: module_hash.clone(),
                         size,
-                        path,
+                        path: path.clone(),
                     },
                 );
 
@@ -55,6 +57,14 @@ impl PostProcessor for PythonProcessor {
                     dep_artifacts.push(dep_artifact);
                     tg.deps.push(dep_abs_path);
                 }
+
+                mat_data.python_artifact = map_from_object(Artifact {
+                    hash: module_hash.clone(),
+                    size,
+                    path,
+                })
+                .map_err(|e| e.to_string())?;
+
                 mat_data.deps_meta = Some(
                     dep_artifacts
                         .iter()

typegraph/node/sdk/src/tg_artifact_upload.ts

@@ -129,7 +129,6 @@ export class ArtifactUploader {
     const artifactMetas = this.getMetas(this.refArtifacts);
 
     const uploadUrls = await this.fetchUploadUrls(artifactMetas);
-
     const results = await Promise.allSettled(
       uploadUrls.map(
         async (url, i) => {