Skip to content

Commit 80cef5f

Browse files
destifodestifo
committed
feat(SDK): add deps tracking deno runtime
1 parent 0445a4e commit 80cef5f

File tree

14 files changed

+129
-16
lines changed

14 files changed

+129
-16
lines changed

‎libs/common/src/typegraph/runtimes/deno.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,11 @@ pub struct FunctionMatData {
1515

1616
#[cfg_attr(feature = "codegen", derive(JsonSchema))]
1717
#[derive(Serialize, Deserialize, Clone, Debug)]
18+
#[serde(rename_all = "camelCase")]
1819
pub struct ModuleMatData {
19-
pub code: String,
20+
pub deno_artifact: IndexMap<String, Value>,
21+
pub deps: Vec<String>,
22+
pub deps_meta: Option<Vec<IndexMap<String, Value>>>,
2023
}
2124

2225
#[cfg_attr(feature = "codegen", derive(JsonSchema))]

‎typegate/tests/runtimes/deno/deno_dep.ts

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// Copyright Metatype OÜ, licensed under the Elastic License 2.0.
2+
// SPDX-License-Identifier: Elastic-2.0
3+
4+
import { Policy, t, typegraph } from "@typegraph/sdk/index.js";
5+
import { DenoRuntime } from "@typegraph/sdk/runtimes/deno.js";
6+
7+
export const tg = await typegraph({
8+
name: "test-deno-dep",
9+
}, (g: any) => {
10+
const deno = DenoRuntime();
11+
const pub = Policy.public();
12+
13+
g.expose({
14+
doAddition: deno.import(
15+
t.struct({ a: t.float(), b: t.float() }),
16+
t.float(),
17+
{
18+
module: "ts/dep/main.ts",
19+
name: "doAddition",
20+
},
21+
).withPolicy(pub),
22+
});
23+
});

‎typegate/tests/runtimes/deno/deno_test.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ Meta.test("Deno runtime: permissions", async (t) => {
109109
});
110110

111111
Meta.test("Deno runtime: use local imports", async (t) => {
112-
const e = await t.engine("runtimes/deno/deno_dep.py");
112+
const e = await t.engine("runtimes/deno/deno_dep.ts");
113113
await t.should("work for local imports", async () => {
114114
await gql`
115115
query {

‎typegraph/core/src/conversion/runtimes.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,11 @@ impl MaterializerConverter for DenoMaterializer {
102102
}
103103
Module(module) => {
104104
let data = serde_json::from_value(json!({
105-
"code": format!("file:{}", module.file),
105+
"denoArtifact": json!({
106+
"path": format!("file:{}", module.file),
107+
}),
108+
"deps": module.deps,
109+
"depsMeta": None::<serde_json::Value>,
106110
}))
107111
.unwrap();
108112
("module".to_string(), data)

‎typegraph/core/src/global_store.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -450,7 +450,7 @@ impl Store {
450450
}
451451
}
452452

453-
pub fn get_deno_module(file: String) -> MaterializerId {
453+
pub fn get_deno_module(file: String, deps: Vec<String>) -> MaterializerId {
454454
if let Some(mat) = with_store(|s| s.deno_modules.get(&file).cloned()) {
455455
mat
456456
} else {
@@ -460,6 +460,7 @@ impl Store {
460460
effect: Effect::Read, // N/A
461461
data: Rc::new(DenoMaterializer::Module(MaterializerDenoModule {
462462
file: file.clone(),
463+
deps: deps.clone(),
463464
}))
464465
.into(),
465466
});

‎typegraph/core/src/runtimes/deno.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ use crate::wit::runtimes as wit;
66
#[derive(Debug)]
77
pub struct MaterializerDenoModule {
88
pub file: String,
9+
pub deps: Vec<String>,
910
}
1011

1112
#[derive(Debug)]

‎typegraph/core/src/runtimes/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,7 @@ impl crate::wit::runtimes::Guest for crate::Lib {
259259
data: wit::MaterializerDenoImport,
260260
effect: wit::Effect,
261261
) -> Result<wit::MaterializerId> {
262-
let module = Store::get_deno_module(data.module);
262+
let module = Store::get_deno_module(data.module, data.deps);
263263
let data = MaterializerDenoImport {
264264
func_name: data.func_name,
265265
module,

‎typegraph/core/src/utils/fs_host.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ pub fn cwd() -> Result<PathBuf, String> {
187187
}
188188

189189
/// Strip given path with `cwd`
190+
#[allow(dead_code)]
190191
pub fn make_relative(path: &Path) -> Result<PathBuf, String> {
191192
path.strip_prefix(cwd()?)
192193
.map_err(|e| e.to_string())

‎typegraph/core/src/utils/postprocess/deno_rt.rs

Lines changed: 81 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,18 @@
33

44
use crate::{global_store::Store, utils::fs_host};
55
use common::typegraph::{
6-
runtimes::deno::ModuleMatData,
6+
runtimes::{deno::ModuleMatData, Artifact},
77
utils::{map_from_object, object_from_map},
88
Materializer, Typegraph,
99
};
1010
use std::path::PathBuf;
1111

12-
use crate::utils::postprocess::{compress_and_encode, PostProcessor};
12+
use crate::utils::postprocess::PostProcessor;
13+
14+
pub struct ResolveModuleOuput {
15+
tg_artifacts: Vec<Artifact>,
16+
tg_deps_paths: Vec<PathBuf>,
17+
}
1318

1419
pub struct DenoProcessor;
1520

@@ -18,7 +23,19 @@ impl PostProcessor for DenoProcessor {
1823
for mat in tg.materializers.iter_mut() {
1924
if mat.name.as_str() == "module" {
2025
match Self::resolve_module(mat)? {
21-
Some(dep_path) => tg.deps.push(fs_host::make_relative(&dep_path)?),
26+
Some(ResolveModuleOuput {
27+
tg_deps_paths: dep_paths,
28+
tg_artifacts: artifacts,
29+
}) => {
30+
for i in 0..artifacts.len() {
31+
let artifact = &artifacts[i];
32+
let dep_path = &dep_paths[i];
33+
tg.deps.push(dep_path.clone());
34+
tg.meta
35+
.artifacts
36+
.insert(artifact.path.clone(), artifact.clone());
37+
}
38+
}
2239
None => continue,
2340
}
2441
}
@@ -28,20 +45,73 @@ impl PostProcessor for DenoProcessor {
2845
}
2946

3047
impl DenoProcessor {
31-
pub fn resolve_module(mat: &mut Materializer) -> Result<Option<PathBuf>, String> {
48+
pub fn resolve_module(mat: &mut Materializer) -> Result<Option<ResolveModuleOuput>, String> {
3249
let mut mat_data: ModuleMatData =
3350
object_from_map(std::mem::take(&mut mat.data)).map_err(|e| e.to_string())?;
34-
let Some(path) = mat_data.code.strip_prefix("file:").to_owned() else {
51+
52+
let prefixed_path = mat_data
53+
.deno_artifact
54+
.get("path")
55+
.unwrap()
56+
.as_str()
57+
.unwrap();
58+
59+
let Some(path) = prefixed_path.strip_prefix("file:").to_owned() else {
3560
return Ok(None);
3661
};
3762

63+
let path = PathBuf::from(path);
64+
3865
// main_path can be either relative or absolute,
3966
// if relative => make it absolute
4067
// fs::canonicalize wouldn't work in this setup
41-
let main_path = fs_host::make_absolute(&PathBuf::from(path))?;
68+
let main_path = fs_host::make_absolute(&path)?;
69+
70+
let mut tg_deps_paths = vec![];
71+
let mut tg_artifacts = vec![];
4272

4373
match fs_host::path_exists(&main_path)? {
44-
true => mat_data.code = compress_and_encode(&main_path)?,
74+
true => {
75+
let (module_hash, size) = fs_host::hash_file(&main_path.clone())?;
76+
77+
let deno_artifact = Artifact {
78+
hash: module_hash.clone(),
79+
size,
80+
path: path.clone(),
81+
};
82+
tg_deps_paths.push(main_path);
83+
84+
let deps = mat_data.deps.clone();
85+
for dep in deps {
86+
let dep_rel_path = PathBuf::from(dep);
87+
let dep_abs_path = fs_host::make_absolute(&dep_rel_path)?;
88+
89+
let (dep_hash, dep_size) = fs_host::hash_file(&dep_abs_path)?;
90+
let dep_artifact = Artifact {
91+
path: dep_rel_path.clone(),
92+
hash: dep_hash,
93+
size: dep_size,
94+
};
95+
tg_artifacts.push(dep_artifact);
96+
tg_deps_paths.push(dep_abs_path);
97+
}
98+
99+
// update post process results
100+
mat_data.deno_artifact = map_from_object(Artifact {
101+
hash: module_hash.clone(),
102+
size,
103+
path,
104+
})
105+
.map_err(|e| e.to_string())?;
106+
107+
mat_data.deps_meta = Some(
108+
tg_artifacts
109+
.iter()
110+
.map(|dep| map_from_object(dep).map_err(|e| e.to_string()))
111+
.collect::<Result<Vec<_>, _>>()?,
112+
);
113+
tg_artifacts.push(deno_artifact);
114+
}
45115
false => {
46116
if !Store::get_codegen_flag() {
47117
return Err(format!(
@@ -54,6 +124,9 @@ impl DenoProcessor {
54124

55125
mat.data = map_from_object(mat_data).map_err(|e| e.to_string())?;
56126

57-
Ok(Some(main_path))
127+
Ok(Some(ResolveModuleOuput {
128+
tg_artifacts,
129+
tg_deps_paths,
130+
}))
58131
}
59132
}

‎typegraph/core/src/utils/postprocess/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ impl PostProcessor for TypegraphPostProcessor {
5656
}
5757
}
5858

59+
#[allow(dead_code)]
5960
pub fn compress_and_encode(main_path: &Path) -> Result<String, String> {
6061
if let Err(e) = fs_host::read_text_file(main_path.display().to_string()) {
6162
return Err(format!("Unable to read {:?}: {}", main_path.display(), e));

‎typegraph/core/src/utils/postprocess/python_rt.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ impl PostProcessor for PythonProcessor {
2121
object_from_map(std::mem::take(&mut mat.data)).map_err(|e| e.to_string())?;
2222
let path = mat_data.python_artifact.get("path").unwrap();
2323
let path: PathBuf = path.as_str().unwrap().into();
24-
// eprint(&format!("***************** {}", path_value));
2524

2625
if tg.meta.artifacts.contains_key(&path) {
2726
continue;

‎typegraph/core/wit/typegraph.wit

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,7 @@ interface runtimes {
271271
record materializer-deno-import {
272272
func-name: string,
273273
module: string,
274+
deps: list<string>,
274275
secrets: list<string>,
275276
}
276277

‎typegraph/node/sdk/src/runtimes/deno.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ export interface DenoFunc {
3434
export interface DenoImport {
3535
name: string;
3636
module: string;
37+
deps?: Array<string>;
3738
secrets?: Array<string>;
3839
effect?: Effect;
3940
}
@@ -88,11 +89,12 @@ export class DenoRuntime extends Runtime {
8889
>(
8990
inp: I,
9091
out: O,
91-
{ name, module, effect = fx.read(), secrets = [] }: DenoImport,
92+
{ name, module, deps = [], effect = fx.read(), secrets = [] }: DenoImport,
9293
): t.Func<I, O, ImportMat> {
9394
const matId = runtimes.importDenoFunction({
9495
funcName: name,
9596
module,
97+
deps,
9698
secrets,
9799
}, effect);
98100
const mat: ImportMat = {
@@ -162,6 +164,7 @@ export class DenoRuntime extends Runtime {
162164
funcName: data.name,
163165
module: data.module,
164166
secrets: data.secrets ?? [],
167+
deps: data.deps ?? [],
165168
}, fx.read()),
166169
);
167170
}

‎typegraph/python/typegraph/runtimes/deno.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,14 +83,17 @@ def import_(
8383
*,
8484
module: str,
8585
name: str,
86+
deps: List[str] = [],
8687
effect: Optional[Effect] = None,
8788
secrets: Optional[List[str]] = None,
8889
):
8990
effect = effect or EffectRead()
9091
secrets = secrets or []
9192
mat_id = runtimes.import_deno_function(
9293
store,
93-
MaterializerDenoImport(func_name=name, module=module, secrets=secrets),
94+
MaterializerDenoImport(
95+
func_name=name, module=module, secrets=secrets, deps=deps
96+
),
9497
effect,
9598
)
9699

0 commit comments

Comments
 (0)