fix: type dedup and substantial filters (#994)

luckasRanarison · web-flow · commit 285a66ff64c0 · 2025-04-03T15:33:22.000+03:00
- Fixes type dedup when using list and optional - Add `run_id` filter for substantial ([MET-870](https://linear.app/metatypedev/issue/MET-870/subs-filter-by-run-id))  #### Migration notes --- - [x] The change comes with new or modified tests - [ ] Hard-to-understand functions have explanatory comments - [ ] End-user documentation is updated to reflect the change  ## Summary by CodeRabbit - **New Features** - Introduced two public endpoints for retrieving structured data as lists and optional objects. - Expanded filtering capabilities to support an additional identifier. - **Improvements** - Enhanced asynchronous processing to ensure operations complete reliably. - Streamlined messaging and error feedback for greater clarity. - Optimized naming consistency across system components. - Enhanced functionality of the `SpecialTerms` type for better expression evaluation.
diff --git a/src/typegate/src/runtimes/substantial/agent.ts b/src/typegate/src/runtimes/substantial/agent.ts
@@ -100,11 +100,9 @@ export class Agent {
     this.workflows = workflows;
 
     this.logger.warn(
-      `Initializing agent to handle ${
-        workflows
-          .map(({ name }) => name)
-          .join(", ")
-      }`,
+      `Initializing agent to handle ${workflows
+        .map(({ name }) => name)
+        .join(", ")}`,
     );
 
     this.pollIntervalHandle = setInterval(async () => {
@@ -267,11 +265,9 @@ export class Agent {
       // A consequence of the above, a workflow is always triggered by gql { start(..) }
       // This can also occur if an event is sent from gql under a runId that is not valid (e.g. due to typo)
       this.logger.warn(
-        `First item in the operation list is not a Start, got "${
-          JSON.stringify(
-            first,
-          )
-        }" instead. Closing the underlying schedule.`,
+        `First item in the operation list is not a Start, got "${JSON.stringify(
+          first,
+        )}" instead. Closing the underlying schedule.`,
       );
 
       await Meta.substantial.storeCloseSchedule(schedDef);
@@ -280,7 +276,7 @@ export class Agent {
 
     const { taskContext } = first.event.kwargs as unknown as StdKwargs;
     try {
-      this.workerManager
+      await this.workerManager
         .triggerStart(
           workflow.name,
           next.run_id,
@@ -322,11 +318,7 @@ export class Agent {
           let result;
           let error;
           try {
-            result = await hostcall(
-              this.hostcallCtx,
-              event.opName,
-              event.json,
-            );
+            result = await hostcall(this.hostcallCtx, event.opName, event.json);
           } catch (err) {
             error = err;
           }
@@ -423,11 +415,9 @@ export class Agent {
     const result = event.type == "SUCCESS" ? event.result : event.error;
 
     this.logger.info(
-      `gracefull completion of "${runId}" (${event.type}): ${
-        JSON.stringify(
-          result,
-        )
-      } started at "${startedAt}"`,
+      `gracefull completion of "${runId}" (${event.type}): ${JSON.stringify(
+        result,
+      )} started at "${startedAt}"`,
     );
 
     this.logger.info(`Append Stop ${runId}`);
@@ -479,11 +469,9 @@ function checkIfRunHasStopped(run: Run) {
     if (op.event.type == "Start") {
       if (life >= 1) {
         logger.error(
-          `bad logs: ${
-            JSON.stringify(
-              run.operations.map(({ event }) => event.type),
-            )
-          }`,
+          `bad logs: ${JSON.stringify(
+            run.operations.map(({ event }) => event.type),
+          )}`,
         );
 
         throw new Error(
@@ -496,11 +484,9 @@ function checkIfRunHasStopped(run: Run) {
     } else if (op.event.type == "Stop") {
       if (life <= 0) {
         logger.error(
-          `bad logs: ${
-            JSON.stringify(
-              run.operations.map(({ event }) => event.type),
-            )
-          }`,
+          `bad logs: ${JSON.stringify(
+            run.operations.map(({ event }) => event.type),
+          )}`,
         );
 
         throw new Error(
diff --git a/src/typegate/src/runtimes/substantial/filter_utils.ts b/src/typegate/src/runtimes/substantial/filter_utils.ts
@@ -22,7 +22,10 @@ type ORD = KVHelper<["eq", "lt", "lte", "gt", "gte"], string>;
 type INCL = KVHelper<["in", "contains"], string>;
 type Terms = ORD & INCL;
 
-type SpecialTerms = KVHelper<["started_at", "ended_at", "status"], Terms>;
+type SpecialTerms = KVHelper<
+  ["run_id", "started_at", "ended_at", "status"],
+  Terms
+>;
 
 export type Expr = Terms & SpecialTerms & AND & OR & NOT;
 
@@ -101,7 +104,6 @@ export async function applyFilter(
   return searchResults;
 }
 
-
 export function evalExpr(
   sResult: SearchItem,
   filter: Expr,
@@ -121,14 +123,12 @@ export function evalExpr(
       const exprList = filter[op];
       if (!Array.isArray(exprList)) {
         // should be unreachable since filter is validated at push
-        throw new Error(
-          `Fatal: array expected at ${path.join(".")}`,
-        );
+        throw new Error(`Fatal: array expected at ${path.join(".")}`);
       }
       const fn = op == "or" ? "some" : "every";
       if (
         !exprList[fn]((subFilter, index) =>
-          evalExpr(sResult, subFilter, [...newPath, `#${index}`])
+          evalExpr(sResult, subFilter, [...newPath, `#${index}`]),
         )
       ) {
         return false;
@@ -142,6 +142,7 @@ export function evalExpr(
       break;
     }
     // Special
+    case "run_id":
     case "status":
     case "started_at":
     case "ended_at": {
@@ -182,7 +183,7 @@ function evalTerm(sResult: SearchItem, terms: Terms, path: Array<string>) {
         return false;
       }
 
-      if (!testCompare(value,  toJS(terms[op]))) {
+      if (!testCompare(value, toJS(terms[op]))) {
         return false;
       }
 
@@ -199,17 +200,13 @@ function evalTerm(sResult: SearchItem, terms: Terms, path: Array<string>) {
     }
     case "contains":
     case "in": {
-      if (
-        !inclusion(value, toJS(terms[op]), op, newPath)
-      ) {
+      if (!inclusion(value, toJS(terms[op]), op, newPath)) {
         return false;
       }
       break;
     }
     default: {
-      throw new Error(
-        `Unknown operator "${op}" at ${path.join(".")}`,
-      );
+      throw new Error(`Unknown operator "${op}" at ${path.join(".")}`);
     }
   }
 
@@ -241,8 +238,8 @@ function ord(l: unknown, r: unknown, cp: keyof ORD, path: Array<string>) {
   }
 
   if (
-    typeof l == "string" && typeof r == "string" ||
-    typeof l == "number" && typeof r == "number"
+    (typeof l == "string" && typeof r == "string") ||
+    (typeof l == "number" && typeof r == "number")
   ) {
     switch (cp) {
       case "lt":
@@ -278,7 +275,9 @@ function inclusion(
   } else if (typeof left == "string" && typeof right == "string") {
     return right.includes(left);
   } else if (
-    typeof left == typeof right && typeof left == "object" && left != null
+    typeof left == typeof right &&
+    typeof left == "object" &&
+    left != null
   ) {
     // Example: { a: { b: 1 } } in { a: { b: 1 }, c: 2 } => true
     const rightV = (right ?? {}) as Record<string, unknown>;
diff --git a/src/typegraph/core/src/utils/postprocess/naming.rs b/src/typegraph/core/src/utils/postprocess/naming.rs
@@ -130,9 +130,8 @@ fn visit_type(cx: &VisitContext, acc: &mut VisitCollector, id: u32) -> anyhow::R
             ));
             let inner_name = visit_type(cx, acc, data.item)?;
             acc.path.pop();
-            // gen_name(cx, acc, id, "optional")
+
             gen_name(cx, acc, id, &format!("{inner_name}_optional"))
-            // format!("{inner_name}_optional").into()
         }
         TypeNode::List { data, .. } => {
             acc.path.push((
@@ -144,13 +143,7 @@ fn visit_type(cx: &VisitContext, acc: &mut VisitCollector, id: u32) -> anyhow::R
             ));
             let inner_name = visit_type(cx, acc, data.items)?;
             acc.path.pop();
-            // gen_name(cx, acc, id, "list")
-            /* if cx.user_named.contains(&data.items) {
-                gen_name(cx, acc, id, &format!("{_inner_name}_list"))
-            } else {
-                format!("{_inner_name}_list").into()
-            } */
-            // format!("{inner_name}_list").into()
+
             gen_name(cx, acc, id, &format!("{inner_name}_list"))
         }
         TypeNode::Object { data, .. } => {
@@ -344,15 +337,31 @@ fn gen_name(cx: &VisitContext, acc: &mut VisitCollector, id: u32, ty_name: &str)
 
         let title;
         let mut last = acc.path.len();
+        let mut wrapped = false;
         loop {
             last -= 1;
             let (last_segment, last_name) = &acc.path[last];
             title = match &last_segment.edge {
                 // we don't include optional and list nodes in
                 // generated names (useless but also, they might be placeholders)
-                Edge::OptionalItem | Edge::ArrayItem => continue,
-                Edge::FunctionInput => format!("{}_input", last_name),
-                Edge::FunctionOutput => format!("{}_output", last_name),
+                Edge::OptionalItem | Edge::ArrayItem => {
+                    wrapped = true;
+                    continue;
+                }
+                Edge::FunctionInput => {
+                    format!(
+                        "{}_input{}",
+                        last_name,
+                        wrapped.then_some(format!("_{ty_name}")).unwrap_or_default()
+                    )
+                }
+                Edge::FunctionOutput => {
+                    format!(
+                        "{}_output{}",
+                        last_name,
+                        wrapped.then_some(format!("_{ty_name}")).unwrap_or_default()
+                    )
+                }
                 Edge::ObjectProp(key) => {
                     join_if_ok!(format!("{last_name}_{key}"), ty_name)
                 }
diff --git a/tests/dedup/tg.ts b/tests/dedup/tg.ts
@@ -51,4 +51,18 @@ await typegraph("materializer-dedup", (g: any) => {
     f1: f.withPolicy(Policy.public()),
     f2: f.withPolicy(Policy.internal()),
   });
+
+  g.expose(
+    {
+      f3: deno.func(t.struct({}), t.list(t.struct({ key: t.string() })), {
+        code: () => [{ key: "value" }],
+      }),
+    },
+    {
+      f4: deno.func(t.struct({}), t.optional(t.struct({ key: t.string() })), {
+        code: () => ({ key: "value" }),
+      }),
+    },
+    Policy.public(),
+  );
 });
diff --git a/tests/introspection/__snapshots__/visibility_complex_test.ts.snap b/tests/introspection/__snapshots__/visibility_complex_test.ts.snap
diff --git a/tests/runtimes/substantial/filter_utils_test.ts b/tests/runtimes/substantial/filter_utils_test.ts
