feat(envs): dynamic variables for envs (#95)

* feat(envs): dynamic env

* feat: dyn env naive (wip)

* feat(envs): dynamic env (wip)

* fix: attempt taskless approach

* refactor: reducer

* fix: dyn env reducer

* feat: dyn env

* test: dyn env, move impl to reducers

* fix: pre-commit, remove unused import

* fix: use hack.ts instead

* chore: bump to 0.2.1

---------

Co-authored-by: Yohe-Am <[email protected]>

Author: michael-0acf4

.gitignore

@@ -2,3 +2,13 @@
 play.*
 examples/**/.ghjk
 .dev
+
+deps/https
+node_analysis_*
+v8_code_cache_*
+dep_analysis_*
+gen
+npm
+deno.land
+jsr.io
+esm.sh

README.md

@@ -28,9 +28,9 @@ ghjk was designed to be an intermediate alternative between [Earthly](https://gi
 
 ```bash
 # stable
-curl -fsSL https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/install.sh | bash
+curl -fsSL https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/install.sh | bash
 # latest (main)
-curl -fsSL https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/install.sh | GHJK_VERSION=main bash/fish/zsh
+curl -fsSL https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/install.sh | GHJK_VERSION=main bash/fish/zsh
 ```
 
 In your project, create a configuration file called `ghjk.ts` that look something like:
@@ -39,11 +39,11 @@ In your project, create a configuration file called `ghjk.ts` that look somethin
 // NOTE: All the calls in your `ghjk.ts` file are ultimately modifying the 'sophon' proxy 
 // object exported here.
 // WARN: always import `hack.ts` file first
-export { sophon } from "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/hack.ts";
+export { sophon } from "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/hack.ts";
 import {
   install, task,
-} from "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/hack.ts";
-import node from "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/ports/node.ts";
+} from "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/hack.ts";
+import node from "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/node.ts";
 
 // install programs (ports) into your env
 install(
@@ -69,9 +69,9 @@ Ghjk is primarily configured through constructs called "environments" or "envs"
 They serve as recipes for making (mostly) reproducable posix shells.
 
 ```ts
-export { sophon } from "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/hack.ts";
-import * as ghjk from "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/hack.ts";
-import * as ports from "https://raw.githubusercontent.com/metatypedev/ghjk/0.2.0/ports/mod.ts";
+export { sophon } from "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/hack.ts";
+import * as ghjk from "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/hack.ts";
+import * as ports from "https://raw.githubusercontent.com/metatypedev/ghjk/v0.2.1/ports/mod.ts";
 
 // top level `install`s go to the `main` env
 ghjk.install(ports.protoc());

deno.jsonc

@@ -1,6 +1,7 @@
 {
   "tasks": {
     "test": "deno test --parallel --unstable-worker-options --unstable-kv -A tests/*",
+    "self": "deno run -A --unstable-kv --unstable-worker-options main.ts ",
     "cache": "deno cache deps/*",
     "check": "deno run -A ./scripts/check.ts",
     "dev": "deno run -A ./scripts/dev.ts"

deno.lock

@@ -0,0 +1,21 @@
+import { file } from "../../hack.ts";
+
+const ghjk = file({
+  defaultEnv: "empty",
+  envs: [{ name: "empty", inherit: false }],
+  defaultBaseEnv: "empty",
+  allowedBuildDeps: [],
+  installs: [],
+  stdDeps: true,
+  enableRuntimes: true,
+  tasks: {},
+});
+
+export const sophon = ghjk.sophon;
+const { env, task } = ghjk;
+
+env("main")
+  .var("A", "A#STATIC")
+  .var("C", ($) => $`echo C [$A, $B]`.text())
+  .var("B", () => "B#DYNAMIC")
+  .onEnter(task(($) => $`echo enter $A, $B, $C`));

examples/env_vars/ghjk.ts

@@ -36,12 +36,13 @@ import * as std_modules from "../modules/std.ts";
 import type { ExecTaskArgs } from "../modules/tasks/deno.ts";
 import { TaskDefHashed, TasksModuleConfig } from "../modules/tasks/types.ts";
 // envs
-import type {
-  EnvRecipe,
-  EnvsModuleConfig,
-  Provision,
-  WellKnownProvision,
+import {
+  type EnvRecipe,
+  type EnvsModuleConfig,
+  type Provision,
+  type WellKnownProvision,
 } from "../modules/envs/types.ts";
+import envsValidators from "../modules/envs/types.ts";
 import modulesValidators from "../modules/types.ts";
 
 const validators = {
@@ -100,7 +101,7 @@ export type TaskDefArgs = {
   desc?: string;
   dependsOn?: string | string[];
   workingDir?: string | Path;
-  vars?: Record<string, string | number>;
+  vars?: Record<string, string | number>; // TODO: add DynEnvValue?
   allowedBuildDeps?: (InstallConfigFat | AllowedPortDep)[];
   installs?: InstallConfigFat | InstallConfigFat[];
   inherit?: EnvParent;
@@ -305,7 +306,7 @@ export class Ghjkfile {
         workingDir,
         `<task:${task.name ?? key}>`,
       );
-      await task.fn(custom$, {
+      return await task.fn(custom$, {
         argv,
         env: Object.freeze(envVars),
         $: custom$,
@@ -653,6 +654,15 @@ export class Ghjkfile {
             const prov: WellKnownProvision = { ty: "posix.envVar", key, val };
             return prov;
           }),
+          ...Object.entries(final.dynVars).map((
+            [key, val],
+          ) => {
+            const prov = { ty: "posix.envVarDyn", key, taskKey: val };
+            return unwrapZodRes(
+              envsValidators.envVarDynProvision.safeParse(prov),
+              prov,
+            );
+          }),
           // env hooks
           ...hooks,
         ],
@@ -886,6 +896,7 @@ type EnvFinalizer = () => {
   installSetId: string;
   inherit: string | string[] | boolean;
   vars: Record<string, string>;
+  dynVars: Record<string, string>;
   desc?: string;
   onEnterHookTasks: string[];
   onExitHookTasks: string[];
@@ -894,6 +905,12 @@ type EnvFinalizer = () => {
 export type EnvDefArgsPartial =
   & { name?: string }
   & Omit<EnvDefArgs, "name">;
+
+export type DynEnvValue =
+  | (() => string | number)
+  | (($_: typeof $) => string | number)
+  | (($_: typeof $) => Promise<string | number>);
+
 //
 // /**
 //  * A version of {@link EnvDefArgs} that has all container
@@ -938,6 +955,7 @@ export class EnvBuilder {
   #file: Ghjkfile;
   #inherit: string | string[] | boolean = true;
   #vars: Record<string, string | number> = {};
+  #dynVars: Record<string, string> = {};
   #desc?: string;
   #onEnterHookTasks: string[] = [];
   #onExitHookTasks: string[] = [];
@@ -958,6 +976,7 @@ export class EnvBuilder {
       vars: Object.fromEntries(
         Object.entries(this.#vars).map(([key, val]) => [key, val.toString()]),
       ),
+      dynVars: this.#dynVars,
       desc: this.#desc,
       onExitHookTasks: this.#onExitHookTasks,
       onEnterHookTasks: this.#onEnterHookTasks,
@@ -991,18 +1010,45 @@ export class EnvBuilder {
   /**
    * Add an environment variable.
    */
-  var(key: string, val: string) {
+  var(key: string, val: string | DynEnvValue) {
     this.vars({ [key]: val });
     return this;
   }
 
   /**
    * Add multiple environment variable.
    */
-  vars(envVars: Record<string, string | number>) {
+  vars(envVars: Record<string, string | number | DynEnvValue>) {
+    const vars = {}, dynVars = {};
+    for (const [k, v] of Object.entries(envVars)) {
+      switch (typeof v) {
+        case "string":
+        case "number":
+          Object.assign(vars, { [k]: v });
+          break;
+        case "function": {
+          const taskKey = this.#file.addTask({
+            ty: "denoFile@v1",
+            fn: v,
+            nonce: k,
+          });
+          Object.assign(dynVars, { [k]: taskKey });
+          break;
+        }
+        default:
+          throw new Error(
+            `environment value of type "${typeof v}" is not supported`,
+          );
+      }
+    }
+
     Object.assign(
       this.#vars,
-      unwrapZodRes(validators.envVars.safeParse(envVars), { envVars }),
+      unwrapZodRes(validators.envVars.safeParse(vars), { envVars: vars }),
+    );
+    Object.assign(
+      this.#dynVars,
+      dynVars,
     );
     return this;
   }

files/mod.ts

@@ -24,7 +24,7 @@ install(
 task(
   "lock-sed",
   async ($) => {
-    const GHJK_VERSION = "0.2.0";
+    const GHJK_VERSION = "0.2.1";
     await sedLock(
       $.path(import.meta.dirname!),
       {
@@ -41,7 +41,7 @@ task(
           ],
           "./README.md": [
             [
-              /(.*\/metatypedev\/ghjk\/)[^/]*(\/.*)/,
+              /(.*\/metatypedev\/ghjk\/v)[^/]*(\/.*)/,
               GHJK_VERSION,
             ],
           ],

ghjk.ts

@@ -30,7 +30,7 @@ type HostCtx = {
   lockedFlagSet: boolean;
 };
 
-const GHJK_VERSION = "0.2.0";
+const GHJK_VERSION = "0.2.1";
 
 export async function cli(args: CliArgs) {
   logger().debug(`ghjk CLI`, GHJK_VERSION);

host/mod.ts

@@ -2,7 +2,7 @@
 
 set -e -u
 
-GHJK_VERSION="${GHJK_VERSION:-v0.2.0}"
+GHJK_VERSION="${GHJK_VERSION:-v0.2.1}"
 GHJK_INSTALLER_URL="${GHJK_INSTALLER_URL:-https://raw.github.com/metatypedev/ghjk/$GHJK_VERSION/install.ts}"
 GHJK_SHARE_DIR="${GHJK_SHARE_DIR:-$HOME/.local/share/ghjk}"
 DENO_VERSION="${DENO_VERSION:-v1.44.2}"

install.sh

@@ -62,6 +62,7 @@ export async function cookPosixEnv(
       case "posix.headerFile":
         includePaths.push(wellKnownProv.absolutePath);
         break;
+      // case "posix.envVarDyn":
       case "posix.envVar":
         if (vars[wellKnownProv.key]) {
           throw new Error(
@@ -84,7 +85,7 @@ export async function cookPosixEnv(
         break;
       default:
         throw Error(
-          `unsupported provision type: ${(wellKnownProv as any).provision}`,
+          `unsupported provision type: ${(wellKnownProv as any).ty}`,
         );
     }
   }));

modules/envs/posix.ts

@@ -1,4 +1,6 @@
 import { unwrapZodRes } from "../../port.ts";
+import { execTask } from "../tasks/exec.ts";
+import { getTasksCtx } from "../tasks/inter.ts";
 import type { GhjkCtx } from "../types.ts";
 import type {
   EnvRecipeX,
@@ -7,7 +9,7 @@ import type {
   WellKnownEnvRecipeX,
   WellKnownProvision,
 } from "./types.ts";
-import { wellKnownProvisionTypes } from "./types.ts";
+import { envVarDynTy, wellKnownProvisionTypes } from "./types.ts";
 import validators from "./types.ts";
 
 export type ProvisionReducerStore = Map<
@@ -31,6 +33,10 @@ export function getProvisionReducerStore(
     store = new Map();
     gcx.blackboard.set(id, store);
   }
+  store?.set(
+    envVarDynTy,
+    installDynEnvReducer(gcx) as ProvisionReducer<Provision, Provision>,
+  );
   return store;
 }
 
@@ -85,3 +91,38 @@ export async function reduceStrangeProvisions(
   };
   return out;
 }
+
+export function installDynEnvReducer(gcx: GhjkCtx) {
+  return async (provisions: Provision[]) => {
+    const output = [];
+    const badProvisions = [];
+    const taskCtx = getTasksCtx(gcx);
+
+    for (const provision of provisions) {
+      const ty = "posix.envVar";
+      const key = provision.taskKey as string;
+
+      const taskGraph = taskCtx.taskGraph;
+      const taskConf = taskCtx.config;
+
+      const targetKey = Object.entries(taskConf.tasks)
+        .filter(([_, task]) => task.key == key)
+        .shift()?.[0];
+
+      if (targetKey) {
+        // console.log("key", key, " maps to target ", targetKey);
+        const results = await execTask(gcx, taskConf, taskGraph, targetKey, []);
+        output.push({ ...provision, ty, val: results[key] as any ?? "" });
+      } else {
+        badProvisions.push(provision);
+      }
+    }
+
+    if (badProvisions.length >= 1) {
+      throw new Error("cannot deduce task from keys", {
+        cause: { badProvisions },
+      });
+    }
+    return output;
+  };
+}

modules/envs/reducer.ts

@@ -21,6 +21,8 @@ export const installProvisionTypes = [
   installProvisionTy,
 ] as const;
 
+export const envVarDynTy = "posix.envVarDyn";
+
 // we separate the posix file types in a separate
 // array in the interest of type inference
 export const wellKnownProvisionTypes = [
@@ -78,9 +80,16 @@ const envsModuleConfig = zod.object({
   message: `no env found under the provided "defaultEnv"`,
 });
 
+const envVarDynProvision = zod.object({
+  ty: zod.literal(envVarDynTy),
+  key: moduleValidators.envVarName,
+  taskKey: zod.string(),
+});
+
 const validators = {
   provision,
   wellKnownProvision,
+  envVarDynProvision,
   envRecipe,
   envsModuleConfig,
   wellKnownEnvRecipe,

modules/envs/types.ts

@@ -108,7 +108,6 @@ export class PortsModule extends ModuleBase<PortsCtx, PortsLockEnt> {
       installSetProvisionTy,
       installSetReducer(gcx) as ProvisionReducer<Provision, Provision>,
     );
-
     return pcx;
   }