feat(sdk): .tgignore file support (#633)

#### Motivation and context

Set what files/folders should be ignored when using the custom
`expand_path` function in an external `.tgignore` file.
`.tgignore` will behave similarly to most .ignore files with basic glob
syntax support.

#### Migration notes

`expand_glob` has been renamed to `expand_path`

### Checklist

- [x] The change come with new or modified tests
- [ ] Hard-to-understand functions have explanatory comments
- [ ] End-user documentation is updated to reflect the change

---------

Signed-off-by: Yohe-Am <[email protected]>
Co-authored-by: Yohe-Am <[email protected]>

Author: michael-0acf4

.github/workflows/tests.yml

@@ -175,59 +175,58 @@ jobs:
           cargo run --locked --package meta-cli -- --help
           cargo test --locked --package meta-cli
 
-  test-docker:
-    needs: changes
-    if: ${{ needs.changes.outputs.typegate == 'true' }}
-    runs-on: ${{ matrix.runner }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - platform: linux/amd64
-            runner: ubuntu-latest
-          # - platform: linux/amd64
-          #   runner: custom-ubuntu-large
-          # FIXME: try macos-14 runner once all actions support it
-          # docker buildx action broken as of 2024-02-09
-
-          # TODO
-          # - platform: linux/arm64
-          #   runner: custom-macos
-    steps:
-      - uses: actions/checkout@v4
-      - uses: docker/setup-buildx-action@v3
-      - uses: scherermichael-oss/action-has-permission@master
-        id: check
-        with:
-          required-permission: write
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - uses: docker/login-action@v3
-        if: steps.check.outputs.has-permission
-        with:
-          registry: docker.io
-          username: zifeo
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build with cache (internal)
-        uses: docker/build-push-action@v5
-        if: steps.check.outputs.has-permission
-        with:
-          file: dev/Dockerfile
-          platforms: ${{ matrix.platform }}
-          push: false
-          cache-from: type=registry,ref=docker.io/zifeo/metatype-cache:ci
-          cache-to: type=registry,ref=docker.io/zifeo/metatype-cache:ci,mode=max
-          # check target runs extra validation steps in the Dockerfile
-          target: check
-      - name: Build without cache (external)
-        uses: docker/build-push-action@v5
-        if: "! steps.check.outputs.has-permission"
-        with:
-          file: dev/Dockerfile
-          platforms: linux/amd64
-          push: false
-          cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/typegate:latest
-          target: check
+  # test-docker:
+  #   needs: changes
+  #   if: ${{ needs.changes.outputs.typegate == 'true' }}
+  #   runs-on: ${{ matrix.runner }}
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       include:
+  #         - platform: linux/amd64
+  #           runner: ubuntu-latest
+  #         # - platform: linux/amd64
+  #         #   runner: custom-ubuntu-large
+  #         # FIXME: try macos-14 runner once all actions support it
+  #         # docker buildx action broken as of 2024-02-09
+  #         # TODO
+  #         # - platform: linux/arm64
+  #         #   runner: custom-macos
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: docker/setup-buildx-action@v3
+  #     - uses: scherermichael-oss/action-has-permission@master
+  #       id: check
+  #       with:
+  #         required-permission: write
+  #       env:
+  #         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  #     - uses: docker/login-action@v3
+  #       if: steps.check.outputs.has-permission
+  #       with:
+  #         registry: docker.io
+  #         username: zifeo
+  #         password: ${{ secrets.DOCKERHUB_TOKEN }}
+  #     - name: Build with cache (internal)
+  #       uses: docker/build-push-action@v5
+  #       if: steps.check.outputs.has-permission
+  #       with:
+  #         file: dev/Dockerfile
+  #         platforms: ${{ matrix.platform }}
+  #         push: false
+  #         cache-from: type=registry,ref=docker.io/zifeo/metatype-cache:ci
+  #         cache-to: type=registry,ref=docker.io/zifeo/metatype-cache:ci,mode=max
+  #         # check target runs extra validation steps in the Dockerfile
+  #         target: check
+  #     - name: Build without cache (external)
+  #       uses: docker/build-push-action@v5
+  #       if: "! steps.check.outputs.has-permission"
+  #       with:
+  #         file: dev/Dockerfile
+  #         platforms: linux/amd64
+  #         push: false
+  #         cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/typegate:latest
+  #         target: check
 
   test-full:
     needs: changes

examples/deploy/.tgignore

@@ -0,0 +1,7 @@
+.git
+node_modules
+
+*.json
+*.y*ml
+.*ignore
+deploy.*

examples/deploy/deploy.mjs

@@ -75,7 +75,7 @@ const artifactsConfig = {
   prismaMigration: {
     globalAction: {
       create: true,
-      reset: false
+      reset: true
     },
     migrationDir: path.join("prisma-migrations", tg.name)
   }

typegate/tests/e2e/self_deploy/.tgignore

@@ -0,0 +1,3 @@
+.tgignore
+self_deploy_*.ts
+*.mjs

typegraph/core/src/utils/fs_host.rs

@@ -10,7 +10,9 @@ use indexmap::IndexMap;
 
 use crate::{
     global_store::Store,
-    wit::metatype::typegraph::host::{expand_glob, get_cwd, read_file, write_file},
+    wit::metatype::typegraph::host::{
+        expand_path as expand_path_host, get_cwd, path_exists, read_file, write_file,
+    },
 };
 
 pub fn read_text_file<P: Into<String>>(path: P) -> Result<String, String> {
@@ -26,7 +28,7 @@ pub fn write_text_file<P: Into<String>>(path: P, text: P) -> Result<(), String>
 }
 
 pub fn common_prefix_paths(paths: &[PathBuf]) -> Option<PathBuf> {
-    if paths.is_empty() {
+    if paths.len() <= 1 {
         return None;
     }
 
@@ -54,6 +56,15 @@ pub fn relativize_paths(paths: &[PathBuf]) -> Result<Vec<PathBuf>, String> {
         return Ok(vec![]);
     }
 
+    // ambiguous case, assume it is under cwd
+    if paths.len() == 1 {
+        let possible_base = cwd()?;
+        return paths[0]
+            .strip_prefix(&possible_base)
+            .map(|stripped| vec![stripped.to_owned()])
+            .map_err(|_| format!("{:?} does not contain path", possible_base.display()));
+    }
+
     if let Some(common_dir) = common_prefix_paths(paths) {
         let ret = paths
             .iter()
@@ -69,10 +80,44 @@ pub fn relativize_paths(paths: &[PathBuf]) -> Result<Vec<PathBuf>, String> {
     Err("Cannot relativize path list if one item is already relative".to_string())
 }
 
+pub fn expand_path(path: &Path, exclude_glob: &[String]) -> Result<Vec<PathBuf>, String> {
+    let exclude_as_regex = exclude_glob
+        .iter()
+        .map(|glob_pattern| {
+            let mut regex_pattern = String::new();
+            for c in glob_pattern.chars() {
+                match c {
+                    '*' => regex_pattern.push_str(".*"),
+                    '?' => regex_pattern.push('.'),
+                    _ => {
+                        if ".()+-[]^$|".contains(c) {
+                            // escape native regex
+                            regex_pattern.push('\\');
+                        }
+                        regex_pattern.push(c);
+                    }
+                }
+            }
+            // test as suffix if glob star is present
+            if glob_pattern.contains('*') {
+                regex_pattern.push('$');
+            }
+            regex_pattern
+        })
+        .collect::<Vec<_>>();
+
+    let ret = expand_path_host(&path.display().to_string(), &exclude_as_regex)?
+        .iter()
+        .map(PathBuf::from)
+        .collect();
+
+    Ok(ret)
+}
+
 pub fn compress<P: Into<String>>(path: P, exclude: Option<Vec<String>>) -> Result<Vec<u8>, String> {
     // Note: each exclude entry is a regex pattern
     let exclude = exclude.unwrap_or_default();
-    let paths = expand_glob(&path.into(), &exclude)?;
+    let paths = expand_path(&PathBuf::from(path.into()), &exclude)?;
     let mut entries = IndexMap::new();
     // eprint("Preparing tarball");
 
@@ -103,11 +148,35 @@ pub fn unpack_base64<P: Into<String>>(tarb64: &str, dest: P) -> Result<(), Strin
 }
 
 pub fn compress_and_encode_base64(path: PathBuf) -> Result<String, String> {
-    let exclude = vec!["node_modules/".to_string(), "\\.git/".to_string()];
-    let bytes = compress(path.display().to_string(), Some(exclude))?;
+    let mut tgignore = load_tg_ignore_file()?;
+    let default = vec!["node_modules".to_string(), ".git".to_string()];
+    tgignore.extend(default);
+
+    let bytes = compress(path.display().to_string(), Some(tgignore))?;
     encode_bytes_to_base_64(bytes).map_err(|e| e.to_string())
 }
 
+/// Search for .tgignore file at `cwd`, if nothing is found, an empty `Vec` is returned
+pub fn load_tg_ignore_file() -> Result<Vec<String>, String> {
+    let file = cwd()?.join(".tgignore").display().to_string();
+
+    match path_exists(&file)? {
+        true => read_text_file(file).map(|content| {
+            content
+                .lines()
+                .filter_map(|line| {
+                    let trimmed = line.trim();
+                    if trimmed.is_empty() || trimmed.starts_with('#') {
+                        return None;
+                    }
+                    Some(line.to_owned())
+                })
+                .collect()
+        }),
+        false => Ok(vec![]),
+    }
+}
+
 /// Returns `get_cwd()` by default, custom `dir` otherwise
 pub fn cwd() -> Result<PathBuf, String> {
     match Store::get_deploy_cwd() {
@@ -116,12 +185,14 @@ pub fn cwd() -> Result<PathBuf, String> {
     }
 }
 
+/// Strip given path with `cwd`
 pub fn make_relative(path: &Path) -> Result<PathBuf, String> {
     path.strip_prefix(cwd()?)
         .map_err(|e| e.to_string())
         .map(|r| r.to_owned())
 }
 
+/// Join given path with `cwd`
 pub fn make_absolute(path: &Path) -> Result<PathBuf, String> {
     match path.is_relative() {
         true => Ok(cwd()?.join(path)),

typegraph/core/wit/typegraph.wit

@@ -557,7 +557,7 @@ interface utils {
 interface host {
     print: func(s: string)
     eprint: func(s: string)
-    expand-glob: func(root: string, exclude: list<string>) -> result<list<string>, string>
+    expand-path: func(root: string, exclude: list<string>) -> result<list<string>, string>
     path-exists: func(path: string) -> result<bool, string>
     read-file: func(path: string) -> result<list<u8>, string>
     write-file: func(path: string, data: list<u8>) -> result<_, string>

typegraph/node/sdk/src/host/host.ts

@@ -26,13 +26,12 @@ function listAllFilesHelper(
   }
 }
 
-export function expandGlob(
+export function expandPath(
   root: string,
   exclude: Array<string>,
 ): Array<string> {
   try {
     const ret = [] as Array<string>;
-    // console.log("[host] received args", root, exclude);
     listAllFilesHelper(root, ret, exclude);
     return ret;
   } catch (err) {

typegraph/python/typegraph/host/host.py

@@ -24,7 +24,7 @@ def print(self, msg: str):
     def eprint(self, msg: str):
         print(msg, file=sys.stderr)
 
-    def expand_glob(self, root: str, exclude: List[str]) -> Result[List[str], str]:
+    def expand_path(self, root: str, exclude: List[str]) -> Result[List[str], str]:
         try:
             result = []
             for path, _, files in os.walk(root):