diff --git a/Cargo.lock b/Cargo.lock index 6b95f7d62e..3cde23d72e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12233,6 +12233,7 @@ dependencies = [ "color-eyre", "common", "enum_dispatch", + "glob", "graphql-parser 0.4.0", "indexmap 2.2.6", "indoc", diff --git a/typegate/deno.lock b/typegate/deno.lock index 4092616585..a246b513de 100644 --- a/typegate/deno.lock +++ b/typegate/deno.lock @@ -19,6 +19,21 @@ "regenerator-runtime": "regenerator-runtime@0.14.1" } }, + "@isaacs/cliui@8.0.2": { + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dependencies": { + "string-width": "string-width@5.1.2", + "string-width-cjs": "string-width@4.2.3", + "strip-ansi": "strip-ansi@7.1.0", + "strip-ansi-cjs": "strip-ansi@6.0.1", + "wrap-ansi": "wrap-ansi@8.1.0", + "wrap-ansi-cjs": "wrap-ansi@7.0.0" + } + }, + "@pkgjs/parseargs@0.11.0": { + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dependencies": {} + }, "@sentry-internal/tracing@7.70.0": { "integrity": "sha512-SpbE6wZhs6QwG2ORWCt8r28o1T949qkWx/KeRTCdK4Ub95PQ3Y3DgnqD8Wz//3q50Wt6EZDEibmz4t067g6PPg==", "dependencies": { @@ -70,12 +85,40 @@ "debug": "debug@4.3.4" } }, + "ansi-regex@5.0.1": { + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dependencies": {} + }, + "ansi-regex@6.0.1": { + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "dependencies": {} + }, + "ansi-styles@4.3.0": { + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "color-convert@2.0.1" + } + }, + "ansi-styles@6.2.1": { + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dependencies": {} + }, "argparse@1.0.10": { "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", "dependencies": { "sprintf-js": "sprintf-js@1.0.3" } }, + "balanced-match@1.0.2": { + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dependencies": {} + }, + "brace-expansion@2.0.1": { + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "balanced-match@1.0.2" + } + }, "buffer-writer@2.0.0": { "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", "dependencies": {} @@ -88,6 +131,16 @@ "integrity": "sha512-kqTg3WWywappJPqtgrdvbA380VoXO2eu9VCV895JgbyHsaErXdyHK9LOZ911OvAk6L0obK7kDk9CGs8+oBawVA==", "dependencies": {} }, + "color-convert@2.0.1": { + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "color-name@1.1.4" + } + }, + "color-name@1.1.4": { + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dependencies": {} + }, "complex.js@2.1.1": { "integrity": "sha512-8njCHOTtFFLtegk6zQo0kkVX1rngygb/KQI6z1qZxlFI3scluC+LVTCFbrkWjBv4vvLlbQ9t88IPMC6k95VTTg==", "dependencies": {} @@ -96,6 +149,14 @@ "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", "dependencies": {} }, + "cross-spawn@7.0.3": { + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dependencies": { + "path-key": "path-key@3.1.1", + "shebang-command": "shebang-command@2.0.0", + "which": "which@2.0.2" + } + }, "debug@4.3.4": { "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "dependencies": { @@ -106,6 +167,18 @@ "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==", "dependencies": {} }, + "eastasianwidth@0.2.0": { + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dependencies": {} + }, + "emoji-regex@8.0.0": { + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dependencies": {} + }, + "emoji-regex@9.2.2": { + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dependencies": {} + }, "escape-latex@1.2.0": { "integrity": "sha512-nV5aVWW1K0wEiUIEdZ4erkGGH8mDxGyxSeqPzRNtWP7ataw+/olFObw7hujFWlVjNsaDFw5VZ5NzVSIqRgfTiw==", "dependencies": {} @@ -114,6 +187,13 @@ "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", "dependencies": {} }, + "foreground-child@3.1.1": { + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "dependencies": { + "cross-spawn": "cross-spawn@7.0.3", + "signal-exit": "signal-exit@4.1.0" + } + }, "format-util@1.0.5": { "integrity": "sha512-varLbTj0e0yVyRpqQhuWV+8hlePAgaoFRhNFj50BNjEIrw1/DphHSObtqwskVCPWNgzwPoQrZAbfa/SBiicNeg==", "dependencies": {} @@ -122,6 +202,16 @@ "integrity": "sha512-pwiTgt0Q7t+GHZA4yaLjObx4vXmmdcS0iSJ19o8d/goUGgItX9UZWKWNnLHehxviD8wU2IWRsnR8cD5+yOJP2Q==", "dependencies": {} }, + "glob@10.3.15": { + "integrity": "sha512-0c6RlJt1TICLyvJYIApxb8GsXoai0KUP7AxKKAtsYXdgJR1mGEUa7DgwShbdk1nly0PYoZj01xd4hzbq3fsjpw==", + "dependencies": { + "foreground-child": "foreground-child@3.1.1", + "jackspeak": "jackspeak@2.3.6", + "minimatch": "minimatch@9.0.4", + "minipass": "minipass@7.1.1", + "path-scurry": "path-scurry@1.11.1" + } + }, "graphql@16.8.1": { "integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==", "dependencies": {} @@ -133,6 +223,21 @@ "debug": "debug@4.3.4" } }, + "is-fullwidth-code-point@3.0.0": { + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dependencies": {} + }, + "isexe@2.0.0": { + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dependencies": {} + }, + "jackspeak@2.3.6": { + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "dependencies": { + "@isaacs/cliui": "@isaacs/cliui@8.0.2", + "@pkgjs/parseargs": "@pkgjs/parseargs@0.11.0" + } + }, "javascript-natural-sort@0.7.1": { "integrity": "sha512-nO6jcEfZWQXDhOiBtG2KvKyEptz7RVbpGP4vTD2hLBdmNQSsCiicO2Ioinv6UI4y9ukqnBpy+XZ9H6uLNgJTlw==", "dependencies": {} @@ -167,6 +272,10 @@ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dependencies": {} }, + "lru-cache@10.2.2": { + "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==", + "dependencies": {} + }, "lru_map@0.3.3": { "integrity": "sha512-Pn9cox5CsMYngeDbmChANltQl+5pi6XmTrraMSzhPmMBbmgcxmqWry0U3PGapCU1yB4/LqCcom7qhHZiF/jGfQ==", "dependencies": {} @@ -185,6 +294,16 @@ "typed-function": "typed-function@4.1.1" } }, + "minimatch@9.0.4": { + "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", + "dependencies": { + "brace-expansion": "brace-expansion@2.0.1" + } + }, + "minipass@7.1.1": { + "integrity": "sha512-UZ7eQ+h8ywIRAW1hIEl2AqdwzJucU/Kp59+8kkZeSvafXhZjul247BvIJjEVFVeON6d7lM46XX1HXCduKAS8VA==", + "dependencies": {} + }, "ms@2.1.2": { "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dependencies": {} @@ -199,6 +318,17 @@ "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==", "dependencies": {} }, + "path-key@3.1.1": { + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dependencies": {} + }, + "path-scurry@1.11.1": { + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dependencies": { + "lru-cache": "lru-cache@10.2.2", + "minipass": "minipass@7.1.1" + } + }, "pg-cloudflare@1.1.1": { "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", "dependencies": {} @@ -276,6 +406,20 @@ "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==", "dependencies": {} }, + "shebang-command@2.0.0": { + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dependencies": { + "shebang-regex": "shebang-regex@3.0.0" + } + }, + "shebang-regex@3.0.0": { + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dependencies": {} + }, + "signal-exit@4.1.0": { + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dependencies": {} + }, "split2@4.2.0": { "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", "dependencies": {} @@ -284,6 +428,34 @@ "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dependencies": {} }, + "string-width@4.2.3": { + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "emoji-regex@8.0.0", + "is-fullwidth-code-point": "is-fullwidth-code-point@3.0.0", + "strip-ansi": "strip-ansi@6.0.1" + } + }, + "string-width@5.1.2": { + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dependencies": { + "eastasianwidth": "eastasianwidth@0.2.0", + "emoji-regex": "emoji-regex@9.2.2", + "strip-ansi": "strip-ansi@7.1.0" + } + }, + "strip-ansi@6.0.1": { + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "ansi-regex@5.0.1" + } + }, + "strip-ansi@7.1.0": { + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dependencies": { + "ansi-regex": "ansi-regex@6.0.1" + } + }, "tiny-emitter@2.1.0": { "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==", "dependencies": {} @@ -300,6 +472,28 @@ "integrity": "sha512-Ii+sehpSfZy+At5nPdnyMhx78fEoPDkR2XW/zimHEL3MyGJQOCQ7WeP20jPYRz7ZCpcKLB21NxuXHF3bxjStBQ==", "dependencies": {} }, + "which@2.0.2": { + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dependencies": { + "isexe": "isexe@2.0.0" + } + }, + "wrap-ansi@7.0.0": { + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "ansi-styles@4.3.0", + "string-width": "string-width@4.2.3", + "strip-ansi": "strip-ansi@6.0.1" + } + }, + "wrap-ansi@8.1.0": { + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dependencies": { + "ansi-styles": "ansi-styles@6.2.1", + "string-width": "string-width@5.1.2", + "strip-ansi": "strip-ansi@7.1.0" + } + }, "xtend@4.0.2": { "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", "dependencies": {} diff --git a/typegate/src/sync/typegraph.ts b/typegate/src/sync/typegraph.ts index f9cd2915a1..c155654877 100644 --- a/typegate/src/sync/typegraph.ts +++ b/typegate/src/sync/typegraph.ts @@ -41,10 +41,7 @@ export class TypegraphStore { ); const data = JSON.stringify([typegraph, encryptedSecrets]); const hash = encodeHex( - await crypto.subtle.digest( - "SHA-256", - new TextEncoder().encode(data), - ), + await crypto.subtle.digest("SHA-256", new TextEncoder().encode(data)), ); const id = { @@ -53,11 +50,7 @@ export class TypegraphStore { uploadedAt: new Date(), }; - await this.#uploadData( - this.bucket, - TypegraphStore.#getKey(this.bucket, id), - data, - ); + await this.#uploadData(this.bucket, TypegraphStore.#getKey(id), data); return id; } @@ -65,7 +58,7 @@ export class TypegraphStore { public async download( id: TypegraphId, ): Promise<[TypeGraphDS, SecretManager]> { - const key = TypegraphStore.#getKey(this.bucket, id); + const key = TypegraphStore.#getKey(id); const data = await this.#downloadData(this.bucket, key); const [typegraph, encryptedSecrets] = JSON.parse(data) as [ TypeGraphDS, @@ -77,10 +70,10 @@ export class TypegraphStore { return [typegraph, secretManager]; } - static #getKey(bucket: string, typegraphId: TypegraphId) { + static #getKey(typegraphId: TypegraphId) { const { name, hash, uploadedAt } = typegraphId; const uploadDate = uploadedAt.toISOString(); - return `${bucket}/typegraphs/${name}/typegraph.json.${uploadDate}.${hash}`; + return `typegraphs/${name}/typegraph.json.${uploadDate}.${hash}`; } async #uploadData(bucket: string, key: string, data: string) { diff --git a/typegate/tests/runtimes/deno/deno_dir.py b/typegate/tests/runtimes/deno/deno_dir.py new file mode 100644 index 0000000000..f3ca1b0ad4 --- /dev/null +++ b/typegate/tests/runtimes/deno/deno_dir.py @@ -0,0 +1,22 @@ +from typegraph.graph.typegraph import Graph +from typegraph.policy import Policy +from typegraph.runtimes.deno import DenoRuntime + +from typegraph import t, typegraph + + +@typegraph() +def deno_dir(g: Graph): + deno = DenoRuntime() + public = Policy.public() + + g.expose( + public, + test_dir=deno.import_( + t.struct({"a": t.float(), "b": t.float()}), + t.float(), + module="ts/dep/main.ts", + deps=["ts/dep"], + name="doAddition", + ), + ) diff --git a/typegate/tests/runtimes/deno/deno_dir.ts b/typegate/tests/runtimes/deno/deno_dir.ts new file mode 100644 index 0000000000..f6f8bb1e13 --- /dev/null +++ b/typegate/tests/runtimes/deno/deno_dir.ts @@ -0,0 +1,25 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { Policy, t, typegraph } from "@typegraph/sdk/index.js"; +import { DenoRuntime } from "@typegraph/sdk/runtimes/deno.js"; + +export const tg = await typegraph( + { + name: "deno_dirs", + }, + (g) => { + const deno = new DenoRuntime(); + const pub = Policy.public(); + + g.expose({ + testDir: deno + .import(t.struct({ a: t.float(), b: t.float() }), t.float(), { + module: "ts/dep/main.ts", + name: "doAddition", + deps: ["ts/dep/nested"], + }) + .withPolicy(pub), + }); + }, +); diff --git a/typegate/tests/runtimes/deno/deno_dir_test.ts b/typegate/tests/runtimes/deno/deno_dir_test.ts new file mode 100644 index 0000000000..9139ee1a8d --- /dev/null +++ b/typegate/tests/runtimes/deno/deno_dir_test.ts @@ -0,0 +1,47 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { gql, Meta } from "../../utils/mod.ts"; + +Meta.test( + { + name: "DenoRuntime - Single Replica: support for dirs when adding deps", + }, + async (t) => { + await t.should( + "work for deps specified with dir on Python SDK", + async () => { + const engine = await t.engine( + "runtimes/deno/deno_dir.py", + ); + + await gql` + query { + test_dir(a: 4, b: 3) + } + ` + .expectData({ + test_dir: 7, + }) + .on(engine); + }, + ); + + await t.should( + "work for deps specified with dir on TypeScript SDK", + async () => { + const engine = await t.engine("runtimes/deno/deno_dir.ts"); + + await gql` + query { + testDir(a: 20, b: 5) + } + ` + .expectData({ + testDir: 25, + }) + .on(engine); + }, + ); + }, +); diff --git a/typegate/tests/runtimes/deno/deno_glob_test.ts b/typegate/tests/runtimes/deno/deno_glob_test.ts new file mode 100644 index 0000000000..ce5783aebe --- /dev/null +++ b/typegate/tests/runtimes/deno/deno_glob_test.ts @@ -0,0 +1,45 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { gql, Meta } from "../../utils/mod.ts"; + +Meta.test( + { + name: "DenoRuntime - Single Replica: support for globs when adding deps", + }, + async (t) => { + await t.should( + "work for deps specified with glob on Python SDK", + async () => { + const engine = await t.engine("runtimes/deno/deno_globs.py"); + + await gql` + query { + test_glob(a: 10, b: 53) + } + ` + .expectData({ + test_glob: 63, + }) + .on(engine); + }, + ); + + await t.should( + "work for deps specified with glob on TypeScript SDK", + async () => { + const engine = await t.engine("runtimes/deno/deno_globs.ts"); + + await gql` + query { + testGlob(a: 10, b: 5) + } + ` + .expectData({ + testGlob: 15, + }) + .on(engine); + }, + ); + }, +); diff --git a/typegate/tests/runtimes/deno/deno_globs.py b/typegate/tests/runtimes/deno/deno_globs.py new file mode 100644 index 0000000000..f0ab6420d7 --- /dev/null +++ b/typegate/tests/runtimes/deno/deno_globs.py @@ -0,0 +1,22 @@ +from typegraph.graph.typegraph import Graph +from typegraph.policy import Policy +from typegraph.runtimes.deno import DenoRuntime + +from typegraph import t, typegraph + + +@typegraph() +def deno_globs(g: Graph): + deno = DenoRuntime() + public = Policy.public() + + g.expose( + public, + test_glob=deno.import_( + t.struct({"a": t.float(), "b": t.float()}), + t.float(), + module="ts/dep/main.ts", + deps=["ts/**/nested/*.ts"], + name="doAddition", + ), + ) diff --git a/typegate/tests/runtimes/deno/deno_globs.ts b/typegate/tests/runtimes/deno/deno_globs.ts new file mode 100644 index 0000000000..92a8508e65 --- /dev/null +++ b/typegate/tests/runtimes/deno/deno_globs.ts @@ -0,0 +1,25 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { Policy, t, typegraph } from "@typegraph/sdk/index.js"; +import { DenoRuntime } from "@typegraph/sdk/runtimes/deno.js"; + +export const tg = await typegraph( + { + name: "deno_globs", + }, + (g) => { + const deno = new DenoRuntime(); + const pub = Policy.public(); + + g.expose({ + testGlob: deno + .import(t.struct({ a: t.float(), b: t.float() }), t.float(), { + module: "ts/dep/main.ts", + name: "doAddition", + deps: ["ts/dep/**/*.ts"], + }) + .withPolicy(pub), + }); + }, +); diff --git a/typegate/tests/runtimes/deno/deno_sync_test.ts b/typegate/tests/runtimes/deno/deno_sync_test.ts index ae2515ff33..624cb339c1 100644 --- a/typegate/tests/runtimes/deno/deno_sync_test.ts +++ b/typegate/tests/runtimes/deno/deno_sync_test.ts @@ -538,3 +538,101 @@ Meta.test( }); }, ); + +Meta.test( + { + name: "DenoRuntime - Sync mode: support for dirs when adding deps", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t) => { + await t.should( + "work for deps specified with dir on Python SDK", + async () => { + const engine = await t.engine( + "runtimes/deno/deno_dir.py", + ); + + await gql` + query { + test_dir(a: 4, b: 3) + } + ` + .expectData({ + test_dir: 7, + }) + .on(engine); + }, + ); + + await t.should( + "work for deps specified with dir on TypeScript SDK", + async () => { + const engine = await t.engine("runtimes/deno/deno_dir.ts"); + + await gql` + query { + testDir(a: 20, b: 5) + } + ` + .expectData({ + testDir: 25, + }) + .on(engine); + }, + ); + }, +); + +Meta.test( + { + name: "DenoRuntime - Sync mode: support for globs when adding deps", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t) => { + await t.should( + "work for deps specified with glob on Python SDK", + async () => { + const engine = await t.engine("runtimes/deno/deno_globs.py"); + + await gql` + query { + test_glob(a: 10, b: 53) + } + ` + .expectData({ + test_glob: 63, + }) + .on(engine); + }, + ); + + await t.should( + "work for deps specified with glob on TypeScript SDK", + async () => { + const engine = await t.engine("runtimes/deno/deno_globs.ts"); + + await gql` + query { + testGlob(a: 10, b: 5) + } + ` + .expectData({ + testGlob: 15, + }) + .on(engine); + }, + ); + }, +); diff --git a/typegate/tests/runtimes/python/python_dir.py b/typegate/tests/runtimes/python/python_dir.py new file mode 100644 index 0000000000..bbb3e64f6f --- /dev/null +++ b/typegate/tests/runtimes/python/python_dir.py @@ -0,0 +1,21 @@ +from typegraph.graph.typegraph import Graph +from typegraph.policy import Policy +from typegraph.runtimes.python import PythonRuntime + +from typegraph import t, typegraph + + +@typegraph() +def python_dir(g: Graph): + public = Policy.public() + python = PythonRuntime() + + g.expose( + test_dir=python.import_( + t.struct({"name": t.string()}), + t.string(), + module="py/hello.py", + deps=["py/nested"], + name="sayHello", + ).with_policy(public), + ) diff --git a/typegate/tests/runtimes/python/python_dir.ts b/typegate/tests/runtimes/python/python_dir.ts new file mode 100644 index 0000000000..e3a80d3234 --- /dev/null +++ b/typegate/tests/runtimes/python/python_dir.ts @@ -0,0 +1,25 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { PythonRuntime } from "@typegraph/sdk/runtimes/python.js"; +import { Policy, t, typegraph } from "@typegraph/sdk/index.js"; + +const tpe = t.struct({ + a: t.string(), + b: t.list(t.either([t.integer(), t.string()])), +}); + +export const tg = await typegraph("python_dirs", (g: any) => { + const python = new PythonRuntime(); + const pub = Policy.public(); + + g.expose({ + testDir: python + .import(t.struct({ input: tpe }), tpe, { + name: "identity", + module: "py/hello.py", + deps: ["py"], + }) + .withPolicy(pub), + }); +}); diff --git a/typegate/tests/runtimes/python/python_dir_test.ts b/typegate/tests/runtimes/python/python_dir_test.ts new file mode 100644 index 0000000000..395c3a64e1 --- /dev/null +++ b/typegate/tests/runtimes/python/python_dir_test.ts @@ -0,0 +1,53 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { gql, Meta } from "../../utils/mod.ts"; + +Meta.test( + { + name: "PythonRuntime - Single Replica: support for dirs when adding deps", + }, + async (t) => { + await t.should( + "work for deps specified with dir on Python SDK", + async () => { + const engine = await t.engine( + "runtimes/python/python_dir.py", + ); + + await gql` + query { + test_dir(name: "Jurgen") + } + ` + .expectData({ + test_dir: "Hello Jurgen", + }) + .on(engine); + }, + ); + + await t.should( + "work for deps specified with dir on TypeScript SDK", + async () => { + const engine = await t.engine("runtimes/python/python_dir.ts"); + + await gql` + query { + testDir(input: { a: "hello", b: [1, 2, "three"] }) { + a + b + } + } + ` + .expectData({ + testDir: { + a: "hello", + b: [1, 2, "three"], + }, + }) + .on(engine); + }, + ); + }, +); diff --git a/typegate/tests/runtimes/python/python_glob_test.ts b/typegate/tests/runtimes/python/python_glob_test.ts new file mode 100644 index 0000000000..ebcf91f0dc --- /dev/null +++ b/typegate/tests/runtimes/python/python_glob_test.ts @@ -0,0 +1,53 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { gql, Meta } from "../../utils/mod.ts"; + +Meta.test( + { + name: "PythonRuntime - Single Replica: support for globs when adding deps", + }, + async (t) => { + await t.should( + "work for deps specified with glob on Python SDK", + async () => { + const engine = await t.engine( + "runtimes/python/python_globs.py", + ); + + await gql` + query { + test_glob(name: "Pep") + } + ` + .expectData({ + test_glob: "Hello Pep", + }) + .on(engine); + }, + ); + + await t.should( + "work for deps specified with glob on TypeScript SDK", + async () => { + const engine = await t.engine("runtimes/python/python_globs.ts"); + + await gql` + query { + testGlob(input: { a: "hello", b: [1, 2, "three"] }) { + a + b + } + } + ` + .expectData({ + testGlob: { + a: "hello", + b: [1, 2, "three"], + }, + }) + .on(engine); + }, + ); + }, +); diff --git a/typegate/tests/runtimes/python/python_globs.py b/typegate/tests/runtimes/python/python_globs.py new file mode 100644 index 0000000000..617d292546 --- /dev/null +++ b/typegate/tests/runtimes/python/python_globs.py @@ -0,0 +1,21 @@ +from typegraph.graph.typegraph import Graph +from typegraph.policy import Policy +from typegraph.runtimes.python import PythonRuntime + +from typegraph import t, typegraph + + +@typegraph() +def python_globs(g: Graph): + public = Policy.public() + python = PythonRuntime() + + g.expose( + test_glob=python.import_( + t.struct({"name": t.string()}), + t.string(), + module="py/hello.py", + deps=["py/nested/*.py"], + name="sayHello", + ).with_policy(public), + ) diff --git a/typegate/tests/runtimes/python/python_globs.ts b/typegate/tests/runtimes/python/python_globs.ts new file mode 100644 index 0000000000..d296f6abda --- /dev/null +++ b/typegate/tests/runtimes/python/python_globs.ts @@ -0,0 +1,25 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { PythonRuntime } from "@typegraph/sdk/runtimes/python.js"; +import { Policy, t, typegraph } from "@typegraph/sdk/index.js"; + +const tpe = t.struct({ + a: t.string(), + b: t.list(t.either([t.integer(), t.string()])), +}); + +export const tg = await typegraph("python_globs", (g: any) => { + const python = new PythonRuntime(); + const pub = Policy.public(); + + g.expose({ + testGlob: python + .import(t.struct({ input: tpe }), tpe, { + name: "identity", + module: "py/hello.py", + deps: ["py/**/*.py"], + }) + .withPolicy(pub), + }); +}); diff --git a/typegate/tests/runtimes/python/python_sync_test.ts b/typegate/tests/runtimes/python/python_sync_test.ts index 17c31a4b9c..a18c1700fb 100644 --- a/typegate/tests/runtimes/python/python_sync_test.ts +++ b/typegate/tests/runtimes/python/python_sync_test.ts @@ -549,3 +549,115 @@ Meta.test( ); }, ); + +Meta.test( + { + name: "PythonRuntime - Sync mode: support for dirs when adding deps", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t) => { + await t.should( + "work for deps specified with dir on Python SDK", + async () => { + const engine = await t.engine( + "runtimes/python/python_dir.py", + ); + + await gql` + query { + test_dir(name: "Jurgen") + } + ` + .expectData({ + test_dir: "Hello Jurgen", + }) + .on(engine); + }, + ); + + await t.should( + "work for deps specified with dir on TypeScript SDK", + async () => { + const engine = await t.engine("runtimes/python/python_dir.ts"); + + await gql` + query { + testDir(input: { a: "hello", b: [1, 2, "three"] }) { + a + b + } + } + ` + .expectData({ + testDir: { + a: "hello", + b: [1, 2, "three"], + }, + }) + .on(engine); + }, + ); + }, +); + +Meta.test( + { + name: "PythonRuntime - Sync mode: support for globs when adding deps", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t) => { + await t.should( + "work for deps specified with glob on Python SDK", + async () => { + const engine = await t.engine( + "runtimes/python/python_globs.py", + ); + + await gql` + query { + test_glob(name: "Pep") + } + ` + .expectData({ + test_glob: "Hello Pep", + }) + .on(engine); + }, + ); + + await t.should( + "work for deps specified with glob on TypeScript SDK", + async () => { + const engine = await t.engine("runtimes/python/python_globs.ts"); + + await gql` + query { + testGlob(input: { a: "hello", b: [1, 2, "three"] }) { + a + b + } + } + ` + .expectData({ + testGlob: { + a: "hello", + b: [1, 2, "three"], + }, + }) + .on(engine); + }, + ); + }, +); diff --git a/typegate/tests/utils/s3.ts b/typegate/tests/utils/s3.ts index d2492a9a4c..453ad4417c 100644 --- a/typegate/tests/utils/s3.ts +++ b/typegate/tests/utils/s3.ts @@ -9,6 +9,7 @@ import { ListObjectsV2Command, S3Client, } from "aws-sdk/client-s3"; +import { connect } from "redis"; export async function tryDeleteBucket(client: S3Client, bucket: string) { while (true) { @@ -63,3 +64,40 @@ export async function hasObject(client: S3Client, bucket: string, key: string) { throw e; } } + +export function generateSyncConfig(bucketName: string) { + const syncConfig = { + redis: { + hostname: "localhost", + port: 6379, + password: "password", + db: 1, + }, + s3: { + endpoint: "http://localhost:9000", + region: "local", + credentials: { + accessKeyId: "minio", + secretAccessKey: "password", + }, + forcePathStyle: true, + }, + s3Bucket: bucketName, + }; + + const redisKey = "typegraph"; + const redisEventKey = "typegraph_event"; + const cleanUp = async () => { + using redis = await connect(syncConfig.redis); + await redis.del(redisKey); + await redis.del(redisEventKey); + + const s3 = new S3Client(syncConfig.s3); + await tryDeleteBucket(s3, syncConfig.s3Bucket); + await createBucket(s3, syncConfig.s3Bucket); + s3.destroy(); + await redis.quit(); + }; + + return { syncConfig, cleanUp }; +} diff --git a/typegraph/core/Cargo.toml b/typegraph/core/Cargo.toml index cc99e4366c..bd1d2489b4 100644 --- a/typegraph/core/Cargo.toml +++ b/typegraph/core/Cargo.toml @@ -24,6 +24,7 @@ sha2 = "0.10.8" paste = "1.0.14" seahash = "4.1.0" ordered-float = "4.2.0" +glob = "0.3.1" [dev-dependencies] insta = { version = "1.34.0", features = ["glob"] } diff --git a/typegraph/core/src/utils/fs_host.rs b/typegraph/core/src/utils/fs_host.rs index 7a13eda4ff..49623fb94b 100644 --- a/typegraph/core/src/utils/fs_host.rs +++ b/typegraph/core/src/utils/fs_host.rs @@ -1,13 +1,18 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 -use std::path::{Path, PathBuf}; +use std::{ + collections::HashSet, + path::{Path, PathBuf}, +}; + +use glob::Pattern as GlobPattern; use crate::{ global_store::Store, wit::metatype::typegraph::host::{ - expand_path as expand_path_host, get_cwd, path_exists as path_exists_host, read_file, - write_file, + eprint, expand_path as expand_path_host, get_cwd, path_exists as path_exists_host, + read_file, write_file, }, }; use common::archive::{ @@ -110,7 +115,6 @@ pub fn expand_path(path: &Path, exclude_glob: &[String]) -> Result, .iter() .map(PathBuf::from) .collect(); - Ok(ret) } @@ -213,3 +217,80 @@ pub fn hash_file(path: &Path) -> Result<(String, u32), String> { pub fn path_exists(path: &Path) -> Result { path_exists_host(&path.to_string_lossy()) } + +pub fn is_glob(path: &str) -> bool { + // dir can also contain wild cards, + path.contains('*') || path.contains('?') +} + +pub fn extract_glob_dirname(path: &str) -> PathBuf { + let path = PathBuf::from(path); + let dirs: Vec<_> = path.components().map(|comp| comp.as_os_str()).collect(); + let mut parent_dir = PathBuf::new(); + let special_chars = &['*', '?', '[', ']']; + + for dir in dirs { + let dir = dir.to_str().unwrap(); + if dir.find(special_chars).is_some() { + break; + } + parent_dir = parent_dir.join(dir); + } + + parent_dir +} + +pub fn expand_glob(path: &str) -> Result, String> { + let abs_path = make_absolute(&PathBuf::from(path))? + .to_string_lossy() + .to_string(); + + let parent_dir = extract_glob_dirname(&abs_path); + let all_files = expand_path(&parent_dir, &[])?; + + let glob_pattern = GlobPattern::new(&abs_path).unwrap(); + + let mut matching_files = vec![]; + for file in all_files { + if glob_pattern.matches(file.to_str().unwrap()) { + matching_files.push(file); + } + } + + Ok(matching_files) +} + +pub fn resolve_globs_dirs(deps: Vec) -> Result, String> { + let mut resolved_deps = HashSet::new(); + for dep in deps { + if is_glob(&dep) { + let abs_path = make_absolute(&PathBuf::from(dep))? + .to_string_lossy() + .to_string(); + + let matching_files = expand_glob(&abs_path).map_err(|err| { + eprint(&format!("Error resolving globs: {:?}", err)); + err + })?; + for file in matching_files { + let rel_path = make_relative(&file)?; + resolved_deps.insert(rel_path); + } + } else { + let all_files = + expand_path(&make_absolute(&PathBuf::from(dep))?, &[]).map_err(|err| { + eprint(&format!( + "Error resolving dependencies and dependency directories: {:?}", + err + )); + err + })?; + for file in all_files { + let rel_path = make_relative(&file)?; + resolved_deps.insert(rel_path); + } + } + } + + Ok(resolved_deps.into_iter().collect()) +} diff --git a/typegraph/core/src/utils/postprocess/deno_rt.rs b/typegraph/core/src/utils/postprocess/deno_rt.rs index 3af51409be..73dda010bd 100644 --- a/typegraph/core/src/utils/postprocess/deno_rt.rs +++ b/typegraph/core/src/utils/postprocess/deno_rt.rs @@ -1,7 +1,10 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 -use crate::{global_store::Store, utils::fs_host}; +use crate::{ + global_store::Store, + utils::fs_host::{self, resolve_globs_dirs}, +}; use common::typegraph::{ runtimes::{deno::ModuleMatData, Artifact}, utils::{map_from_object, object_from_map}, @@ -65,7 +68,6 @@ impl DenoProcessor { let mut tg_deps_paths = vec![]; let mut tg_artifacts = vec![]; - match fs_host::path_exists(&main_path)? { true => { let (module_hash, size) = fs_host::hash_file(&main_path.clone())?; @@ -78,8 +80,11 @@ impl DenoProcessor { tg_deps_paths.push(main_path); let deps = mat_data.deps.clone(); - for dep in deps { - let dep_rel_path = PathBuf::from(dep); + + // resolve globs and dirs + let resolved_deps = resolve_globs_dirs(deps)?; + + for dep_rel_path in resolved_deps { let dep_abs_path = fs_host::make_absolute(&dep_rel_path)?; let (dep_hash, dep_size) = fs_host::hash_file(&dep_abs_path)?; diff --git a/typegraph/core/src/utils/postprocess/python_rt.rs b/typegraph/core/src/utils/postprocess/python_rt.rs index e3dabfc76b..e02d9ab7a8 100644 --- a/typegraph/core/src/utils/postprocess/python_rt.rs +++ b/typegraph/core/src/utils/postprocess/python_rt.rs @@ -1,7 +1,7 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 -use crate::utils::fs_host; +use crate::utils::fs_host::{self, resolve_globs_dirs}; use common::typegraph::{ runtimes::{python::ModuleMatData, Artifact}, utils::{map_from_object, object_from_map}, @@ -39,8 +39,9 @@ impl PostProcessor for PythonProcessor { let deps = mat_data.deps.clone(); let mut dep_artifacts = vec![]; - for dep in deps { - let dep_rel_path = PathBuf::from(dep); + let resolved_deps = resolve_globs_dirs(deps)?; + + for dep_rel_path in resolved_deps { let dep_abs_path = fs_host::make_absolute(&dep_rel_path)?; if let Entry::Vacant(entry) = tg.meta.artifacts.entry(dep_rel_path.clone()) { diff --git a/typegraph/python/typegraph/host/host.py b/typegraph/python/typegraph/host/host.py index 415640fba3..145a500360 100644 --- a/typegraph/python/typegraph/host/host.py +++ b/typegraph/python/typegraph/host/host.py @@ -1,12 +1,13 @@ # Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. # SPDX-License-Identifier: MPL-2.0 -from typing import List -from typegraph.gen import imports -from typegraph.gen.types import Err, Ok, Result import os import re import sys +from typing import List + +from typegraph.gen import imports +from typegraph.gen.types import Err, Ok, Result def has_match(text: str, items: List[str]) -> bool: @@ -27,11 +28,14 @@ def eprint(self, msg: str): def expand_path(self, root: str, exclude: List[str]) -> Result[List[str], str]: try: result = [] + if os.path.isfile(root): + result.append(root) for path, _, files in os.walk(root): for name in files: file_path = os.path.join(path, name) if not has_match(file_path, exclude): result.append(file_path) + return Ok(result) except Exception as e: return Err(str(e))