diff --git a/docker-compose.yaml b/docker-compose.yaml index 7c3efe79..8f7c9fad 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -55,6 +55,21 @@ services: start_period: 5s networks: - datalayer + datalayer-graphql-config: + build: + context: . + dockerfile: scripts/hasura/Dockerfile + env_file: + - .env + environment: + HASURA_ENDPOINT: http://datalayer-graphql-api:8080 + HASURA_ADMIN_SECRET: ${DATALAYER_HASURA_ADMIN_SECRET:-secret} + HASURA_SCHEMA: public + depends_on: + datalayer-graphql-api: + condition: service_healthy + networks: + - datalayer indexer-postgres-db: image: postgres:16 restart: always diff --git a/package.json b/package.json index 68e8a44d..c8883059 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "author": "Wonderland", "type": "module", "scripts": { + "api:configure": "pnpm run --filter @grants-stack-indexer/hasura-config-scripts api:configure", "build": "turbo run build", "check-types": "turbo run check-types", "clean": "turbo run clean", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5b7d7ed1..447e0cef 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -274,6 +274,22 @@ importers: specifier: 3.15.0 version: 3.15.0 + scripts/hasura: + dependencies: + axios: + specifier: 1.7.7 + version: 1.7.7 + dotenv: + specifier: 16.4.5 + version: 16.4.5 + zod: + specifier: 3.23.8 + version: 3.23.8 + devDependencies: + tsx: + specifier: 4.19.2 + version: 4.19.2 + scripts/migrations: dependencies: "@grants-stack-indexer/repository": diff --git a/scripts/hasura/Dockerfile b/scripts/hasura/Dockerfile new file mode 100644 index 00000000..4b9785d6 --- /dev/null +++ b/scripts/hasura/Dockerfile @@ -0,0 +1,24 @@ +FROM node:20-slim +ENV PNPM_HOME="/pnpm" +ENV PATH="$PNPM_HOME:$PATH" +RUN corepack enable + +WORKDIR /app + +# Copy root workspace files first +COPY package.json pnpm-workspace.yaml pnpm-lock.yaml tsconfig.base.json tsconfig.build.json tsconfig.json ./ + +# Copy the package's files +COPY scripts/hasura scripts/hasura/ + +# Install dependencies +RUN pnpm install --frozen-lockfile + +# Set working directory to the hasura package +WORKDIR /app/scripts/hasura + +# Build the project +RUN pnpm build + +# Run the configure script +CMD ["pnpm", "api:configure"] \ No newline at end of file diff --git a/scripts/hasura/README.md b/scripts/hasura/README.md new file mode 100644 index 00000000..028ea0bb --- /dev/null +++ b/scripts/hasura/README.md @@ -0,0 +1,121 @@ +# Hasura Configuration Scripts + +This directory contains scripts to configure Hasura metadata, including: + +- Tracking tables +- Setting up relationships between tables +- Configuring public permissions +- Tracking custom functions + +## Setup + +1. Install dependencies: + +```bash +pnpm install +``` + +2. Configure environment variables in the root `.env` file: + +```env +HASURA_ENDPOINT=http://localhost:8082 # Your Hasura endpoint +HASURA_ADMIN_SECRET=secret # Your Hasura admin secret +HASURA_SCHEMA=public # Your database schema +``` + +## Usage + +Run the configuration script: + +```bash +pnpm api:configure +``` + +This will: + +1. Clear existing metadata +2. Track all tables in the database +3. Create relationships between tables +4. Track custom functions +5. Set up public permissions for SELECT operations with a limit of 50 rows + +## Tables Configured + +The script will configure the following tables: + +- projects +- pending_project_roles +- project_roles +- rounds +- pending_round_roles +- round_roles +- applications +- applications_payouts +- donations +- legacy_projects + +## Relationships + +The script sets up the following relationships: + +### Array Relationships (One-to-Many) + +#### Projects + +- Has many applications +- Has many projectRoles +- Has many rounds + +#### Rounds + +- Has many applications +- Has many roundRoles + +#### Applications + +- Has many applicationsPayouts + +### Object Relationships (Many-to-One) + +#### Project Roles + +- Belongs to project + +#### Rounds + +- Belongs to project + +#### Round Roles + +- Belongs to round + +#### Applications + +- Belongs to project +- Belongs to round + +#### Applications Payouts + +- Belongs to application + +## Custom Functions + +The script tracks the following custom functions: + +- search_projects + +## Development + +```bash +# Run type checking +pnpm check-types + +# Run linting +pnpm lint + +# Run tests +pnpm test + +# Format code +pnpm format:fix +``` diff --git a/scripts/hasura/package.json b/scripts/hasura/package.json new file mode 100644 index 00000000..be240d49 --- /dev/null +++ b/scripts/hasura/package.json @@ -0,0 +1,34 @@ +{ + "name": "@grants-stack-indexer/hasura-config-scripts", + "version": "1.0.0", + "description": "Scripts to configure Hasura metadata", + "license": "MIT", + "author": "Wonderland", + "type": "module", + "directories": { + "src": "src" + }, + "files": [ + "package.json" + ], + "scripts": { + "api:configure": "tsx src/configure-hasura.script.ts", + "build": "tsc -p tsconfig.build.json", + "check-types": "tsc --noEmit -p ./tsconfig.json", + "clean": "rm -rf dist/", + "format": "prettier --check \"{src,test}/**/*.{js,ts,json}\"", + "format:fix": "prettier --write \"{src,test}/**/*.{js,ts,json}\"", + "lint": "eslint \"{src,test}/**/*.{js,ts,json}\"", + "lint:fix": "pnpm lint --fix", + "test": "vitest run --config vitest.config.ts --passWithNoTests", + "test:cov": "vitest run --config vitest.config.ts --coverage --passWithNoTests" + }, + "dependencies": { + "axios": "1.7.7", + "dotenv": "16.4.5", + "zod": "3.23.8" + }, + "devDependencies": { + "tsx": "4.19.2" + } +} diff --git a/scripts/hasura/src/configure-hasura.script.ts b/scripts/hasura/src/configure-hasura.script.ts new file mode 100644 index 00000000..daa2f0e2 --- /dev/null +++ b/scripts/hasura/src/configure-hasura.script.ts @@ -0,0 +1,71 @@ +import { configDotenv } from "dotenv"; +import { z } from "zod"; + +import { CustomFunction, HasuraMetadataApi } from "./internal.js"; + +configDotenv(); + +const DEFAULT_PUBLIC_FETCH_LIMIT = 50; + +const envSchema = z.object({ + HASURA_ENDPOINT: z.string().url(), + HASURA_ADMIN_SECRET: z.string().min(1), + HASURA_SCHEMA: z.string().min(1).default("public"), + HASURA_PUBLIC_FETCH_LIMIT: z.coerce.number().int().min(1).default(DEFAULT_PUBLIC_FETCH_LIMIT), +}); + +// Tables to track +const tables = [ + "projects", + "pending_project_roles", + "project_roles", + "rounds", + "pending_round_roles", + "round_roles", + "applications", + "applications_payouts", + "donations", + "legacy_projects", +] as const; + +type Tables = typeof tables; + +// Custom functions to track +const customFunctions: CustomFunction[] = [ + { + name: "search_projects", + schema: "public", + }, +]; + +async function configureHasura(): Promise { + // Parse and validate environment variables + const env = envSchema.parse(process.env); + + const hasuraApi = new HasuraMetadataApi({ + endpoint: env.HASURA_ENDPOINT, + adminSecret: env.HASURA_ADMIN_SECRET, + schema: env.HASURA_SCHEMA, + fetchLimit: env.HASURA_PUBLIC_FETCH_LIMIT, + }); + + await hasuraApi.clearMetadata(); + + for (const table of tables) { + await hasuraApi.trackTable(table); + await hasuraApi.setSelectPermission(table); + } + + await hasuraApi.createSuggestedRelationships(Array.from(tables)); + + for (const func of customFunctions) { + await hasuraApi.trackFunction(func); + } + + console.log("✅ Hasura configured"); +} + +configureHasura().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/scripts/hasura/src/exceptions/hasuraApi.exception.ts b/scripts/hasura/src/exceptions/hasuraApi.exception.ts new file mode 100644 index 00000000..94962782 --- /dev/null +++ b/scripts/hasura/src/exceptions/hasuraApi.exception.ts @@ -0,0 +1,6 @@ +export class HasuraApiException extends Error { + constructor(message: string) { + super(message); + this.name = "HasuraApiException"; + } +} diff --git a/scripts/hasura/src/exceptions/index.ts b/scripts/hasura/src/exceptions/index.ts new file mode 100644 index 00000000..c6817d00 --- /dev/null +++ b/scripts/hasura/src/exceptions/index.ts @@ -0,0 +1,2 @@ +export * from "./hasuraApi.exception.js"; +export * from "./network.exception.js"; diff --git a/scripts/hasura/src/exceptions/network.exception.ts b/scripts/hasura/src/exceptions/network.exception.ts new file mode 100644 index 00000000..69c02650 --- /dev/null +++ b/scripts/hasura/src/exceptions/network.exception.ts @@ -0,0 +1,6 @@ +export class NetworkException extends Error { + constructor(message: string) { + super(message); + this.name = "NetworkException"; + } +} diff --git a/scripts/hasura/src/internal.ts b/scripts/hasura/src/internal.ts new file mode 100644 index 00000000..3259c7e4 --- /dev/null +++ b/scripts/hasura/src/internal.ts @@ -0,0 +1,3 @@ +export * from "./types/index.js"; +export * from "./exceptions/index.js"; +export * from "./services/index.js"; diff --git a/scripts/hasura/src/services/hasura.api.ts b/scripts/hasura/src/services/hasura.api.ts new file mode 100644 index 00000000..db78ee82 --- /dev/null +++ b/scripts/hasura/src/services/hasura.api.ts @@ -0,0 +1,304 @@ +import assert from "assert"; +import axios, { AxiosError, AxiosInstance, isAxiosError } from "axios"; + +import type { CustomFunction, HasuraConfig, RelationshipConfig } from "../internal.js"; +import { HasuraApiException, NetworkException } from "../internal.js"; + +type SuggestedRelationship = { + type: "array" | "object"; + from: { + table: { + name: string; + schema: string; + }; + columns: string[]; + constraint_name?: string; + }; + to: { + table: { + name: string; + schema: string; + }; + columns: string[]; + constraint_name?: string; + }; +}; + +/** + * A class to interact with the Hasura Metadata API for managing database configurations, + * including tracking tables, setting up relationships, and configuring permissions. + * Provides methods to programmatically update Hasura's metadata through its HTTP API. + * + * refer to: https://hasura.io/docs/2.0/api-reference/metadata-api/index/ + * + * @template Tables - An array of table names. + */ +export class HasuraMetadataApi { + private adminSecret: string; + private endpoint: string; + private schema: string; + private fetchLimit: number; + private axiosInstance: AxiosInstance; + + constructor(config: HasuraConfig) { + this.adminSecret = config.adminSecret; + this.endpoint = config.endpoint; + this.schema = config.schema; + this.fetchLimit = config.fetchLimit; + this.axiosInstance = axios.create({ + baseURL: this.endpoint, + headers: { + "Content-Type": "application/json", + "X-Hasura-Admin-Secret": this.adminSecret, + }, + }); + } + + /** + * Clears the metadata in Hasura. + * + * @returns {Promise} + * @throws {HasuraApiException} If the metadata clearing fails. + * @throws {NetworkException} If there is a network error. + */ + async clearMetadata(): Promise { + try { + await this.axiosInstance.post("/v1/metadata", { + type: "clear_metadata", + args: {}, + }); + console.log("✅ Metadata cleared"); + } catch (err) { + this.handleError(err, "clear metadata"); + } + } + + /** + * Tracks a table in Hasura. + * + * @param {Tables[number]} tableName - The name of the table to track. + * @returns {Promise} + * @throws {HasuraApiException} If the table tracking fails. + * @throws {NetworkException} If there is a network error. + */ + async trackTable(tableName: Tables[number]): Promise { + try { + await this.axiosInstance.post("/v1/metadata", { + type: "pg_track_table", + args: { + source: "default", + table: { + name: tableName, + schema: this.schema, + }, + }, + }); + console.log(`✅ Tracked table: ${tableName}`); + } catch (err) { + this.handleError(err, `track ${tableName} table`); + } + } + + /** + * Retrieves suggested relationships to create between tables in Hasura. + * + * @param {Tables[number][]} tables - The tables to suggest relationships for. + * @returns {Promise} + * @throws {HasuraApiException} If the relationship suggestion fails. + * @throws {NetworkException} If there is a network error. + */ + async suggestRelationships(tables: Tables[number][]): Promise { + try { + const { data } = await this.axiosInstance.post<{ + relationships: SuggestedRelationship[]; + }>("/v1/metadata", { + type: "pg_suggest_relationships", + args: { + omit_tracked: true, + source: "default", + tables: tables, + }, + }); + + return data.relationships; + } catch (err) { + this.handleError(err, `suggest relationships for ${tables.join(", ")}`); + } + } + + /** + * Creates suggested relationships between tables in Hasura. + * + * @param {Tables[number][]} tables - The tables to create relationships for. + * @returns {Promise} + * @throws {HasuraApiException} If the relationship creation fails. + * @throws {NetworkException} If there is a network error. + */ + async createSuggestedRelationships(tables: Tables[number][]): Promise { + const relationships = await this.suggestRelationships(tables); + console.log(`Fetched ${relationships.length} relationships`); + for (const relationship of relationships) { + assert( + relationship.from.columns.length === relationship.to.columns.length, + "Number of columns in from and to tables must be the same", + ); + const payload = { + name: relationship.to.table.name, + table: relationship.from.table, + source: "default", + using: { + manual_configuration: { + remote_table: relationship.to.table, + source: "default", + column_mapping: Object.fromEntries( + relationship.from.columns.map((col, i) => [ + col, + relationship.to.columns[i]!, + ]), + ), + }, + }, + }; + + if (relationship.type === "array") { + await this.createArrayRelationship(payload); + } else { + await this.createObjectRelationship(payload); + } + } + } + + /** + * Creates an array relationship in Hasura. + * + * @param {RelationshipConfig} relationship - The relationship configuration. + * @returns {Promise} + * @throws {HasuraApiException} If the array relationship creation fails. + * @throws {NetworkException} If there is a network error. + */ + async createArrayRelationship(relationship: RelationshipConfig): Promise { + try { + await this.axiosInstance.post("/v1/metadata", { + type: "pg_create_array_relationship", + args: relationship, + }); + console.log( + `✅ Created array relationship: ${relationship.name} for ${relationship.table.name}`, + ); + } catch (err) { + this.handleError(err, `create array relationship ${relationship.name}`); + } + } + + /** + * Creates an object relationship in Hasura. + * + * @param {RelationshipConfig} relationship - The relationship configuration. + * @returns {Promise} + * @throws {HasuraApiException} If the object relationship creation fails. + * @throws {NetworkException} If there is a network error. + */ + async createObjectRelationship(relationship: RelationshipConfig): Promise { + try { + await this.axiosInstance.post("/v1/metadata", { + type: "pg_create_object_relationship", + args: relationship, + }); + console.log( + `✅ Created object relationship: ${relationship.name} for ${relationship.table.name}`, + ); + } catch (err) { + this.handleError(err, `create object relationship ${relationship.name}`); + } + } + + /** + * Sets a select permission for public role for a table in Hasura. + * + * @param {Tables[number]} tableName - The name of the table to set the permission for. + * @returns {Promise} + * @throws {HasuraApiException} If the select permission creation fails. + * @throws {NetworkException} If there is a network error. + */ + async setSelectPermission(tableName: Tables[number]): Promise { + try { + await this.axiosInstance.post("/v1/metadata", { + type: "pg_create_select_permission", + args: { + table: { + name: tableName, + schema: this.schema, + }, + source: "default", + role: "public", + permission: { + limit: this.fetchLimit, + allow_aggregations: true, + columns: "*", + filter: {}, + }, + }, + }); + console.log(`✅ Set select permission for ${tableName}`); + } catch (err) { + this.handleError(err, `set select permission for ${tableName}`); + } + } + + /** + * Tracks a custom function in Hasura. + * + * @param {CustomFunction} func - The function configuration. + * @returns {Promise} + * @throws {HasuraApiException} If the custom function tracking fails. + * @throws {NetworkException} If there is a network error. + */ + async trackFunction(func: CustomFunction): Promise { + try { + await this.axiosInstance.post("/v1/metadata", { + type: "pg_track_function", + args: { + function: { + name: func.name, + schema: func.schema, + }, + source: "default", + }, + }); + console.log(`✅ Tracked function: ${func.name}`); + } catch (err) { + this.handleError(err, `track custom function ${func.name}`); + } + } + + /** + * Handles errors from the Hasura API. + * + * @param {unknown} err - The error to handle. + * @param {string} operation - The operation that failed. + * @returns {never} + */ + private handleError(err: unknown, operation: string): never { + const error = err as Error | AxiosError; + let errorMessage = `❌ Failed to ${operation}`; + + if (isAxiosError<{ error: string; code: string }>(error)) { + if (!error.response) { + // Network error + throw new NetworkException(`Network error while ${operation}: ${error.message}`); + } + if (error.response.status >= 500) { + // Server error + throw new HasuraApiException( + `Server error while ${operation}: ${error.response.data || error.message}`, + ); + } + // Other API errors + errorMessage += `: ${error.response.data.error}`; + } else { + errorMessage += `: ${error.message}`; + } + + throw new HasuraApiException(errorMessage); + } +} diff --git a/scripts/hasura/src/services/index.ts b/scripts/hasura/src/services/index.ts new file mode 100644 index 00000000..f094c37c --- /dev/null +++ b/scripts/hasura/src/services/index.ts @@ -0,0 +1 @@ +export * from "./hasura.api.js"; diff --git a/scripts/hasura/src/types/hasura.types.ts b/scripts/hasura/src/types/hasura.types.ts new file mode 100644 index 00000000..4f887b4d --- /dev/null +++ b/scripts/hasura/src/types/hasura.types.ts @@ -0,0 +1,32 @@ +export type HasuraConfig = { + endpoint: string; + adminSecret: string; + schema: string; + fetchLimit: number; +}; + +export type ManualConfiguration = { + remote_table: { + name: Tables[number]; + schema: string; + }; + source: string; + column_mapping: Record; +}; + +export type RelationshipConfig = { + name: string; + table: { + name: Tables[number]; + schema: string; + }; + using: { + manual_configuration: ManualConfiguration; + }; + source: string; +}; + +export type CustomFunction = { + name: string; + schema: string; +}; diff --git a/scripts/hasura/src/types/index.ts b/scripts/hasura/src/types/index.ts new file mode 100644 index 00000000..3a15ee16 --- /dev/null +++ b/scripts/hasura/src/types/index.ts @@ -0,0 +1 @@ +export * from "./hasura.types.js"; diff --git a/scripts/hasura/tsconfig.build.json b/scripts/hasura/tsconfig.build.json new file mode 100644 index 00000000..32768e3e --- /dev/null +++ b/scripts/hasura/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.build.json", + "compilerOptions": { + "composite": true, + "declarationMap": true, + "declaration": true, + "outDir": "dist" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "test"] +} diff --git a/scripts/hasura/tsconfig.json b/scripts/hasura/tsconfig.json new file mode 100644 index 00000000..21c1c5be --- /dev/null +++ b/scripts/hasura/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*", "test/**/*"] +} diff --git a/scripts/hasura/vitest.config.ts b/scripts/hasura/vitest.config.ts new file mode 100644 index 00000000..fcafa05d --- /dev/null +++ b/scripts/hasura/vitest.config.ts @@ -0,0 +1,22 @@ +import path from "path"; +import { configDefaults, defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + globals: true, // Use Vitest's global API without importing it in each file + environment: "node", // Use the Node.js environment + include: ["test/**/*.spec.ts"], // Include test files + exclude: ["node_modules", "dist"], // Exclude certain directories + coverage: { + provider: "v8", + reporter: ["text", "json", "html"], // Coverage reporters + exclude: ["node_modules", "dist", ...configDefaults.exclude], // Files to exclude from coverage + }, + }, + resolve: { + alias: { + // Setup path alias based on tsconfig paths + "@": path.resolve(__dirname, "src"), + }, + }, +});