Split BigQuery export to Cloud Run Job (#37)

* split to cloud run job

* some docs

* lint

* linter off

* path moved

* clean before build

* cleanup

* no wrapper subscription

* local test

* cleanup

* wrapped subscription events

Author: max-ostapenko

.github/workflows/linter.yaml

@@ -33,3 +33,4 @@ jobs:
           VALIDATE_JSCPD: false
           VALIDATE_JAVASCRIPT_PRETTIER: false
           VALIDATE_MARKDOWN_PRETTIER: false
+          VALIDATE_CHECKOV: false

Makefile

@@ -1,5 +1,11 @@
 .PHONY: *
 
+clean:
+	rm -rf ./infra/dataform-trigger/node_modules
+	rm -rf ./infra/dataform-export/node_modules
+	rm -rf ./infra/bigquery-export/node_modules
+	rm -rf infra/tf/tmp/*
+
 tf_plan:
 	terraform -chdir=infra/tf init -upgrade && terraform -chdir=infra/tf plan
 

infra/README.md

@@ -1,12 +1,10 @@
 # Infrastructure for the HTTP Archive data pipeline
 
-## Cloud function for triggering Dataform workflows
+## Cloud Function for triggering Dataform workflows
 
 [dataformTrigger](https://console.cloud.google.com/functions/details/us-central1/dataformTrigger?env=gen2&authuser=7&project=httparchive) Cloud Run Function
 
-This function may be triggered by a PubSub message or Cloud Scheduler and triggers a Dataform workflow based on the trigger configuration provided.
-
-### Trigger configuration
+This function may be triggered by a PubSub message or Cloud Scheduler and invokes a Dataform workflow based on the provided configuration.
 
 Trigger types:
 
@@ -16,7 +14,7 @@ Trigger types:
 
 See [available trigger configurations](https://github.com/HTTPArchive/dataform/blob/main/src/index.js#L4).
 
-Request body example with trigger name:
+Request body example:
 
 ```json
 {
@@ -26,7 +24,7 @@ Request body example with trigger name:
 }
 ```
 
-Trigger for local development:
+Request example for local development:
 
 ```bash
 curl -X POST http://localhost:8080/ \
@@ -38,25 +36,23 @@ curl -X POST http://localhost:8080/ \
   }'
 ```
 
-## Cloud Function for report data exports
+## Cloud Function for triggering data exports
 
 [exportReport](https://console.cloud.google.com/functions/details/us-central1/bqExport?env=gen2&authuser=7&project=httparchive) Cloud Run Function
 
-This function exports reports data to GCS or Firestore.
+This function triggers a job to export data to GCS or Firestore.
 
-### Export configuration
+Request body example:
 
 ```json
 {
-  "message": {
-    "protoPayload": {
-      "serviceData": {
-        "jobCompletedEvent": {
-          "job": {
-            "jobConfiguration": {
-              "query": {
-                "query": "/* {\"dataform_trigger\": \"report_cwv_tech_complete\", \"date\": \"2024-11-01\", \"name\": \"technologies\", \"type\": \"dict\"} *\/"
-              }
+  "protoPayload": {
+    "serviceData": {
+      "jobCompletedEvent": {
+        "job": {
+          "jobConfiguration": {
+            "query": {
+              "query": "/* {\"dataform_trigger\": \"report_cwv_tech_complete\", \"date\": \"2024-11-01\", \"name\": \"technologies\", \"type\": \"dict\"} *\/"
             }
           }
         }
@@ -66,21 +62,19 @@ This function exports reports data to GCS or Firestore.
 }
 ```
 
-Trigger for local development:
+Request example for local development:
 
 ```bash
 curl -X POST http://localhost:8080/ \
   -H "Content-Type: application/json" \
   -d '{
-  "message": {
-    "protoPayload": {
-      "serviceData": {
-        "jobCompletedEvent": {
-          "job": {
-            "jobConfiguration": {
-              "query": {
-                "query": "/* {\"dataform_trigger\": \"report_complete\", \"date\": \"2024-11-01\", \"name\": \"bytesTotal\", \"type\": \"timeseries\"} *\/"
-              }
+  "protoPayload": {
+    "serviceData": {
+      "jobCompletedEvent": {
+        "job": {
+          "jobConfiguration": {
+            "query": {
+              "query": "/* {\"dataform_trigger\": \"report_complete\", \"date\": \"2024-11-01\", \"name\": \"bytesTotal\", \"type\": \"timeseries\"} *\/"
             }
           }
         }
@@ -96,15 +90,13 @@ or
 curl -X POST http://localhost:8080/ \
   -H "Content-Type: application/json" \
   -d '{
-  "message": {
-    "protoPayload": {
-      "serviceData": {
-        "jobCompletedEvent": {
-          "job": {
-            "jobConfiguration": {
-              "query": {
-                "query": "/* {\"dataform_trigger\": \"report_cwv_tech_complete\", \"date\": \"2024-11-01\", \"name\": \"adoption\", \"type\": \"report\"} *\/"
-              }
+  "protoPayload": {
+    "serviceData": {
+      "jobCompletedEvent": {
+        "job": {
+          "jobConfiguration": {
+            "query": {
+              "query": "/* {\"dataform_trigger\": \"report_cwv_tech_complete\", \"date\": \"2024-11-01\", \"name\": \"lighthouse\", \"type\": \"report\"} *\/"
             }
           }
         }
@@ -114,6 +106,25 @@ curl -X POST http://localhost:8080/ \
 }'
 ```
 
+## Cloud Run Job for exporting data
+
+[exportData](https://console.cloud.google.com/run/detail/us-central1/export-data?authuser=7&project=httparchive) Cloud Run Job
+
+This job exports data to GCS or Firestore based on the provided configuration.
+
+Input parameters:
+
+- `EXPORT_CONFIG` - JSON string with the export configuration.
+
+Example values:
+
+```plaintext
+{"dataform_trigger":"report_cwv_tech_complete","name":"technologies","type":"dict"}
+{"dataform_trigger":"report_cwv_tech_complete","date":"2024-11-01","name":"page_weight","type":"report"}
+{"dataform_trigger":"report_complete","name":"bytesTotal","type":"timeseries"}
+{"dataform_trigger":"report_complete","date":"2024-11-01","name":"bytesTotal","type":"histogram"}
+```
+
 ## Monitoring
 
 The issues within the pipeline are being tracked using the following alerts:
@@ -134,6 +145,15 @@ npm run start
 
 Then, in a separate terminal, run the command with the test trigger payload.
 
+## Build
+
+Building a container image for the `bigquery-export` Cloud Run Job:
+
+```bash
+cd infra/bigquery-export
+npm run buildpack
+```
+
 ## Deployment
 
 From project root directory run:

infra/bigquery-export/Dockerfile

@@ -0,0 +1,13 @@
+# checkov:skip=CKV_DOCKER_3:Ensure that a user for the container has been created
+FROM node:20-slim
+
+WORKDIR /usr/src/app
+
+COPY . .
+
+# Clean up the node_modules directory
+RUN rm -rf node_modules
+
+RUN npm ci --only=production
+
+CMD ["node", "index.js"]

infra/dataform-export/bigquery.js infra/bigquery-export/bigquery.js

@@ -15,7 +15,9 @@ export class FirestoreBatch {
   constructor () {
     this.firestore = new Firestore()
     this.bigquery = new BigQueryExport()
-    this.firestore.settings({ databaseId: 'tech-report-apis-prod' })
+    this.firestore.settings({
+      databaseId: 'tech-report-apis-prod'
+    })
     this.batchSize = 500
     this.maxConcurrentBatches = 200
   }

infra/dataform-export/firestore.js infra/bigquery-export/firestore.js

@@ -0,0 +1,34 @@
+import { ReportsExporter, TechReportsExporter } from './reports.js'
+
+const exportConfig = process.env.EXPORT_CONFIG && JSON.parse(process.env.EXPORT_CONFIG)
+
+async function main (exportConfig) {
+  if (!exportConfig) {
+    throw new Error('No config received')
+  }
+
+  const eventName = exportConfig.dataform_trigger
+  if (!eventName) {
+    throw new Error('No trigger name found')
+  }
+
+  if (eventName === 'report_complete') {
+    console.info('Report export')
+    console.log(exportConfig)
+    const reports = new ReportsExporter()
+    await reports.export(exportConfig)
+  } else if (eventName === 'report_cwv_tech_complete') {
+    console.info('Tech Report export')
+    console.log(exportConfig)
+    const techReports = new TechReportsExporter()
+    await techReports.export(exportConfig)
+  } else {
+    throw new Error('Bad Request: unknown trigger name')
+  }
+  return 'OK'
+}
+
+await main(exportConfig).catch((error) => {
+  console.error(error)
+  process.exit(1)
+})