-
Notifications
You must be signed in to change notification settings - Fork 4
/
Copy pathopenfn-e5d0c9ec-bad5-48c8-add9-970b3e248d82-state.json
482 lines (482 loc) · 111 KB
/
openfn-e5d0c9ec-bad5-48c8-add9-970b3e248d82-state.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
{
"id": "e5d0c9ec-bad5-48c8-add9-970b3e248d82",
"name": "social-sciences",
"description": "Data integration workflows for WCS social sciences data. Used to transfer BNS & NRGT data from KoboToolbox forms to the WCS Programs MS SQL database.\n",
"concurrency": null,
"inserted_at": "2025-03-10T07:28:21Z",
"updated_at": "2025-04-15T12:29:35Z",
"project_credentials": {
"[email protected]": {
"id": "8f098581-1fba-449e-b7e5-49d7e8edc66b",
"name": "Rediet Googlesheet",
"owner": "[email protected]"
},
"[email protected]": {
"id": "f9c29826-894a-4f02-a120-f6bab5d16224",
"name": "WCS Kobo",
"owner": "[email protected]"
},
"[email protected]": {
"id": "50000511-d991-4efd-b41e-030dc4d9373a",
"name": "WCS Kobo Raw",
"owner": "[email protected]"
},
"[email protected]": {
"id": "b5fa2c78-00ab-4795-92d0-1078e7789aaa",
"name": "WCS MSSQL Test DB",
"owner": "[email protected]"
},
"[email protected]": {
"id": "4eb4c277-68ae-4d71-a569-3584b2cc4719",
"name": "AK Asana",
"owner": "[email protected]"
},
"[email protected]": {
"id": "950c9c55-fb89-4d8c-ab24-e2ca04a38ba7",
"name": "WCS Kobo Raw with URL",
"owner": "[email protected]"
},
"[email protected]": {
"id": "89b65a0e-e01d-4760-b021-1778799e9b2e",
"name": "WCS Kobo account",
"owner": "[email protected]"
},
"[email protected]": {
"id": "dfc9700d-b427-45f3-b91a-e4059e8a663f",
"name": "Asana API Token - AK WCS Grievances",
"owner": "[email protected]"
},
"[email protected]": {
"id": "97be9e14-c281-4d59-8336-972c3ee30b87",
"name": "Invalid WCS MSSQL Test DB",
"owner": "[email protected]"
}
},
"scheduled_deletion": null,
"history_retention_period": 180,
"dataclip_retention_period": null,
"retention_policy": "retain_all",
"collections": {
"mydata": {
"id": "18cc32b8-cb96-40d9-83cb-1d940b63b725",
"name": "mydata"
}
},
"workflows": {
"x_A2-Process-Forms-AUTO": {
"id": "32090694-7fd3-4131-b68c-f2372fa9623b",
"name": "x_A2 Process Forms AUTO",
"inserted_at": "2025-04-11T12:54:52.719019Z",
"lock_version": 7,
"triggers": {
"webhook": {
"enabled": false,
"id": "ec90e89e-9fd3-4d26-ab9f-df0dc01a0629",
"type": "webhook"
}
},
"jobs": {
"A2-Process-Forms-AUTO": {
"id": "ea64cfd3-401d-4c36-a708-a4aa89ea4f05",
"name": "A2 Process Forms AUTO",
"body": "get(`${state.data.url}`, {}, state => {\n state.formDefinition = state.data; // keeping form definition for data dictionary\n const tablesToBeCreated = [];\n const { survey, choices } = state.data.content;\n if (survey.length === 0) {\n console.log(\n 'No survey available or defined to analyze. Please check the Kobo form deployment status'\n );\n return state;\n }\n // PREFIX HANDLER\n const prefix1 = state.references[0].prefix1 || 'WCS';\n const prefix2 = state.references[0].prefix2 || '';\n const tableId = state.references[0].tableId;\n const uuidColumnName = 'generated_uuid';\n const prefixes = [prefix1, prefix2].join('_');\n // END OF PREFIX HANDLER\n\n const multiSelectIds = [];\n\n const mapType = {\n calculate: 'varchar(100)',\n date: 'date',\n decimal: 'float4',\n end: 'date',\n integer: 'int4',\n select_one: 'select_one',\n start: 'date',\n text: 'text',\n today: 'date',\n jsonb: 'jsonb',\n select_multiple: 'select_multiple',\n geopoint: 'text',\n };\n\n const discards = [\n 'begin_group',\n 'begin_repeat',\n 'end_group',\n 'end_repeat',\n 'note',\n ];\n\n // Camelize columns and table name\n function toCamelCase(str) {\n if (!str) return '';\n let underscores = [];\n let i = 0;\n while (str[i] === '_') {\n underscores.push(str[i]);\n i++;\n }\n let words = str.match(/[0-9a-zA-Z\\u00C0-\\u00FF]+/gi);\n if (!words) return '';\n words = words\n .map(word => {\n return word.charAt(0).toUpperCase() + word.substr(1).toLowerCase();\n })\n .join('');\n return `${underscores.join('')}${words}${underscores.join('')}`;\n }\n\n function questionsToColumns(questions) {\n var form = questions.filter(elt => !discards.includes(elt.type));\n\n form.forEach(obj => (obj.type = mapType[obj.type] || 'text'));\n\n form.forEach(obj => {\n // List of reserved keys in postgresql and their transformations\n if (obj.name === 'group') {\n obj.name = 'kobogroup';\n }\n if (obj.name == 'end') {\n obj.name = 'form_date__end';\n }\n if (obj.name == 'column') {\n obj.name = 'column_name';\n }\n if (obj.name == 'date') {\n obj.name = 'date_value';\n }\n if (obj.type === 'select_one') {\n obj.type = 'int4';\n obj.select_one = true;\n delete obj.default;\n }\n });\n\n form.forEach(q => {\n if (q.name === 'gps') {\n form.push({ name: 'latitude', type: 'float4' });\n form.push({ name: 'longitude', type: 'float4' });\n }\n });\n\n form = form.map(x => {\n let name = toCamelCase(x.name) || toCamelCase(x.$autoname);\n name = x.select_one\n ? `${prefixes}${toCamelCase(x.select_from_list_name)}ID_${name}`\n : name;\n return {\n ...x,\n name: `${name.split(/-/).join('_')}`,\n findValue: x.select_one || x.type === 'select_multiple' || false,\n required: x.required,\n };\n });\n\n const parentColumn =\n // questions[0].path.length > 1\n questions[0].depth > 1\n ? `${questions[0].path.slice(-2, -1)[0]}_uuid`\n : `${tableId}_uuid`;\n\n if (questions[0].depth > 0)\n form.push({ name: toCamelCase(parentColumn), type: 'text' });\n\n form.push(\n { name: 'AnswerId', type: 'text' },\n { name: toCamelCase(uuidColumnName), type: 'varchar(100)', unique: true }\n );\n\n return form;\n }\n\n function standardColumns(tableName) {\n // prettier-ignore\n return [\n // { name: `${prefix1}${tableName}ID`, type: 'int4', required: true, identity: true },\n // { name: `${prefix1}${tableName}Name`, type: 'varchar(255)', required: false },\n // { name: `${prefix1}${tableName}ExtCode`, type: 'varchar(50)', required: true, default: '' },\n { name: `${prefixes}${tableName}Code`, type: 'varchar(255)', required: false },\n { name: `${prefixes}${tableName}Description`, type: 'varchar(255)', required: false },\n { name: `${prefixes}OrganizationID_Owner`, type: 'int4', required: true, default: 1 },\n { name: `${prefixes}SecuritySettingID_Row`, type: 'int4', required: true, default: 1 },\n { name: 'Archive', type: 'BIT', required: true, default: '0' },\n { name: 'IsPublic', type: 'BIT', required: true, default: '0' },\n { name: 'CRDate', type: 'timestamp', required: true, default: 'NOW()' },\n { name: 'LMDate', type: 'timestamp', required: true, default: 'NOW()' },\n { name: 'UserID_CR', type: 'int4', required: true, default: -1 },\n { name: 'UserID_LM', type: 'int4', required: true, default: -1 },\n { name: 'CRIPAddress', type: 'varchar(32)', required: true, default: '' },\n { name: 'LMIPAddress', type: 'varchar(32)', required: true, default: '' },\n ];\n }\n\n function customColumns(tableName) {\n // prettier-ignore\n return [\n { name: `${prefixes}${tableName}ID`, type: 'int4', required: true, identity: true },\n { name: `${prefixes}${tableName}Name`, type: 'varchar(255)', required: false },\n { name: `${prefixes}${tableName}ExtCode`, type: 'varchar(50)', required: true, default: '' },\n ];\n }\n\n function processPath(question, i, arr) {\n let path = [];\n if (i === 0) {\n path = [];\n } else {\n let parent = arr.find(question => question.name === arr[i - 1].path[0]);\n if (parent && parent.type === 'begin_group') {\n path = [[arr[i - 1].path, question.name].join('/')];\n } else {\n path = i === 0 ? [] : [...arr[i - 1].path, question.name];\n }\n }\n return path;\n }\n\n function buildLookupTableColumns(prefixes, q, i, arr) {\n const path = processPath(q, i, arr);\n return [\n {\n name: `${prefixes}${toCamelCase(q.select_from_list_name)}ID`,\n type: 'int4',\n identity: true,\n required: q.required,\n depth: path.length,\n select_multiple: q.type === 'select_multiple' ? true : false,\n path,\n rule: 'DO_NOT_MAP',\n parentColumn: q.name,\n },\n {\n name: `${prefixes}${toCamelCase(q.select_from_list_name)}Name`,\n type: 'varchar(100)',\n required: q.required,\n depth: path.length,\n select_multiple: q.type === 'select_multiple' ? true : false,\n path,\n parentColumn: q.name,\n },\n {\n name: `${prefixes}${toCamelCase(q.select_from_list_name)}ExtCode`,\n type: 'varchar(100)',\n required: q.required,\n unique: true,\n depth: q.type === 'select_multiple' ? 3 : 0,\n select_multiple: q.type === 'select_multiple' ? true : false,\n path: i === 0 ? [] : [...arr[i - 1].path, q.name],\n parentColumn: q.name,\n },\n ];\n }\n\n // prettier-ignore\n function addLookupTable(tables, lookupTableName, prefixes, q, i, formName, arr) {\n tables.push({\n name: lookupTableName,\n columns: buildLookupTableColumns(prefixes, q, i, arr),\n defaultColumns: standardColumns(toCamelCase(q.select_from_list_name)),\n formName,\n depth: q.type === 'select_multiple' ? 1 : q.depth,\n lookupTable: q.type === 'select_multiple' ? true : undefined,\n select_from_list_name: toCamelCase(q.select_from_list_name),\n ReferenceUuid: `${prefixes}${toCamelCase(q.select_from_list_name)}ExtCode`,\n });\n tablesToBeCreated.push(lookupTableName)\n }\n\n function buildForeignTables(questions) {\n const foreignTables = [];\n questions.forEach(q => {\n if (q.select_one) {\n foreignTables.push({\n table: `${prefixes}${toCamelCase(q.select_from_list_name)}`,\n id: `${prefixes}${toCamelCase(q.select_from_list_name)}ID`,\n reference: `${prefixes}${toCamelCase(\n q.select_from_list_name\n )}ID_${toCamelCase(q.name)}`,\n });\n }\n });\n return foreignTables;\n }\n\n function buildTablesFromSelect(questions, formName, tables) {\n questions.forEach((q, i, arr) => {\n if (q.type === 'select_multiple') {\n multiSelectIds.push(q.name);\n const getType = name => survey.find(s => s.name === name).type; // return the type of a question\n\n let suffix = q.path.slice(-1)[0];\n if (suffix && getType(suffix) === 'begin_group') suffix = undefined;\n\n const lookupTableName = `${prefixes}${toCamelCase(\n q.select_from_list_name\n )}`;\n\n const junctionTableName = `${prefixes}${toCamelCase(\n suffix || tableId\n )}${toCamelCase(q.select_from_list_name)}`;\n\n // prettier-ignore\n const parentTableName = `${prefixes}${tableId}${toCamelCase(suffix)}`;\n // prettier-ignore\n const parentTableReferenceColumn = `${prefixes}${toCamelCase(suffix || tableId)}ID`;\n\n if (!tables.find(t => t.name === junctionTableName)) {\n // console.log('junctiontable', junctionTableName);\n const path = processPath(q, i, arr);\n tables.push({\n name: junctionTableName,\n dependencies: 3,\n columns: [\n {\n name: `${prefixes}${toCamelCase(q.select_from_list_name)}ID`,\n type: 'select_multiple',\n required: q.required,\n referent: lookupTableName,\n refersToLookup: true,\n depth: path.length,\n path,\n },\n {\n name: parentTableReferenceColumn,\n type: 'select_multiple',\n required: q.required,\n referent: parentTableName,\n refersToLookup: false,\n depth: path.length,\n path,\n },\n ],\n defaultColumns: [\n // prettier-ignore\n ...[\n { name: `${prefixes}${toCamelCase(q.select_from_list_name)}Name`, type: 'varchar(255)', required: false },\n { name: `${prefixes}${toCamelCase(q.select_from_list_name)}ExtCode`, type: 'varchar(50)', required: true, default: '' },\n ],\n ...standardColumns(toCamelCase(q.select_from_list_name)),\n ],\n foreignTables: [\n {\n table: lookupTableName,\n id: `${lookupTableName}ID`,\n },\n {\n table: parentTableName,\n id: `${prefixes}${toCamelCase(suffix || tableId)}ID`,\n },\n ],\n formName,\n depth: 1,\n select_multiple: true,\n select_from_list_name: toCamelCase(q.select_from_list_name),\n });\n tablesToBeCreated.push(junctionTableName);\n }\n }\n\n if (['select_one', 'select_multiple'].includes(q.type)) {\n // Use list_name to name select_table\n const lookupTableName = `${prefixes}${toCamelCase(\n q.select_from_list_name\n )}`;\n if (!tablesToBeCreated.includes(lookupTableName)) {\n // console.log('lookup', lookupTableName);\n //prettier-ignore\n addLookupTable(tables, lookupTableName, prefixes, q, i, formName, arr);\n }\n }\n });\n return tables;\n }\n\n function tablesFromQuestions(questions, formName, tables) {\n const backwardsFirstBegin = questions\n .reverse()\n .findIndex(item => item.type === 'begin_repeat');\n\n const lastBegin =\n backwardsFirstBegin !== -1\n ? questions.length - backwardsFirstBegin - 1\n : false;\n\n const tName = `${prefixes}${tableId}`;\n\n if (lastBegin) {\n const firstEndAfterLastBegin =\n questions\n .reverse()\n .slice(lastBegin)\n .findIndex(item => item.type === 'end_repeat') + lastBegin;\n\n // Remove the deepest repeat group from the 'questions' array, parse it\n // and push it to the 'tables' array, and call tablesFromQuestions with\n // the remaining questions.\n const group = questions.splice(\n lastBegin,\n firstEndAfterLastBegin - lastBegin + 1\n );\n\n const tableName = toCamelCase(\n group[0].path\n .slice(-1)\n .pop()\n .split(/\\s|-|'/)\n .join('_')\n .replace('.', '')\n );\n const name = `${prefixes}${tableId}${tableName}`;\n\n tables.push({\n name,\n dependencies: 2,\n columns: questionsToColumns(\n group.filter(q => q.type !== 'select_multiple')\n ),\n defaultColumns: [\n // prettier-ignore\n ...[ { name: `${tName}ID`, type: 'int4', required: false } ],\n ...customColumns(tableName),\n ...standardColumns(tableName),\n ],\n foreignTables: [\n ...[\n {\n table: tName,\n id: `${tName}ID`,\n },\n ],\n ...buildForeignTables(group),\n ],\n formName,\n depth: group[0].depth,\n });\n tablesToBeCreated.push(name);\n\n return tablesFromQuestions(questions, formName, tables);\n }\n\n tables.push(\n {\n // This is the main table to hold submissions for this Kobo form.\n name: tName,\n dependencies: 1,\n columns: [\n // Note that we do not create columns for select multiple Qs. Answers\n // to select multiple Qs will appear as records in a junction table.\n ...questionsToColumns(\n questions.filter(q => q.type !== 'select_multiple')\n ),\n ...[\n {\n name: 'Payload',\n type: 'jsonb',\n depth: 0,\n path: [],\n },\n ],\n ],\n defaultColumns: [\n ...customColumns(tableId),\n ...standardColumns(tableId),\n ],\n foreignTables: buildForeignTables(questions),\n formName,\n depth: 0,\n },\n {\n name: `${prefix1}_KoboDataset`,\n // This is a table that must exist in all DBs that will hold submission data from any form.\n columns: [\n {\n name: 'FormName',\n type: 'text',\n depth: 0,\n path: [],\n },\n {\n name: 'DatasetId',\n type: 'varchar(100)',\n depth: 0,\n path: [],\n unique: true,\n },\n {\n name: 'LastUpdated',\n type: 'timestamp',\n depth: 0,\n path: [],\n },\n ],\n defaultColumns: [\n {\n name: `${prefix1}ID`,\n type: 'int4',\n required: true,\n identity: true,\n },\n {\n name: `${prefix1}Name`,\n type: 'varchar(255)',\n required: false,\n },\n {\n name: `${prefix1}ExtCode`,\n type: 'varchar(50)',\n required: true,\n default: '',\n },\n {\n name: `${prefix1}Code`,\n type: 'varchar(255)',\n required: false,\n },\n {\n name: `${prefix1}Description`,\n type: 'varchar(255)',\n required: false,\n },\n {\n name: `${prefix1}OrganizationID_Owner`,\n type: 'int4',\n required: true,\n default: 1,\n },\n {\n name: `${prefix1}SecuritySettingID_Row`,\n type: 'int4',\n required: true,\n default: 1,\n },\n { name: 'Archive', type: 'BIT', required: true, default: '0' },\n { name: 'IsPublic', type: 'BIT', required: true, default: '0' },\n {\n name: 'CRDate',\n type: 'timestamp',\n required: true,\n default: 'NOW()',\n },\n {\n name: 'LMDate',\n type: 'timestamp',\n required: true,\n default: 'NOW()',\n },\n { name: 'UserID_CR', type: 'int4', required: true, default: -1 },\n { name: 'UserID_LM', type: 'int4', required: true, default: -1 },\n {\n name: 'CRIPAddress',\n type: 'varchar(32)',\n required: true,\n default: '',\n },\n {\n name: 'LMIPAddress',\n type: 'varchar(32)',\n required: true,\n default: '',\n },\n ],\n formName,\n depth: 0,\n }\n );\n tablesToBeCreated.push(tName);\n\n return tables;\n }\n\n // We build a dictionary of different select_one/select_multiple questions\n // and the different values they hold ===================================\n function createSeeds(choicesArr) {\n const obj = {};\n\n choicesArr.forEach(c => {\n const table = `${prefixes}${toCamelCase(c.list_name)}`;\n if (!obj[table]) obj[table] = [];\n if (!obj[table].includes(c.name)) obj[table].push(c.name);\n });\n\n const arr = [];\n\n Object.keys(obj).forEach(table => {\n arr.push({\n table: table,\n externalId: `${table}ExtCode`,\n records: [...obj[table]],\n });\n });\n\n return arr;\n }\n\n let depth = 0;\n\n survey.forEach((q, i, arr) => {\n switch (q.type) {\n case 'begin_group':\n arr[i] = {\n ...q,\n depth,\n path: i === 0 ? [] : [...arr[i - 1].path, q.name],\n };\n break;\n\n case 'begin_repeat':\n depth++;\n arr[i] = {\n ...q,\n depth,\n path: i === 0 ? [] : [...arr[i - 1].path, q.name],\n };\n break;\n\n case 'end_repeat':\n arr[i] = {\n ...q,\n depth,\n path: i === 0 ? [] : [...arr[i - 1].path.slice(0, -1)],\n };\n depth--;\n break;\n\n case 'end_group':\n arr[i] = {\n ...q,\n depth,\n path: i === 0 ? [] : [...arr[i - 1].path.slice(0, -1)],\n };\n break;\n\n default:\n arr[i] = {\n ...q,\n depth,\n path: i === 0 ? [] : [...arr[i - 1].path],\n };\n break;\n }\n });\n\n const seeds = createSeeds(choices);\n const lookupTables = buildTablesFromSelect(survey, state.data.name, []);\n let tables = tablesFromQuestions(survey, state.data.name, []).reverse();\n tables = lookupTables.concat(tables);\n\n // Given the initial input of a \"Kobo form definition\", we return...\n return {\n ...state,\n tableId, // this is unique per form and used to identify the main \"submissions table\" for the form\n tables, // this is a list of tables (main table, lookup tables, junction tables, etc.) to create in the db\n seeds, // this is a list of records (grouped by table) to insert at build time, not form submission time (runtime)\n prefix1, // this is a constant used in various places\n prefix2, // this is a constant used in various places\n prefixes, // this is `{prefix1}_{prefix2}`\n uuidColumnName, // this is a constant used identify unique ID columns in the db\n multiSelectIds, // this is an array of the 'list_name' of every select_multiple question\n data: {}, // we clear data\n response: {}, // we clear response\n };\n});\n\n// Sort the tables by dependencies so that we can create them in the correct order\nfn(state => ({\n ...state,\n tables: state.tables.sort((a, b) =>\n !b.hasOwnProperty('dependencies')\n ? 1\n : a.dependencies > b.dependencies\n ? 1\n : -1\n ),\n}));\n\n// Print out a \"DROP STATEMENT\" for each table in the list of tables.\nfn(state => {\n console.log('====================DROP STATEMENT====================');\n console.log('Use this to clean database from created tables...');\n\n const { tables } = state;\n\n const query = `DROP TABLE ${tables.map(t => t.name).reverse()};`;\n\n console.log(`query: ${query}`);\n console.log('====================END DROP STATEMENT====================');\n\n return state;\n});\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": null
},
"A4-Generate-OpenFn-Job-script-AUTO---OPENFN-JOB": {
"id": "ddcb3bd8-a268-4f74-bae5-8b1c203244cb",
"name": "A4 Generate OpenFn Job script AUTO - OPENFN JOB",
"body": "// {\n// ...state,\n// tableId, // this is unique per form and used to identify the main \"submissions table\" for the form\n// tables, // this is a list of tables (main table, lookup tables, junction tables, etc.) to create in the db\n// seeds, // this is a list of records (grouped by table) to insert at build time, not form submission time (runtime)\n// prefix1, // this is a constant used in various places\n// prefix2, // this is a constant used in various places\n// prefixes, // this is `{prefix1}_{prefix2}`\n// uuidColumnName, // this is a constant used identify unique ID columns in the db\n// multiSelectIds, // this is an array of the 'list_name' of every select_multiple question\n// data: {}, // we clear data\n// response: {}, // we clear response\n// };\n\n// Pluck projectId out of state for convenience, filter out tables that were populated at build time.\nfn(state => {\n const { projectId } = state.configuration;\n\n return {\n ...state,\n projectId,\n tables: state.tables\n .filter(t => !t.ReferenceUuid) // filter out tables that were seeded\n .filter(t => t.columns.length > 0) // filter out tables with no columns\n .filter(t => t.name !== `${state.prefixes}_Untitled`), // filter out bad test data\n };\n});\n\nfn(state => {\n // Create the first operation in our expression.\n var expression = `fn(state => {\n const multiSelectIds = [\"${state.multiSelectIds.join('\", \"')}\"];\n\n function generateUuid(body, uuid) {\n for (const property in body) {\n if (Array.isArray(body[property]) && body !== null) {\n body['__generatedUuid'] = uuid;\n body[property].forEach((thing, i, arr) => {\n if (thing !== null) {\n thing['__parentUuid'] = uuid;\n let newUuid = uuid + '-' + (i + 1);\n thing['__generatedUuid'] = newUuid;\n for (const property in thing) {\n if (Array.isArray(thing[property])) {\n generateUuid(thing, newUuid);\n }\n }\n }\n });\n }\n }\n }\n\n generateUuid(\n state.data.body,\n state.data.body._id+'-'+state.data.body._xform_id_string\n );\n\n const { body } = state.data;\n const { _id, _xform_id_string } = body;\n state.data = { ...state.data, ...body };\n\n return { ...state, _id, _xform_id_string };\n}); \\n`;\n\n function toCamelCase(str) {\n const words = str.split('_'); // we split using '_'. With regex we would use: \"match(/[a-z]+/gi)\"\n if (!words) return '';\n return words\n .map(word => {\n return word.charAt(0).toUpperCase() + word.substr(1).toLowerCase();\n })\n .join('');\n }\n\n // Iterate through every table and create an operation to upsert (or upsertMany) records for that table.\n for (const table of state.tables) {\n const { columns, name, depth, select_multiple, lookupTable } = table;\n var paths = [];\n\n for (const column of columns) {\n // Handling master parent table\n if (name === `${state.prefix1}_KoboDataset`) {\n const values = {\n FormName: \"dataValue('formName')\",\n DatasetId: \"dataValue('_xform_id_string')\",\n LastUpdated: 'new Date().toISOString()',\n };\n for (x in values) paths.push(values[x]);\n break;\n }\n // end of master parent table\n\n const currentPath = column.path;\n\n paths.push(\n (currentPath && currentPath.length > 0\n ? currentPath.join('/') + '/'\n : '') + column.$autoname\n );\n }\n\n var mapKoboToPostgres = {}; // This is the jsonBody that should be given to our upsert\n\n // We generate findValue function (fn) for those that needs it.\n function generateFindValue(question, relation, searchAttr, searchVal) {\n const inferredUUid =\n // if not junction, but yes select_one or many...\n !question.referent && question.findValue\n ? toCamelCase(\n question.select_from_list_name.replace(`${state.tableId}_`, '')\n )\n : question.name;\n\n const suffixedUUid = !inferredUUid.endsWith('ID')\n ? `${inferredUUid}ID`\n : `${inferredUUid}`;\n\n const finalUUid = !suffixedUUid.startsWith(state.prefixes)\n ? `${state.prefixes}${suffixedUUid}`\n : suffixedUUid;\n\n searchAttr = searchAttr.replace('ID', '');\n\n if (question.referent) {\n // if it's in a junction\n if (question.refersToLookup) {\n // and it refers to lookup\n searchAttr = `${question.referent}ExtCode`;\n searchVal = `x`;\n }\n // if it doesn't we don't touch it!\n } else {\n searchAttr = !searchAttr.includes(state.prefixes)\n ? `${state.prefixes}${searchAttr}ExtCode`\n : `${searchAttr}ExtCode`;\n\n searchVal =\n question.depth > 0\n ? `x['${searchVal}']`\n : `dataValue('${searchVal}')`;\n }\n\n return `await findValue({uuid: '${finalUUid.toLowerCase()}', relation: '${relation.replace(\n 'ID',\n ''\n )}', where: { ${searchAttr}: ${searchVal} }})(state)`;\n }\n\n // FROM HERE WE ARE BUILDING MAPPINGS\n columns.forEach((col, i) => {\n if (col.rule !== 'DO_NOT_MAP') {\n if (col.findValue) {\n mapKoboToPostgres[col.name] = generateFindValue(\n col,\n `${state.prefixes}${toCamelCase(col.select_from_list_name)}`,\n `${toCamelCase(col.select_from_list_name)}`,\n paths[i]\n );\n } else if (col.name === 'Latitude') {\n mapKoboToPostgres[col.name] = `state => state.data.gps.split(' ')[0]`;\n } else if (col.name === 'Longitude') {\n mapKoboToPostgres[col.name] = `state => state.data.gps.split(' ')[1]`;\n } else if (col.name === 'Payload') {\n // Here we use an expression, rather than a function, to take the ======\n // original, unaltered body of the Kobo submission as JSON.\n mapKoboToPostgres.Payload = `state.data.body`;\n } else if (col.referent) {\n // If we see a referent, this is a column in a junction table.\n if (col.refersToLookup) {\n // If refersToLookup is true, then this column refers to the lookup table\n mapKoboToPostgres[col.name] = generateFindValue(\n col,\n col.referent,\n `${col.select_from_list_name}`,\n 'x'\n );\n // if If refersToLookup is false, this column refers to the main submission table\n // TODO: Mamadou, please confirm the line below. Should it change to \"findValue\"?\n } else {\n mapKoboToPostgres[col.name] = generateFindValue(\n col,\n col.referent,\n 'AnswerId',\n 'state._id'\n );\n }\n } else if (col.depth > 0) {\n mapKoboToPostgres[col.name] = `x['${paths[i]}']`;\n } else {\n mapKoboToPostgres[col.name] =\n name !== `${state.prefix1}_KoboDataset`\n ? col.parentColumn\n ? `dataValue('${col.path.join('/')}')`\n : `dataValue('${paths[i]}')`\n : `${paths[i]}`;\n }\n\n if (col.name === 'AnswerId') {\n mapKoboToPostgres[col.name] = `state._id`;\n }\n if (col.name === 'GeneratedUuid') {\n if (depth > 0) mapKoboToPostgres[col.name] = `x['__generatedUuid']`;\n else mapKoboToPostgres[col.name] = `dataValue('__generatedUuid')`;\n }\n }\n });\n\n // =====================================================================\n\n // We generate a mapping variable that we are going=======\n // to use inside our operation============================\n const mapObject = `const mapping = ${JSON.stringify(\n mapKoboToPostgres,\n null,\n 2\n ).replace(/\"/g, '')}`;\n // =======================================================\n\n // We build a set of statements for when depth > 0========\n const path = columns[0].path.join('/');\n\n let statements = null;\n // console.log('select', select_multiple);\n // console.log('name', depth);\n // if table is a table referencing a select multiple table.\n if (select_multiple || lookupTable) {\n statements = `if (state.data['${path}']) { \\n\n const array = state.data['${path}'].split(' '); \\n\n const mapping = []; \\n \n for ( let x of array ) { \\n\n mapping.push(${JSON.stringify(\n mapKoboToPostgres,\n null,\n 2\n ).replace(/\"/g, '')}); \\n\n } \\n\n `;\n } else {\n // statements = `if (state.data['${path}']) { \\n\n // const array = state.data['${path}'].split(' '); \\n\n // const mapping = []; \\n\n // for ( let x of array ) { \\n\n // mapping.push(${JSON.stringify(mapKoboToPostgres, null, 2).replace(\n // /\"/g,\n // ''\n // )}); \\n\n // } \\n\n // }`;\n statements = `const dataArray = state.data.body['${path}'] || [] \\n\n const mapping = []; \\n\n for (let x of dataArray) { \\n\n mapping.push(${JSON.stringify(mapKoboToPostgres, null, 2).replace(\n /\"/g,\n ''\n )}) \\n\n }`;\n }\n // =======================================================\n\n const opFirstLineNoDepth = `fn(async state => {\\n ${mapObject} \\n`;\n const opFirstLineDepth = `fn(async state => {\\n ${statements} \\n`;\n // const alterSOpeningSelect = `fn(async state => {\\n ${selectStatement} \\n`;\n const opLastLine = `})`;\n\n function wrapper(column, mapping) {\n let prefix = '';\n const depth = column.depth;\n /* if (select_multiple || lookupTable) {\n prefix += mapping + `)(state); \\n${alterSClosing} \\n`;\n return prefix;\n } else */ if (depth > 1) {\n // console.log('Im here');\n let closingPar = 0; // hold how many brackets we need to close\n for (var i = 0; i < depth - 1; i++) {\n if (i === 0 && column.path[i]) {\n // We generate \"body.something\" only for the first 'each'\n prefix += `each('$.${column.path[i]}[*]', `;\n closingPar++;\n } else if (column.path[i]) {\n prefix += `each(dataPath('${column.path[i]}[*]'), `;\n closingPar++;\n }\n }\n // prefix += mapping;\n prefix +=\n mapping +\n (select_multiple || lookupTable\n ? `)(state); } \\n return state; \\n${opLastLine} \\n`\n : `)(state); \\n${opLastLine} \\n`);\n for (var i = 0; i < closingPar; i++) {\n prefix += ')';\n }\n\n return prefix;\n }\n return mapping;\n }\n\n const operation = `return ${depth > 0 ? 'upsertMany' : 'upsert'}`;\n\n var uuid =\n name === `${state.prefix1}_KoboDataset`\n ? '\"DatasetId\"'\n : table.select_multiple\n ? `[\"${columns[0].name}\", \"${columns[1].name}\"]`\n : `'${toCamelCase(state.uuidColumnName)}'`;\n\n let mapping =\n depth > 0 || select_multiple\n ? `${opFirstLineDepth} ${operation}('${name}', ${uuid}, `\n : `${opFirstLineNoDepth} ${operation}('${name}', ${uuid}, `;\n\n if (columns[0].depth > 0 || select_multiple) {\n mapping += `() => mapping, {setNull: [\"''\", \"'undefined'\"]}`;\n } else {\n mapping += `mapping, {setNull: [\"''\", \"'undefined'\"]}`;\n }\n // END OF BUILDING MAPPINGS (state)\n\n expression +=\n wrapper(columns[0], mapping) +\n (columns[0].depth > 1\n ? '\\n'\n : select_multiple || lookupTable\n ? `)(state); } \\n return state; \\n${opLastLine} \\n`\n : `)(state); \\n${opLastLine} \\n`);\n }\n\n state.triggerCriteria = {\n tableId: `${state.prefixes}${state.tableId}`,\n };\n\n return { ...state, expression };\n});\n\n// Get existing triggers for this project.\nfn(state => {\n return request(\n {\n method: 'get',\n path: 'triggers',\n params: {\n project_id: state.projectId,\n },\n },\n next => ({ ...next, triggers: next.data })\n )(state);\n});\n\n// Get existing jobs for this project.\nfn(state => {\n return request(\n {\n method: 'get',\n path: 'jobs',\n params: {\n project_id: state.projectId,\n },\n },\n next => ({ ...next, jobs: next.data.filter(job => !job.archived) })\n )(state);\n});\n\n// Create or update the trigger to detect submissions from this form.\nfn(state => {\n const { triggers, prefixes, tableId, triggerCriteria, projectId } = state;\n const triggerNames = triggers.map(t => t.name);\n\n const name = `auto/${prefixes}${tableId}`;\n const criteria = triggerCriteria;\n const triggerIndex = triggerNames.indexOf(name);\n\n const trigger = {\n project_id: projectId,\n name,\n type: 'message',\n criteria,\n };\n\n if (triggerIndex === -1) {\n console.log('Inserting trigger.');\n return request(\n {\n method: 'post',\n path: 'triggers',\n data: { trigger },\n },\n next => ({ ...next, triggers: [...next.triggers, next.data] })\n )(state);\n }\n\n console.log('Trigger already existing.');\n return state;\n});\n\n// Create or update the job for handling submissions from this form.\nfn(state => {\n const { expression, prefixes, tableId, jobs, triggers, projectId } = state;\n\n console.log('Inserting/updating job: ', `auto/${prefixes}${tableId}`);\n\n const jobNames = jobs.map(j => j.name);\n const triggersName = triggers.map(t => t.name);\n const name = `auto/${prefixes}${tableId}`;\n const jobIndex = jobNames.indexOf(name); // We check if there is a job with that name.\n const triggerIndex = triggersName.indexOf(name);\n const triggerId = triggers[triggerIndex].id;\n\n const method = jobIndex !== -1 ? 'put' : 'post';\n const path = method === 'put' ? `jobs/${jobs[jobIndex].id}` : 'jobs/';\n\n const job = {\n adaptor: 'mssql',\n adaptor_version: 'v2.6.11',\n expression,\n name,\n project_id: projectId,\n trigger_id: triggerId, // we (1) create a trigger first; (2) get the id ; (3) assign it here!\n };\n\n return request({ method, path, data: { job } })(state);\n});\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": null
},
"A3-Generate-SQL-to-setup-DB-AUTO---MSSQL": {
"id": "6946277d-c9cd-45bd-917b-a3f9de1fab2b",
"name": "A3 Generate SQL to setup DB AUTO - MSSQL",
"body": "// Here we set default options for the SQL adaptor. Setting execute or writeSql\n// below will set the standard behavior of all SQL functions below unless overwritten.\n\n//SET execute: true if you want to SQL script to be auto-executed in the DB linked to this job \n//SET execute: false if you do NOT want to execure the SQL script, and only wnat to generate the script (see \"writeSql\")\nfn(state => ({ ...state, execute: false, writeSql: true }));\n\n// Creates tables in the db.\neach(\n '$.tables[*]',\n fn(state => {\n const { execute, writeSql } = state;\n const { name, defaultColumns } = state.data;\n\n function convertToMssqlTypes(col) {\n col.type = col.referent\n ? 'int'\n : col.type === 'select_one' ||\n col.type === 'select_multiple' ||\n col.type === 'text' ||\n col.type === 'jsonb'\n ? 'nvarchar(max)'\n : col.type.includes('varchar')\n ? col.type.replace('varchar', 'nvarchar')\n : col.type === 'int4' || col.type === 'float4'\n ? col.type.substring(0, col.type.length - 1)\n : col.type === 'timestamp'\n ? 'datetime'\n : col.type;\n\n if (col.type === 'datetime') col.default = 'GETDATE()';\n }\n\n function insert(name, columns, execute, writeSql, state) {\n columns.forEach(col => convertToMssqlTypes(col));\n return insertTable(name, state => columns, {\n writeSql,\n execute,\n })(state).then(state => {\n if (defaultColumns) {\n let foreignKeyQueries = [];\n if (state.data.foreignTables) {\n const { foreignTables } = state.data;\n for (let ft of foreignTables) {\n const { table, id, reference } = ft;\n foreignKeyQueries.push(`ALTER TABLE ${name} WITH CHECK ADD CONSTRAINT FK_${name}_${\n reference ? reference : id\n } FOREIGN KEY (${reference ? reference : id})\n REFERENCES ${table} (${id});\n ALTER TABLE ${name} CHECK CONSTRAINT FK_${name}_${\n reference ? reference : id\n };`);\n }\n }\n // Creating foreign keys constraints to standard WCS DB and fields\n return sql({\n query: state =>\n `ALTER TABLE ${name} WITH CHECK ADD CONSTRAINT FK_${name}_OrganizationID_Owner FOREIGN KEY (${\n state.prefixes\n }OrganizationID_Owner)\n REFERENCES WCSPROGRAMS_Organization (WCSPROGRAMS_OrganizationID);\n ALTER TABLE ${name} CHECK CONSTRAINT FK_${name}_OrganizationID_Owner;\n ALTER TABLE ${name} WITH CHECK ADD CONSTRAINT FK_${name}_SecuritySettingID_Row FOREIGN KEY (${\n state.prefixes\n }SecuritySettingID_Row)\n REFERENCES WCSPROGRAMS_SecuritySetting (WCSPROGRAMS_SecuritySettingID);\n ALTER TABLE ${name} CHECK CONSTRAINT FK_${name}_SecuritySettingID_Row;\n ${foreignKeyQueries.join('\\n')}\n `,\n options: {\n writeSql: true, // Keep to true to log query (otherwise make it false).\n execute: false, // keep to false to not alter DB\n },\n })(state);\n }\n return state;\n });\n }\n\n function modify(name, newColumns, execute, writeSql, state) {\n if (newColumns && newColumns.length > 0) {\n console.log('Existing table found in mssql --- Updating.');\n // Note: Specify options here (e.g {writeSql: false, execute: true})\n return modifyTable(name, state => newColumns, {\n writeSql, // Keep to true to log query (otherwise make it false).\n execute, // keep to false to not alter DB\n })(state);\n } else {\n console.log('No new columns to add.');\n return state;\n }\n }\n\n if (name !== `${state.prefixes}_Untitled`) {\n let mergedColumns = state.data.columns;\n if (state.data.defaultColumns)\n mergedColumns = [...state.data.columns, ...state.data.defaultColumns];\n\n return describeTable(name.toLowerCase(), {\n writeSql: true, // Keep to true to log query.\n execute, // Keep to true to execute query.\n })(state)\n .then(resp => {\n const { rows } = resp.response.body;\n if (resp.response.body.rowCount === 0) {\n console.log('No matching table found in mssql --- Inserting.');\n const columns = mergedColumns.filter(x => x.name !== undefined);\n\n // change this line to 'return insert(name, columns, true, writeSql, state);' to override 'execute: false' at top\n return insert(name, columns, execute, writeSql, state);\n } else {\n const columnNames = rows.map(x => x.column_name.toLowerCase());\n\n console.log('----------------------');\n const newColumns = mergedColumns.filter(\n x =>\n x.name !== undefined &&\n !columnNames.includes(x.name.toLowerCase())\n );\n newColumns.forEach(col => convertToMssqlTypes(col));\n console.log(newColumns);\n\n // change this line to 'return modify(name, newColumns, true, writeSql, state);' to override 'execute: false' at top\n return modify(name, newColumns, execute, writeSql, state);\n }\n })\n .catch(() => {\n // If describeTable does NOT get executed because they've turned off execute,\n // we should write the SQL for all the insert statements without executing them.\n const columns = mergedColumns.filter(x => x.name !== undefined);\n return insert(name, columns, execute, writeSql, state);\n });\n }\n return state;\n })\n);\n\n// Adds \"seeds\" to the lookup tables—rows that can be referenced in submissions.\neach(\n '$.seeds[*]',\n fn(state => {\n const { writeSql, execute, data } = state;\n const { table, externalId, records } = data;\n return upsertMany(\n table, // table name\n externalId, // external ID column name\n state => {\n // array of records to upsert\n return records.map(r => ({\n [externalId]: r,\n [`${table}Name`]: r,\n }));\n },\n { writeSql, execute, logValues: true } // options\n )(state);\n })\n);\n\n// Prints out SQL statements for manual inspection and work.\nfn(state => {\n console.log('----------------------');\n console.log('Logging queries.');\n for (query of state.queries) console.log(query);\n console.log('----------------------');\n\n return state;\n});\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": null
},
"A5-Prepare-Form-for-Data-Dictionary-AUTO---MSSQL": {
"id": "ef1f89eb-6b95-48d8-b7f9-1e5ddd1dd044",
"name": "A5 Prepare Form for Data Dictionary AUTO - MSSQL",
"body": "fn(state => {\n const KoboToolBox_Forms = [\n {\n name: 'form_name',\n type: 'nvarchar(100)',\n },\n {\n name: 'date_created',\n type: 'date',\n },\n {\n name: 'date_modified',\n type: 'date',\n },\n {\n name: 'form_owner',\n type: 'nvarchar(100)',\n },\n {\n name: 'languages',\n type: 'nvarchar(100)',\n },\n {\n name: 'form_id',\n type: 'nvarchar(100)',\n unique: true,\n },\n {\n name: 'form_group',\n type: 'nvarchar(100)',\n },\n {\n name: 'table_id',\n type: 'nvarchar(100)',\n },\n ];\n\n const KoboToolBox_Questions = [\n {\n name: 'question_id',\n type: 'nvarchar(100)',\n unique: true,\n },\n {\n name: 'form_id',\n type: 'nvarchar(100)',\n },\n {\n name: 'analytics_label',\n type: 'nvarchar(max)',\n },\n {\n name: 'question_name',\n type: 'nvarchar(max)',\n },\n {\n name: 'label',\n type: 'nvarchar(max)',\n },\n {\n name: 'question_type',\n type: 'nvarchar(100)',\n },\n {\n name: 'list_id',\n type: 'nvarchar(100)',\n },\n {\n name: 'question_constraint',\n type: 'nvarchar(max)',\n },\n ];\n\n const KoboToolBox_Choices = [\n {\n name: 'choice_id',\n type: 'nvarchar(100)',\n unique: true,\n },\n {\n name: 'list_id',\n type: 'nvarchar(100)',\n },\n {\n name: 'list_name',\n type: 'nvarchar(100)',\n },\n {\n name: 'choice_name',\n type: 'nvarchar(100)',\n },\n {\n name: 'choice_label',\n type: 'nvarchar(max)',\n },\n {\n name: 'form_uid',\n type: 'nvarchar(100)',\n },\n ];\n\n const MetadataForms = [\n {\n name: 'KoboToolBox_Forms',\n columns: KoboToolBox_Forms,\n },\n {\n name: 'KoboToolBox_Questions',\n columns: KoboToolBox_Questions,\n },\n {\n name: 'KoboToolBox_Choices',\n columns: KoboToolBox_Choices,\n },\n ];\n\n return { ...state, MetadataForms };\n});\n\neach(\n '$.MetadataForms[*]',\n fn(state => {\n const { name, columns } = state.data;\n \n function insert(name, columns, execute, writeSql, state) {\n columns.forEach(col =>\n col.type === 'select_one' || col.type === 'select_multiple'\n ? (col.type = 'nvarchar(max)')\n : col.type\n );\n return insertTable(name, state => columns, {\n writeSql,\n execute,\n })(state);\n }\n\n function modify(name, newColumns, execute, writeSql, state) {\n newColumns.forEach(col =>\n col.type === 'select_one' || col.type === 'select_multiple'\n ? (col.type = 'nvarchar(max)')\n : col.type\n );\n if (newColumns && newColumns.length > 0) {\n console.log('Existing table found in mssql --- Updating.');\n // Note: Specify options here (e.g {writeSql: false, execute: true})\n return modifyTable(name, state => newColumns, {\n writeSql, // Keep to true to log query (otherwise make it false).\n execute, // keep to false to not alter DB\n })(state);\n } else {\n console.log('No new columns to add.');\n return state;\n }\n }\n\n // Note: Specify options here\n const execute = false;\n const writeSql = true;\n\n return describeTable(name.toLowerCase(), {\n writeSql: true,\n execute,\n })(state)\n .then(mssqlColumn => {\n const { rows } = mssqlColumn.response.body;\n if (mssqlColumn.response.body.rowCount === 0) {\n console.log('No matching table found in mssql --- Inserting.');\n\n const cols = columns.filter(x => x.name !== undefined);\n return insert(name, cols, execute, writeSql, state);\n } else {\n const columnNames = rows.map(x => x.column_name);\n\n console.log('----------------------');\n const newColumns = columns.filter(\n x =>\n x.name !== undefined &&\n !columnNames.includes(x.name.toLowerCase())\n );\n\n console.log(newColumns);\n return modify(name, newColumns, execute, writeSql, state);\n }\n })\n .catch(() => {\n return insert(name, columns, execute, writeSql, state);\n });\n })\n);\n\nfn(state => {\n const { openfnInboxUrl } = state.configuration;\n const data = {\n type: 'Form Definition',\n formDefinition: state.formDefinition,\n prefixes: state.prefixes,\n prefix2: state.prefix2,\n tableId: state.tableId,\n };\n console.log('Sending form definition to OpenFN inbox.');\n http.post({\n url: openfnInboxUrl,\n data,\n maxContentLength: Infinity,\n maxBodyLength: Infinity,\n })(state);\n\n return state;\n});\n\nfn(state => {\n console.log('----------------------');\n console.log('Logging queries.');\n for (query of state.queries) console.log(query);\n console.log('----------------------');\n return state;\n});\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": null
}
},
"edges": {
"webhook->A2-Process-Forms-AUTO": {
"enabled": true,
"id": "e24c3cbb-792a-46a0-b9e6-c7657b997884",
"target_job_id": "ea64cfd3-401d-4c36-a708-a4aa89ea4f05",
"source_trigger_id": "ec90e89e-9fd3-4d26-ab9f-df0dc01a0629",
"condition_type": "js_expression",
"condition_label": "Form Changed",
"condition_expression": "state.data.formUpdate == true"
},
"A2-Process-Forms-AUTO->A4-Generate-OpenFn-Job-script-AUTO---OPENFN-JOB": {
"enabled": true,
"id": "25cd6c6d-95d0-4979-840f-b630f6383e8d",
"target_job_id": "ddcb3bd8-a268-4f74-bae5-8b1c203244cb",
"source_job_id": "ea64cfd3-401d-4c36-a708-a4aa89ea4f05",
"condition_type": "on_job_success"
},
"A2-Process-Forms-AUTO->A3-Generate-SQL-to-setup-DB-AUTO---MSSQL": {
"enabled": true,
"id": "8fd938dd-68d7-4d7c-8dd2-71242cb8059c",
"target_job_id": "6946277d-c9cd-45bd-917b-a3f9de1fab2b",
"source_job_id": "ea64cfd3-401d-4c36-a708-a4aa89ea4f05",
"condition_type": "on_job_success"
},
"A2-Process-Forms-AUTO->A5-Prepare-Form-for-Data-Dictionary-AUTO---MSSQL": {
"enabled": true,
"id": "88500546-709d-49ec-b022-6e2fa89342bb",
"target_job_id": "ef1f89eb-6b95-48d8-b7f9-1e5ddd1dd044",
"source_job_id": "ea64cfd3-401d-4c36-a708-a4aa89ea4f05",
"condition_type": "on_job_success"
}
}
},
"Monitor-Forms-Shared": {
"id": "cc58f66e-143e-43da-8f62-8225b9258bd5",
"name": "Monitor Forms Shared",
"inserted_at": "2025-04-11T13:54:16.271665Z",
"lock_version": 26,
"triggers": {
"cron": {
"enabled": false,
"id": "f2d20419-c117-42b7-8c01-7366b82060ef",
"type": "cron",
"cron_expression": "0 0 * * *"
}
},
"jobs": {
"FS1---Get-Forms": {
"id": "6e03d604-ca33-4547-b6d1-5cdbd7b2fe1b",
"name": "FS1 - Get Forms",
"body": "//Check Kobo account for forms with these matching keywords\ngetForms({}, state => {\n //ALL KEYWORDS:\n //const keywords = ['price', 'prix', 'bns', 'nrgt', 'grm', 'feedback'];\n\n //BNS KEYWORDS ONLY\n const keywords = ['price', 'prix', 'bns', 'nrgt'];\n\n const checkForKeyWords = name => {\n return keywords.some(keyword => name.toLowerCase().includes(keyword));\n };\n\n state.activeForms = state.data.results\n .filter(form => checkForKeyWords(form.name))\n .filter(form => form.deployment__active);\n\n state.archivedForms = state.data.results\n .filter(form => checkForKeyWords(form.name))\n .filter(form => !form.deployment__active);\n \n console.log('# of activeForms ::', state.activeForms ? state.activeForms.length : null );\n console.log('# of archivedForms ::', state.archivedForms ? state.archivedForms.length : null );\n\n state.data = {};\n state.references = [];\n return state;\n});\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": "950c9c55-fb89-4d8c-ab24-e2ca04a38ba7"
},
"FS2---Get-List-from-Sheets": {
"id": "baf76774-7aff-41e6-a2f8-db557595e423",
"name": "FS2 - Get List from Sheets",
"body": "getValues(\n '1xexwj6HKJGHJM-sOzIOGqPHolgpdnp0GmjMIXwoV7OE', //googlesheet id\n 'wcs-bns-DEPLOYED!A:O' //range of columns in sheet\n);\nfn(state => {\n const { activeForms, archivedForms, data } = state;\n const [headers, ...sheetsData] = data.values;\n const sheetsUids = sheetsData.map(row => row[0]);\n console.log('Ignoring headers', headers);\n\n state.formsToCreate = activeForms.filter(\n form => !sheetsUids.includes(form.uid)\n );\n\n state.formsToUpdate = archivedForms\n .filter(form => sheetsUids.includes(form.uid))\n .map(form => {\n const rowIndex = sheetsData.findIndex(row => {\n return row[0] === form.uid;\n });\n if (rowIndex !== -1) {\n return { ...form, rowIndex };\n }\n console.log(form.uid, 'Could not be found in google sheet');\n });\n\n return state;\n});\n\nfn(state => {\n const { data, references, response, ...remainingState } = state;\n\n return remainingState;\n});\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": "8f098581-1fba-449e-b7e5-49d7e8edc66b"
},
"FS3---Update-Spreadsheet-with-New-Forms-Shared": {
"id": "dec99f73-9804-4cd6-ac33-bd8ac65338d0",
"name": "FS3 - Update Spreadsheet with New Forms Shared",
"body": "//Compare new forms in Kobo with GoogleSheet list to see if new forms were shared in Kobo\nfn(state => {\n const { formsToCreate, formsToUpdate } = state;\n const keywords = ['price', 'prix', 'bns', 'nrgt', 'grm', 'feedback'];\n\n const tagMapping = {\n price: 'bns_price',\n prix: 'bns_price',\n bns: 'bns_survey',\n nrgt: 'nrgt_current',\n grm: 'grm',\n feedback: 'grm',\n };\n\n const createTagName = name => {\n let tag = '';\n const keyword = keywords.find(keyword =>\n name.toLowerCase().includes(keyword)\n );\n\n if (keyword) {\n tag = tagMapping[keyword] || keyword;\n }\n return tag;\n };\n\n const containsGRMFeedback = name =>\n !name.toLowerCase().includes('grm', 'feedback');\n\n const instance = name =>\n containsGRMFeedback(name) ? 'ADD MANUALLY @Admin!' : '';\n\n const projectId = name =>\n containsGRMFeedback(name) ? 'ADD MANUALLY @Admin!' : '';\n\n const grmID = name => (containsGRMFeedback(name) ? 'GRM ID. XX' : '');\n\n const workspaceName = name =>\n containsGRMFeedback(name) ? 'Grievances' : 'ConSoSci';\n \n state.formLastModified = form => form.date_modified; \n\n const sheetRowMap = form => [\n form.uid,\n form.name,\n createTagName(form.name),\n form.owner__username,\n instance(form.name),\n //projectId(form.name), //for GRM only\n //grmID(form.name), //for GRM only\n form.deployment__active ? 'deployed' : 'archived', //deployment status //if we assume only deployed forms will be fetched\n 'ConSoSci', //openfn project space --> OLD dynamic mapping: //workspaceName(form.name),\n `https://kf.kobotoolbox.org/#/forms/${form.url}/summary`, //form.url,\n form.date_modified, //kobo_form_date_modified\n form.date_created, //kobo_form_date_created\n new Date().toISOString(), //row_date_modified\n false, //auto_sync checkbox\n //job code template\n `\"{id: '${form.uid}', tag: '${createTagName(form.name)}', name: '${\n form.name\n }', owner: '${form.owner__username}', instance: '${instance(form.name)}'},\"`,\n ];\n\n state.rowValuesToCreate = formsToCreate.map(form => sheetRowMap(form));\n state.rowValuesToUpdate = formsToUpdate.map(form => ({\n range: `wcs-bns-DEPLOYED!A${form.rowIndex + 2}:N${form.rowIndex + 2}`,\n values: [sheetRowMap(form)],\n }));\n state.rowValuesToArchive = formsToUpdate.map(form => sheetRowMap(form));\n\n console.log('# of new forms detected:: ', state.rowValuesToCreate.length);\n console.log('Forms to add to the master sheet:: ', state.rowValuesToCreate);\n return state;\n});\n\n//if new Kobo form shared, adding to the \"Deployed\"\" Sheet...\nappendValues({\n spreadsheetId: '1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY', //sheet id\n range: 'wcs-bns-DEPLOYED!A:O', //range of columns in sheet\n values: state => state.rowValuesToCreate,\n});\n\n//updating rows in Sheet where forms are archived\neach(\n '$.rowValuesToUpdate[*]',\n batchUpdateValues({\n spreadsheetId: '1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY', //sheet id\n range: state => state.data.range, //range of columns in sheet\n values: state => state.data.values,\n })\n);\n\n//also adding archived rows to \"Archived\" Sheet...\nappendValues({\n spreadsheetId: '1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY', //sheet id\n range: 'wcs-bns-ARCHIVED!A:O', //range of columns in sheet\n values: state => state.rowValuesToArchive,\n});\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": "8f098581-1fba-449e-b7e5-49d7e8edc66b"
},
"FS4---Notify-in-Asana": {
"id": "b72847b2-dd6a-42d7-a9d8-e87a6a227695",
"name": "FS4 - Notify in Asana",
"body": "//This job will add a task to Asana if a new Kobo form was shared\nfn(state => {\n console.log('formLastModifiedDate:: ', state.formLastModified); \n const dueDate = new Date(new Date().getTime() + 5 * 24 * 60 * 60 * 1000)\n .toISOString()\n .split('T')[0];\n\n state.asanaTasks = state.activeForms.map(form => {\n return {\n name: `New form added to OpenFn: ${form.name}`,\n approval_status: 'pending',\n projects: ['1198901998266253'],\n assignee_section: '1207247884457665', //OLD General Section: '1203181218738601',\n assignee: '1208302456826465',\n due_on: dueDate,\n notes: `New form added to OpenFn: ${form.name} (uid: ${form.uid}). Please review the Google Sheet to (1) update columns L, N, & O, and (2) update column E (look for cells where it says \"ADD MANUALLY\" to add any values missing e.g., \"Instance\"): https://docs.google.com/spreadsheets/d/1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY/edit#gid=1559623602`,\n };\n });\n\n state.archivedFormsTasks = state.archivedForms.map(form => {\n return {\n name: `Form archived: ${form.name}`,\n projects: ['1198901998266253'],\n assignee_section: '1207247884457665', //OLD General Section: '1203181218738601',\n assignee: '1208302456826465',\n due_on: dueDate,\n notes: `Kobo form was archived: ${form.name} (uid: ${form.uid}). Please review the Google Sheet to (1) confirm this is correct, (2) remove from the \"Deployed\" sheet if you want to remove from the OpenFn Sync, and (3) update notes in the \"Archived\" sheet: https://docs.google.com/spreadsheets/d/1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY/edit#gid=1559623602`,\n };\n });\n\n console.log('# of New Form Asana Tasks to add:: ', state.asanaTasks.length);\n console.log('New form alert tasks to upsert:: ', state.asanaTasks);\n console.log(\n '# of Archibed Form Asana Tasks to add:: ',\n state.archivedFormsTasks.length\n );\n console.log(\n 'Archived form alert tasks to upsert:: ',\n state.archivedFormsTasks\n );\n return state;\n});\n\n//upsert Asana task if new form shared notification needed\neach(\n '$.asanaTasks[*]',\n upsertTask('1198901998266253', {\n //project_id\n externalId: 'name',\n data: state => state.data,\n })\n);\n\neach(\n '$.archivedFormsTasks[*]',\n upsertTask('1198901998266253', {\n //project_id\n externalId: 'name',\n data: state => state.data,\n })\n);\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": "dfc9700d-b427-45f3-b91a-e4059e8a663f"
}
},
"edges": {
"cron->FS1---Get-Forms": {
"enabled": true,
"id": "21dbd800-bb57-4164-8b76-82d2b650ea37",
"target_job_id": "6e03d604-ca33-4547-b6d1-5cdbd7b2fe1b",
"source_trigger_id": "f2d20419-c117-42b7-8c01-7366b82060ef",
"condition_type": "always"
},
"FS1---Get-Forms->FS2---Get-List-from-Sheets": {
"enabled": true,
"id": "7d5485c9-82c3-40cd-9fed-66f9a294cbc5",
"target_job_id": "baf76774-7aff-41e6-a2f8-db557595e423",
"source_job_id": "6e03d604-ca33-4547-b6d1-5cdbd7b2fe1b",
"condition_type": "on_job_success"
},
"FS2---Get-List-from-Sheets->FS3---Update-Spreadsheet-with-New-Forms-Shared": {
"enabled": true,
"id": "e61859ca-20ba-4a4d-88de-bb6dcf35dd23",
"target_job_id": "dec99f73-9804-4cd6-ac33-bd8ac65338d0",
"source_job_id": "baf76774-7aff-41e6-a2f8-db557595e423",
"condition_type": "on_job_success"
},
"FS3---Update-Spreadsheet-with-New-Forms-Shared->FS4---Notify-in-Asana": {
"enabled": false,
"id": "51b306a5-092c-4e9a-af08-ba9208b4eebe",
"target_job_id": "b72847b2-dd6a-42d7-a9d8-e87a6a227695",
"source_job_id": "dec99f73-9804-4cd6-ac33-bd8ac65338d0",
"condition_type": "on_job_success"
}
}
},
"1.1-Get-BNS-FormsList-(Ongoing)": {
"id": "89b68b57-226e-4696-819f-ab79dc382280",
"name": "1.1 Get BNS FormsList (Ongoing)",
"inserted_at": "2025-04-14T14:40:02.168805Z",
"lock_version": 24,
"triggers": {
"cron": {
"enabled": false,
"id": "b6c56639-4b31-4812-91fe-f327bbf219c1",
"type": "cron",
"cron_expression": "0 */3 * * *"
}
},
"jobs": {
"Get-Forms-List": {
"id": "3936a1b5-88ca-4dda-9021-d1f8d4803dd6",
"name": "Get Forms List",
"body": "//== Job to be used for getting a list of \"deployed\" Kobo forms from sheets to auto-sync ==//\n// This can be run on-demand at any time by clicking \"run\" or modify manualCursor below //\ngetValues(\n '1xexwj6HKJGHJM-sOzIOGqPHolgpdnp0GmjMIXwoV7OE',\n 'wcs-bns-DEPLOYED!A:L', //get Deployed forms list from Sheet\n state => {\n const [headers, ...values] = state.data.values;\n\n const mapHeaderToValue = value => {\n return headers.reduce((obj, header) => {\n obj[header] = value[headers.indexOf(header)];\n return obj;\n }, {});\n };\n\n state.sheetsData = values\n .filter(\n item =>\n item.includes('TRUE') //return forms where auto-sync = TRUE\n //&& item.includes('bns_survey', 'nrgt_current') \n )\n .map(item => mapHeaderToValue(item));\n\n return state;\n }\n);\n\nfn(state => {\n const { sheetsData } = state;\n\n // Set a manual cursor if you'd like to only fetch data after this date...\n //e.g., '2023-01-01T23:51:45.491+01:00'\n const manualCursor = '2024-12-06T00:00:00.000Z'; //lastUsed: 2024-11-21T00:00:00.000Z\n console.log('manualCursor defined?', manualCursor);\n \n //...otherwise the job will use this dynamicCursor\n const dynamicCursor = getTodayISODate(); \n\n function getTodayISODate() {\n const today = new Date();\n today.setUTCHours(0, 0, 0, 0); // Set hours, minutes, seconds, and milliseconds to 0\n return today.toISOString(); // Convert to ISO string\n }\n \n //UNCOMMENT FOR FUTURE\n // const cursorValue = dynamicCursor || manualCursor ; \n const cursorValue = manualCursor ; \n \n // const cursorValue = manualCursor ; \n console.log('Cursor value to use in query:', cursorValue);\n\n const formsList = sheetsData.map(survey => ({\n formId: survey.uid,\n tag: survey.tag,\n name: survey.name \n })); \n \n console.log('# of deployed forms detected in Sheet:: ', formsList.length);\n console.log('List of forms to auto-sync:: ', JSON.stringify(formsList, null, 2)); \n\n state.data = {\n surveys: sheetsData.map(survey => ({\n formId: survey.uid,\n tag: survey.tag,\n name: survey.name,\n owner: survey.owner,\n url: `https://kf.kobotoolbox.org/api/v2/assets/${survey.uid}/data/?format=json`,\n query: `&query={\"start\":{\"$gte\":\"${cursorValue}\"}}`,\n })),\n };\n return state;\n});\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": "8f098581-1fba-449e-b7e5-49d7e8edc66b"
},
"Get-BNS-NRGT-Forms": {
"id": "6a095185-628e-4e8f-bee4-e39c601c2b30",
"name": "Get BNS NRGT Forms",
"body": "// Here we fetch submissions for all \"Deployed\" forms in GoogleSheet\n// NOTE: See linked job \"[BNS-1A] 1.Get FormsList (Ongoing)\" for cursor & GoogleSheet query logic\n//**********************************************************//\n\nfn(state => {\n state.surveySubmissions = [];\n state.errors = [];\n state.globalIndex = 0;\n console.log('surveys ::', JSON.stringify(state.data.surveys, null, 2));\n return state;\n});\n\neach('$.data.surveys[*]', state => {\n const { url, query, tag, formId, name, owner } = state.data;\n console.log('Sending GET to ::', `${url}${query}`);\n\n return get(`${url}${query}`)(state)\n .then(state => {\n const results = state.data.results.map(submission => {\n const uniqueIndex = state.globalIndex++;\n return {\n i: uniqueIndex,\n // Here we append the tags defined above to the Kobo form submission data\n form: tag,\n formName: name,\n formOwner: owner,\n body: submission,\n };\n });\n\n state.surveySubmissions.push(...results);\n const count = results.length;\n console.log(`Fetched ${count} submissions from ${formId} (${tag}).`);\n //Once we fetch the data, we want to post each individual Kobo survey\n //back to the OpenFn inbox to run through the jobs =========================\n return state;\n })\n .catch(err => {\n state.errors.push({\n formId,\n message: err.message,\n });\n console.log(`Error fetching submissions from ${formId}::`, err.message);\n return state;\n });\n});\n\neach(\n '$.surveySubmissions[*]',\n post(\n \"https://app.openfn.org/i/b3f86593-f37e-4139-80b8-852b9d3c49f4\",\n state => {\n const {i, formName} = state.data;\n const count = state.surveySubmissions.length;\n console.log(`Posting ${i} of ${count} from ${formName}`);\n return { body: state.data };\n }\n )\n); \n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": "89b65a0e-e01d-4760-b021-1778799e9b2e"
}
},
"edges": {
"cron->Get-Forms-List": {
"enabled": true,
"id": "c3fd057e-7257-4ba3-924f-7e90170a0d10",
"target_job_id": "3936a1b5-88ca-4dda-9021-d1f8d4803dd6",
"source_trigger_id": "b6c56639-4b31-4812-91fe-f327bbf219c1",
"condition_type": "always"
},
"Get-Forms-List->Get-BNS-NRGT-Forms": {
"enabled": true,
"id": "900d23a9-5f96-447d-8d45-444ebff8b56b",
"target_job_id": "6a095185-628e-4e8f-bee4-e39c601c2b30",
"source_job_id": "3936a1b5-88ca-4dda-9021-d1f8d4803dd6",
"condition_type": "on_job_success"
}
}
},
"1.2-Get-BNS-FormsList-(Historical)": {
"id": "d064ad3d-f301-4071-9af3-77ede0dab5f6",
"name": "1.2 Get BNS FormsList (Historical)",
"inserted_at": "2025-04-14T14:40:17.778719Z",
"lock_version": 49,
"triggers": {
"webhook": {
"enabled": false,
"id": "8c2a16dd-f19c-4e71-8ef6-3f06e9806b43",
"type": "webhook"
}
},
"jobs": {
"Get-FormsList": {
"id": "31883ef6-acc3-4cc3-bfca-647c18897496",
"name": "Get FormsList",
"body": "getValues(\n '1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY',\n 'wcs-bns-DEPLOYED!A:O', //get Deployed forms list from Sheet\n state => {\n const [headers, ...values] = state.data.values;\n\n const mapHeaderToValue = value => {\n return headers.reduce((obj, header) => {\n obj[header] = value[headers.indexOf(header)];\n return obj;\n }, {});\n };\nconsole.log(\"deployedData\", values);\n\n state.deployedData = values\n .map(item => mapHeaderToValue(item))\n .filter(item => item['historical_sync'] === 'TRUE');\n return state;\n }\n);\n//== Job to be used for getting a list of \"archived\" Kobo forms from sheets to auto-sync ==//\n// This can be run on-demand at any time by clicking \"run\" //\ngetValues(\n '1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY',\n 'wcs-bns-ARCHIVED!A:O', //get Deployed forms list from Sheet\n state => {\n const [headers, ...values] = state.data.values;\n\n const mapHeaderToValue = value => {\n return headers.reduce((obj, header) => {\n obj[header] = value[headers.indexOf(header)];\n return obj;\n }, {});\n };\nconsole.log(\"archivedData\", values);\n\n\n state.archivedData = values\n .map(item => mapHeaderToValue(item))\n .filter(item => item['historical_sync'] === 'TRUE');\n\n return state;\n }\n);\n\nfn(state => {\n const { archivedData, deployedData } = state;\n\n // Set a manual cursor if you'd like to only fetch data after this date...\n //e.g., '2023-01-01T23:51:45.491+01:00'\n // const manualCursor = ''; //lastUsed: 2024-04-01T00:00:00.000Z\n // console.log('manualCursor defined?', manualCursor);\n //...otherwise the job will use this dynamicCursor\n // const dynamicCursor = getTodayISODate();\n\n // function getTodayISODate() {\n // const today = new Date();\n // today.setUTCHours(0, 0, 0, 0); // Set hours, minutes, seconds, and milliseconds to 0\n // return today.toISOString(); // Convert to ISO string\n // }\n\n // const cursorValue = manualCursor || dynamicCursor;\n // console.log('Cursor value to use in query:', cursorValue);\n const combinedData = [...deployedData, ...archivedData];\n const formsList = combinedData.map(survey => ({\n formId: survey.uid,\n tag: survey.tag,\n name: survey.name,\n }));\n\n console.log('# of forms detected in Sheet:: ', formsList.length);\n console.log(\n 'List of forms to re-sync:: ',\n JSON.stringify(formsList, null, 2)\n );\n\n state.data = {\n surveys: combinedData.map(survey => ({\n formId: survey.uid,\n tag: survey.tag,\n name: survey.name,\n owner: survey.owner,\n url: `https://kf.kobotoolbox.org/api/v2/assets/${survey.uid}/data/?format=json`,\n //query: `&query={\"end\":{\"$gte\":\"${cursorValue}\"}}`, //get ALL forms for historical job\n })),\n };\n return state;\n});\n\n//Clear final state\nfn(state => {\n delete state.references;\n delete state.response;\n return state;\n}); \n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": "8f098581-1fba-449e-b7e5-49d7e8edc66b"
},
"Get-Forms": {
"id": "e8e4e659-d4f0-4fd3-b109-9d96d885505f",
"name": "Get Forms",
"body": "// Here we fetch submissions for all \"Archived\" forms in GoogleSheet\n// NOTE: See linked job \"[BNS-1B] 1.Get FormsList (Historical)\" for GoogleSheet query logic\n//**********************************************************//\neach(dataPath('surveys[*]'), state => {\n const { url, tag, formId, name, owner } = state.data;\n const query = \"&query={\\\"start\\\":{\\\"$gte\\\":\\\"2021-10-29T00:00:00.000Z\\\"}}\"\n return get(`${url}${query}`, {}, state => {\n console.log(state.data.result)\n state.data.submissions = state.data.results.map((submission, i) => {\n return {\n i,\n // Here we append the tags defined above to the Kobo form submission data\n form: tag,\n formName: name,\n formOwner: owner,\n body: submission,\n };\n });\n const count = state.data.submissions.length;\n console.log('Finding historical forms to resync...');\n console.log(`Fetched ${count} submissions from ${formId} (${tag}).`);\n //Once we fetch the data, we want to post each individual Kobo survey\n //back to the OpenFn inbox to run through the jobs =========================\n return each(dataPath('submissions[*]'), state => {\n console.log(`Posting ${state.data.i + 1} of ${count}...`);\n return post(\"https://app.openfn.org/i/b3f86593-f37e-4139-80b8-852b9d3c49f4\", {\n body: state => state.data,\n })(state);\n })(state);\n })(state)\n});\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": "89b65a0e-e01d-4760-b021-1778799e9b2e"
}
},
"edges": {
"Get-FormsList->Get-Forms": {
"enabled": true,
"id": "eaf51415-d4e7-41bf-9d38-9d6e5765f78a",
"target_job_id": "e8e4e659-d4f0-4fd3-b109-9d96d885505f",
"source_job_id": "31883ef6-acc3-4cc3-bfca-647c18897496",
"condition_type": "on_job_success"
},
"webhook->Get-FormsList": {
"enabled": true,
"id": "c9db77c0-2540-40d5-8709-60899c6b4780",
"target_job_id": "31883ef6-acc3-4cc3-bfca-647c18897496",
"source_trigger_id": "8c2a16dd-f19c-4e71-8ef6-3f06e9806b43",
"condition_type": "always"
}
}
},
"x_A1-Generate-Jobs-DB-Tables-and-Dictionary-AUTO": {
"id": "130c93c1-371a-4883-bc7e-073da26dc263",
"name": "x_A1 Generate Jobs DB Tables and Dictionary AUTO",
"inserted_at": "2025-04-15T07:37:26.821998Z",
"lock_version": 10,
"triggers": {
"cron": {
"enabled": false,
"id": "6b373324-0767-4bc2-a668-1d4e26bd6e9c",
"type": "cron",
"cron_expression": "0 */3 * * *"
}
},
"jobs": {
"A1-Generate-Jobs-DB-Tables-and-Dictionary-AUTO": {
"id": "f5836ca3-5263-4137-93e6-1b0eb51cb5e8",
"name": "A1 Generate Jobs DB Tables and Dictionary AUTO",
"body": "get('https://kf.kobotoolbox.org/api/v2/assets/?format=json', {}, state => {\n console.log(`Previous cursor: ${state.lastEnd}`);\n // Set a manual cursor if you'd like to only fetch form after a certain date\n const manualCursor = '2019-05-25T14:32:43.325+01:00';\n\n // ===========================================================================\n // == FOR ADMINS: Update the below `manualFormList` to designate which Kobo forms to sync ==//\n\n const manualFormList = [\n //==================== Forms must be shared with the account openfn_kobo====================//\n //=== WCS Camera trap metadata =====\n // {\n // uid: 'axDXRTMWEkhrQDYQ9K3YdT', // Form name: 1. Project and cameras\n // p1: 'WCSPROGRAMS',\n // p2: 'CameraKobo',\n // tableId: 'Project' // Result is Run 062f3e43-46fe-7332-99e3-07cce87ddbdf\n // },\n {\n uid: 'axfD6ntJyhfD2mxAGuVRSE', // Form name: 2. Deployments\n p1: 'WCSPROGRAMS',\n p2: 'CameraKobo',\n tableId: 'Deployment' // Result is Run 062f3e47-ed7f-7fc5-b367-4c0f9b530053\n },\n // {\n // uid: 'a4yYjawjdbpHcckBx8m8AP', // Form name: 3. Retrieval\n // p1: 'WCSPROGRAMS',\n // p2: 'CameraKobo',\n // tableId: 'Retrieval' // Result is Run 062f3e43-10bc-7697-9417-11c931b14c8f\n // },\n // {\n // uid: 'a9F5e7wMMopSm85Abw3LTN', // Form name: 4. Images\n // p1: 'WCSPROGRAMS',\n // p2: 'CameraKobo',\n // tableId: 'Image'\n // },\n //=== WCS Socio Economic Database =====\n // {\n // uid: 'aukhdejQU76K33caCkF4rP',\n // p1: 'WCSPROGRAMS',\n // p2: 'SocioEco',\n // tableId: 'SocioEcoSurvey'\n // },\n //==== SharksRays ===============//\n // {\n // uid: 'aaknL3DQQgkgZ8iay89X5P',\n // p1: 'WCSPROGRAMS',\n // p2: '',\n // tableId: 'SharksRays',\n // },\n // {\n // uid: 'aStMvYShWXZsKYa7AyN6sr',\n // p1: 'WCSPROGRAMS',\n // p2: '',\n // tableId: 'SharksRays',\n // },\n // {\n // uid: 'aQeXAtEkgg8PGwxDiCUnPW',\n // p1: 'WCSPROGRAMS',\n // p2: '',\n // tableId: 'SharksRays',\n // },\n //=== Trillion Trees forms =====\n // {\n // uid: 'aHPGTtrrLB4k3xDA9UZipu',\n // p1: 'WCSPROGRAMS',\n // p2: '',\n // tableId: 'Site',\n // },\n // {\n // uid: 'a8ffyF7HgbFUEnYBppEL79',\n // p1: 'WCSPROGRAMS',\n // p2: '',\n // tableId: 'Land',\n // },\n //=================================\n // {\n // uid: 'avLpvrukkvuFzCHacjHdRs',\n // p1: 'WCS',\n // p2: 'Vegetation',\n // tableId: 'VegetationClassficationAndTreeMeasurementForm'},\n\n //{ uid: 'apZrpKcK78xzrPcAfRrfac', p1: 'OpenFn', p2: 'Sharks', tableId: 'SharkRaysMay4Test'},\n //{ uid: 'azg4rJb2Kk8DT2upSPyYjB', p1: 'WCS', p2: 'Livestock', tableId: 'LivestockProduction'},\n //{ uid: 'aDgPJqN4SAYohZ4ZueEeYU', p1: 'WCS', p2: 'Arcadia', tableId: 'ArcadiaDataCollection'},\n //{ uid: 'a7Dx4vpFcj7ziwaKE4682U', p1: 'WCS', p2: 'Vegetation', tableId: 'VegetationClassficationAndTreeMeasurementForm'},\n //{ uid: 'apZrpKcK78xzrPcAfRrfac', p1: 'WCS', p2: 'SR', tableId: 'SharkAndRaysTraining'}\n ];\n\n state.data.forms = state.data.results\n // Filter the response from Kobo to show only those forms we want to update.\n .filter(form => manualFormList.map(x => x.uid).includes(form.uid))\n .filter(form => {\n // Note: If a form in manualFormList was not present in the list during\n // the last run of the job (formsWatched), then we always trigger an\n // update for that form.\n if (!state.formsWatched) {\n return true;\n }\n if (!state.formsWatched.find(f => f.uid === form.uid)) {\n console.log(`New form ${form.uid} (${form.name}) added to watch list.`);\n return true;\n }\n // Note: If a form is not NEW to the watch list, then we only trigger an\n // update if it has been modified more recently than the greatest\n // last-modified date across all forms from our last run.\n return form.date_modified > (state.lastEnd || manualCursor);\n })\n // Map those forms so that we can post each to the inbox later.\n .map(form => {\n const url = form.url.split('?').join('?');\n const manualSpec = manualFormList.find(f => f.uid === form.uid);\n return {\n formId: form.uid,\n tag: manualSpec.surveyTable || form.name,\n //tag: form.name,\n url,\n prefix1: manualSpec.p1,\n prefix2: manualSpec.p2 || '',\n tableId: manualSpec.tableId,\n lastModified: form.date_modified,\n };\n });\n\n // Set lastEnd to the greatest date_modified value for all forms we care about.\n const lastEnd = state.data.results\n .filter(item => item.date_modified)\n .map(s => s.date_modified)\n .sort((a, b) => new Date(b.date) - new Date(a.date))[0];\n\n console.log(\n 'Detected changes for:',\n JSON.stringify(\n state.data.forms.map(f => f.url),\n null,\n 2\n )\n );\n\n return { ...state, lastEnd, formsWatched: manualFormList };\n});\n\neach(dataPath('forms[*]'), state => {\n const form = state.data;\n return post(\n state.configuration.openfnInboxUrl,\n { body: { ...form, formUpdate: true } },\n state => {\n console.log('Sent ', form.tag, ' for handling:');\n console.log(form);\n return state;\n }\n )(state);\n});\n\n// Clear everything from state but the required cursors.\nalterState(state => ({\n lastEnd: state.lastEnd,\n formsWatched: state.formsWatched,\n}));\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": null
}
},
"edges": {
"cron->A1-Generate-Jobs-DB-Tables-and-Dictionary-AUTO": {
"enabled": true,
"id": "2e483885-f409-4328-8a61-e1d7fb41a0b0",
"target_job_id": "f5836ca3-5263-4137-93e6-1b0eb51cb5e8",
"source_trigger_id": "6b373324-0767-4bc2-a668-1d4e26bd6e9c",
"condition_type": "always"
}
}
},
"Test-Martin": {
"id": "d2006259-f990-406f-b63d-38ad3ad834f4",
"name": "Test Martin",
"inserted_at": "2025-04-15T12:24:55.378744Z",
"lock_version": 1,
"triggers": {
"webhook": {
"enabled": false,
"id": "1779d825-f58f-481f-8c2f-3b6338dad750",
"type": "webhook"
}
},
"jobs": {
"New-job": {
"id": "3438fb91-916d-45a9-824d-8bb11baf5e4a",
"name": "New job",
"body": "// Check out the Job Writing Guide for help getting started:\n// https://docs.openfn.org/documentation/jobs/job-writing-guide\n",
"adaptor": "@openfn/language-common@latest",
"project_credential_id": null
}
},
"edges": {
"webhook->New-job": {
"enabled": true,
"id": "89641e06-7eea-4768-8313-4ef5e889a7e7",
"target_job_id": "3438fb91-916d-45a9-824d-8bb11baf5e4a",
"source_trigger_id": "1779d825-f58f-481f-8c2f-3b6338dad750",
"condition_type": "always"
}
}
},
"2.-Sync-BNS-and-NRGT-Forms": {
"id": "83fd9ae8-4dc6-4dd9-bc9b-f5940c5d8ef5",
"name": "2. Sync BNS and NRGT Forms",
"inserted_at": "2025-04-15T14:06:22.931233Z",
"lock_version": 82,
"triggers": {
"webhook": {
"enabled": true,
"id": "b3f86593-f37e-4139-80b8-852b9d3c49f4",
"type": "webhook"
}
},
"jobs": {
"Triage-jobs": {
"id": "65e96922-0db6-4823-88ea-64601641c902",
"name": "Triage jobs",
"body": "// Check out the Job Writing Guide for help getting started:\n// https://docs.openfn.org/documentation/jobs/job-writing-guide\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": "b5fa2c78-00ab-4795-92d0-1078e7789aaa"
},
"BNS-2A-BNS-Price": {
"id": "7280bfa9-2a67-4c6e-8a3a-f7f21bcb26ae",
"name": "BNS-2A BNS Price",
"body": "// NOTE: This data cleaning operation returns state, modified as needed.\nfn(state => {\n //try {\n const { body, formName, instance, formOwner } = state.data;\n const { _submission_time, _id, _xform_id_string } = body;\n let cleanedSubmission = {};\n\n for (const key in body) {\n switch (body[key]) {\n case 'yes':\n cleanedSubmission[key] = 1;\n break;\n\n case 'no':\n cleanedSubmission[key] = 0;\n break;\n\n default:\n cleanedSubmission[key] = body[key];\n break;\n }\n }\n\n cleanedSubmission.instance = instance;\n\n const landscapeMap = {\n Ndoki: 'ndoki',\n 'Lac Télé': 'lac_tele',\n Ituri: 'ituri',\n Kahuzi: 'kahuzi',\n MTKB: 'kahuzi',\n 'Cross River': 'crossriver',\n Soariake: 'soariake',\n Ankarea: 'ankarea',\n ABS: 'baie_antongil',\n 'Nosy Be': 'tandavandriva',\n Makira: 'makira',\n 'BNS Ndoki Prix 2020': 'ndoki',\n PNMD: 'pnmd',\n\n //formName: landscapeValue,\n //other values\n };\n\n return {\n ...state,\n landscapeMap,\n formName,\n formOwner,\n data: {\n ...cleanedSubmission,\n durableUUID: `${_submission_time}-${_xform_id_string}-${_id}`,\n datasetId: `${formName}-${_xform_id_string}`,\n end: cleanedSubmission.end.slice(0, 10),\n },\n };\n /* } catch (error) {\n state.connection.close();\n throw error;\n }*/\n});\n\n// Refactor this for scale so it doesn't perform a no-op delete 9/10 times.\n// Maybe check result of previous op, then only delete if it was an update.\nsql({\n query: state =>\n `DELETE FROM WCSPROGRAMS_KoboBnsPrice where AnswerId = '${state.data._id}'`,\n});\n\nfn(state => {\n const { good } = state.data;\n if (!good || good.length === 0) {\n return state;\n }\n\n const data = good.map((g, i) => ({\n // Id: state.data._id,\n Id: i + 1,\n AnswerId: state.data._id,\n DatasetUuidId: state.data.datasetId,\n Surveyor: state.data.surveyor,\n Village: state.data.village,\n Gs: g[`good/name`],\n Price: g[`good/price`],\n LastUpdate: new Date().toISOString(),\n //Landscape: state.landscapeMap[state.data.formName], //see L24 for mappings. We want to use formName to look-up a new value\n Landscape: () => {\n for (let val in state.landscapeMap)\n if (state.formName.includes(val)) return state.landscapeMap[val];\n return '';\n },\n SurveyDate: state.data.today,\n }));\n // console.log('data', data);\n return insertMany('WCSPROGRAMS_KoboBnsPrice', data)(state);\n});\n\nfn(state => {\n console.log('DatasetName ::', state.formName);\n console.log('DatasetOwner ::', state.formOwner);\n console.log('form submission id ::', state.data['_id']);\n console.log('DatasetUuidId ::', state.data['datasetId']);\n //console.log('data to upload ::', state.data);\n return state;\n});\n\nupsert('WCSPROGRAMS_KoboData', 'DatasetUuidId', {\n //AnswerId: dataValue('durableUUID'),\n DatasetName: state => state.formName,\n DatasetOwner: state => state.formOwner,\n Landscape: dataValue('landscape'),\n DatasetUuidId: dataValue('datasetId'),\n DatasetYear: new Date().getFullYear(),\n LastSubmissionTime: dataValue('_submission_time'),\n LastCheckedTime: dataValue('_submission_time'),\n LastUpdateTime: new Date().toISOString(),\n KoboManaged: true,\n Tags: dataValue('_tags'),\n Citation: dataValue('instance'),\n});\n\nfn(state => {\n console.log('data uploaded ::', state.data);\n return state;\n});\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": "b5fa2c78-00ab-4795-92d0-1078e7789aaa"
},
"BNS-2B-BNS-Survey": {
"id": "196697f9-b581-4397-8487-280df2d47529",
"name": "BNS 2B BNS Survey",
"body": "// NOTE: This data cleaning operation returns state, modified as needed.\n\nfn(state => {\n try {\n const { body, formName, formOwner, instance } = state.data;\n const { _submission_time, _id, _xform_id_string } = body;\n\n let cleanedSubmission = {};\n\n for (const key in body) {\n switch (body[key]) {\n case 'yes':\n cleanedSubmission[key] = 1;\n break;\n\n case 'no':\n cleanedSubmission[key] = 0;\n break;\n\n default:\n cleanedSubmission[key] = body[key];\n break;\n }\n }\n\n // NOTE: This assumes all device-collected geo data follows specific lat, log data format\n if (cleanedSubmission.gps_method === 'device') {\n cleanedSubmission['gps/lat'] =\n cleanedSubmission.geo && cleanedSubmission.geo.split(' ')[0];\n cleanedSubmission['gps/long'] =\n cleanedSubmission.geo && cleanedSubmission.geo.split(' ')[1];\n } else if (\n Math.abs(parseFloat(cleanedSubmission['gps/lat'])) > 90 ||\n Math.abs(parseFloat(cleanedSubmission['gps/long'])) > 180\n ) {\n console.log(\n `WARNING: Discarding invalid manual GPS entry: 'gps/lat': ${cleanedSubmission['gps/lat']}; 'gps/long': ${cleanedSubmission['gps/long']}`\n );\n delete cleanedSubmission['gps/lat'];\n delete cleanedSubmission['gps/long'];\n }\n\n cleanedSubmission.durableUUID = `${_submission_time}-${_xform_id_string}-${_id}`; //survey uuid\n cleanedSubmission.datasetId = `${formName}-${_xform_id_string}`; //dataset uuid\n cleanedSubmission.instance = instance;\n state.data = cleanedSubmission;\n\n state.landscapeMap = {\n tns: 'ndoki',\n ltlt: 'lac_tele',\n mamabay: 'makira',\n mtkb: 'kahuzi',\n };\n\n // Cleaning datasetId if formName is 'BNS Cross River 2017-2020'==============\n if (formName.replace(/\\s/g, '') === 'BNSCrossRiver2017-2020') {\n state.data.datasetId = `${state.data.datasetId}${\n body.today.split('-')[0]\n }`;\n }\n\n // ===========================================================================\n\n // ===========================================================================\n // NOTE: These job mappings assume a specific Kobo form metadata naming syntax!\n // 'NR' and 'BNS matrix' questions should follow the naming conventions below\n // See Docs to learn more about the assumptions made here.\n // ===========================================================================\n // If a partner creates a form with slightly different field names, this\n // section will need to be updated by WCS. If future forms are being designed,\n // we'd recommend using a repeat group that allows the partner to select the\n // type of 'nr' or 'matrix' they're reporting on. The current approach treats\n // the form field names in Kobo _AS_ data themselves.\n state.nr = Object.keys(state.data)\n .filter(key => key.startsWith('nr/'))\n .map(key => ({\n DatasetUuidId: state.data.datasetId,\n AnswerId: state.data._id,\n Id: state.data._id,\n LastUpdate: new Date().toISOString(),\n Nr: key.substring(3),\n NrCollect: state.data[key],\n }));\n\n const matrix = Object.keys(state.data)\n .filter(key => key.includes('bns_matrix_'))\n .map(key => {\n const item = key.substring(\n key.lastIndexOf('bns_matrix_') + 'bns_matrix_'.length,\n key.lastIndexOf('_')\n );\n return {\n Dataset_Id: state.data.datasetId, //DatasetUuidId\n DatasetUuidId: state.data.datasetId,\n //Id: state.data._id,\n AnswerId: state.data._id,\n gs: item.replace(/_/g, ' '),\n have:\n state.data[\n `hh_assets/bns_matrix_${item}/bns_matrix_${item}_possess`\n ] || state.data[`bns_matrix_${item}/bns_matrix_${item}_possess`],\n necessary:\n state.data[\n `hh_assets/bns_matrix_${item}/bns_matrix_${item}_necessary`\n ] || state.data[`bns_matrix_${item}/bns_matrix_${item}_necessary`],\n quantity:\n state.data[\n `hh_assets/bns_matrix_${item}/bns_matrix_${item}_number`\n ] || state.data[`bns_matrix_${item}/bns_matrix_${item}_number`],\n };\n });\n\n state.matrix = matrix.filter(\n (x, i) => matrix.findIndex(y => y.gs == x.gs) == i\n );\n // ===========================================================================\n // console.log(\n // 'The bns_matrix',\n // JSON.stringify(state.matrix, null, 2),\n // `contains ${state.matrix.length} items.`\n // );\n console.log('instance: ', instance);\n return {...state, formOwner, formName};\n } catch (error) {\n state.connection.close();\n throw error;\n }\n});\n\nupsert('WCSPROGRAMS_KoboBnsAnswer', 'AnswerId', {\n DatasetUuidId: dataValue('datasetId'),\n //Id: dataValue('durableUUID'), //Q: does not exist, to add for consistency?\n SubmissionUuid: dataValue('_uuid'),\n AnswerId: dataValue('_id'),\n LastUpdate: new Date().toISOString(),\n SurveyDate: state => {\n const date = state.data.today || state.data._submission_time;\n const year = Number(date.trim().split('-')[0]);\n const formName = dataValue('formName');\n if (year <= 2010) return Number(formName.trim().split(' ').at(-1));\n return year;\n },\n Landscape: state => {\n var landscape = dataValue('landscape')(state);\n return state.landscapeMap[landscape] || landscape;\n },\n Surveyor: dataValue('surveyor'),\n Participant: dataValue('participant'),\n Arrival: dataValue('arrival'),\n District: dataValue('district'),\n Village: dataValue('village'),\n HhId: dataValue('hh_id'),\n BenefProject: dataValue('benef_project'),\n HhTypeControl: state => (state.data.hh_type === 'control' ? 1 : 0),\n HhTypeOrgBenef: state => (state.data.hh_type === 'wcs_benef' ? 1 : 0),\n HhTypeOtherBenef: state => (state.data.hh_type === 'other_benef' ? 1 : 0),\n ExplainProject: dataValue('explain_project'),\n KnowPa: dataValue('know_PA'),\n BenefPa: dataValue('benef_PA'),\n ExplainBenefPa: dataValue('explain_benef_PA'),\n Livelihood1: dataValue('livelihoods/l1'),\n Livelihood2: dataValue('livelihoods/l2'),\n Livelihood3: dataValue('livelihoods/l3'),\n Livelihood4: dataValue('livelihoods/l4'),\n BnsPlus: dataValue('bns_plus'),\n});\n\n// Refactor this for scale so it doesn't perform a no-op delete 9/10 times.\n// Maybe check result of previous op, then only delete if it was an update.\nsql({\n query: state =>\n `DELETE FROM WCSPROGRAMS_KoboBnsAnswerhhmembers where AnswerId = '${state.data._id}'`,\n});\n\ninsert('WCSPROGRAMS_KoboBnsAnswerhhmembers', {\n //insert hh head first\n DatasetUuidId: dataValue('datasetId'),\n Id: '0',\n //Id: state => state.data.hh_members.length,\n AnswerId: dataValue('_id'),\n Head: dataValue('gender_head') ? '1' : '0',\n Gender: dataValue('gender_head'),\n Ethnicity: dataValue('ethnicity_head'),\n Birth: state => {\n var birth = dataValue('birth_head')(state);\n return birth ? parseInt(birth.substring(0, 4)) : null;\n },\n LastUpdate: new Date().toISOString(),\n});\n\nfn(state => {\n if (state.data.hh_members) {\n return insertMany(\n 'WCSPROGRAMS_KoboBnsAnswerhhmembers',\n (\n state //then insert other members\n ) =>\n state.data.hh_members.map((member, i) => ({\n DatasetUuidId: state.data.datasetId,\n // Id: state.data._id,\n Id: i + 1,\n AnswerId: state.data._id,\n Head: '0',\n Gender:\n member[`hh_members/gender`] || member[`hh_members/gender_001`],\n Ethnicity: member[`hh_members/ethnicity`],\n Birth: parseInt(member[`hh_members/birth`].substring(0, 4)),\n LastUpdate: new Date().toISOString(),\n }))\n )(state);\n }\n\n console.log('No household members found.');\n return state;\n});\n\n// Refactor this for scale so it doesn't perform a no-op delete 9/10 times.\n// Maybe check result of previous op, then only delete if it was an update.\nsql({\n query: state =>\n `DELETE FROM WCSPROGRAMS_KoboBnsAnswernr where AnswerId = '${state.data._id}'`,\n});\n\nfn(state => {\n if (state.nr && state.nr.length > 0) {\n return insertMany('WCSPROGRAMS_KoboBnsAnswernr', state => state.nr)(state);\n }\n\n console.log('No natural resource found.');\n return state;\n});\n\n// Refactor this for scale so it doesn't perform a no-op delete 9/10 times.\n// Maybe check result of previous op, then only delete if it was an update.\n//sql({ query: state => `DELETE FROM WCSPROGRAMS_KoboBnsAnswergs where AnswerId = '${state.data._id}'` }); //ERROR: AnswerId does not exist\nsql({\n query: state =>\n `DELETE FROM WCSPROGRAMS_KoboBnsAnswerGS where AnswerId = '${state.data._id}'`,\n});\n\nfn(state => {\n if (state.matrix && state.matrix.length > 0) {\n return insertMany(\n 'WCSPROGRAMS_KoboBnsAnswerGS',\n state => state.matrix\n )(state);\n }\n\n console.log('No matrix found.');\n return state;\n});\n\nupsert('WCSPROGRAMS_KoboBnsAnswergps', 'AnswerId', {\n DatasetUuidId: dataValue('datasetId'), //Q: Add new column\n AnswerId: dataValue('_id'),\n Id: dataValue('_id'),\n Geom: dataValue('_geolocation'),\n Lat: state => {\n return dataValue('gps/lat')(state)\n ? dataValue('gps/lat')(state)\n : state.data._geolocation[0] || undefined;\n },\n Long: state => {\n return dataValue('gps/long')(state)\n ? dataValue('gps/long')(state)\n : state.data._geolocation[1] || undefined;\n },\n LastUpdate: new Date().toISOString(),\n});\n\nfn(state => {\n console.log('DatasetName ::', state.formName);\n console.log('DatasetOwner ::', state.formOwner);\n console.log('form submission id ::', state.data['_id']);\n console.log('DatasetUuidId ::', state.data['datasetId']);\n //console.log('data to upload ::', state.data);\n return state;\n})\n\nupsert('WCSPROGRAMS_KoboData', 'DatasetUuidId', {\n //renamed from DatasetUuid\n //AnswerId: dataValue('_id'), //KoboData = 1 Dataset (not 1 survey)\n DatasetName: state => state.formName,\n DatasetOwner: state => state.formOwner,\n Landscape: dataValue('landscape'),\n DatasetUuidId: dataValue('datasetId'),\n Citation: dataValue('instance'),\n DatasetYear: state => {\n const date = state.data.today || state.data._submission_time;\n const year = Number(date.trim().split('-')[0]);\n const formName = dataValue('formName')(state);\n const yearToReturn = year;\n if (year <= 2010) yearToReturn = Number(formName.trim().split(' ').at(-1));\n console.log(yearToReturn);\n return yearToReturn;\n // const formName = dataValue('formName')(state);\n // if (formName === 'BNS Cross River 2017-2020') {\n return state.data.body.today.split('-')[0];\n // }\n //const year = dataValue('body.today');\n //console.log(year);\n return new Date().getFullYear(); // Here we don't want the date of today we want the year of the value today\n //console.log(Date(year).getFullYear());\n },\n LastSubmissionTime: dataValue('_submission_time'),\n LastCheckedTime: dataValue('_submission_time'),\n LastUpdateTime: new Date().toISOString(),\n KoboManaged: true,\n Tags: dataValue('_tags'),\n});\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": "b5fa2c78-00ab-4795-92d0-1078e7789aaa"
},
"BNS-2C-NRGT---Historical-Version": {
"id": "856aa8dd-0b53-4dbd-8af8-972315bc5879",
"name": "BNS-2C NRGT - Historical Version",
"body": "// NOTE: This data cleaning operation returns state, modified as needed.\nalterState(state => {\n try {\n const { body, formName, instance } = state.data;\n const { _submission_time, _id, _xform_id_string } = body;\n let cleanedSubmission = {};\n\n for (const key in body) {\n switch (body[key]) {\n case 'yes':\n cleanedSubmission[key] = 1;\n break;\n\n case 'no':\n cleanedSubmission[key] = 0;\n break;\n\n default:\n cleanedSubmission[key] = body[key];\n break;\n }\n }\n\n state.landscapeMap = {\n tns: 'ndoki',\n mamabay: 'makira',\n mtkb: 'kahuzi',\n lactele: 'lac_tele',\n };\n\n cleanedSubmission.durableUUID = `${_submission_time}-${_xform_id_string}-${_id}`;\n cleanedSubmission.datasetId = `${formName}-${_xform_id_string}`;\n cleanedSubmission.instance = instance;\n state.data = cleanedSubmission;\n return state;\n } catch (error) {\n state.connection.close();\n throw error;\n }\n});\n\nupsert('WCSPROGRAMS_KoboNrgtNrgtanswer', 'AnswerId', {\n DatasetUuidId: dataValue('datasetId'),\n AnswerId: dataValue('_id'),\n Landscape: state => {\n var landscape = dataValue('landscape')(state);\n return state.landscapeMap[landscape] || landscape;\n },\n GovGroup: dataValue('gov_group'),\n Jurisdiction: dataValue('jurisdiction'),\n Objective: dataValue('objective'),\n Members: dataValue('members'),\n Women: dataValue('women'),\n LastUpdate: new Date().toISOString(),\n});\n\nupsert('WCSPROGRAMS_KoboNrgtNrgtanswergs', 'AnswerId', {\n DatasetUuidId: dataValue('datasetId'),\n Id: dataValue('_id'),\n AnswerId: dataValue('_id'),\n SurveyDate: dataValue('today'),\n Code: dataValue('code'),\n Gender: dataValue('gender'),\n Member: dataValue('member'),\n Legitimacy: dataValue('legitimacy'),\n Accountability: dataValue('accountability'),\n Transparency: dataValue('transparency'),\n Participation: dataValue('participation'),\n Fairness: dataValue('fairness'),\n KnowledgeSkills: dataValue('knowledge_skills'),\n Resources: dataValue('resources'),\n InstutionalFramework: dataValue('institutional_framework'),\n Motivation: dataValue('motivation'),\n EnactDecision: dataValue('enact_decision'),\n HeldAccountable: dataValue('held_accountable'),\n Diversity: dataValue('diversity'),\n LastUpdate: new Date().toISOString(),\n});\n\nupsert('WCSPROGRAMS_KoboData', 'DatasetUuidId', {\n //AnswerId: dataValue('_id'),\n DatasetName: state.data.formName,\n DatasetOwner: state.data.formOwner,\n DatasetUuidId: dataValue('datasetId'),\n Citation: dataValue('instance'),\n DatasetYear: new Date().getFullYear(),\n LastSubmissionTime: dataValue('_submission_time'),\n LastCheckedTime: dataValue('_submission_time'),\n LastUpdateTime: new Date().toISOString(),\n KoboManaged: true,\n Tags: dataValue('_tags'),\n});\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": "b5fa2c78-00ab-4795-92d0-1078e7789aaa"
},
"BNS-2D-NRGT-2019": {
"id": "c0cef1cd-ad7c-4aeb-8f70-dc6f7f0b4658",
"name": "BNS-2D NRGT 2019",
"body": "// NOTE: This data cleaning operation returns state, modified as needed.\nalterState(state => {\n try {\n const { body, formName, instance } = state.data;\n const { _submission_time, _id, _xform_id_string } = body;\n let cleanedSubmission = {};\n\n for (const key in body) {\n switch (body[key]) {\n case 'yes':\n cleanedSubmission[key] = 1;\n break;\n\n case 'no':\n cleanedSubmission[key] = 0;\n break;\n\n default:\n cleanedSubmission[key] = body[key];\n break;\n }\n }\n\n state.landscapeMap = {\n tns: 'ndoki',\n mamabay: 'makira',\n mtkb: 'kahuzi',\n };\n\n cleanedSubmission.durableUUID = `${_submission_time}-${_xform_id_string}-${_id}`;\n cleanedSubmission.datasetId = `${formName}-${_xform_id_string}`;\n cleanedSubmission.instance = instance;\n state.data = cleanedSubmission;\n return state;\n } catch (error) {\n state.connection.close();\n throw error;\n }\n});\n\nupsert('WCSPROGRAMS_KoboNrgtNrgtanswer', 'AnswerId', {\n DatasetUuidId: dataValue('datasetId'),\n AnswerId: dataValue('_id'),\n Landscape: state => {\n var landscape = dataValue('landscape')(state);\n return state.landscapeMap[landscape] || landscape;\n },\n Surveyor: dataValue('surveyor'),\n GovGroup: dataValue('gov_group'),\n SurveyDate: state => {\n const date = state.data.today || state.data._submission_time\n if (Number(date.split('-')[0]) >= 2014 ) {\n return date\n } \n return 2019 \n // If the time/date is not properly set on the device used to collect the data, the year of \"today\" will be 2000. \n // With the code above we are replacing any 2000 by 2019:\n },\n LastUpdate: new Date().toISOString(),\n});\nupsert('WCSPROGRAMS_KoboNrgtNrgtanswergs', 'AnswerId', {\n // upsert('WCSPROGRAMS_KoboNrgtNrgtanswergs', 'DatasetUuidId', {\n DatasetUuidId: dataValue('datasetId'),\n Id: dataValue('_id'),\n AnswerId: dataValue('_id'),\n SurveyDate: state => {\n const date = state.data.today || state.data._submission_time\n if (Number(date.split('-')[0]) >= 2014 ) {\n return date\n } \n return 2019\n // If the time/date is not properly set on the device used to collect the data, the year of \"today\" will be 2000. \n // With the code above we are replacing any 2000 by 2019:\n },\n Gender: dataValue('gender'),\n Member: dataValue('member'),\n Objective: dataValue('objective'),\n Legitimacy: dataValue('legitimacy'),\n Accountability: dataValue('accountability'),\n Transparency: dataValue('transparency'),\n Participation: dataValue('participation'),\n Fairness: dataValue('fairness'),\n Diversity: dataValue('diversity'),\n KnowledgeSkills: dataValue('knowledge_skills'),\n Resources: dataValue('resources'),\n InstutionalFramework: dataValue('framework'),\n Motivation: dataValue('motivation'),\n Power: dataValue('power'),\n LastUpdate: new Date().toISOString(),\n});\n\nupsert('WCSPROGRAMS_KoboData', 'DatasetUuidId', {\n //AnswerId: dataValue('_id'),\n DatasetName: state.data.formName,\n DatasetOwner: state.data.formOwner,\n DatasetUuidId: dataValue('datasetId'),\n Citation: dataValue('instance'),\n DatasetYear: new Date().getFullYear(),\n LastSubmissionTime: dataValue('_submission_time'),\n LastCheckedTime: dataValue('_submission_time'),\n LastUpdateTime: new Date().toISOString(),\n KoboManaged: true,\n Tags: dataValue('_tags'),\n});\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": "b5fa2c78-00ab-4795-92d0-1078e7789aaa"
},
"BNS-2E-NRGT---Historical-Version-2-2022": {
"id": "55d15411-6a56-4614-8e25-f314f3381334",
"name": "BNS-2E NRGT - Historical Version 2 2022",
"body": "fn(state => {\n try {\n const { body, formName, instance } = state.data;\n const { _submission_time, _id, _xform_id_string, group_scores } = body;\n let cleanedSubmission = {};\n\n for (const key in body) {\n switch (body[key]) {\n case 'yes':\n cleanedSubmission[key] = 1;\n break;\n\n case 'no':\n cleanedSubmission[key] = 0;\n break;\n\n default:\n cleanedSubmission[key] = body[key];\n break;\n }\n }\n\n state.landscapeMap = {\n tns: 'ndoki',\n mamabay: 'makira',\n mtkb: 'kahuzi',\n lactele: 'lac_tele',\n };\n\n cleanedSubmission.durableUUID = `${_submission_time}-${_xform_id_string}-${_id}`;\n cleanedSubmission.datasetId = `${formName}-${_xform_id_string}`;\n cleanedSubmission.instance = instance;\n cleanedSubmission.group_scores = group_scores;\n \n state.data = cleanedSubmission;\n\n return state;\n } catch (error) {\n state.connection.close();\n throw error;\n }\n});\n\n\nupsert('WCSPROGRAMS_KoboNrgtNrgtanswer', 'AnswerId', {\n DatasetUuidId: dataValue('datasetId'),\n AnswerId: dataValue('_id'),\n Landscape: state => {\n var landscape = dataValue('landscape')(state);\n return state.landscapeMap[landscape] || landscape;\n },\n GovGroup: dataValue('gov_group'),\n Jurisdiction: dataValue('jurisdiction'),\n Objective: dataValue('objective'),\n Members: dataValue('members'),\n Women: dataValue('women'),\n LastUpdate: new Date().toISOString(),\n});\n\n/*\nupsert('WCSPROGRAMS_KoboNrgtNrgtanswergs', 'AnswerId', {\n DatasetUuidId: dataValue('datasetId'),\n Id: dataValue('_id'),\n AnswerId: dataValue('_id'),\n SurveyDate: dataValue('today'),\n Code: dataValue('code'),\n Gender: dataValue('gender'),\n Member: dataValue('member'),\n Legitimacy: dataValue('legitimacy'),\n Accountability: dataValue('accountability'),\n Transparency: dataValue('transparency'),\n Participation: dataValue('participation'),\n Fairness: dataValue('fairness'),\n KnowledgeSkills: dataValue('knowledge_skills'),\n Resources: dataValue('resources'),\n InstutionalFramework: dataValue('institutional_framework'),\n Motivation: dataValue('motivation'),\n EnactDecision: dataValue('enact_decision'),\n HeldAccountable: dataValue('held_accountable'),\n Diversity: dataValue('diversity'),\n LastUpdate: new Date().toISOString(),\n});\n*/\n\nupsertMany(\n 'WCSPROGRAMS_KoboNrgtNrgtanswergs', \n 'AnswerId', \n state => state.data.group_scores.map(x => ({\n AnswerId: state.data._id,\n Id: state.data._id,\n DatasetUuidId: state.data.datasetId,\n Accountability: x[\"group_scores/accountability\"],\n Code: x[\"group_scores/code\"],\n Diversity: x[\"group_scores/diversity\"],\n EnactDecision: x[\"group_scores/enact_decision\"],\n Fairness: x[\"group_scores/fairness\"],\n Gender: x[\"group_scores/gender\"],\n HeldAccountable: x[\"group_scores/held_accountable\"],\n InstutionalFramework: x[\"group_scores/institutional_framework\"],\n KnowledgeSkills: x[\"group_scores/knowledge_skills\"],\n Legitimacy: x[\"group_scores/legitimacy\"],\n Member: (x[\"group_scores/member\"] === \"yes\"),\n Motivation: x[\"group_scores/motivation\"],\n Participation: x[\"group_scores/participation\"],\n Resources: x[\"group_scores/resources\"],\n SurveyDate: x[\"group_scores/survey_date\"],\n Transparency: x[\"group_scores/transparency\"],\n LastUpdate: new Date().toISOString()\n }))\n);\n\n\nupsert('WCSPROGRAMS_KoboData', 'DatasetUuidId', {\n DatasetName: state.data.formName,\n DatasetOwner: state.data.formOwner,\n DatasetUuidId: dataValue('datasetId'),\n Citation: dataValue('instance'),\n DatasetYear: new Date().getFullYear(),\n LastSubmissionTime: dataValue('_submission_time'),\n LastCheckedTime: dataValue('_submission_time'),\n LastUpdateTime: new Date().toISOString(),\n KoboManaged: true,\n Tags: dataValue('_tags'),\n});\n",
"adaptor": "@openfn/[email protected]",
"project_credential_id": "b5fa2c78-00ab-4795-92d0-1078e7789aaa"
}
},
"edges": {
"webhook->Triage-jobs": {
"enabled": true,
"id": "b758386e-7c93-44dc-8393-cdea13e31c83",
"target_job_id": "65e96922-0db6-4823-88ea-64601641c902",
"source_trigger_id": "b3f86593-f37e-4139-80b8-852b9d3c49f4",
"condition_type": "always"
},
"Triage-jobs->BNS-2A-BNS-Price": {
"enabled": true,
"id": "a4d1312a-c209-497a-8e19-462feccddeb3",
"target_job_id": "7280bfa9-2a67-4c6e-8a3a-f7f21bcb26ae",
"source_job_id": "65e96922-0db6-4823-88ea-64601641c902",
"condition_type": "js_expression",
"condition_label": "BNS Price",
"condition_expression": "state.data.form === \"bns_price\" && \nstate.data.body?.survey_type && \nstate.data.body.survey_type !== \"practice\""
},
"Triage-jobs->BNS-2B-BNS-Survey": {
"enabled": true,
"id": "b541e65c-79a6-4214-88c6-653133d65210",
"target_job_id": "196697f9-b581-4397-8487-280df2d47529",
"source_job_id": "65e96922-0db6-4823-88ea-64601641c902",
"condition_type": "js_expression",
"condition_label": "BNS Survey",
"condition_expression": "state.data.form == \"bns_survey\" && \nstate.data.body?.survey_type && \nstate.data.body.survey_type !== \"practice\""
},
"Triage-jobs->BNS-2C-NRGT---Historical-Version": {
"enabled": true,
"id": "f19030bd-1907-409b-84c4-9d0f41b358f4",
"target_job_id": "856aa8dd-0b53-4dbd-8af8-972315bc5879",
"source_job_id": "65e96922-0db6-4823-88ea-64601641c902",
"condition_type": "js_expression",
"condition_label": "NRGT Historical",
"condition_expression": "state.data.form == \"nrgt_historical\" && state.data.formName != \"NRGT Kahuzi Biega 2019\""
},
"Triage-jobs->BNS-2D-NRGT-2019": {
"enabled": true,
"id": "8d13c9cc-6efd-436b-8d4f-f655e89283cc",
"target_job_id": "c0cef1cd-ad7c-4aeb-8f70-dc6f7f0b4658",
"source_job_id": "65e96922-0db6-4823-88ea-64601641c902",
"condition_type": "js_expression",
"condition_label": "NRGT Current",
"condition_expression": "state.data.form == \"nrgt_current\" && \nstate.data.body?.survey_type && \nstate.data.body.survey_type !== \"practice\""
},
"Triage-jobs->BNS-2E-NRGT---Historical-Version-2-2022": {
"enabled": true,
"id": "07ac5d9c-cc2e-480a-82a2-1cf46d4bad22",
"target_job_id": "55d15411-6a56-4614-8e25-f314f3381334",
"source_job_id": "65e96922-0db6-4823-88ea-64601641c902",
"condition_type": "js_expression",
"condition_label": "NGRT Historical",
"condition_expression": "state.data.form == \"nrgt_historical\" && state.data.formName == \"NRGT Kahuzi Biega 2019\""
}
}
}
},
"requires_mfa": false
}