Skip to content

Commit c18a6ae

Browse files
steveonimpolidori
steveoni
and
mpolidori
committed
[ODP-414] add email notification when download-flow fails (#641)
* add email notification when flow fails * update error notification * empty * remove pgclient version * update libpq --------- Co-authored-by: Michael Polidori <[email protected]>
1 parent 482c947 commit c18a6ae

File tree

6 files changed

+279
-108
lines changed

6 files changed

+279
-108
lines changed

‎ckan-backend-dev/ckan/Dockerfile.dev

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,8 @@ RUN apk --update add build-base libxslt-dev python3-dev
4040

4141
# Fixing security vulnerabilities
4242
RUN apk --update add git
43-
RUN apk add postgresql15-client=15.10-r0
44-
RUN apk add libpq=15.10-r0
43+
RUN apk add postgresql15-client
44+
RUN apk add libpq
4545
RUN pip install --force-reinstall -v "Pillow==11.0.0"
4646

4747
RUN apk add --virtual .build-deps \

‎ckan-backend-dev/src/ckanext-wri/ckanext/wri/logic/action/datapusher_download.py

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import ckanext.s3filestore.uploader as uploader
77
from ckan.lib.mailer import mail_recipient
88
from ckan.common import config
9+
from .datapusher_download_zip import fetch_dataset_name
910

1011
import datetime
1112
import requests
@@ -177,7 +178,11 @@ def download_request(context: Context, data_dict: dict[str, Any]):
177178
task["state"] = "error"
178179
task["last_updated"] = (str(datetime.datetime.utcnow()),)
179180
p.toolkit.get_action("task_status_update")(context, task)
180-
send_error([email], resource_title)
181+
dataset_name = fetch_dataset_name({
182+
"entity_id": res_id,
183+
"entity_type": "resource"
184+
})
185+
send_error([email], resource_title, dataset_name)
181186
raise p.toolkit.ValidationError(error)
182187

183188
try:
@@ -199,7 +204,11 @@ def download_request(context: Context, data_dict: dict[str, Any]):
199204
task["state"] = "error"
200205
task["last_updated"] = (str(datetime.datetime.utcnow()),)
201206
p.toolkit.get_action("task_status_update")(context, task)
202-
send_error([email], resource_title)
207+
dataset_name = fetch_dataset_name({
208+
"entity_id": res_id,
209+
"entity_type": "resource"
210+
})
211+
send_error([email], resource_title, dataset_name)
203212
raise p.toolkit.ValidationError(error)
204213

205214
value = {"job_id": r.json()["id"]}
@@ -250,7 +259,11 @@ def download_callback(context: Context, data_dict: dict[str, Any]):
250259
url = data_dict.get("url")
251260
send_email(emails, url, download_filename)
252261
else:
253-
send_error(emails, download_filename)
262+
dataset_name = fetch_dataset_name({
263+
"entity_id": entity_id,
264+
"entity_type": "resource"
265+
})
266+
send_error(emails, download_filename, dataset_name)
254267
log.error(error)
255268

256269

@@ -282,23 +295,39 @@ def send_email(emails: list[str], url: str, download_filename: str):
282295
)
283296

284297

285-
ERROR_EMAIL_HTML = '''
298+
ERROR_EMAIL_HTML = """
286299
<html>
287300
<body>
288-
<p>An error happened while preparing the file you requested for download. Please, try again.</p>
301+
<p>
302+
You recently requested the below data from the World Resources Institute Data Explorer.
303+
Our systems encountered an error during the packaging of this data and we are unable to deliver your files at this time.
304+
</p>
305+
306+
<b>
307+
{}
308+
</b>
309+
</br>
310+
<b>
311+
<a target="_blank" href="{}/datasets/{}">Dataset link</a>
312+
</b>
313+
314+
<p>
315+
This may be a temporary issue but more likely represents some misconfiguration in our systems.
316+
Please reach out to <a href="mailto:[email protected]">[email protected]</a> to request immediate support.
317+
</p>
289318
<br>
290319
<a target="_blank" href="{}">{}</a>
291320
</body>
292321
</html>
293322
294-
'''
323+
"""
295324

296325

297-
def send_error(emails: list[str], resource_title):
326+
def send_error(emails: list[str], resource_title, dataset_name):
298327
odp_url = config.get('ckanext.wri.odp_url')
299328
for email in emails:
300329
mail_recipient("", email,
301330
"WRI - Failed to process file ({})".format(resource_title),
302331
"",
303-
ERROR_EMAIL_HTML.format(odp_url, odp_url)
332+
ERROR_EMAIL_HTML.format(dataset_name,odp_url,dataset_name,odp_url, odp_url),
304333
)

‎ckan-backend-dev/src/ckanext-wri/ckanext/wri/logic/action/datapusher_download_subset.py

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import ckanext.s3filestore.uploader as uploader
77
from ckan.lib.mailer import mail_recipient
88
from ckan.common import config
9+
from .datapusher_download_zip import fetch_dataset_name
910

1011
import datetime
1112
import requests
@@ -255,7 +256,11 @@ def subset_download_request(context: Context, data_dict: dict[str, Any]):
255256
task["state"] = "error"
256257
task["last_updated"] = (str(datetime.datetime.utcnow()),)
257258
p.toolkit.get_action("task_status_update")(context, task)
258-
send_error([email], "Subset of data")
259+
dataset_name = fetch_dataset_name({
260+
"entity_id": id if provider == "datastore" else dataset_id,
261+
"entity_type": "resource" if provider == "datastore" else "dataset"
262+
})
263+
send_error([email], "Subset of data", dataset_name)
259264
raise p.toolkit.ValidationError(error)
260265

261266
try:
@@ -277,7 +282,11 @@ def subset_download_request(context: Context, data_dict: dict[str, Any]):
277282
task["state"] = "error"
278283
task["last_updated"] = (str(datetime.datetime.utcnow()),)
279284
p.toolkit.get_action("task_status_update")(context, task)
280-
send_error([email], "Subset of data")
285+
dataset_name = fetch_dataset_name({
286+
"entity_id": id if provider == "datastore" else dataset_id,
287+
"entity_type": "resource" if provider == "datastore" else "dataset"
288+
})
289+
send_error([email], "Subset of data", dataset_name)
281290
raise p.toolkit.ValidationError(error)
282291

283292
value = {"job_id": r.json()["id"]}
@@ -317,12 +326,17 @@ def subset_download_callback(context: Context, data_dict: dict[str, Any]):
317326
value = json.loads(task["value"])
318327
emails = value.get("emails", [])
319328
download_filename = value.get("download_filename")
329+
320330

321331
if state == "complete":
322332
url = data_dict.get("url")
323333
send_email(emails, url, download_filename)
324334
else:
325-
send_error(emails, download_filename)
335+
dataset_name = fetch_dataset_name({
336+
"entity_id": entity_id,
337+
"entity_type": data_dict.get("entity_type", "resource")
338+
})
339+
send_error(emails, download_filename, dataset_name)
326340
log.error(error)
327341

328342

@@ -357,7 +371,23 @@ def send_email(emails: list[str], url: str, download_filename: str):
357371
ERROR_EMAIL_HTML = """
358372
<html>
359373
<body>
360-
<p>An error happened while preparing the file you requested for download. Please, try again.</p>
374+
<p>
375+
You recently requested the below data from the World Resources Institute Data Explorer.
376+
Our systems encountered an error during the packaging of this data and we are unable to deliver your files at this time.
377+
</p>
378+
379+
<b>
380+
{}
381+
</b>
382+
</br>
383+
<b>
384+
<a target="_blank" href="{}/datasets/{}">Dataset link</a>
385+
</b>
386+
387+
<p>
388+
This may be a temporary issue but more likely represents some misconfiguration in our systems.
389+
Please reach out to <a href="mailto:[email protected]">[email protected]</a> to request immediate support.
390+
</p>
361391
<br>
362392
<a target="_blank" href="{}">{}</a>
363393
</body>
@@ -366,13 +396,13 @@ def send_email(emails: list[str], url: str, download_filename: str):
366396
"""
367397

368398

369-
def send_error(emails: list[str], resource_title):
399+
def send_error(emails: list[str], resource_title, dataset_name):
370400
odp_url = config.get("ckanext.wri.odp_url")
371401
for email in emails:
372402
mail_recipient(
373403
"",
374404
email,
375405
"WRI - Failed to process file ({})".format(resource_title),
376406
"",
377-
ERROR_EMAIL_HTML.format(odp_url, odp_url),
407+
ERROR_EMAIL_HTML.format(dataset_name,odp_url,dataset_name,odp_url, odp_url),
378408
)

‎ckan-backend-dev/src/ckanext-wri/ckanext/wri/logic/action/datapusher_download_zip.py

Lines changed: 59 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,8 @@ def build_download_filename(dataset_id: str, context) -> str:
5656
},
5757
)
5858
download_filename = (
59-
dataset_dict.get("title")
60-
or dataset_dict.get("name")
59+
dataset_dict.get("name")
60+
or dataset_dict.get("title")
6161
or dataset_dict.get("id")
6262
or "file"
6363
)
@@ -274,6 +274,42 @@ def zipped_download_callback(context: Context, data_dict: dict[str, Any]):
274274
send_error(emails, download_filename)
275275
log.error(error)
276276

277+
def send_error_callback(context: Context, data_dict: dict[str, Any]):
278+
entity_id = data_dict.get("entity_id")
279+
key = data_dict.get("key")
280+
task = p.toolkit.get_action("task_status_show")(
281+
context,
282+
{"entity_id": entity_id, "task_type": "download_zipped", "key": key},
283+
)
284+
285+
if not task:
286+
raise logic.NotFound("Task not found")
287+
288+
value = json.loads(task["value"])
289+
emails = value.get("emails", [])
290+
download_filename = value.get("download_filename")
291+
dataset_name = fetch_dataset_name({
292+
"entity_id": entity_id,
293+
"entity_type": data_dict.get("entity_type", "dataset")
294+
})
295+
send_error(emails, download_filename, dataset_name)
296+
297+
298+
def fetch_dataset_name(data_dict):
299+
entity_id = data_dict.get("entity_id")
300+
entity_type = data_dict.get("entity_type")
301+
if entity_type == "dataset":
302+
dataset_dict = p.toolkit.get_action("package_show")(
303+
{"ignore_auth": True}, {"id": entity_id}
304+
)
305+
return dataset_dict.get("name")
306+
else:
307+
resource_dict = p.toolkit.get_action("resource_show")(
308+
{"ignore_auth": True}, {"id": entity_id}
309+
)
310+
return resource_dict.get("package_id")
311+
312+
277313

278314
FILE_EMAIL_HTML = """
279315
<html>
@@ -306,7 +342,23 @@ def send_email(emails: list[str], url: str, download_filename: str):
306342
ERROR_EMAIL_HTML = """
307343
<html>
308344
<body>
309-
<p>An error happened while preparing the file you requested for download. Please, try again.</p>
345+
<p>
346+
You recently requested the below data from the World Resources Institute Data Explorer.
347+
Our systems encountered an error during the packaging of this data and we are unable to deliver your files at this time.
348+
</p>
349+
350+
<b>
351+
{}
352+
</b>
353+
</br>
354+
<b>
355+
<a target="_blank" href="{}/datasets/{}">Dataset link</a>
356+
</b>
357+
358+
<p>
359+
This may be a temporary issue but more likely represents some misconfiguration in our systems.
360+
Please reach out to <a href="mailto:[email protected]">[email protected]</a> to request immediate support.
361+
</p>
310362
<br>
311363
<a target="_blank" href="{}">{}</a>
312364
</body>
@@ -315,13 +367,14 @@ def send_email(emails: list[str], url: str, download_filename: str):
315367
"""
316368

317369

318-
def send_error(emails: list[str], resource_title):
370+
def send_error(emails: list[str], resource_title,dataset_name: str=None):
319371
odp_url = config.get("ckanext.wri.odp_url")
372+
datasetName = dataset_name if dataset_name else resource_title
320373
for email in emails:
321374
mail_recipient(
322375
"",
323376
email,
324-
"WRI - Failed to process file ({})".format(resource_title),
377+
"Your WRI data file could not be delivered ({})".format(resource_title),
325378
"",
326-
ERROR_EMAIL_HTML.format(odp_url, odp_url),
379+
ERROR_EMAIL_HTML.format(datasetName,odp_url,datasetName,odp_url, odp_url),
327380
)

‎ckan-backend-dev/src/ckanext-wri/ckanext/wri/plugin.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
from ckanext.wri.logic.action.datapusher_download_zip import (
1313
zipped_download_request,
1414
zipped_download_callback,
15+
send_error_callback,
1516
)
1617
import ckanext.wri.logic.validators as wri_validators
1718
from ckan import model, logic, authz
@@ -270,7 +271,8 @@ def get_actions(self):
270271
"package_show": package_show,
271272
"package_update": package_update,
272273
"download_event_create": download_event_create,
273-
"download_event_list": get_download_events
274+
"download_event_list": get_download_events,
275+
"prefect_send_error_callback": send_error_callback,
274276
}
275277

276278
# IPermissionLabels

0 commit comments

Comments
 (0)