Skip to content

[ODP-414] add email notification when download-flow fails #641

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Feb 26, 2025
Merged

[ODP-414] add email notification when download-flow fails #641

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0c48323
add email notification when flow fails
steveoni Feb 21, 2025
d760745
update error notification
steveoni Feb 24, 2025
bed4710
empty
steveoni Feb 24, 2025
daa9e19
remove pgclient version
steveoni Feb 24, 2025
7eb2288
update libpq
steveoni Feb 24, 2025
0e0f414
Merge branch 'dev' into flowemail
mpolidori Feb 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ckan-backend-dev/ckan/Dockerfile.dev
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@ RUN apk --update add build-base libxslt-dev python3-dev

# Fixing security vulnerabilities
RUN apk --update add git
RUN apk add postgresql15-client=15.10-r0
RUN apk add libpq=15.10-r0
RUN apk add postgresql15-client
RUN apk add libpq
RUN pip install --force-reinstall -v "Pillow==11.0.0"

RUN apk add --virtual .build-deps \
Expand Down
45 changes: 37 additions & 8 deletions ckan-backend-dev/src/ckanext-wri/ckanext/wri/logic/action/datapusher_download.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import ckanext.s3filestore.uploader as uploader
from ckan.lib.mailer import mail_recipient
from ckan.common import config
from .datapusher_download_zip import fetch_dataset_name

import datetime
import requests
Expand Down Expand Up @@ -177,7 +178,11 @@ def download_request(context: Context, data_dict: dict[str, Any]):
task["state"] = "error"
task["last_updated"] = (str(datetime.datetime.utcnow()),)
p.toolkit.get_action("task_status_update")(context, task)
send_error([email], resource_title)
dataset_name = fetch_dataset_name({
"entity_id": res_id,
"entity_type": "resource"
})
send_error([email], resource_title, dataset_name)
raise p.toolkit.ValidationError(error)

try:
Expand All @@ -199,7 +204,11 @@ def download_request(context: Context, data_dict: dict[str, Any]):
task["state"] = "error"
task["last_updated"] = (str(datetime.datetime.utcnow()),)
p.toolkit.get_action("task_status_update")(context, task)
send_error([email], resource_title)
dataset_name = fetch_dataset_name({
"entity_id": res_id,
"entity_type": "resource"
})
send_error([email], resource_title, dataset_name)
raise p.toolkit.ValidationError(error)

value = {"job_id": r.json()["id"]}
Expand Down Expand Up @@ -250,7 +259,11 @@ def download_callback(context: Context, data_dict: dict[str, Any]):
url = data_dict.get("url")
send_email(emails, url, download_filename)
else:
send_error(emails, download_filename)
dataset_name = fetch_dataset_name({
"entity_id": entity_id,
"entity_type": "resource"
})
send_error(emails, download_filename, dataset_name)
log.error(error)


Expand Down Expand Up @@ -282,23 +295,39 @@ def send_email(emails: list[str], url: str, download_filename: str):
)


ERROR_EMAIL_HTML = '''
ERROR_EMAIL_HTML = """
<html>
<body>
<p>An error happened while preparing the file you requested for download. Please, try again.</p>
<p>
You recently requested the below data from the World Resources Institute Data Explorer.
Our systems encountered an error during the packaging of this data and we are unable to deliver your files at this time.
</p>

<b>
{}
</b>
</br>
<b>
<a target="_blank" href="{}/datasets/{}">Dataset link</a>
</b>

<p>
This may be a temporary issue but more likely represents some misconfiguration in our systems.
Please reach out to <a href="mailto:[email protected]">[email protected]</a> to request immediate support.
</p>
<br>
<a target="_blank" href="{}">{}</a>
</body>
</html>

'''
"""


def send_error(emails: list[str], resource_title):
def send_error(emails: list[str], resource_title, dataset_name):
odp_url = config.get('ckanext.wri.odp_url')
for email in emails:
mail_recipient("", email,
"WRI - Failed to process file ({})".format(resource_title),
"",
ERROR_EMAIL_HTML.format(odp_url, odp_url)
ERROR_EMAIL_HTML.format(dataset_name,odp_url,dataset_name,odp_url, odp_url),
)
42 changes: 36 additions & 6 deletions ckan-backend-dev/src/ckanext-wri/ckanext/wri/logic/action/datapusher_download_subset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import ckanext.s3filestore.uploader as uploader
from ckan.lib.mailer import mail_recipient
from ckan.common import config
from .datapusher_download_zip import fetch_dataset_name

import datetime
import requests
Expand Down Expand Up @@ -255,7 +256,11 @@ def subset_download_request(context: Context, data_dict: dict[str, Any]):
task["state"] = "error"
task["last_updated"] = (str(datetime.datetime.utcnow()),)
p.toolkit.get_action("task_status_update")(context, task)
send_error([email], "Subset of data")
dataset_name = fetch_dataset_name({
"entity_id": id if provider == "datastore" else dataset_id,
"entity_type": "resource" if provider == "datastore" else "dataset"
})
send_error([email], "Subset of data", dataset_name)
raise p.toolkit.ValidationError(error)

try:
Expand All @@ -277,7 +282,11 @@ def subset_download_request(context: Context, data_dict: dict[str, Any]):
task["state"] = "error"
task["last_updated"] = (str(datetime.datetime.utcnow()),)
p.toolkit.get_action("task_status_update")(context, task)
send_error([email], "Subset of data")
dataset_name = fetch_dataset_name({
"entity_id": id if provider == "datastore" else dataset_id,
"entity_type": "resource" if provider == "datastore" else "dataset"
})
send_error([email], "Subset of data", dataset_name)
raise p.toolkit.ValidationError(error)

value = {"job_id": r.json()["id"]}
Expand Down Expand Up @@ -317,12 +326,17 @@ def subset_download_callback(context: Context, data_dict: dict[str, Any]):
value = json.loads(task["value"])
emails = value.get("emails", [])
download_filename = value.get("download_filename")


if state == "complete":
url = data_dict.get("url")
send_email(emails, url, download_filename)
else:
send_error(emails, download_filename)
dataset_name = fetch_dataset_name({
"entity_id": entity_id,
"entity_type": data_dict.get("entity_type", "resource")
})
send_error(emails, download_filename, dataset_name)
log.error(error)


Expand Down Expand Up @@ -357,7 +371,23 @@ def send_email(emails: list[str], url: str, download_filename: str):
ERROR_EMAIL_HTML = """
<html>
<body>
<p>An error happened while preparing the file you requested for download. Please, try again.</p>
<p>
You recently requested the below data from the World Resources Institute Data Explorer.
Our systems encountered an error during the packaging of this data and we are unable to deliver your files at this time.
</p>

<b>
{}
</b>
</br>
<b>
<a target="_blank" href="{}/datasets/{}">Dataset link</a>
</b>

<p>
This may be a temporary issue but more likely represents some misconfiguration in our systems.
Please reach out to <a href="mailto:[email protected]">[email protected]</a> to request immediate support.
</p>
<br>
<a target="_blank" href="{}">{}</a>
</body>
Expand All @@ -366,13 +396,13 @@ def send_email(emails: list[str], url: str, download_filename: str):
"""


def send_error(emails: list[str], resource_title):
def send_error(emails: list[str], resource_title, dataset_name):
odp_url = config.get("ckanext.wri.odp_url")
for email in emails:
mail_recipient(
"",
email,
"WRI - Failed to process file ({})".format(resource_title),
"",
ERROR_EMAIL_HTML.format(odp_url, odp_url),
ERROR_EMAIL_HTML.format(dataset_name,odp_url,dataset_name,odp_url, odp_url),
)
65 changes: 59 additions & 6 deletions ckan-backend-dev/src/ckanext-wri/ckanext/wri/logic/action/datapusher_download_zip.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ def build_download_filename(dataset_id: str, context) -> str:
},
)
download_filename = (
dataset_dict.get("title")
or dataset_dict.get("name")
dataset_dict.get("name")
or dataset_dict.get("title")
or dataset_dict.get("id")
or "file"
)
Expand Down Expand Up @@ -274,6 +274,42 @@ def zipped_download_callback(context: Context, data_dict: dict[str, Any]):
send_error(emails, download_filename)
log.error(error)

def send_error_callback(context: Context, data_dict: dict[str, Any]):
entity_id = data_dict.get("entity_id")
key = data_dict.get("key")
task = p.toolkit.get_action("task_status_show")(
context,
{"entity_id": entity_id, "task_type": "download_zipped", "key": key},
)

if not task:
raise logic.NotFound("Task not found")

value = json.loads(task["value"])
emails = value.get("emails", [])
download_filename = value.get("download_filename")
dataset_name = fetch_dataset_name({
"entity_id": entity_id,
"entity_type": data_dict.get("entity_type", "dataset")
})
send_error(emails, download_filename, dataset_name)


def fetch_dataset_name(data_dict):
entity_id = data_dict.get("entity_id")
entity_type = data_dict.get("entity_type")
if entity_type == "dataset":
dataset_dict = p.toolkit.get_action("package_show")(
{"ignore_auth": True}, {"id": entity_id}
)
return dataset_dict.get("name")
else:
resource_dict = p.toolkit.get_action("resource_show")(
{"ignore_auth": True}, {"id": entity_id}
)
return resource_dict.get("package_id")



FILE_EMAIL_HTML = """
<html>
Expand Down Expand Up @@ -306,7 +342,23 @@ def send_email(emails: list[str], url: str, download_filename: str):
ERROR_EMAIL_HTML = """
<html>
<body>
<p>An error happened while preparing the file you requested for download. Please, try again.</p>
<p>
You recently requested the below data from the World Resources Institute Data Explorer.
Our systems encountered an error during the packaging of this data and we are unable to deliver your files at this time.
</p>

<b>
{}
</b>
</br>
<b>
<a target="_blank" href="{}/datasets/{}">Dataset link</a>
</b>

<p>
This may be a temporary issue but more likely represents some misconfiguration in our systems.
Please reach out to <a href="mailto:[email protected]">[email protected]</a> to request immediate support.
</p>
<br>
<a target="_blank" href="{}">{}</a>
</body>
Expand All @@ -315,13 +367,14 @@ def send_email(emails: list[str], url: str, download_filename: str):
"""


def send_error(emails: list[str], resource_title):
def send_error(emails: list[str], resource_title,dataset_name: str=None):
odp_url = config.get("ckanext.wri.odp_url")
datasetName = dataset_name if dataset_name else resource_title
for email in emails:
mail_recipient(
"",
email,
"WRI - Failed to process file ({})".format(resource_title),
"Your WRI data file could not be delivered ({})".format(resource_title),
"",
ERROR_EMAIL_HTML.format(odp_url, odp_url),
ERROR_EMAIL_HTML.format(datasetName,odp_url,datasetName,odp_url, odp_url),
)
4 changes: 3 additions & 1 deletion ckan-backend-dev/src/ckanext-wri/ckanext/wri/plugin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from ckanext.wri.logic.action.datapusher_download_zip import (
zipped_download_request,
zipped_download_callback,
send_error_callback,
)
import ckanext.wri.logic.validators as wri_validators
from ckan import model, logic, authz
Expand Down Expand Up @@ -270,7 +271,8 @@ def get_actions(self):
"package_show": package_show,
"package_update": package_update,
"download_event_create": download_event_create,
"download_event_list": get_download_events
"download_event_list": get_download_events,
"prefect_send_error_callback": send_error_callback,
}

# IPermissionLabels
Expand Down
Loading
Loading