Merge pull request #25 from AfricasVoices/fetch-contacts

Fetch contacts

Author: as2388

fetch_contacts/Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.6-slim
+
+# Install the tools we need.
+RUN apt-get update && apt-get install -y git
+RUN pip install pipenv
+
+# Set working directory
+WORKDIR /app
+
+# Install project dependencies.
+ADD Pipfile.lock /app
+ADD Pipfile /app
+RUN pipenv sync
+
+# Copy the rest of the project
+ADD fetch_contacts.py /app
+
+# Make a directory for intermediate data
+RUN mkdir /data
+
+# Environment variables need to be set when constructing this container e.g. via
+# docker run or docker container create. Use docker-run.sh to set these automatically.
+CMD pipenv run python fetch_contacts.py --server "$SERVER" "$TOKEN" "$USER" /data/phone-uuid-table.json /data/output.json

fetch_contacts/Pipfile

@@ -0,0 +1,13 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+rapidpro-python = ">=2.4"
+CoreDataModules = {git = "https://www.github.com/AfricasVoices/CoreDataModules", editable = true, ref = "v0.5.3"}
+
+[dev-packages]
+
+[requires]
+python_version = "3.6"

fetch_contacts/Pipfile.lock

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -e
+
+IMAGE_NAME=rapid-pro-fetch-contacts
+
+# Check that the correct number of arguments were provided.
+if [ $# -ne 5 ]; then
+    echo "Usage: sh docker-run.sh <server> <token> <user> <phone-uuid-table> <output-json>"
+    exit
+fi
+
+# Assign the program arguments to bash variables.
+SERVER=$1
+TOKEN=$2
+USER=$3
+PHONE_UUID_TABLE=$4
+OUTPUT_JSON=$5
+
+# Build an image for this pipeline stage.
+docker build -t "$IMAGE_NAME" .
+
+# Create a container from the image that was just built.
+container="$(docker container create --env SERVER="$SERVER" --env TOKEN="$TOKEN" --env USER="$USER" "$IMAGE_NAME")"
+
+function finish {
+    # Tear down the container when done.
+    docker container rm "$container" >/dev/null
+}
+trap finish EXIT
+
+# Copy input data into the container
+docker cp "$PHONE_UUID_TABLE" "$container:/data/phone-uuid-table.json"
+
+# Run the image as a container.
+docker start -a -i "$container"
+
+# Copy the output data back out of the container
+mkdir -p "$(dirname "$PHONE_UUID_TABLE")"
+docker cp "$container:/data/phone-uuid-table.json" "$PHONE_UUID_TABLE"
+
+mkdir -p "$(dirname "$OUTPUT_JSON")"
+docker cp "$container:/data/output.json" "$OUTPUT_JSON"

fetch_contacts/docker-run.sh

@@ -0,0 +1,64 @@
+import argparse
+import os
+import time
+
+from core_data_modules.traced_data import TracedData, Metadata
+from core_data_modules.traced_data.io import TracedDataJsonIO
+from core_data_modules.util import PhoneNumberUuidTable, IOUtils
+from temba_client.v2 import TembaClient
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Downloads contacts from RapidPro")
+    parser.add_argument("--server", help="Address of RapidPro server. Defaults to http://localhost:8000.",
+                        nargs="?", default="http://localhost:8000")
+    parser.add_argument("token", help="RapidPro API Token")
+    parser.add_argument("user", help="Identifier of user launching this program, for use in TracedData Metadata")
+    parser.add_argument("phone_uuid_table_path", metavar="phone-uuid-table-path",
+                        help="JSON file containing an existing phone number <-> UUID lookup table. "
+                             "This file will be updated with the new phone numbers which are found by this process")
+    parser.add_argument("json_output_path", metavar="json-output-path",
+                        help="Path to serialized TracedData JSON file")
+
+    args = parser.parse_args()
+    server = args.server
+    token = args.token
+    user = args.user
+    phone_uuid_path = args.phone_uuid_table_path
+    json_output_path = args.json_output_path
+
+    rapid_pro = TembaClient(server, token)
+
+    # Load the existing phone number <-> UUID table.
+    if not os.path.exists(phone_uuid_path):
+        raise FileNotFoundError("No such phone uuid table file '{}'. "
+                                "To create a new, empty UUID table, "
+                                "run $ echo \"{{}}\" > <target-json-file>".format(phone_uuid_path))
+    with open(phone_uuid_path, "r") as f:
+        phone_uuids = PhoneNumberUuidTable.load(f)
+
+    # Download all contacts
+    print("Fetching contacts...")
+    start = time.time()
+    contacts = rapid_pro.get_contacts().all(retry_on_rate_exceed=True)
+    print("Fetched {} contacts ({}s)".format(len(contacts), time.time() - start))
+
+    # Convert contacts to TracedData
+    traced_contacts = []
+    for contact in contacts:
+        contact_dict = dict()
+        contact_dict["avf_phone_id"] = phone_uuids.add_phone(contact.urns[0])
+        contact_dict.update(contact.fields)
+
+        traced_contacts.append(TracedData(
+            contact_dict,
+            Metadata(user, Metadata.get_call_location(), time.time())
+        ))
+
+    # Write the UUIDs out to a file
+    with open(phone_uuid_path, "w") as f:
+        phone_uuids.dump(f)
+
+    # Output TracedData to JSON
+    IOUtils.ensure_dirs_exist_for_file(json_output_path)
+    with open(json_output_path, "w") as f:
+        TracedDataJsonIO.export_traced_data_iterable_to_json(traced_contacts, f, pretty_print=True)