Skip to content

Commit 40628e7

Browse files
DanielMwendwaDanielMwendwa
authored
Merge pull request #91 from AfricasVoices/mno-shell-scripts
add shell scripts in mno analysis tools dir
2 parents 537da9a + cde5626 commit 40628e7

5 files changed

+321
-1
lines changed

‎mno_analysis_tools/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,4 @@ RUN pipenv sync
2626
ADD fetch_raw_messages.py /app
2727
ADD compute_window_of_downtime.py /app
2828
ADD compute_messages_per_period.py /app
29-
ADD compute_msg_difference_btwn_periods.py /app
29+
ADD compute_msg_difference_btwn_two_firebase_time_periods.py /app

‎mno_analysis_tools/docker-run-compute-messages-per-period.sh

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
IMAGE_NAME=compute-messages-per-period
6+
7+
while [[ $# -gt 0 ]]; do
8+
case "$1" in
9+
--profile-memory)
10+
PROFILE_MEMORY=true
11+
MEMORY_PROFILE_OUTPUT_PATH="$2"
12+
shift 2;;
13+
--)
14+
shift
15+
break;;
16+
*)
17+
break;;
18+
esac
19+
done
20+
21+
# Check that the correct number of arguments were provided.
22+
if [[ $# -ne 7 ]]; then
23+
echo "Usage: ./docker-run-compute-window-of-downtime.sh
24+
<raw_messages_file_path> <target_operator> <target_message_direction>
25+
<start_date> <end_date> <time_frame> <output_dir>"
26+
exit
27+
fi
28+
29+
# Assign the program arguments to bash variables.
30+
RAW_MESSAGES_FILE_PATH=$1
31+
TARGET_OPERATOR=$2
32+
TARGET_MESSAGE_DIRECTION=$3
33+
START_DATE=$4
34+
END_DATE=$5
35+
TIME_FRAME=$6
36+
OUTPUT_DIR=$7
37+
38+
# Build an image for this pipeline stage.
39+
docker build --build-arg INSTALL_MEMORY_PROFILER="$PROFILE_MEMORY" -t "$IMAGE_NAME" .
40+
41+
# Create a container from the image that was just built.
42+
if [[ "$PROFILE_MEMORY" = true ]]; then
43+
PROFILE_MEMORY_CMD="mprof run -o /data/memory.prof"
44+
fi
45+
46+
if [[ $TARGET_MESSAGE_DIRECTION == "in" ]]
47+
then
48+
MSG_DIRECTION="incoming"
49+
else
50+
MSG_DIRECTION="outgoing"
51+
fi
52+
53+
CMD="pipenv run $PROFILE_MEMORY_CMD python -u compute_messages_per_period.py \
54+
/data/raw_messages.json /data/${MSG_DIRECTION}_msg.json \
55+
\"$TARGET_OPERATOR\" \"$TARGET_MESSAGE_DIRECTION\" \"$START_DATE\" \"$END_DATE\" \"$TIME_FRAME\"
56+
"
57+
58+
container="$(docker container create -w /app "$IMAGE_NAME" /bin/bash -c "$CMD")"
59+
echo "Created container $container"
60+
container_short_id=${container:0:7}
61+
62+
function finish {
63+
# Tear down the container when done.
64+
docker container rm "$container" >/dev/null
65+
}
66+
trap finish EXIT
67+
68+
# Copy input data into the container
69+
docker cp "$RAW_MESSAGES_FILE_PATH" "$container:/data/raw_messages.json"
70+
71+
# Run the container
72+
echo "Starting container $container_short_id"
73+
docker start -a -i "$container"
74+
75+
# Copy the output data back out of the container
76+
echo "Copying $container_short_id:/data/. -> $OUTPUT_DIR"
77+
mkdir -p "$OUTPUT_DIR"
78+
docker cp "$container:/data/." "$OUTPUT_DIR"
79+
80+
if [[ "$PROFILE_MEMORY" = true ]]; then
81+
echo "Copying $container_short_id:/data/memory.prof -> $MEMORY_PROFILE_OUTPUT_PATH"
82+
mkdir -p "$(dirname "$MEMORY_PROFILE_OUTPUT_PATH")"
83+
docker cp "$container:/data/memory.prof" "$MEMORY_PROFILE_OUTPUT_PATH"
84+
fi

‎mno_analysis_tools/docker-run-compute-msg-difference-btwn-two-firebase-time-periods.sh

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
IMAGE_NAME=compute-msg-difference-btwn-two-firebase-time-periods
6+
7+
while [[ $# -gt 0 ]]; do
8+
case "$1" in
9+
--profile-memory)
10+
PROFILE_MEMORY=true
11+
MEMORY_PROFILE_OUTPUT_PATH="$2"
12+
shift 2;;
13+
--)
14+
shift
15+
break;;
16+
*)
17+
break;;
18+
esac
19+
done
20+
21+
# Check that the correct number of arguments were provided.
22+
if [[ $# -lt 6 ]]; then
23+
echo "Usage: ./docker-run-compute-msg-difference-btwn-periods.sh
24+
<raw_messages_file_path> <target_operator> <target_message_direction>
25+
<start_date> <end_date> <output_dir> <time_frame>"
26+
exit
27+
fi
28+
29+
# Assign the program arguments to bash variables.
30+
RAW_MESSAGES_FILE_PATH=$1
31+
TARGET_OPERATOR=$2
32+
TARGET_MESSAGE_DIRECTION=$3
33+
START_DATE=$4
34+
END_DATE=$5
35+
OUTPUT_DIR=$6
36+
TIME_FRAME=${7:-00:00:10}
37+
38+
# Build an image for this pipeline stage.
39+
docker build --build-arg INSTALL_MEMORY_PROFILER="$PROFILE_MEMORY" -t "$IMAGE_NAME" .
40+
41+
# Create a container from the image that was just built.
42+
if [[ "$PROFILE_MEMORY" = true ]]; then
43+
PROFILE_MEMORY_CMD="mprof run -o /data/memory.prof"
44+
fi
45+
46+
if [[ $TARGET_MESSAGE_DIRECTION == "in" ]]
47+
then
48+
MSG_DIRECTION="incoming"
49+
else
50+
MSG_DIRECTION="outgoing"
51+
fi
52+
53+
CMD="pipenv run $PROFILE_MEMORY_CMD python -u compute_msg_difference_btwn_two_firebase_time_periods.py \
54+
/data/raw_messages.json /data/${MSG_DIRECTION}_msg_diff_per_period.json \
55+
\"$TARGET_OPERATOR\" \"$TARGET_MESSAGE_DIRECTION\" \"$START_DATE\" \"$END_DATE\" -t \"$TIME_FRAME\"
56+
"
57+
58+
container="$(docker container create -w /app "$IMAGE_NAME" /bin/bash -c "$CMD")"
59+
echo "Created container $container"
60+
container_short_id=${container:0:7}
61+
62+
function finish {
63+
# Tear down the container when done.
64+
docker container rm "$container" >/dev/null
65+
}
66+
trap finish EXIT
67+
68+
# Copy input data into the container
69+
docker cp "$RAW_MESSAGES_FILE_PATH" "$container:/data/raw_messages.json"
70+
71+
# Run the container
72+
echo "Starting container $container_short_id"
73+
docker start -a -i "$container"
74+
75+
# Copy the output data back out of the container
76+
echo "Copying $container_short_id:/data/. -> $OUTPUT_DIR"
77+
mkdir -p "$OUTPUT_DIR"
78+
docker cp "$container:/data/." "$OUTPUT_DIR"
79+
80+
if [[ "$PROFILE_MEMORY" = true ]]; then
81+
echo "Copying $container_short_id:/data/memory.prof -> $MEMORY_PROFILE_OUTPUT_PATH"
82+
mkdir -p "$(dirname "$MEMORY_PROFILE_OUTPUT_PATH")"
83+
docker cp "$container:/data/memory.prof" "$MEMORY_PROFILE_OUTPUT_PATH"
84+
fi

‎mno_analysis_tools/docker-run-compute-window-of-downtime.sh

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
IMAGE_NAME=compute-window-of-downtime
6+
7+
while [[ $# -gt 0 ]]; do
8+
case "$1" in
9+
--profile-memory)
10+
PROFILE_MEMORY=true
11+
MEMORY_PROFILE_OUTPUT_PATH="$2"
12+
shift 2;;
13+
--)
14+
shift
15+
break;;
16+
*)
17+
break;;
18+
esac
19+
done
20+
21+
# Check that the correct number of arguments were provided.
22+
if [[ $# -ne 6 ]]; then
23+
echo "Usage: ./docker-run-compute-window-of-downtime.sh
24+
<raw_messages_file_path> <target_operator> <target_message_direction>
25+
<start_date> <end_date> <output_dir>"
26+
exit
27+
fi
28+
29+
# Assign the program arguments to bash variables.
30+
RAW_MESSAGES_FILE_PATH=$1
31+
TARGET_OPERATOR=$2
32+
TARGET_MESSAGE_DIRECTION=$3
33+
START_DATE=$4
34+
END_DATE=$5
35+
OUTPUT_DIR=$6
36+
37+
# Build an image for this pipeline stage.
38+
docker build --build-arg INSTALL_MEMORY_PROFILER="$PROFILE_MEMORY" -t "$IMAGE_NAME" .
39+
40+
# Create a container from the image that was just built.
41+
if [[ "$PROFILE_MEMORY" = true ]]; then
42+
PROFILE_MEMORY_CMD="mprof run -o /data/memory.prof"
43+
fi
44+
45+
if [[ $TARGET_MESSAGE_DIRECTION == "in" ]]
46+
then
47+
MSG_DIRECTION="incoming"
48+
else
49+
MSG_DIRECTION="outgoing"
50+
fi
51+
52+
CMD="pipenv run $PROFILE_MEMORY_CMD python -u compute_window_of_downtime.py \
53+
/data/raw_messages.json /data/${MSG_DIRECTION}_msg_downtime.json \
54+
\"$TARGET_OPERATOR\" \"$TARGET_MESSAGE_DIRECTION\" \"$START_DATE\" \"$END_DATE\"
55+
"
56+
57+
container="$(docker container create -w /app "$IMAGE_NAME" /bin/bash -c "$CMD")"
58+
echo "Created container $container"
59+
container_short_id=${container:0:7}
60+
61+
function finish {
62+
# Tear down the container when done.
63+
docker container rm "$container" >/dev/null
64+
}
65+
trap finish EXIT
66+
67+
# Copy input data into the container
68+
docker cp "$RAW_MESSAGES_FILE_PATH" "$container:/data/raw_messages.json"
69+
70+
# Run the container
71+
echo "Starting container $container_short_id"
72+
docker start -a -i "$container"
73+
74+
# Copy the output data back out of the container
75+
echo "Copying $container_short_id:/data/. -> $OUTPUT_DIR"
76+
mkdir -p "$OUTPUT_DIR"
77+
docker cp "$container:/data/." "$OUTPUT_DIR"
78+
79+
if [[ "$PROFILE_MEMORY" = true ]]; then
80+
echo "Copying $container_short_id:/data/memory.prof -> $MEMORY_PROFILE_OUTPUT_PATH"
81+
mkdir -p "$(dirname "$MEMORY_PROFILE_OUTPUT_PATH")"
82+
docker cp "$container:/data/memory.prof" "$MEMORY_PROFILE_OUTPUT_PATH"
83+
fi

‎mno_analysis_tools/docker-run-fetch-raw-messages.sh

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
IMAGE_NAME=fetch-raw-messages
6+
7+
while [[ $# -gt 0 ]]; do
8+
case "$1" in
9+
--profile-memory)
10+
PROFILE_MEMORY=true
11+
MEMORY_PROFILE_OUTPUT_PATH="$2"
12+
shift 2;;
13+
--)
14+
shift
15+
break;;
16+
*)
17+
break;;
18+
esac
19+
done
20+
21+
# Check that the correct number of arguments were provided.
22+
if [[ $# -ne 3 ]]; then
23+
echo "Usage: ./docker-run-fetch-raw-messages.sh
24+
[--profile-memory <profile-output-path>]
25+
<domain> <token> <output-dir>"
26+
exit
27+
fi
28+
29+
# Assign the program arguments to bash variables.
30+
DOMAIN=$1
31+
TOKEN=$2
32+
OUTPUT_DIR=$3
33+
34+
# Build an image for this pipeline stage.
35+
docker build --build-arg INSTALL_MEMORY_PROFILER="$PROFILE_MEMORY" -t "$IMAGE_NAME" .
36+
37+
# Create a container from the image that was just built.
38+
if [[ "$PROFILE_MEMORY" = true ]]; then
39+
PROFILE_MEMORY_CMD="mprof run -o /data/memory.prof"
40+
fi
41+
42+
CMD="pipenv run $PROFILE_MEMORY_CMD python -u fetch_raw_messages.py \
43+
\"$DOMAIN\" \"$TOKEN\" /data/raw_messages.json
44+
"
45+
46+
container="$(docker container create -w /app "$IMAGE_NAME" /bin/bash -c "$CMD")"
47+
echo "Created container $container"
48+
container_short_id=${container:0:7}
49+
50+
function finish {
51+
# Tear down the container when done.
52+
docker container rm "$container" >/dev/null
53+
}
54+
trap finish EXIT
55+
56+
# Run the container
57+
echo "Starting container $container_short_id"
58+
docker start -a -i "$container"
59+
60+
# Copy the output data back out of the container
61+
echo "Copying $container_short_id:/data/. -> $OUTPUT_DIR"
62+
mkdir -p "$OUTPUT_DIR"
63+
docker cp "$container:/data/." "$OUTPUT_DIR"
64+
65+
if [[ "$PROFILE_MEMORY" = true ]]; then
66+
echo "Copying $container_short_id:/data/memory.prof -> $MEMORY_PROFILE_OUTPUT_PATH"
67+
mkdir -p "$(dirname "$MEMORY_PROFILE_OUTPUT_PATH")"
68+
docker cp "$container:/data/memory.prof" "$MEMORY_PROFILE_OUTPUT_PATH"
69+
fi

0 commit comments

Comments
 (0)