Enable aws-nuke to delete specified AWS resources (#283)

Author: phanikumv

.circleci/integration-tests/Dockerfile

@@ -1,6 +1,7 @@
 FROM quay.io/astronomer/ap-airflow:2.2.5 as staging
 
 ENV AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=False
+ENV AWS_NUKE_VERSION=v2.17.0
 
 USER root
 
@@ -31,7 +32,13 @@ ENV HADOOP_LIBRARY_VERSION=hadoop-2.10.1
 # install AWS CLI
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
     && unzip awscliv2.zip \
-    && ./aws/install
+    && ./aws/install;
+
+# install AWS nuke
+RUN wget --quiet  \
+    "https://github.com/rebuy-de/aws-nuke/releases/download/${AWS_NUKE_VERSION}/aws-nuke-${AWS_NUKE_VERSION}-linux-amd64.tar.gz" \
+    && tar -xzvf aws-nuke-${AWS_NUKE_VERSION}-linux-amd64.tar.gz -C /usr/local/bin \
+    && mv /usr/local/bin/aws-nuke-${AWS_NUKE_VERSION}-linux-amd64 /usr/local/bin/aws-nuke
 
 # install eksctl
 RUN curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp \
@@ -64,6 +71,7 @@ RUN mkdir -p ${AIRFLOW_HOME}/dags
 COPY . .
 RUN cp -r example_* ${AIRFLOW_HOME}/dags
 RUN cp master_dag.py  ${AIRFLOW_HOME}/dags/
+RUN cp nuke-config.yml  ${AIRFLOW_HOME}/dags/
 
 USER astro
 
@@ -101,7 +109,13 @@ ENV HADOOP_LIBRARY_VERSION=hadoop-2.10.1
 # install AWS CLI
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
     && unzip awscliv2.zip \
-    && ./aws/install
+    && ./aws/install;
+
+# install AWS nuke
+RUN wget --quiet  \
+    "https://github.com/rebuy-de/aws-nuke/releases/download/${AWS_NUKE_VERSION}/aws-nuke-${AWS_NUKE_VERSION}-linux-amd64.tar.gz" \
+    && tar -xzvf aws-nuke-${AWS_NUKE_VERSION}-linux-amd64.tar.gz -C /usr/local/bin \
+    && mv /usr/local/bin/aws-nuke-${AWS_NUKE_VERSION}-linux-amd64 /usr/local/bin/aws-nuke
 
 # install eksctl
 RUN curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp \
@@ -134,5 +148,6 @@ RUN mkdir -p ${AIRFLOW_HOME}/dags
 COPY . .
 RUN cp -r example_* ${AIRFLOW_HOME}/dags
 RUN cp master_dag.py  ${AIRFLOW_HOME}/dags/
+RUN cp nuke-config.yml  ${AIRFLOW_HOME}/dags/
 
 USER astro

.circleci/integration-tests/master_dag.py

@@ -94,6 +94,7 @@ def prepare_dag_dependency(task_info, execution_time):
         {"s3_sensor_dag": "example_s3_sensor"},
         {"redshift_sql_dag": "example_async_redshift_sql"},
         {"redshift_cluster_mgmt_dag": "example_async_redshift_cluster_management"},
+        {"aws_nuke_dag": "example_aws_nuke"},
     ]
     amazon_trigger_tasks, ids = prepare_dag_dependency(amazon_task_info, "{{ ds }}")
     dag_run_ids.extend(ids)

.circleci/integration-tests/nuke-config.yml

@@ -0,0 +1,30 @@
+---
+regions:
+  - us-east-2
+  - us-east-1
+  - us-west-1
+  - us-west-2
+  - ap-south-1
+  - ap-northeast-2
+  - ap-northeast-3
+  - ap-southeast-1
+  - ap-southeast-2
+  - ap-northeast-1
+  - ca-central-1
+  - eu-central-1
+  - eu-west-1
+  - eu-west-2
+  - eu-west-3
+  - eu-north-1
+  - sa-east-1
+account-blocklist:
+  - '999999999999'
+resource-types:
+  targets:
+    - EMRCluster
+    - EC2Instance
+    - EKSCluster
+    - S3MultipartUpload
+    - RedshiftCluster
+accounts:
+  '475538383708':  # aws-oss-main

astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py

@@ -0,0 +1,42 @@
+import os
+from datetime import datetime, timedelta
+
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+from airflow.operators.dummy import DummyOperator
+
+AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID", "**********")
+AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", "***********")
+AWS_DEFAULT_REGION = os.environ.get("AWS_DEFAULT_REGION", "us-east-2")
+AWS_CONN_ID = os.environ.get("ASTRO_AWS_CONN_ID", "aws_default")
+EXECUTION_TIMEOUT = int(os.getenv("EXECUTION_TIMEOUT", 6))
+
+default_args = {
+    "execution_timeout": timedelta(hours=EXECUTION_TIMEOUT),
+}
+
+with DAG(
+    dag_id="example_aws_nuke",
+    start_date=datetime(2022, 1, 1),
+    schedule_interval="30 20 * * *",
+    catchup=False,
+    default_args=default_args,
+    tags=["example", "aws-nuke"],
+) as dag:
+    start = DummyOperator(task_id="start")
+
+    set_aws_config = BashOperator(
+        task_id="aws_config",
+        bash_command=f"aws configure set aws_access_key_id {AWS_ACCESS_KEY_ID}; "
+        f"aws configure set aws_secret_access_key {AWS_SECRET_ACCESS_KEY}; "
+        f"aws configure set default.region {AWS_DEFAULT_REGION}; ",
+    )
+
+    execute_aws_nuke = BashOperator(
+        task_id="execute_aws_nuke",
+        bash_command="aws-nuke -c /usr/local/airflow/dags/nuke-config.yml --profile default --force --no-dry-run; ",
+    )
+
+    end = DummyOperator(task_id="end")
+
+    start >> set_aws_config >> execute_aws_nuke >> end