Merge pull request #207 from cliveseldon/metaflow_aws

Fix metaflow local/remote decision and update to 0.5.1 release

Author: ukclivecox

docs/examples/metaflow/README.ipynb

@@ -53,13 +53,7 @@
    "id": "b2a246ca",
    "metadata": {},
    "source": [
-    "## Run Flow locally to deploy to Docker\n",
-    "\n",
-    "To run the workflow with a local Docker deployment use the flag:\n",
-    "\n",
-    "```\n",
-    "--tempo-on-docker true\n",
-    "```\n"
+    "## Run Flow locally to deploy to Docker"
    ]
   },
   {
@@ -105,8 +99,6 @@
     "\n",
     "### Setup AWS Metaflow Support\n",
     "\n",
-    "Note at present this is required even for a local run as artifacts are stored on S3.\n",
-    "\n",
     "[Install Metaflow with remote AWS support](https://docs.metaflow.org/metaflow-on-aws/metaflow-on-aws).\n",
     "\n",
     "### Seldon Requirements\n",
@@ -313,7 +305,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.10"
+   "version": "3.7.9"
   }
  },
  "nbformat": 4,

docs/examples/metaflow/README.md

@@ -32,13 +32,6 @@ conda config --add channels conda-forge
 
 ## Run Flow locally to deploy to Docker
 
-To run the workflow with a local Docker deployment use the flag:
-
-```
---tempo-on-docker true
-```
-
-
 
 ```python
 !python src/irisflow.py --environment=conda run 
@@ -61,8 +54,6 @@ We will now run our flow on AWS Batch and will launch Tempo artifacts onto a rem
 
 ### Setup AWS Metaflow Support
 
-Note at present this is required even for a local run as artifacts are stored on S3.
-
 [Install Metaflow with remote AWS support](https://docs.metaflow.org/metaflow-on-aws/metaflow-on-aws).
 
 ### Seldon Requirements

docs/examples/metaflow/src/irisflow.py

@@ -162,19 +162,22 @@ def deploy_tempo_remote(self, classifier):
         print(self.client_model.predict(np.array([[1, 2, 3, 4]])))
 
     @conda(libraries={"numpy": "1.19.5"})
-    @pip(libraries={"mlops-tempo": "0.5.0", "conda_env": "2.4.2"})
+    @pip(libraries={"mlops-tempo": "0.5.1", "conda_env": "2.4.2"})
     @step
     def tempo(self):
         """
         Create Tempo artifacts locally and saved to S3 within the workflow bucket.
         Then either deploy locally to Docker or deploy to a remote Kubernetes cluster based on the
         --tempo-on-docker parameter
         """
-        classifier, remote = self.create_tempo_artifacts()
+        from tempo.metaflow.utils import running_aws_batch
 
-        if remote:
+        classifier, s3_active = self.create_tempo_artifacts()
+        if s3_active and running_aws_batch(self.tempo):
+            print("Deploying to remote k8s cluster")
             self.deploy_tempo_remote(classifier)
         else:
+            print("Deploying to local Docker")
             self.deploy_tempo_local(classifier)
 
         self.next(self.end)

tempo/metaflow/utils.py

@@ -1,6 +1,7 @@
 from typing import Any
 
-from metaflow import S3, FlowSpec, IncludeFile
+from metaflow import S3, FlowSpec, IncludeFile, Step
+from metaflow.plugins.aws.batch.batch_decorator import BatchDecorator
 
 
 def save_artifact(model: Any, filename: str):
@@ -178,3 +179,22 @@ def save_pipeline_with_conda(pipeline, folder: str, conda_env: IncludeFile):
     with open(conda_env_path, "w") as f:
         f.write(conda_env)
     save(pipeline)
+
+
+def running_aws_batch(step: Step) -> bool:
+    """
+    Test if a Step is running on AWS batch
+    Parameters
+    ----------
+    step The step to test
+
+    Returns
+    -------
+    True if flow is running on AWS Batch
+
+    """
+    running_on_aws_batch = False
+    for deco in step.decorators:  # pylint: disable=maybe-no-member
+        if isinstance(deco, BatchDecorator):
+            running_on_aws_batch = True
+    return running_on_aws_batch

tempo/version.py

@@ -1 +1 @@
-__version__ = "0.5.0"
+__version__ = "0.5.1"