Add Google Cloud support with GCPCluster (#131)

* Add VMCluster class and Digital Ocean implementation

* Fix worker startup command

* spec work on gcp provider

* updates

* checkpoint

* Working state

* Refactoring

* working setup

* Use correct images

* Add more config options, docs and a RAPIDS example

* Use correct images

* Add more config options, docs and a RAPIDS example

* clean up

* Add Packer support

* fix importerror

* credential handling

* better messaging

* Add autoshutdown to EC2Cluster

* more updates

* add worker args

* more updates

* Fix init order and test sync cluster

* Skip tests with missing deps

* Flake8

* Start on docs refactor

* More docs refactor

* More documentation

* Add flake8 to precommit hooks

* Fix linting

* Black

* update yaml spec and add defaults

* move default values to config

* Change worker command to module, add worker/scheduler options and refactor

* Refactor mixins and siplify code

* Refactor EC2Cluster

* Change black version

* Run black 20.8b1

* Print black version

* Force black to 20.8b1

* Shuffle AzureML docs from merge to new location

* Add GPU docs

* swtich to worker_class

* Add package import

* fix imports and update google dependecy requirements

* Migrate to GitHub Actions (#155)

* Migrate to GitHub Actions

* Fix environment file name

* Quiet conda

* Success message

* Exclude versioneer from flake8

* add pytest-asycnio/timeout to ci env

* Skip sync test without credentials

* Test imports

* Skip all auth failures and fix linting

* Slight refactors, start docs and add non RAPIDS tests

* Black

* Fix pytest-asyncio versions

* Add custom exception for missing credentials

* Add missing skip without credentials call

* Add docs and fix tests

* Remove fixture

* Fix get cloud init test

Co-authored-by: Jacob Tomlinson <[email protected]>
Co-authored-by: Jacob Tomlinson <[email protected]>

Author: quasiben

ci/environment-3.7.yml

@@ -27,6 +27,9 @@ dependencies:
   - tblib
   - tornado >=5
   - zict >=0.1.3
+  # `event_loop_policy` change See https://github.com/dask/distributed/pull/4212
+  - pytest-asyncio >=0.14.0
+  - pytest-timeout
   - pip:
       - git+https://github.com/dask/dask
       - git+https://github.com/dask/distributed

ci/environment-3.8.yml

@@ -27,6 +27,9 @@ dependencies:
   - tblib
   - tornado >=5
   - zict >=0.1.3
+  # `event_loop_policy` change See https://github.com/dask/distributed/pull/4212
+  - pytest-asyncio >=0.14.0
+  - pytest-timeout
   - pip:
       - git+https://github.com/dask/dask
       - git+https://github.com/dask/distributed

ci/scripts/test_imports.sh

@@ -18,3 +18,4 @@ test_import () {
 test_import "aws"               "import dask_cloudprovider.aws"
 test_import "azure"             "import dask_cloudprovider.azure"
 test_import "digitalocean"      "import dask_cloudprovider.digitalocean"
+test_import "gcp"               "import dask_cloudprovider.gcp"

dask_cloudprovider/aws/ec2.py

@@ -51,7 +51,7 @@ def __init__(
         self.region = region
         self.bootstrap = bootstrap
         self.ami = ami
-        self.docker_image = docker_image
+        self.docker_image = docker_image or self.config.get("docker_image")
         self.instance_type = instance_type
         self.gpu_instance = gpu_instance
         self.vpc = vpc
@@ -311,6 +311,7 @@ def __init__(
         subnet_id=None,
         security_groups=None,
         filesystem_size=None,
+        docker_image=None,
         **kwargs,
     ):
         self.boto_session = aiobotocore.get_session()
@@ -354,7 +355,7 @@ def __init__(
             "region": self.region,
             "bootstrap": self.bootstrap,
             "ami": self.ami,
-            "docker_image": self.docker_image,
+            "docker_image": docker_image or self.config.get("docker_image"),
             "instance_type": self.instance_type,
             "gpu_instance": self.gpu_instance,
             "vpc": self.vpc,

dask_cloudprovider/cloudprovider.yaml

@@ -41,6 +41,7 @@ cloudprovider:
     # worker_command: "dask-worker" # The command for workers to run. If the instance_type is a GPU instance dask-cuda-worker will be used.
     # ami: "" # AMI ID to use for all instances. Defaults to latest Ubuntu 20.04 image.
     instance_type: "t2.micro" # Instance type for all workers
+    docker_image: "daskdev/dask:latest" # docker image to use
     # vpc: "" # VPC id for instances to join. Defaults to default VPC.
     # subnet_id: "" # Subnet ID for instances to. Defaults to all subnets in default VPC.
     # security_groups: [] # Security groups for instances. Will create a minimal Dask security group by default.
@@ -66,3 +67,14 @@ cloudprovider:
     region: "nyc3" # Region to launch Droplets in
     size: "s-1vcpu-1gb" # Droplet size to launch, default is 1GB RAM, 1 vCPU
     image: "ubuntu-20-04-x64" # Operating System image to use
+
+  gcp:
+    source_image: "projects/ubuntu-os-cloud/global/images/ubuntu-minimal-1804-bionic-v20201014" # the gcp image to use for all instances
+    zone: "us-east1-c" # the zone of of where to launch the instances
+    projectid: "" # name of the google cloud project
+    machine_type: "n1-standard-1" # size of the machine type to use
+    filesystem_size: 50 # amount in GBs of hard drive space to allocate
+    ngpus: "" # number of GPUs to use
+    gpu_type: "" # type of gpus to use: nvidia-tesla-k80, nvidia-tesla-p100, nvidia-tesla-t4
+    docker_image: "daskdev/dask:latest" # docker image to use
+    auto_shutdown: true # Shutdown instances automatically if the scheduler or worker services time out.

dask_cloudprovider/gcp/__init__.py

@@ -0,0 +1 @@
+from .instances import GCPCluster