[New Scheduler] Etcd installation & Implements EtcdClient

ansible/environments/docker-machine/hosts.j2.ini

@@ -39,6 +39,11 @@ invoker1                    ansible_host={{ docker_machine_ip }}
 [elasticsearch:children]
 db
 
+[etcd]
+etcd0            ansible_host={{ docker_machine_ip }}
+{% if mode is defined and 'HA' in mode %}
+etcd1            ansible_host={{ docker_machine_ip }}
+
 ; define variables
 [all:vars]
 ansible_connection=ssh

ansible/environments/local/hosts.j2.ini

@@ -39,3 +39,6 @@ db
 
 [apigateway]
 172.17.0.1          ansible_host=172.17.0.1 ansible_connection=local
+
+[etcd]
+etcd0            ansible_host=172.17.0.1 ansible_connection=local

ansible/environments/vagrant/hosts

@@ -27,3 +27,6 @@ invoker0            ansible_host=172.17.0.1 ansible_connection=local
 
 [apigateway]
 172.17.0.1          ansible_host=172.17.0.1 ansible_connection=local
+
+[etcd]
+etcd0            ansible_host=172.17.0.1 ansible_connection=local

ansible/etcd.yml

@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# This playbook deploys Openwhisk Invokers.
+
+- hosts: etcd
+  roles:
+  - etcd

ansible/group_vars/all

@@ -51,6 +51,7 @@ whisk:
   feature_flags:
     require_api_key_annotation: "{{ require_api_key_annotation | default(true) | lower }}"
     require_response_payload: "{{ require_response_payload | default(true) | lower }}"
+  cluster_name: "{{ whisk_cluster_name | default('whisk') }}"
 
 ##
 # configuration parameters related to support runtimes (see org.apache.openwhisk.core.entity.ExecManifest for schema of the manifest).
@@ -424,3 +425,28 @@ user_events: "{{ user_events_enabled | default(false) | lower }}"
 
 durationChecker:
     timeWindow: "{{ duration_checker_time_window | default('1 d') }}"
+
+etcd:
+  version: "{{ etcd_version | default('v3.4.0') }}"
+  client:
+    port: 2379
+  server:
+    port: 2480
+  cluster:
+    token: "{{ etcd_cluster_token | default('openwhisk-etcd-token') }}"
+  dir:
+    data: "{{ etcd_data_dir | default('') }}"
+  lease:
+    timeout: "{{ etcd_lease_timeout | default(1) }}"
+  loglevel: "{{ etcd_log_level | default('info') }}"
+  quota_backend_bytes: "{{ etcd_quota_backend_bytes | default(0) }}"
+  snapshot_count: "{{ etcd_snapshot_count | default(100000) }}"
+  auto_compaction_retention: "{{ etcd_auto_compaction_retention | default(1) }}"
+  auto_compaction_mode: "{{ etcd_auto_compaction_mode | default('periodic') }}"
+  pool_threads: "{{ etcd_pool_threads | default(10) }}"
+
+etcd_connect_string: "{% set ret = [] %}\
+                      {% for host in groups['etcd'] %}\
+                        {{ ret.append( hostvars[host].ansible_host + ':' + ((etcd.client.port+loop.index-1)|string) ) }}\
+                      {% endfor %}\
+                      {{ ret | join(',') }}"

ansible/roles/etcd/tasks/clean.yml

@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# Remove etcd containers.
+
+- name: remove etcd
+  docker_container:
+    name: etcd{{ groups['etcd'].index(inventory_hostname) }}
+    keep_volumes: True
+    state: absent
+  ignore_errors: True

ansible/roles/etcd/tasks/deploy.yml

@@ -0,0 +1,70 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# This role will install etcd in group 'etcd' in the environment inventory
+
+- name: "Set the name of the etcd node"
+  set_fact:
+    name: "etcd{{ groups['etcd'].index(inventory_hostname) }}"
+
+- name: "set the volume_dir"
+  set_fact:
+    volume_dir: "{{ etcd.dir.data }}/etcd{{ groups['etcd'].index(inventory_hostname) }}:/etcd-data"
+  when: etcd_data_dir is defined
+
+
+- name: "Set the cluster of the etcd cluster"
+  set_fact:
+    cluster: "{% set etcdhosts = [] %}
+              {% for host in groups['etcd'] %}
+                  {{ etcdhosts.append('etcd' + ((loop.index-1)|string) + '=' + 'http://' + hostvars[host].ansible_host + ':' + ((2480+loop.index-1)|string) ) }}
+              {% endfor %}
+              {{ etcdhosts | join(',') }}"
+
+- name: (re)start etcd
+  docker_container:
+    name: etcd{{ groups['etcd'].index(inventory_hostname) }}
+    image: quay.io/coreos/etcd:{{ etcd.version }}
+    state: started
+    recreate: true
+    restart_policy: "{{ docker.restart.policy }}"
+    volumes: "{{volume_dir | default([])}}"
+    command: "/usr/local/bin/etcd \
+               --data-dir=/etcd-data --name '{{ name }}' \
+               --initial-advertise-peer-urls http://{{ ansible_host }}:{{ etcd.server.port + groups['etcd'].index(inventory_hostname) }} \
+               --advertise-client-urls http://{{ ansible_host }}:{{ etcd.client.port + groups['etcd'].index(inventory_hostname) }} \
+               --listen-peer-urls http://0.0.0.0:{{ etcd.server.port + groups['etcd'].index(inventory_hostname) }} \
+               --listen-client-urls http://0.0.0.0:{{ etcd.client.port + groups['etcd'].index(inventory_hostname) }} \
+               --initial-cluster {{ cluster }} \
+               --initial-cluster-state new --initial-cluster-token {{ etcd.cluster.token }} \
+               --quota-backend-bytes {{ etcd.quota_backend_bytes }} \
+               --snapshot-count {{ etcd.snapshot_count }} \
+               --auto-compaction-retention {{ etcd.auto_compaction_retention }} \
+               --auto-compaction-mode {{ etcd.auto_compaction_mode }} \
+               --log-level {{ etcd.loglevel }}"
+    ports:
+      - "{{ etcd.client.port + groups['etcd'].index(inventory_hostname) }}:{{ etcd.client.port + groups['etcd'].index(inventory_hostname) }}"
+      - "{{ etcd.server.port + groups['etcd'].index(inventory_hostname) }}:{{ etcd.server.port + groups['etcd'].index(inventory_hostname) }}"
+    pull: "{{ etcd.pull_etcd | default(true) }}"
+
+- name: wait until etcd in this host is up and running
+  uri:
+    url: "http://{{ ansible_host }}:{{ etcd.client.port + groups['etcd'].index(inventory_hostname) }}/health"
+  register: result
+  until: result.status == 200
+  retries: 12
+  delay: 5

ansible/roles/etcd/tasks/main.yml

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# This role will install etcd in group 'etcd' in the environment inventory
+# In deploy mode it will deploy etcd containers.
+# In clean mode it will remove etcd containers.
+
+- import_tasks: deploy.yml
+  when: mode == "deploy"
+
+- import_tasks: clean.yml
+  when: mode == "clean"

common/scala/build.gradle

@@ -73,6 +73,9 @@ dependencies {
     //for mesos
     compile "com.adobe.api.platform.runtime:mesos-actor:0.0.17"
 
+    // for etcd
+    compile("com.ibm.etcd:etcd-java:0.0.13")
+
     //tracing support
     compile "io.opentracing:opentracing-api:0.31.0"
     compile "io.opentracing:opentracing-util:0.31.0"

common/scala/src/main/resources/application.conf

@@ -383,6 +383,11 @@ whisk {
         local-image-prefix = "whisk"
     }
 
+    # cluster name related etcd configuration
+    cluster {
+        name = "whisk"
+    }
+
     user-events {
         enabled = false
     }

common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala

@@ -265,6 +265,9 @@ object ConfigKeys {
   val controller = s"whisk.controller"
   val controllerActivation = s"$controller.activation"
 
+  val etcd = "whisk.etcd"
+  val etcdPoolThreads = "whisk.etcd.pool.threads"
+
   val activationStore = "whisk.activation-store"
   val elasticSearchActivationStore = s"$activationStore.elasticsearch"
   val activationStoreWithFileStorage = s"$activationStore.with-file-storage"
@@ -286,4 +289,6 @@ object ConfigKeys {
   val parameterStorage = "whisk.parameter-storage"
 
   val azBlob = "whisk.azure-blob"
+
+  val whiskClusterName = "whisk.cluster.name"
 }

common/scala/src/main/scala/org/apache/openwhisk/core/entity/CreationId.scala

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.openwhisk.core.entity
+
+import org.apache.openwhisk.core.entity.size._
+import org.apache.openwhisk.http.Messages
+import spray.json.DefaultJsonProtocol.StringJsonFormat
+import spray.json.{JsObject, _}
+
+import scala.util.{Failure, Success, Try}
+
+protected[openwhisk] case class CreationId private (val asString: String) extends AnyVal {
+  override def toString: String = asString
+  def toJsObject: JsObject = JsObject("creationId" -> asString.toJson)
+}
+
+protected[core] object CreationId {
+
+  protected[core] trait CreationIdGenerator {
+    def make(): CreationId = CreationId.generate()
+  }
+
+  /** Checks if the current character is hexadecimal */
+  private def isHexadecimal(c: Char) = c.isDigit || c == 'a' || c == 'b' || c == 'c' || c == 'd' || c == 'e' || c == 'f'
+
+  /**
+   * Parses an creation id from a string.
+   *
+   * @param id the creation id as string
+   * @return CreationId instance
+   */
+  def parse(id: String): Try[CreationId] = {
+    val length = id.length
+    if (length != 32) {
+      Failure(new IllegalArgumentException(Messages.creationIdLengthError(SizeError("Creation id", length.B, 32.B))))
+    } else if (!id.forall(isHexadecimal)) {
+      Failure(new IllegalArgumentException(Messages.creationIdIllegal))
+    } else {
+      Success(new CreationId(id))
+    }
+  }
+
+  /**
+   * Generates a random creation id using java.util.UUID factory.
+   *
+   * Uses fast path to generate the CreationId without additional requirement checks.
+   *
+   * @return new CreationId
+   */
+  protected[core] def generate(): CreationId = new CreationId(UUIDs.randomUUID().toString.filterNot(_ == '-'))
+
+  protected[core] implicit val serdes: RootJsonFormat[CreationId] = new RootJsonFormat[CreationId] {
+    def write(d: CreationId) = JsString(d.toString)
+
+    def read(value: JsValue): CreationId = {
+      val parsed = value match {
+        case JsString(s) => CreationId.parse(s)
+        case JsNumber(n) => CreationId.parse(n.toString)
+        case _           => Failure(DeserializationException(Messages.creationIdIllegal))
+      }
+
+      parsed match {
+        case Success(cid)                         => cid
+        case Failure(t: IllegalArgumentException) => deserializationError(t.getMessage)
+        case Failure(_)                           => deserializationError(Messages.creationIdIllegal)
+      }
+    }
+  }
+}