Rewrite etcd role for proper lifecycle

Previously, docker could not be restarted on a node in a cluster and
have v2plugin remaining working due to a conflict because the container
was never stopped, the container name killed now matches that start name
for the container.

Previously, docker could not be shut down on multiple nodes without
getting etcd in a state where it could not start because the start/stop
script was dynamically managing the cluster membership as part of
service start/stop state.

Etcd is a HA database, starting or stopping should not depend on a
single node, the old code checked the first node for peers and used
that to determine if the cluster was being initialized or was already
active. Instead, just configure etcd statically with all the member
nodes in configuration.

To improve readability, much of the old etcd start/stop scripts
template to produce urls for etcd configuration was jinja macros,
redid that as role default variables using composition instead.

Introduced some naming for the etcd cluster that drives systemd service
naming

etcd member names are by default based on the hostname instead of
numbers that can be out of order with the machines.

The systemd unit was updated to handle:
* docker stop
* docker kill
* systemctl stop docker
* systemctl restart docker
When stopping docker, etcd will be stopped first, and when docker
starts, it will always trigger etcd to start.

Also, slow down restarts a little to avoid getting blocked by systemd
for restarting too many times and requiring manual intervention

v2plugin updated to be wantedby instead of PartOf so it starts when
docker is "start"ed and not just restarted

Drive-by:
* Add names to some plays and tasks to improve output
* Removed etcd peer port 7001 as it is not used

Signed-off-by: Chris Plock [email protected]

Author: chrisplo

group_vars/all

@@ -30,3 +30,5 @@ service_vip: "{{ netmaster_ip }}"
 # netmaster_ip:
 
 host_capability: "can-run-user-containers, storage"
+
+etcd_peer_advertise_interface: "{{ control_interface }}"

roles/contiv_network/files/v2plugin.service

@@ -1,11 +1,11 @@
 [Unit]
 Description=enable v2plugin after etcd 
 After=docker.service etcd.service
-PartOf=etcd.service
 
 [Service]
 Type=oneshot
 ExecStart=/usr/bin/v2plugin.sh
 StandardOutput=journal
+
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target etcd.service

roles/etcd/defaults/main.yml

@@ -1,14 +1,56 @@
 ---
 # role variable for the etcd service
 
+etcd_version: v2.3.8
+etcd_cluster_name: contiv
+
+etcd_proxy: "{{ etcd_proxies_group in groups
+                 and inventory_hostname in groups[etcd_proxies_group] }}"
+etcd_member: "{{ etcd_members_group in groups
+                 and inventory_hostname in groups[etcd_members_group] }}"
+etcd_service_name: "etcd{{ etcd_proxy|bool|ternary('-proxy','') }}-{{
+    etcd_cluster_name}}"
+etcd_docker_name: "{{ etcd_service_name }}"
+
+etcd_node_name: "{{ inventory_hostname_short }}"
 etcd_client_port1: 2379
 etcd_client_port2: 4001
+
 etcd_peer_port1: 2380
-etcd_peer_port2: 7001
-etcd_peers_group: "service-master"
-etcd_rule_comment: "contiv_etcd traffic"
-etcd_version: "v2.3.8"
+
+etcd_members_group: netplugin-master
+etcd_proxies_group: netplugin-worker
+etcd_initial_cluster_token: "etcd-{{ etcd_cluster_name }}"
+
+etcd_rule_comment: "{{ etcd_cluster_name }}_etcd traffic"
+
 etcd_heartbeat_interval: 1000
 etcd_election_timeout: 10000
-etcd_data_dir: /var/lib/etcd
+
+etcd_data_dir: /var/lib/etcd/data
+etcd_wal_dir: /var/lib/etcd/wal
+
 etcd_reset_state: false
+
+# Etcd has "advertise" urls for the other nodes to use if it's behind a proxy
+# Also, it needs to "listen" on IPs and ports for peers to talk to
+etcd_peer_advertise_interface: "{{ ansible_default_ipv4.interface }}"
+etcd_peer_advertise_address: "{{
+    hostvars[inventory_hostname]['ansible_' + etcd_peer_advertise_interface]['ipv4']['address'] }}"
+etcd_peer_listen_address: "{{ etcd_peer_advertise_address }}"
+etcd_peer_advertise_urls:
+  - "http://{{ etcd_peer_advertise_address }}:{{ etcd_peer_port1 }}"
+etcd_peer_listen_urls:
+  - "http://{{ etcd_peer_listen_address }}:{{ etcd_peer_port1 }}"
+
+etcd_client_advertise_address: "{{ etcd_peer_advertise_address }}"
+etcd_client_listen_address: 0.0.0.0
+etcd_client_advertise_urls:
+  - "http://{{ etcd_client_advertise_address }}:{{ etcd_client_port1 }}"
+  - "http://{{ etcd_client_advertise_address }}:{{ etcd_client_port2 }}"
+etcd_client_listen_urls:
+  - "http://{{ etcd_client_listen_address }}:{{ etcd_client_port1 }}"
+  - "http://{{ etcd_client_listen_address }}:{{ etcd_client_port2 }}"
+
+etcd_systemd_restart_delay_sec: 2
+etcd_systemd_restart: always

roles/etcd/files/etcd.service

@@ -1,43 +1,56 @@
 ---
 # This role contains tasks for configuring and starting etcd service
 
-- name: download etcdctl {{ etcd_version }}
-  get_url:
-    validate_certs: "{{ validate_certs }}"
-    url: https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz
-    dest: /tmp/etcd-{{ etcd_version }}-linux-amd64.tar.gz
-  tags:
-    - prebake-for-dev
-
-- name: install etcdctl
-  shell: >
-      tar vxzf /tmp/etcd-{{ etcd_version }}-linux-amd64.tar.gz && \
-      mv etcd-{{ etcd_version }}-linux-amd64/etcd* /usr/bin
-  tags:
-    - prebake-for-dev
-
-- name: install etcd {{ etcd_version }}
-  shell: docker pull quay.io/coreos/etcd:{{ etcd_version }}
-  tags:
-    - prebake-for-dev
-
-- name: setup iptables for etcd
-  shell: >
-      ( iptables -L INPUT | grep "{{ etcd_rule_comment }} ({{ item }})" ) || \
-      iptables -I INPUT 1 -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "{{ etcd_rule_comment }} ({{ item }})"
-  become: true
-  with_items:
-    - "{{ etcd_client_port1 }}"
-    - "{{ etcd_client_port2 }}"
-    - "{{ etcd_peer_port1 }}"
-    - "{{ etcd_peer_port2 }}"
-
-- name: copy the etcd start/stop script
-  template: src=etcd.j2 dest=/usr/bin/etcd.sh mode=u=rwx,g=rx,o=rx
-
-- name: copy systemd units for etcd
-  copy: src=etcd.service dest=/etc/systemd/system/etcd.service
-
-- name: start etcd
-  systemd: name=etcd daemon_reload=yes state=started enabled=yes
-
+- block:
+    - name: download etcdctl {{ etcd_version }}
+      get_url:
+        validate_certs: "{{ validate_certs }}"
+        url: https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz
+        dest: /tmp/etcd-{{ etcd_version }}-linux-amd64.tar.gz
+      tags:
+        - prebake-for-dev
+
+    - name: install etcdctl binaries
+      shell: >
+          tar vxzf /tmp/etcd-{{ etcd_version }}-linux-amd64.tar.gz &&
+          mv etcd-{{ etcd_version }}-linux-amd64/etcd* /usr/bin
+      tags:
+        - prebake-for-dev
+
+    - name: pull etcd container {{ etcd_version }}
+      shell: docker pull quay.io/coreos/etcd:{{ etcd_version }}
+      tags:
+        - prebake-for-dev
+
+    - name: setup iptables for etcd
+      shell: >
+          ( iptables -L INPUT | grep "{{ etcd_rule_comment }} ({{ item }})" ) || \
+          iptables -I INPUT 1 -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "{{ etcd_rule_comment }} ({{ item }})"
+      become: true
+      with_items:
+        - "{{ etcd_client_port1 }}"
+        - "{{ etcd_client_port2 }}"
+        - "{{ etcd_peer_port1 }}"
+
+    - name: Set facts from role defaults for sharing with other hosts
+      set_fact:
+        etcd_node_name: "{{ etcd_node_name }}"
+        etcd_peer_advertise_urls: "{{ etcd_peer_advertise_urls }}"
+
+    - name: template etcd docker environment file
+      template:
+        src: etcd_env_file.j2
+        dest: "/etc/{{ etcd_service_name }}.env"
+
+    - name: template systemd units for etcd
+      template:
+        src: etcd.service.j2
+        dest: "/etc/systemd/system/{{ etcd_service_name }}.service"
+
+    - name: start etcd server or proxy
+      systemd:
+        name: "{{ etcd_service_name }}"
+        daemon_reload: yes
+        state: started
+        enabled: yes
+  when: etcd_member|bool or etcd_proxy|bool