diff --git a/install/HowtoSetupContiv.md b/install/HowtoSetupContiv.md index 4cf223219..37b9fce0d 100644 --- a/install/HowtoSetupContiv.md +++ b/install/HowtoSetupContiv.md @@ -34,7 +34,7 @@ COMMANDS: GLOBAL OPTIONS: --consul-endpoints value, --consul value a comma-delimited list of netplugin consul endpoints [$CONTIV_NETPLUGIN_CONSUL_ENDPOINTS] --ctrl-ip value set netplugin control ip for control plane communication (default: ) [$CONTIV_NETPLUGIN_CONTROL_IP] - --etcd-endpoints value, --etcd value a comma-delimited list of netplugin etcd endpoints [$CONTIV_NETPLUGIN_ETCD_ENDPOINTS] + --etcd-endpoints value, --etcd value a comma-delimited list of netplugin etcd endpoints (default: http://127.0.0.1:2379) [$CONTIV_NETPLUGIN_ETCD_ENDPOINTS] --fwdmode value, --forward-mode value set netplugin forwarding network mode, options: [bridge, routing] [$CONTIV_NETPLUGIN_FORWARD_MODE] --host value, --host-label value set netplugin host to identify itself (default: ) [$CONTIV_NETPLUGIN_HOST] --log-level value set netplugin log level, options: [DEBUG, INFO, WARN, ERROR] (default: "INFO") [$CONTIV_NETPLUGIN_LOG_LEVEL] @@ -73,7 +73,7 @@ COMMANDS: GLOBAL OPTIONS: --consul-endpoints value, --consul value a comma-delimited list of netmaster consul endpoints [$CONTIV_NETMASTER_CONSUL_ENDPOINTS] - --etcd-endpoints value, --etcd value a comma-delimited list of netmaster etcd endpoints [$CONTIV_NETMASTER_ETCD_ENDPOINTS] + --etcd-endpoints value, --etcd value a comma-delimited list of netmaster etcd endpoints (default: http://127.0.0.1:2379) [$CONTIV_NETMASTER_ETCD_ENDPOINTS] --external-address value, --listen-url value set netmaster external address to listen on, used for general API service (default: "0.0.0.0:9999") [$CONTIV_NETMASTER_EXTERNAL_ADDRESS] --fwdmode value, --forward-mode value set netmaster forwarding network mode, options: [bridge, routing] [$CONTIV_NETMASTER_FORWARD_MODE] --infra value, --infra-type value set netmaster infra type, options [aci, default] (default: "default") [$CONTIV_NETMASTER_INFRA] diff --git a/install/k8s/README.md b/install/k8s/README.md index 4ef83e3cf..c1c330d12 100644 --- a/install/k8s/README.md +++ b/install/k8s/README.md @@ -9,17 +9,17 @@ Install kubernetes 1.6 or higher using http://kubernetes.io/docs/getting-started * Replace all instances of `__NETMASTER_IP__` in contiv/contiv.yaml with the master IP. * Replace `__VLAN_IF__` with the data plane interface. If there is no requirement to create vlan based networks there is no need for a seperate data interface and `__VLAN_IF__` can be set to "". If vlan based networks are to be created then a separate data interface is mandatory which can be set appropriately. -* Optional: Replace the contiv version(v0.1-11-30-2016.20-08-20.UTC) with the desired release/test version. +* Optional: Replace the contiv version(1.1.7) with the desired release/test version. * Optional ACI only steps: - Replace __APIC_xxx__ fields with their corresponding values. - Default value for __APIC_EPG_BRIDGE_DOMAIN__ is "not_specified" - Default value for __APIC_CONTRACTS_UNRESTRICTED_MODE__ is "no" - Password based authentication: When using password based authentication, APIC_CERT_DN fields must be deleted. - - Certificate based authentication: - - When using certificate based authentication, APIC_PASSWORD can be empty. - - Copy the certificate to a file named aci.key. - - Create a secret by running the following on the management node - ```sh + - Certificate based authentication: + - When using certificate based authentication, APIC_PASSWORD can be empty. + - Copy the certificate to a file named aci.key. + - Create a secret by running the following on the management node + ```sh kubectl create secret generic aci.key --from-file= -n kube-system ``` * On the management node, run diff --git a/install/k8s/contiv/contiv.yaml b/install/k8s/contiv/contiv.yaml index af828b761..67614ac93 100644 --- a/install/k8s/contiv/contiv.yaml +++ b/install/k8s/contiv/contiv.yaml @@ -8,18 +8,21 @@ metadata: name: contiv-config namespace: kube-system data: + contiv_mode: kubernetes + contiv_fwdmode: routing + contiv_netmode: vxlan # The location of your cluster store. This is set to the # avdertise-client value below from the contiv-etcd service. # Change it to an external etcd/consul instance if required. - cluster_store: "etcd://__NETMASTER_IP__:6666" + contiv_etcd: "http://__NETMASTER_IP__:6666" # The CNI network configuration to install on each node. - cni_config: |- + contiv_cni_config: |- { "cniVersion": "0.1.0", "name": "contiv-net", "type": "contivk8s" } - config: |- + contiv_k8s_config: |- { "K8S_API_SERVER": "https://__NETMASTER_IP__:6443", "K8S_CA": "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", @@ -29,7 +32,7 @@ data: } --- -# This manifest installs the Contiv etcd on the kubeadm master. +# This manifest installs the Contiv etcd on the kubeadm master. # If using an external etcd instance, this can be deleted. This uses a DaemonSet # to force it to run on the master even when the master isn't schedulable, and uses # nodeSelector to ensure it only runs on the master. @@ -95,31 +98,46 @@ spec: # container programs network policy and routes on each # host. - name: contiv-netplugin - image: contiv/netplugin:v1.0.0-alpha-01-28-2017.10-23-11.UTC - args: - - -pkubernetes + image: contiv/netplugin:1.1.7 env: - - name: VLAN_IF + - name: CONTIV_ROLE + value: netplugin + - name: CONTIV_NETPLUGIN_VLAN_UPLINKS value: __VLAN_IF__ - - name: VTEP_IP + - name: CONTIV_NETPLUGIN_MODE + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_mode + - name: CONTIV_NETPLUGIN_VTEP_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: CONTIV_ETCD + - name: CONTIV_NETPLUGIN_ETCD_ENDPOINTS valueFrom: configMapKeyRef: name: contiv-config - key: cluster_store + key: contiv_etcd - name: CONTIV_CNI_CONFIG valueFrom: configMapKeyRef: name: contiv-config - key: cni_config - - name: CONTIV_CONFIG + key: contiv_cni_config + - name: CONTIV_K8S_CONFIG valueFrom: configMapKeyRef: name: contiv-config - key: config + key: contiv_k8s_config + - name: CONTIV_NETPLUGIN_FORWARD_MODE + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_fwdmode + - name: CONTIV_NETPLUGIN_NET_MODE + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_netmode securityContext: privileged: true volumeMounts: @@ -135,6 +153,9 @@ spec: - mountPath: /var/contiv name: var-contiv readOnly: false + - mountPath: /var/log/contiv + name: var-log-contiv + readOnly: false - mountPath: /etc/kubernetes/pki name: etc-kubernetes-pki readOnly: false @@ -174,6 +195,9 @@ spec: - name: etc-cni-dir hostPath: path: /etc/cni/net.d/ + - name: var-log-contiv + hostPath: + path: /var/log/contiv --- # This manifest deploys the Contiv API Server on Kubernetes. @@ -209,16 +233,35 @@ spec: hostPID: true containers: - name: contiv-netmaster - image: contiv/netplugin:v1.0.0-alpha-01-28-2017.10-23-11.UTC - args: - - -m - - -pkubernetes + image: contiv/netplugin:1.1.7 env: - - name: CONTIV_ETCD + - name: CONTIV_ROLE + value: netmaster + - name: CONTIV_NETMASTER_MODE + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_mode + - name: CONTIV_NETMASTER_ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_etcd + - name: CONTIV_K8S_CONFIG + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_k8s_config + - name: CONTIV_NETMASTER_FORWARD_MODE valueFrom: configMapKeyRef: name: contiv-config - key: cluster_store + key: contiv_fwdmode + - name: CONTIV_NETMASTER_NET_MODE + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_netmode securityContext: privileged: true volumeMounts: @@ -231,6 +274,9 @@ spec: - mountPath: /var/run name: var-run readOnly: false + - mountPath: /var/log/contiv + name: var-log-contiv + readOnly: false - mountPath: /var/contiv name: var-contiv readOnly: false @@ -240,6 +286,7 @@ spec: - mountPath: /opt/cni/bin name: cni-bin-dir readOnly: false + volumes: # Used by contiv-netmaster - name: etc-openvswitch @@ -260,4 +307,7 @@ spec: - name: cni-bin-dir hostPath: path: /opt/cni/bin + - name: var-log-contiv + hostPath: + path: /var/log/contiv --- diff --git a/install/k8s/contiv/contiv_aci.yaml b/install/k8s/contiv/contiv_aci.yaml index 3cc76e6d9..06af03cb1 100644 --- a/install/k8s/contiv/contiv_aci.yaml +++ b/install/k8s/contiv/contiv_aci.yaml @@ -8,18 +8,21 @@ metadata: name: contiv-config namespace: kube-system data: + contiv_mode: kubernetes + contiv_fwdmode: routing + contiv_netmode: vxlan # The location of your cluster store. This is set to the # avdertise-client value below from the contiv-etcd service. # Change it to an external etcd/consul instance if required. - cluster_store: "etcd://__NETMASTER_IP__:6666" + contiv_etcd: "http://__NETMASTER_IP__:6666" # The CNI network configuration to install on each node. - cni_config: |- + contiv_cni_config: |- { "cniVersion": "0.1.0", "name": "contiv-net", "type": "contivk8s" } - config: |- + contiv_k8s_config: |- { "K8S_API_SERVER": "https://__NETMASTER_IP__:6443", "K8S_CA": "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", @@ -29,7 +32,7 @@ data: } --- -# This manifest installs the Contiv etcd on the kubeadm master. +# This manifest installs the Contiv etcd on the kubeadm master. # If using an external etcd instance, this can be deleted. This uses a DaemonSet # to force it to run on the master even when the master isn't schedulable, and uses # nodeSelector to ensure it only runs on the master. @@ -95,31 +98,46 @@ spec: # container programs network policy and routes on each # host. - name: contiv-netplugin - image: contiv/netplugin:v1.0.0-alpha-01-28-2017.10-23-11.UTC - args: - - -pkubernetes + image: contiv/netplugin:1.1.7 env: - - name: VLAN_IF + - name: CONTIV_ROLE + value: netplugin + - name: CONTIV_NETPLUGIN_VLAN_UPLINKS value: __VLAN_IF__ - - name: VTEP_IP + - name: CONTIV_NETPLUGIN_MODE + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_mode + - name: CONTIV_NETPLUGIN_VTEP_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: CONTIV_ETCD + - name: CONTIV_NETPLUGIN_ETCD_ENDPOINTS valueFrom: configMapKeyRef: name: contiv-config - key: cluster_store + key: contiv_etcd - name: CONTIV_CNI_CONFIG valueFrom: configMapKeyRef: name: contiv-config - key: cni_config - - name: CONTIV_CONFIG + key: contiv_cni_config + - name: CONTIV_K8S_CONFIG + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_k8s_config + - name: CONTIV_NETPLUGIN_FORWARD_MODE + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_fwdmode + - name: CONTIV_NETPLUGIN_NET_MODE valueFrom: configMapKeyRef: name: contiv-config - key: config + key: contiv_netmode securityContext: privileged: true volumeMounts: @@ -135,6 +153,9 @@ spec: - mountPath: /var/contiv name: var-contiv readOnly: false + - mountPath: /var/log/contiv + name: var-log-contiv + readOnly: false - mountPath: /etc/kubernetes/pki name: etc-kubernetes-pki readOnly: false @@ -174,6 +195,9 @@ spec: - name: etc-cni-dir hostPath: path: /etc/cni/net.d/ + - name: var-log-contiv + hostPath: + path: /var/log/contiv --- # This manifest deploys the Contiv API Server on Kubernetes. @@ -209,25 +233,52 @@ spec: hostPID: true containers: - name: contiv-netmaster - image: contiv/netplugin:v1.0.0-alpha-01-28-2017.10-23-11.UTC - args: - - -m - - -pkubernetes + image: contiv/netplugin:1.1.7 env: - - name: CONTIV_ETCD + - name: CONTIV_ROLE + value: netmaster + - name: CONTIV_NETMASTER_INFRA + value: aci + - name: CONTIV_NETMASTER_MODE + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_mode + - name: CONTIV_NETMASTER_ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_etcd + - name: CONTIV_K8S_CONFIG valueFrom: configMapKeyRef: name: contiv-config - key: cluster_store + key: contiv_k8s_config + - name: CONTIV_NETMASTER_FORWARD_MODE + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_fwdmode + - name: CONTIV_NETMASTER_NET_MODE + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_netmode securityContext: privileged: true volumeMounts: - mountPath: /etc/openvswitch name: etc-openvswitch readOnly: false + - mountPath: /lib/modules + name: lib-modules + readOnly: false - mountPath: /var/run name: var-run readOnly: false + - mountPath: /var/log/contiv + name: var-log-contiv + readOnly: false - mountPath: /var/contiv name: var-contiv readOnly: false @@ -237,11 +288,15 @@ spec: - mountPath: /opt/cni/bin name: cni-bin-dir readOnly: false + volumes: # Used by contiv-netmaster - name: etc-openvswitch hostPath: path: /etc/openvswitch + - name: lib-modules + hostPath: + path: /lib/modules - name: var-run hostPath: path: /var/run @@ -254,6 +309,11 @@ spec: - name: cni-bin-dir hostPath: path: /opt/cni/bin + - name: var-log-contiv + hostPath: + path: /var/log/contiv +--- + --- # This manifest installs contiv-aci-gw container on # each master and worker node in a Kubernetes cluster. @@ -275,7 +335,7 @@ spec: spec: hostNetwork: true containers: - # Runs aci-gw container on each Kubernetes node. + # Runs aci-gw container on each Kubernetes node. - name: contiv-aci-gw image: contiv/aci-gw env: diff --git a/install/k8s/contiv/contiv_devtest.yaml b/install/k8s/contiv/contiv_devtest.yaml index d262cdfce..3a8516018 100644 --- a/install/k8s/contiv/contiv_devtest.yaml +++ b/install/k8s/contiv/contiv_devtest.yaml @@ -8,18 +8,23 @@ metadata: name: contiv-config namespace: kube-system data: + contiv_mode: kubernetes + contiv_fwdmode: routing + contiv_netmode: vxlan # The location of your cluster store. This is set to the # avdertise-client value below from the contiv-etcd service. # Change it to an external etcd/consul instance if required. - cluster_store: "etcd://__NETMASTER_IP__:6666" + # this is not required for dev test, etcd or consul endpoints + # will be passed in in testing + # contiv_etcd: "http://__NETMASTER_IP__:6666" # The CNI network configuration to install on each node. - cni_config: |- + contiv_cni_config: |- { "cniVersion": "0.1.0", "name": "contiv-net", "type": "contivk8s" } - config: |- + contiv_k8s_config: |- { "K8S_API_SERVER": "https://__NETMASTER_IP__:6443", "K8S_CA": "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", @@ -29,7 +34,7 @@ data: } --- -# This manifest installs the Contiv etcd on the kubeadm master. +# This manifest installs the Contiv etcd on the kubeadm master. # If using an external etcd instance, this can be deleted. This uses a DaemonSet # to force it to run on the master even when the master isn't schedulable, and uses # nodeSelector to ensure it only runs on the master. @@ -96,30 +101,46 @@ spec: # host. - name: contiv-netplugin image: contiv/netplugin:k8s_devtest - args: - - -pkubernetes env: - - name: VLAN_IF + - name: CONTIV_ROLE + value: netplugin + - name: CONTIV_NETPLUGIN_VLAN_UPLINKS value: __VLAN_IF__ - - name: VTEP_IP + - name: CONTIV_NETPLUGIN_MODE + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_mode + - name: CONTIV_NETPLUGIN_VTEP_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: CONTIV_ETCD + # set in testing codes + # - name: CONTIV_NETPLUGIN_ETCD_ENDPOINTS + # valueFrom: + # configMapKeyRef: + # name: contiv-config + # key: contiv_etcd + - name: CONTIV_CNI_CONFIG valueFrom: configMapKeyRef: name: contiv-config - key: cluster_store - - name: CONTIV_CNI_CONFIG + key: contiv_cni_config + - name: CONTIV_K8S_CONFIG valueFrom: configMapKeyRef: name: contiv-config - key: cni_config - - name: CONTIV_CONFIG + key: contiv_k8s_config + - name: CONTIV_NETPLUGIN_FORWARD_MODE valueFrom: configMapKeyRef: name: contiv-config - key: config + key: contiv_fwdmode + - name: CONTIV_NETPLUGIN_NET_MODE + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_netmode securityContext: privileged: true volumeMounts: @@ -135,6 +156,9 @@ spec: - mountPath: /var/contiv name: var-contiv readOnly: false + - mountPath: /var/log/contiv + name: var-log-contiv + readOnly: false - mountPath: /etc/kubernetes/pki name: etc-kubernetes-pki readOnly: false @@ -150,6 +174,9 @@ spec: - mountPath: /contiv/bin name: contiv-bin-dir readOnly: false + - mountPath: /contiv/scripts/ + name: contiv-scripts-dir + readOnly: false volumes: # Used by contiv-netplugin - name: etc-openvswitch @@ -177,9 +204,15 @@ spec: - name: etc-cni-dir hostPath: path: /etc/cni/net.d/ + - name: var-log-contiv + hostPath: + path: /var/log/contiv - name: contiv-bin-dir hostPath: path: /opt/gopath/bin + - name: contiv-scripts-dir + hostPath: + path: /opt/gopath/src/github.com/contiv/netplugin/scripts/netContain/scripts/ --- # This manifest deploys the Contiv API Server on Kubernetes. @@ -216,15 +249,35 @@ spec: containers: - name: contiv-netmaster image: contiv/netplugin:k8s_devtest - args: - - -m - - -pkubernetes env: - - name: CONTIV_ETCD + - name: CONTIV_ROLE + value: netmaster + - name: CONTIV_NETMASTER_MODE valueFrom: configMapKeyRef: name: contiv-config - key: cluster_store + key: contiv_mode + # set in testing codes + # - name: CONTIV_NETMASTER_ETCD_ENDPOINTS + # valueFrom: + # configMapKeyRef: + # name: contiv-config + # key: contiv_etcd + - name: CONTIV_K8S_CONFIG + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_k8s_config + - name: CONTIV_NETMASTER_FORWARD_MODE + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_fwdmode + - name: CONTIV_NETMASTER_NET_MODE + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_netmode securityContext: privileged: true volumeMounts: @@ -237,6 +290,9 @@ spec: - mountPath: /var/run name: var-run readOnly: false + - mountPath: /var/log/contiv + name: var-log-contiv + readOnly: false - mountPath: /var/contiv name: var-contiv readOnly: false @@ -249,6 +305,9 @@ spec: - mountPath: /contiv/bin name: contiv-bin-dir readOnly: false + - mountPath: /contiv/scripts/ + name: contiv-scripts-dir + readOnly: false volumes: # Used by contiv-netmaster - name: etc-openvswitch @@ -269,7 +328,13 @@ spec: - name: cni-bin-dir hostPath: path: /opt/cni/bin + - name: var-log-contiv + hostPath: + path: /var/log/contiv - name: contiv-bin-dir hostPath: path: /opt/gopath/bin + - name: contiv-scripts-dir + hostPath: + path: /opt/gopath/src/github.com/contiv/netplugin/scripts/netContain/scripts/ --- diff --git a/install/v2plugin/README.md b/install/v2plugin/README.md index 928ccb52b..2e7d442ac 100644 --- a/install/v2plugin/README.md +++ b/install/v2plugin/README.md @@ -16,30 +16,54 @@ Contiv plugin config options should be specified if it is different from default docker plugin install contiv/v2plugin: ARG1=VALUE1 ARG2=VALUE2 ... ``` -ARG : DESCRIPTION : DEFAULT VALUE ---------------:-----------------------------------------------:---------------------- -iflist : VLAN uplink interface used by OVS : "" -cluster_store : Etcd or Consul cluster store url : etcd://localhost:2379 -ctrl_ip : Local IP address to be used by netplugin : none - for control communication -vtep_ip : Local VTEP IP address to be used by netplugin : none -plugin_role : In 'master' role, plugin runs netmaster : master - and netplugin -listen_url : Netmaster url to listen http requests on : ":9999" -control_url : Netmaster url for control messages : ":9999" -dbg_flag : To enable debug mode, set to '-debug' : "" -fwd_mode : Forwarding mode : "" - +ARG : DESCRIPTION : DEFAULT VALUE +----------------------------------:---------------------------------------------------------------------------:---------------------- +CONTIV_ROLE : contiv net service net, options: [netmaster, netplugin] : "netplugin" +CONTIV_LOG_DIR : contiv log file directory : "/var/log/contiv" +CONTIV_NETPLUGIN_CONSUL_ENDPOINTS : a comma-delimited list of netplugin consul endpoints : "" +CONTIV_NETPLUGIN_ETCD_ENDPOINTS : a comma-delimited list of netplugin etcd endpoints : "http://127.0.0.1:2379" +CONTIV_NETPLUGIN_VLAN_UPLINKS : a comma-delimited list of netplugin VLAN uplink interfaces used by OVS : "" +CONTIV_NETPLUGIN_VTEP_IP : netplugin vtep ip for vxlan communication : +CONTIV_NETPLUGIN_CONTROL_IP : netplugin control ip for control plane communication : +CONTIV_NETPLUGIN_FORWARD_MODE : netplugin forwarding network mode, options: [bridge, routing] : "" +CONTIV_NETPLUGIN_HOST : netplugin host to identify itself : +CONTIV_NETPLUGIN_LOG_LEVEL : netplugin log level, options: [DEBUG, INFO, WARN, ERROR] : "INFO" +CONTIV_NETPLUGIN_MODE : netplugin mode, options: [docker, kubernetes, swarm-mode] : "" +CONTIV_NETPLUGIN_NET_MODE : netplugin network mode, options: [vlan, vxlan] : "" +CONTIV_NETPLUGIN_SYSLOG_URL : netplugin syslog url in format protocol://ip:port : "udp://127.0.0.1:514" +CONTIV_NETPLUGIN_USE_SYSLOG : netplugin send log to syslog if flag is provided : +CONTIV_NETPLUGIN_USE_JSON_LOG : netplugin log format to json if flag is provided : +CONTIV_NETPLUGIN_VXLAN_PORT : netplugin VXLAN port : 4789 +CONTIV_NETMASTER_CONSUL_ENDPOINTS : a comma-delimited list of netmaster consul endpoints : "" +CONTIV_NETMASTER_ETCD_ENDPOINTS : a comma-delimited list of netmaster etcd endpoints : "" +CONTIV_NETMASTER_FORWARD_MODE : netmaster forwarding network mode, options: [bridge, routing] : "" +CONTIV_NETMASTER_EXTERNAL_ADDRESS : netmaster external address to listen on, used for general API service : "0.0.0.0:9999" +CONTIV_NETMASTER_INTERNAL_ADDRESS : netmaster internal address to listen on, used for RPC and leader election : : +CONTIV_NETMASTER_INFRA : netmaster infra type, options [aci, default] : "default" +CONTIV_NETMASTER_LOG_LEVEL : netmaster log level, options: [DEBUG, INFO, WARN, ERROR] : "INFO" +CONTIV_NETMASTER_MODE : netmaster mode, options: [docker, kubernetes, swarm-mode] : "" +CONTIV_NETMASTER_PLUGIN_NAME : netmaster plugin name for docker v2 plugin : netplugin +CONTIV_NETMASTER_NET_MODE : netmaster network mode, options: [vlan, vxlan] : "" +CONTIV_NETMASTER_SYSLOG_URL : netmaster syslog url in format protocol://ip:port : "udp://127.0.0.1:514" +CONTIV_NETMASTER_USE_SYSLOG : netmaster send log to syslog if flag is provided : +CONTIV_NETMASTER_USE_JSON_LOG : netmaster log format to json if flag is provided : ``` ### docker store Docker certified contiv plugin is avaliable on [Docker Store](https://store.docker.com/plugins/803eecee-0780-401a-a454-e9523ccf86b3?tab=description). ``` -docker plugin install store/contiv/v2plugin: iflist= fwd_mode= +docker plugin install store/contiv/v2plugin: \ +CONTIV_ROLE=netmaster CONTIV_NETPLUGIN_VLAN_UPLINKS= \ +CONTIV_NETPLUGIN_FORWARD_MODE=bridge CONTIV_NETPLUGIN_MODE=docker CONTIV_NETPLUGIN_NET_MODE=vlan \ +CONTIV_NETMASTER_FORWARD_MODE=bridge CONTIV_NETMASTER_MODE=docker CONTIV_NETMASTER_NET_MODE=vlan ``` ### docker hub Developer release of v2plugin from contiv repo is also pushed to docker hub +Please update mode, forward mode, net mode according to your deployment. + ``` -docker plugin install contiv/v2plugin: iflist= fwd_mode= +docker plugin install contiv/v2plugin: CONTIV_ROLE=netmaster CONTIV_NETPLUGIN_VLAN_UPLINKS= \ +CONTIV_NETPLUGIN_FORWARD_MODE=bridge CONTIV_NETPLUGIN_MODE=docker CONTIV_NETPLUGIN_NET_MODE=vlan \ +CONTIV_NETMASTER_FORWARD_MODE=bridge CONTIV_NETMASTER_MODE=docker CONTIV_NETMASTER_NET_MODE=vlan ``` ### vagrant dev/demo setup To create a plugin from [contiv repo](https://github.com/contiv/netplugin), enable v2plugin and run docker in swarm-mode, use the Makefile target demo-v2plugin @@ -49,8 +73,11 @@ make demo-v2plugin ## Contiv plugin-roles Contiv plugin runs both netplugin and netmaster by default. Contiv v2plugin can be run with only netplugin by setting the plugin_role to worker. +Please update mode, forward mode, net mode according to your deployment. ``` -docker plugin install contiv/v2plugin: iflist= plugin_role=worker fwd_mode= +docker plugin install contiv/v2plugin: CONTIV_ROLE=netmaster CONTIV_NETPLUGIN_VLAN_UPLINKS= \ +CONTIV_NETPLUGIN_FORWARD_MODE=bridge CONTIV_NETPLUGIN_MODE=docker CONTIV_NETPLUGIN_NET_MODE=vlan \ +CONTIV_NETMASTER_FORWARD_MODE=bridge CONTIV_NETMASTER_MODE=docker CONTIV_NETMASTER_NET_MODE=vlan ``` ## Contiv plugin swarm-mode workflow (recommended and default for v2plugin) @@ -70,11 +97,14 @@ docker plugin install contiv/v2plugin: iflist= plugin_role=master iflist= fwd_mode= + docker plugin install contiv/v2plugin: CONTIV_ROLE=netmaster CONTIV_NETPLUGIN_VLAN_UPLINKS= \ +CONTIV_NETPLUGIN_FORWARD_MODE=bridge CONTIV_NETPLUGIN_MODE=docker CONTIV_NETPLUGIN_NET_MODE=vlan \ +CONTIV_NETMASTER_FORWARD_MODE=bridge CONTIV_NETMASTER_MODE=docker CONTIV_NETMASTER_NET_MODE=vlan ( allow/grant the install permissions when prompted ) # on worker nodes, install plugin with 'worker' role - docker plugin install contiv/v2plugin: plugin_role=worker iflist= fwd_mode= + docker plugin install contiv/v2plugin: CONTIV_ROLE=netplugin CONTIV_NETPLUGIN_VLAN_UPLINKS= \ +CONTIV_NETPLUGIN_FORWARD_MODE=bridge CONTIV_NETPLUGIN_MODE=docker CONTIV_NETPLUGIN_NET_MODE=vlan # to see if the plugin is installed and enabled docker plugin ls @@ -83,7 +113,8 @@ docker plugin install contiv/v2plugin: iflist= ctrl_ip=192.168.2.10 control_url=192.168.2.10:9999 iflist=eth2,eth3 fwd_mode=bridge + docker plugin install contiv/v2plugin: CONTIV_ROLE=netplugin CONTIV_NETPLUGIN_VLAN_UPLINKS=eth2,eth3 \ +CONTIV_NETPLUGIN_FORWARD_MODE=bridge CONTIV_NETPLUGIN_MODE=docker CONTIV_NETPLUGIN_NET_MODE=vlan ``` 4. Debug logs ``` @@ -130,15 +161,15 @@ docker plugin install contiv/v2plugin: iflist= plugin-mode=docker iflist= fwd_mode= - ( allow/grant the install permissions when prompted ) - # on node where netmaster needs to run, install plugin with 'master' role - docker plugin install contiv/v2plugin: plugin_role=master iflist= fwd_mode= + docker plugin install contiv/v2plugin: CONTIV_ROLE=netmaster CONTIV_NETPLUGIN_VLAN_UPLINKS= \ +CONTIV_NETPLUGIN_FORWARD_MODE=bridge CONTIV_NETPLUGIN_MODE=docker CONTIV_NETPLUGIN_NET_MODE=vlan \ +CONTIV_NETMASTER_ETCD_ENDPOINTS=http://127.0.0.1:2379 CONTIV_NETMASTER_FORWARD_MODE=bridge CONTIV_NETMASTER_MODE=docker CONTIV_NETMASTER_NET_MODE=vlan ( allow/grant the install permissions when prompted ) # on all other nodes, install plugin with 'worker' role - docker plugin install contiv/v2plugin: plugin_role=worker iflist= fwd_mode= + docker plugin install contiv/v2plugin: CONTIV_ROLE=netplugin CONTIV_NETPLUGIN_VLAN_UPLINKS=eth2,eth3 \ +CONTIV_NETPLUGIN_FORWARD_MODE=bridge CONTIV_NETPLUGIN_MODE=docker CONTIV_NETPLUGIN_NET_MODE=vlan # to see if the plugin is installed properly and enabled docker plugin ls diff --git a/install/v2plugin/config.template b/install/v2plugin/config.template index 7d13350a6..60af35a82 100644 --- a/install/v2plugin/config.template +++ b/install/v2plugin/config.template @@ -8,110 +8,237 @@ }, "env": [ { - "Description": "To enable debug mode, set to '-debug'", - "Name": "dbg_flag", + "Description": "contiv net service net, options: [netmaster, netplugin]", + "Name": "CONTIV_ROLE", "Settable": [ "value" ], - "Value": "" + "Value": "netplugin" }, { - "Description": "Change the directory where the logs are saved", - "Name": "log_dir", + "Description": "contiv log file directory", + "Name": "CONTIV_LOG_DIR", "Settable": [ "value" ], "Value": "/var/log/contiv" }, { - "Description": "VLAN uplink interface used by OVS", - "Name": "iflist", + "Description": "a comma-delimited list of netplugin consul endpoints", + "Name": "CONTIV_NETPLUGIN_CONSUL_ENDPOINTS", "Settable": [ "value" ], "Value": "" }, { - "Description": "Etcd or Consul cluster store url", - "Name": "cluster_store", + "Description": "a comma-delimited list of netplugin etcd endpoints", + "Name": "CONTIV_NETPLUGIN_ETCD_ENDPOINTS", "Settable": [ "value" ], - "Value": "etcd://localhost:2379" + "Value": "http://127.0.0.1:2379" }, { - "Description": "Plugin mode set to docker or swarm-mode", - "Name": "plugin_mode", + "Description": "a comma-delimited list of netplugin VLAN uplink interfaces used by OVS", + "Name": "CONTIV_NETPLUGIN_VLAN_UPLINKS", "Settable": [ "value" ], - "Value": "swarm-mode" + "Value": "" }, { - "Description": "Local IP address to be used by netplugin for control communication", - "Name": "ctrl_ip", + "Description": "netplugin vtep ip for vxlan communication", + "Name": "CONTIV_NETPLUGIN_VTEP_IP", "Settable": [ "value" ], - "Value": "none" + "Value": "" }, { - "Description": "Local VTEP IP address to be used by netplugin", - "Name": "vtep_ip", + "Description": "netplugin control ip for control plane communication", + "Name": "CONTIV_NETPLUGIN_CONTROL_IP", "Settable": [ "value" ], - "Value": "none" + "Value": "" }, { - "Description": "Vxlan UDP port number used for encapsulating vxlan packets", - "Name": "vxlan_port", + "Description": "netplugin forwarding network mode, options: [bridge, routing]", + "Name": "CONTIV_NETPLUGIN_FORWARD_MODE", "Settable": [ "value" ], - "Value": "8742" + "Value": "" }, { - "Description": "In 'master' role, plugin runs netmaster and netplugin", - "Name": "plugin_role", + "Description": "netplugin host to identify itself", + "Name": "CONTIV_NETPLUGIN_HOST", "Settable": [ "value" ], - "Value": "master" + "Value": "" }, { - "Description": "Netmaster url to listen http requests on", - "Name": "listen_url", + "Description": "netplugin log level, options: [DEBUG, INFO, WARN, ERROR]", + "Name": "CONTIV_NETPLUGIN_LOG_LEVEL", "Settable": [ "value" ], - "Value": ":9999" + "Value": "INFO" }, { - "Description": "Netmaster url for control messages", - "Name": "control_url", + "Description": "netplugin mode, options: [docker, kubernetes, swarm-mode]", + "Name": "CONTIV_NETPLUGIN_MODE", "Settable": [ "value" ], - "Value": ":9999" + "Value": "" + }, + { + "Description": "netplugin network mode, options: [vlan, vxlan]", + "Name": "CONTIV_NETPLUGIN_NET_MODE", + "Settable": [ + "value" + ], + "Value": "" }, { - "Description": "Network Driver name for requests to dockerd. Should be same as name:tag of the plugin", - "Name": "plugin_name", + "Description": "netplugin syslog url in format protocol://ip:port", + "Name": "CONTIV_NETPLUGIN_SYSLOG_URL", "Settable": [ "value" ], - ## Do not change the default value, this will be replaced with $CONTIV_V2PLUGIN_NAME - "Value": "__CONTIV_V2PLUGIN_NAME__" + "Value": "udp://127.0.0.1:514" }, { - "Description": "Forwarding mode for netplugin", - "Name": "fwd_mode", + "Description": "netplugin send log to syslog if flag is provided", + "Name": "CONTIV_NETPLUGIN_USE_SYSLOG", "Settable": [ "value" ], "Value": "" - } + }, + { + "Description": "netplugin log format to json if flag is provided", + "Name": "CONTIV_NETPLUGIN_USE_JSON_LOG", + "Settable": [ + "value" + ], + "Value": "" + }, + { + "Description": "netplugin VXLAN port", + "Name": "CONTIV_NETPLUGIN_VXLAN_PORT", + "Settable": [ + "value" + ], + "Value": "4789" + }, + { + "Description": "a comma-delimited list of netmaster consul endpoints", + "Name": "CONTIV_NETMASTER_CONSUL_ENDPOINTS", + "Settable": [ + "value" + ], + "Value": "" + }, + { + "Description": "a comma-delimited list of netmaster etcd endpoints", + "Name": "CONTIV_NETMASTER_ETCD_ENDPOINTS", + "Settable": [ + "value" + ], + "Value": "http://127.0.0.1:2379" + }, + { + "Description": "netmaster forwarding network mode, options: [bridge, routing]", + "Name": "CONTIV_NETMASTER_FORWARD_MODE", + "Settable": [ + "value" + ], + "Value": "" + }, + { + "Description": "netmaster external address to listen on, used for general API service", + "Name": "CONTIV_NETMASTER_EXTERNAL_ADDRESS", + "Settable": [ + "value" + ], + "Value": "0.0.0.0:9999" + }, + { + "Description": "netmaster internal address to listen on, used for RPC and leader election", + "Name": "CONTIV_NETMASTER_INTERNAL_ADDRESS", + "Settable": [ + "value" + ], + "Value": "" + }, + { + "Description": "netmaster infra type, options [aci, default]", + "Name": "CONTIV_NETMASTER_INFRA", + "Settable": [ + "value" + ], + "Value": "default" + }, + { + "Description": "netmaster log level, options: [DEBUG, INFO, WARN, ERROR]", + "Name": "CONTIV_NETMASTER_LOG_LEVEL", + "Settable": [ + "value" + ], + "Value": "INFO" + }, + { + "Description": "netmaster mode, options: [docker, kubernetes, swarm-mode]", + "Name": "CONTIV_NETMASTER_MODE", + "Settable": [ + "value" + ], + "Value": "" + }, + { + "Description": "netmaster plugin name for docker v2 plugin", + "Name": "CONTIV_NETMASTER_PLUGIN_NAME", + "Settable": [ + "value" + ], + "Value": "netplugin" + }, + { + "Description": "netmaster network mode, options: [vlan, vxlan]", + "Name": "CONTIV_NETMASTER_NET_MODE", + "Settable": [ + "value" + ], + "Value": "" + }, + { + "Description": "netmaster syslog url in format protocol://ip:port", + "Name": "CONTIV_NETMASTER_SYSLOG_URL", + "Settable": [ + "value" + ], + "Value": "udp://127.0.0.1:514" + }, + { + "Description": "netmaster send log to syslog if flag is provided", + "Name": "CONTIV_NETMASTER_USE_SYSLOG", + "Settable": [ + "value" + ], + "Value": "" + }, + { + "Description": "netmaster log format to json if flag is provided", + "Name": "CONTIV_NETMASTER_USE_JSON_LOG", + "Settable": [ + "value" + ], + "Value": "" + }, ], "mounts": [ { diff --git a/install/v2plugin/startcontiv.sh b/install/v2plugin/startcontiv.sh index d1792b9a2..e9561a6a1 100755 --- a/install/v2plugin/startcontiv.sh +++ b/install/v2plugin/startcontiv.sh @@ -4,125 +4,107 @@ # run a cluster store like etcd or consul set -e +echo "INFO: Starting contiv net with ARGS:" +echo "$@" +echo "INFO: Starting contiv net with ENV:" +/usr/bin/env | grep CONTIV_ + +# this is different between k8s and v2plugin because v2plugin have netmaster +# in one container +if [ -z "$CONTIV_ROLE" ]; then + CONTIV_ROLE="netplugin" +elif [ "$CONTIV_ROLE" != "netmaster" ] && [ "$CONTIV_ROLE" != "netplugin" ]; then + echo "CRITICAL: ENV CONTIV_ROLE must be in [netmaster, netplugin]" + echo "CRITICAL: Unknown contiv role" + exit 1 +fi +echo "INFO: Starting contiv net as role: $CONTIV_ROLE" -if [ "$log_dir" == "" ]; then - log_dir="/var/log/contiv" +# setting up logs +if [ -z "$CONTIV_LOG_DIR" ]; then + CONTIV_LOG_DIR="/var/log/contiv" fi -mkdir -p $log_dir -BOOTUP_LOGFILE="$log_dir/plugin_bootup.log" +mkdir -p "$CONTIV_LOG_DIR" +echo "INFO: Logging contiv net under: $CONTIV_LOG_DIR" +BOOTUP_LOGFILE="$CONTIV_LOG_DIR/plugin_bootup.log" # Redirect stdout and stdin to BOOTUP_LOGFILE exec 1<&- # Close stdout exec 2<&- # Close stderr exec 1<>$BOOTUP_LOGFILE # stdout read and write to logfile instead of console exec 2>&1 # redirect stderr to where stdout is (logfile) -mkdir -p $log_dir -mkdir -p /var/run/openvswitch -mkdir -p /etc/openvswitch +mkdir -p "$CONTIV_LOG_DIR" /var/run/openvswitch /etc/openvswitch -echo "V2 Plugin logs" > $BOOTUP_LOGFILE +# setting up ovs +# TODO: this is the same code in ovsInit.sh, needs to reduce the duplication +set -uo pipefail -if [ $iflist == "" ]; then - echo "iflist is empty. Host interface(s) should be specified to use vlan mode" >> $BOOTUP_LOGFILE -fi -if [ $ctrl_ip != "none" ]; then - ctrl_ip_cfg="--ctrl-ip=$ctrl_ip" -fi -if [ $vtep_ip != "none" ]; then - vtep_ip_cfg="--vtep-ip=$vtep_ip" -fi -if [ $listen_url != ":9999" ]; then - listen_url_cfg="-listen-url=$listen_url" -fi -if [ $control_url != ":9999" ]; then - control_url_cfg="-control-url=$control_url" -fi -if [ $vxlan_port != "4789" ]; then - vxlan_port_cfg="--vxlan-port=$vxlan_port" -fi +modprobe openvswitch || (echo "CRITICAL: Failed to load kernel module openvswitch" && exit 1 ) +echo "INFO: Loaded kernel module openvswitch" -if [[ "$cluster_store" =~ ^etcd://.+ ]]; then - store_arg="--etcd-endpoints $(echo $cluster_store | sed s/etcd/http/)" +if [ -d "/etc/openvswitch" ]; then + if [ -f "/etc/openvswitch/conf.db" ]; then + echo "INFO: The Open vSwitch database exists" + else + echo "INFO: The Open VSwitch database doesn't exist" + echo "INFO: Creating the Open VSwitch database..." + ovsdb-tool create /etc/openvswitch/conf.db /usr/share/openvswitch/vswitch.ovsschema + fi else - store_arg="--consul-endpoints $(echo $cluster_store | sed s/consul/http/)" + echo "CRITICAL: Open vSwitch is not mounted from host" + exit 1 fi -set -e - -echo "Loading OVS" >> $BOOTUP_LOGFILE -(modprobe openvswitch) || (echo "Load ovs FAILED!!! " >> $BOOTUP_LOGFILE) - -echo " Cleaning up ovsdb files" >> $BOOTUP_LOGFILE -rm -rf /var/run/openvswitch/* -rm -rf /etc/openvswitch/conf.db -rm -rf /etc/openvswitch/.conf.db.~lock~ -echo " Creating OVS DB" >> $BOOTUP_LOGFILE -(ovsdb-tool create /etc/openvswitch/conf.db /usr/share/openvswitch/vswitch.ovsschema) || (while true; do sleep 1; done) +echo "INFO: Starting ovsdb-server..." +ovsdb-server --remote=punix:/var/run/openvswitch/db.sock \ + --remote=db:Open_vSwitch,Open_vSwitch,manager_options \ + --private-key=db:Open_vSwitch,SSL,private_key \ + --certificate=db:Open_vSwitch,SSL,certificate \ + --bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert \ + --log-file=$CONTIV_LOG_DIR/ovs-db.log -vsyslog:info -vfile:info \ + --pidfile --detach /etc/openvswitch/conf.db + +echo "INFO: Starting ovs-vswitchd" +ovs-vswitchd -v --pidfile --detach --log-file=$CONTIV_LOG_DIR/ovs-vswitchd.log \ + -vconsole:err -vsyslog:info -vfile:info & + +retry=0 +while [[ $(ovsdb-client list-dbs | grep -c Open_vSwitch) -eq 0 ]] ; do + if [[ ${retry} -eq 5 ]]; then + echo "CRITICAL: Failed to start ovsdb in 5 seconds." + exit 1 + else + echo "INFO: Waiting for ovsdb to start..." + sleep 1 + ((retry+=1)) + fi +done -echo " Starting OVSBD server " >> $BOOTUP_LOGFILE -ovsdb-server --remote=punix:/var/run/openvswitch/db.sock --remote=db:Open_vSwitch,Open_vSwitch,manager_options --private-key=db:Open_vSwitch,SSL,private_key --certificate=db:Open_vSwitch,SSL,certificate --bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert --log-file=$log_dir/ovs-db.log -vsyslog:dbg -vfile:dbg --pidfile --detach /etc/openvswitch/conf.db >> $BOOTUP_LOGFILE -echo " Starting ovs-vswitchd " >> $BOOTUP_LOGFILE -ovs-vswitchd -v --pidfile --detach --log-file=$log_dir/ovs-vswitchd.log -vconsole:err -vsyslog:info -vfile:info & +echo "INFO: Setting OVS manager (tcp)..." ovs-vsctl set-manager tcp:127.0.0.1:6640 -ovs-vsctl set-manager ptcp:6640 -echo "Started OVS, logs in $log_dir" >> $BOOTUP_LOGFILE +echo "INFO: Setting OVS manager (ptcp)..." +ovs-vsctl set-manager ptcp:6640 +# starting services set +e - -if [ $plugin_role == "master" ]; then - if [ -z "$fwd_mode" ]; then - echo "fwd_mode is not set, plugin cannot be enabled" - exit 1 - fi - echo "Starting Netmaster " >> $BOOTUP_LOGFILE +if [ "$CONTIV_ROLE" = "netmaster" ]; then while true ; do - echo "/netmaster $dbg_flag -plugin-name=$plugin_name -cluster-mode=$plugin_mode -cluster-store=$cluster_store $listen_url_cfg $control_url_cfg" >> $BOOTUP_LOGFILE - /netmaster $dbg_flag -plugin-name=$plugin_name -cluster-mode=$plugin_mode -cluster-store=$cluster_store $listen_url_cfg $control_url_cfg &> $log_dir/netmaster.log - echo "CRITICAL : Net Master has exited, Respawn in 5s" >> $BOOTUP_LOGFILE - mv $log_dir/netmaster.log $log_dir/netmaster.log.lastrun + echo "INFO: Starting contiv netmaster" + set -x + /contiv/bin/netmaster "$@" &>> "$CONTIV_LOG_DIR/netmaster.log" + set +x + echo "ERROR: Contiv netmaster has exited, restarting in 5s" sleep 5 - echo "Restarting Netmaster " >> $BOOTUP_LOGFILE done & - - set -e - echo "Waiting for netmaster to be ready for connections" - # wait till netmaster starts to listen - for i in $(seq 1 10); do - [ "$(curl -s -o /dev/null -w '%{http_code}' $control_url)" != "000" ] \ - && break - sleep 1 - done - if [ "$i" -ge "10" ]; then - echo "netmaster port not open (needed to set forwarding mode), plugin failed" - exit 1 - fi - sleep 1 - echo "Netmaster ready for connections, setting forward mode to $fwd_mode" - /netctl --netmaster http://$control_url global set --fwd-mode "$fwd_mode" - echo "Forward mode is set" -else - echo "Not starting netmaster as plugin role is" $plugin_role >> $BOOTUP_LOGFILE fi -if [[ "$fwd_mode" == "bridge" ]]; then - network_mode=vlan -else - network_mode=vxlan -fi - -echo "Starting Netplugin" while true ; do + echo "INFO: Starting contiv netplugin" set -x - /netplugin $dbg_flag --plugin-mode=$plugin_mode $vxlan_port_cfg \ - --vlan-if=$iflist $store_arg $ctrl_ip_cfg $vtep_ip_cfg \ - --netmode $network_mode --fwdmode $fwd_mode &> $log_dir/netplugin.log + /contiv/bin/netplugin "$@" &>> "$CONTIV_LOG_DIR/netplugin.log" set +x - echo "CRITICAL : Net Plugin has exited, Respawn in 5" - mv $log_dir/netplugin.log $log_dir/netplugin.log.lastrun + echo "ERROR: Contiv netplugin has exited, restarting in 5s" sleep 5 - echo "Restarting Netplugin" -done & - -while true; do sleep 1; done +done diff --git a/scripts/get-contiv-diags b/scripts/get-contiv-diags index 550a23561..05d833fcc 100755 --- a/scripts/get-contiv-diags +++ b/scripts/get-contiv-diags @@ -93,7 +93,7 @@ end def copy_contiv_logs(tmpdir) system("mkdir -p #{tmpdir}/contiv-logs/") - system("cp /var/contiv/log/* #{tmpdir}/contiv-logs/") + system("cp /var/log/contiv/* #{tmpdir}/contiv-logs/") system("bzip2 #{tmpdir}/contiv-logs/*") end @@ -300,13 +300,13 @@ def write_json(filename, data) end if ARGV.length < 1 || ARGV[0] == "-h" then - puts "get-diags.rb [-h] " + puts "get-diags.rb [-h] " puts "TKT# is ticket given by case worker" puts "to capture contiv install logs, this must be run in contiv install dir" puts " install logs are located in @INSTALL_ROOT@/config/*.log" exit 1 end - + tkt = ARGV[0] puts "TKT = #{tkt}" puts "If running on docker+swarm, make sure DOCKER_HOST is set first" diff --git a/scripts/netContain/contivInit.sh b/scripts/netContain/contivInit.sh index ef55d7b14..d408d9fd1 100755 --- a/scripts/netContain/contivInit.sh +++ b/scripts/netContain/contivInit.sh @@ -7,7 +7,7 @@ set -x source /contiv/scripts/contivRc if [ $IS_NETMASTER == 1 ]; then - /contiv/bin/netmaster 2>/var/log/contiv/netmaster.errlog 1>/var/log/contiv/netmaster.log & + /contiv/bin/netmaster 2>/var/log/contiv/netmaster.errlog 1>/var/log/contiv/netmaster.log & fi echo "$NETMASTER_IP netmaster" >/etc/hosts @@ -15,7 +15,7 @@ echo "0.0.0.0 localhost" >>/etc/hosts export no_proxy="0.0.0.0, $NETMASTER_IP" if [ not $CONTIV_FWD_MODE == "routing" ]; then - CONTIV_FWD_MODE="bridge" + CONTIV_FWD_MODE="bridge" fi /contiv/bin/netplugin --netmode vlan --plugin-mode docker --vtep-ip $VTEP_IP --vlan-if $VLAN_IF --fwdmode $CONTIV_FWD_MODE 2>/var/log/contiv/netplugin.errlog 1>/var/log/contiv/netplugin.log & diff --git a/scripts/netContain/scripts/contivNet.sh b/scripts/netContain/scripts/contivNet.sh index fdcf881fd..af5fa1cd2 100755 --- a/scripts/netContain/scripts/contivNet.sh +++ b/scripts/netContain/scripts/contivNet.sh @@ -1,22 +1,11 @@ #!/bin/bash #Initialize contiv container. Start OVS and netplugin +set -e -cstore="$CONTIV_ETCD" -vtep_ip="$VTEP_IP" -vlan_if="$VLAN_IF" - -set -euo pipefail - -reinit=false -plugin="docker" -netmaster=false -netplugin=true -debug="" -cleanup=false -vtep_ip_param="" -vlan_if_param="" -control_url=":9999" -listen_url=":9999" +echo "INFO: Starting contiv net with ARGS:" +echo "$@" +echo "INFO: Starting contiv net with ENV:" +/usr/bin/env | grep CONTIV_ # These files indicate if the netmaster/netplugin process needs to be restarted touch /tmp/restart_netmaster @@ -29,126 +18,91 @@ touch /tmp/restart_netplugin #Needed for netplugin to connect with OVS, This needs to be #fixed as well. netplugin should have OVS locally. -echo "0.0.0.0 localhost" >>/etc/hosts - -while getopts ":xmp:v:i:c:drl:o:t:" opt; do - case $opt in - m) - netmaster=true - netplugin=false - ;; - v) - vtep_ip=$OPTARG - netplugin=true - ;; - i) - vlan_if=$OPTARG - netplugin=true - ;; - c) - cstore=$OPTARG - ;; - p) - plugin=$OPTARG - ;; - r) - cleanup=true - ;; - x) - reinit=true - ;; - d) - debug="--log-level debug" - ;; - l) - listen_url=$OPTARG - ;; - o) - control_url=$OPTARG - ;; - :) - echo "An argument required for $OPTARG was not passed" - ;; - ?) - echo "Invalid option supplied" - ;; - esac -done - -if [ $cleanup == false ] && [ $netplugin == true ]; then - echo "Initializing OVS" - /contiv/scripts/ovsInit.sh - echo "Initialized OVS" -fi - -if [ $cleanup == true ] || [ $reinit == true ]; then - ovs-vsctl del-br contivVlanBridge || true - ovs-vsctl del-br contivVxlanBridge || true - for p in $(ifconfig | grep vport | awk '{print $1}'); do - ip link delete $p type veth - done - rm -f /opt/cni/bin/contivk8s || true - rm -f /etc/cni/net.d/1-contiv.conf || true +echo "0.0.0.0 localhost" >> /etc/hosts + +if [ -z "$CONTIV_ROLE" ]; then + echo "CRITICAL: ENV CONTIV_ROLE must be set" + echo "CRITICAL: Unknown contiv role" + exit 1 +elif [ "$CONTIV_ROLE" != "netmaster" ] && [ "$CONTIV_ROLE" != "netplugin" ]; then + echo "CRITICAL: ENV CONTIV_ROLE must be in [netmaster, netplugin]" + echo "CRITICAL: Unknown contiv role" + exit 1 fi +echo "INFO: Running contiv as $CONTIV_ROLE" -if [ $cleanup == true ]; then - exit 0 +if [ ! -z "$CONTIV_MODE" ]; then + if [ "$CONTIV_ROLE" = "netmaster" ] && [ -z "$CONTIV_NETMASTER_MODE" ] ; then + CONTIV_NETMASTER_MODE="$CONTIV_ROLE" + elif [ "$CONTIV_ROLE" = "netplugin" ] && [ -z "$CONTIV_NETPLUGIN_MODE" ] ; then + CONTIV_NETPLUGIN_MODE="$CONTIV_ROLE" + fi +elif [ ! -z "$CONTIV_NETMASTER_MODE" ]; then + CONTIV_MODE="$CONTIV_NETMASTER_MODE" +elif [ ! -z "$CONTIV_NETPLUGIN_MODE" ]; then + CONTIV_MODE="$CONTIV_NETPLUGIN_MODE" +else + echo "CRITICAL: ENV CONTIV_MODE or CONTIV_NETMASTER_MODE or CONTIV_NETPLUGIN_MODE must be set" + echo "CRITICAL: Unknown contiv mode" + exit 1 fi +echo "INFO: Running contiv in mode $CONTIV_MODE" -if [ $netplugin == false ] && [ $netmaster == false ]; then - echo "No netmaster or netplugin options were specified" - exit 1 -fi +set -uo pipefail -mkdir -p /opt/contiv/ -mkdir -p /var/contiv/log/ +mkdir -p /opt/contiv/ /var/log/contiv -if [ "$plugin" == "kubernetes" ]; then - mkdir -p /opt/contiv/config - mkdir -p /var/contiv/config - echo ${CONTIV_CONFIG} >/var/contiv/config/contiv.json - cp /var/contiv/config/contiv.json /opt/contiv/config/contiv.json +if [ -d /var/contiv/log ]; then + # /var/contiv/log/ is deprecated, move all data to /var/log/contiv + cp -a /var/contiv/log/* /var/log/contiv/ + echo "INFO: Copied contiv log from /var/contiv/log (deprecated) to /var/log/contiv" +fi - if [ $netplugin == true ]; then - mkdir -p /opt/cni/bin - cp /contiv/bin/contivk8s /opt/cni/bin/ - mkdir -p /etc/cni/net.d/ - echo ${CONTIV_CNI_CONFIG} >/etc/cni/net.d/1-contiv.conf - fi +if [ "$CONTIV_ROLE" = "netplugin" ]; then + echo "INFO: Initializing OVS" + /contiv/scripts/ovsInit.sh + echo "INFO: Initialized OVS" fi -if [ $netmaster == true ]; then - echo "Starting netmaster " - while true; do - if [ -f /tmp/restart_netmaster ]; then - if [ "$cstore" != "" ]; then - /contiv/bin/netmaster $debug -cluster-mode $plugin -cluster-store $cstore -listen-url $listen_url -control-url $control_url || true - else - /contiv/bin/netmaster $debug -cluster-mode $plugin -listen-url $listen_url -control-url $control_url || true - fi - echo "CRITICAL : Netmaster has exited. Trying to respawn in 5s" - fi - sleep 5 - done -elif [ $netplugin == true ]; then - echo "Starting netplugin" - if [[ "$cluster_store" =~ ^etcd://.+ ]]; then - store_arg="--etcd-endpoints $(echo $cluster_store | sed s/etcd/http/)" - elif [[ "$cluster_store" =~ ^consul://.+ ]]; then - store_arg="--consul-endpoints $(echo $cluster_store | sed s/consul/http/)" +if [ "$CONTIV_MODE" = "kubernetes" ]; then + echo "INFO: Setting kubernetes configs" + mkdir -p /opt/contiv/config + mkdir -p /var/contiv/config + echo ${CONTIV_K8S_CONFIG} > /var/contiv/config/contiv.json + set -x + cp /var/contiv/config/contiv.json /opt/contiv/config/contiv.json + set +x + if [ "$CONTIV_ROLE" = "netplugin" ]; then + mkdir -p /opt/cni/bin + cp /contiv/bin/contivk8s /opt/cni/bin/ + mkdir -p /etc/cni/net.d/ + set -x + echo ${CONTIV_CNI_CONFIG} > /etc/cni/net.d/1-contiv.conf + set +x fi +fi - while true; do - if [ -f /tmp/restart_netplugin ]; then - if [ "$vtep_ip" != "" ]; then - vtep_ip_param="--vtep-ip" - fi - if [ "$vlan_if" != "" ]; then - vlan_if_param="--vlan-if" - fi - /contiv/bin/netplugin $debug $store_arg $vtep_ip_param $vtep_ip $vlan_if_param $vlan_if --plugin-mode $plugin || true - echo "CRITICAL : Netplugin has exited. Trying to respawn in 5s" - fi - sleep 5 - done +set +e +if [ "$CONTIV_ROLE" = "netmaster" ]; then + while true; do + echo "INFO: Starting contiv netmaster" + if [ -f /tmp/restart_netmaster ]; then + set -x + /contiv/bin/netmaster "$@" + set +x + echo "ERROR: Contiv netmaster has exited, restarting in 5s" + fi + sleep 5 + done +elif [ "$CONTIV_ROLE" = "netplugin" ]; then + while true; do + echo "INFO: Starting contiv netplugin" + if [ -f /tmp/restart_netplugin ]; then + set -x + /contiv/bin/netplugin "$@" + set +x + echo "ERROR: Contiv netplugin has exited, restarting in 5s" + fi + sleep 5 + done fi diff --git a/scripts/netContain/scripts/ovsInit.sh b/scripts/netContain/scripts/ovsInit.sh index d448fecba..7eac1a9db 100755 --- a/scripts/netContain/scripts/ovsInit.sh +++ b/scripts/netContain/scripts/ovsInit.sh @@ -3,12 +3,11 @@ set -euo pipefail -modprobe openvswitch +modprobe openvswitch || (echo "CRITICAL: Failed to load kernel module openvswitch" && exit 1 ) +echo "INFO: Loaded kernel module openvswitch" mkdir -p /var/run/openvswitch -mkdir -p /var/contiv/log/ - -sleep 2 +mkdir -p /var/log/contiv/ if [ -d "/etc/openvswitch" ]; then if [ -f "/etc/openvswitch/conf.db" ]; then @@ -29,11 +28,11 @@ ovsdb-server --remote=punix:/var/run/openvswitch/db.sock \ --private-key=db:Open_vSwitch,SSL,private_key \ --certificate=db:Open_vSwitch,SSL,certificate \ --bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert \ - --log-file=/var/contiv/log/ovs-db.log -vsyslog:info -vfile:info \ + --log-file=/var/log/contiv/ovs-db.log -vsyslog:info -vfile:info \ --pidfile --detach /etc/openvswitch/conf.db echo "INFO: Starting ovs-vswitchd" -ovs-vswitchd -v --pidfile --detach --log-file=/var/contiv/log/ovs-vswitchd.log \ +ovs-vswitchd -v --pidfile --detach --log-file=/var/log/contiv/ovs-vswitchd.log \ -vconsole:err -vsyslog:info -vfile:info & retry=0 diff --git a/test/systemtests/kubeadm_test.go b/test/systemtests/kubeadm_test.go index dbe871455..a0c31a8fa 100755 --- a/test/systemtests/kubeadm_test.go +++ b/test/systemtests/kubeadm_test.go @@ -24,8 +24,8 @@ const ( k8sMasterNode = "k8master" netmasterRestartFile = "/tmp/restart_netmaster" netpluginRestartFile = "/tmp/restart_netplugin" - netmasterLogLocation = "/var/contiv/log/netmaster.log" - netpluginLogLocation = "/var/contiv/log/netplugin.log" + netmasterLogLocation = "/var/log/contiv/netmaster.log" + netpluginLogLocation = "/var/log/contiv//netplugin.log" ) func (s *systemtestSuite) NewK8sPodExec(n *node) *kubePod { @@ -500,7 +500,8 @@ func (k *kubePod) startNetplugin(args string) error { } logrus.Infof("Starting netplugin on %s", k.node.Name()) - startNetpluginCmd := k.node.suite.basicInfo.BinPath + `/netplugin --vlan-if=` + k.node.suite.hostInfo.HostDataInterfaces + k.commonArgs() + args + ` > ` + netpluginLogLocation + ` 2>&1` + startNetpluginCmd := (k.node.suite.basicInfo.BinPath + `/netplugin --vlan-if=` + + k.node.suite.hostInfo.HostDataInterfaces + k.commonArgs() + args + ` > ` + netpluginLogLocation + ` 2>&1`) return k.podExecBG(podName, startNetpluginCmd, "kube-system") } @@ -554,7 +555,8 @@ func (k *kubePod) startNetmaster(args string) error { if k.node.suite.basicInfo.AciMode == "on" { infraType = " --infra aci " } - netmasterStartCmd := k.node.suite.basicInfo.BinPath + `/netmaster` + infraType + k.commonArgs() + args + ` > ` + netmasterLogLocation + ` 2>&1` + netmasterStartCmd := (k.node.suite.basicInfo.BinPath + `/netmaster` + + infraType + k.commonArgs() + args + ` > ` + netmasterLogLocation + ` 2>&1`) return k.podExecBG(podName, netmasterStartCmd, "kube-system") } @@ -680,7 +682,7 @@ func (k *kubePod) checkForNetpluginErrors() error { // NOTE: Checking for error here could result in Error code: 123 // Err code 123 might be the case when grep results in no output - fatalCheckCmd := `ls /var/contiv/log/net* | xargs -r -I % grep --text -A 5 "panic\|fatal" %` + fatalCheckCmd := `ls /var/log/contiv/net* | xargs -r -I % grep --text -A 5 "panic\|fatal" %` out, _ := k.podExec(podName, fatalCheckCmd, "kube-system") if out != "" { errStr := fmt.Sprintf("fatal error in netplugin logs on %s\n", k.node.Name()) @@ -689,7 +691,7 @@ func (k *kubePod) checkForNetpluginErrors() error { return errors.New(errStr) } - errCheckCmd := `ls /var/contiv/log/net* | xargs -r -I {} grep --text "error" {}` + errCheckCmd := `ls /var/log/contiv/net* | xargs -r -I {} grep --text "error" {}` out, _ = k.podExec(podName, errCheckCmd, "kube-system") if out != "" { logrus.Errorf("error output in netplugin logs on %s: \n", k.node.Name()) @@ -708,8 +710,8 @@ func (k *kubePod) rotateLog(processName string) error { return err } - oldLogFile := fmt.Sprintf("/var/contiv/log/%s.log", processName) - newLogFilePrefix := fmt.Sprintf("/var/contiv/log/_%s", processName) + oldLogFile := fmt.Sprintf("/var/log/contiv/%s.log", processName) + newLogFilePrefix := fmt.Sprintf("/var/log/contiv/_%s", processName) rotateLogCmd := `echo` + " `date +%s` " + `| xargs -I {} mv ` + oldLogFile + ` ` + newLogFilePrefix + `-{}.log` _, err = k.podExec(podName, rotateLogCmd, "kube-system") return err diff --git a/utils/configs.go b/utils/configs.go index 0202e641e..0fd53c260 100644 --- a/utils/configs.go +++ b/utils/configs.go @@ -60,7 +60,7 @@ func BuildDBFlags(binary string) []cli.Flag { cli.StringFlag{ Name: "etcd-endpoints, etcd", EnvVar: fmt.Sprintf("CONTIV_%s_ETCD_ENDPOINTS", binUpper), - Usage: fmt.Sprintf("a comma-delimited list of %s etcd endpoints", binLower), + Usage: fmt.Sprintf("a comma-delimited list of %s etcd endpoints (default: http://127.0.0.1:2379)", binLower), }, cli.StringFlag{ Name: "consul-endpoints, consul", @@ -185,7 +185,9 @@ func ValidateDBOptions(binary string, ctx *cli.Context) (*DBConfigs, error) { if etcdURLs != "" && consulURLs != "" { return nil, fmt.Errorf("ambiguous %s db endpoints, both etcd and consul specified: etcd: %s, consul: %s", binary, etcdURLs, consulURLs) } else if etcdURLs == "" && consulURLs == "" { - return nil, fmt.Errorf("unknown %s db endpoints, neither etcd or consul endpoints are specified", binary) + // if neither etcd or consul is set, try etcd at http://127.0.0.1:2379 + storeDriver = "etcd" + storeURLs = "http://127.0.0.1:2379" } else if etcdURLs != "" { storeDriver = "etcd" storeURLs = etcdURLs @@ -254,8 +256,8 @@ func ValidateNetworkOptions(binary string, ctx *cli.Context) (*NetworkConfigs, e // FlattenFlags concatenate slices of flags into one slice func FlattenFlags(flagSlices ...[]cli.Flag) []cli.Flag { - flags := flagSlices[0] - for _, slice := range flagSlices[1:] { + var flags []cli.Flag + for _, slice := range flagSlices { flags = append(flags, slice...) } return flags diff --git a/vagrant/mesos-cni/Vagrantfile b/vagrant/mesos-cni/Vagrantfile index 6082defe6..96649892f 100644 --- a/vagrant/mesos-cni/Vagrantfile +++ b/vagrant/mesos-cni/Vagrantfile @@ -30,7 +30,7 @@ echo "export CONTIV_CLUSTER_STORE_URL=$8" >> /etc/profile.d/envvar.sh source /etc/profile.d/envvar.sh if [[ $# -gt 10 ]] && [[ ${11} != "" ]]; then - shift; shift; shift; shift; shift; shift; shift; shift; shift; shift + shift 10 echo "export $@" >> /etc/profile.d/envvar.sh fi diff --git a/vagrant/nomad-docker/Vagrantfile b/vagrant/nomad-docker/Vagrantfile index 7724200fa..ca1171ada 100755 --- a/vagrant/nomad-docker/Vagrantfile +++ b/vagrant/nomad-docker/Vagrantfile @@ -31,7 +31,7 @@ echo "export GO15VENDOREXPERIMENT=1" >> /etc/profile.d/envvar.sh source /etc/profile.d/envvar.sh if [[ $# -gt 10 ]] && [[ ${11} != "" ]]; then - shift; shift; shift; shift; shift; shift; shift; shift; shift; shift + shift 10 echo "export $@" >> /etc/profile.d/envvar.sh fi