contiv-vpp.yaml

---
# Source: contiv-vpp/templates/vpp.yaml
# Contiv-VPP deployment YAML file. This deploys Contiv VPP networking on a Kuberntes cluster.
# The deployment consists of the following components:
#   - contiv-etcd - deployed on k8s master
#   - contiv-vswitch - deployed on each k8s node
#   - contiv-ksr - deployed on k8s master

###########################################################
#  Configuration
###########################################################

# This config map contains contiv-agent configuration. The most important part is the IPAMConfig,
# which may be updated in case the default IPAM settings do not match your needs.
# NodeConfig may be used in case your nodes have more than 1 VPP interface. In that case, one
# of them needs to be marked as the main inter-node interface, and the rest of them can be
# configured with any IP addresses (the IPs cannot conflict with the main IPAM config).
apiVersion: v1
kind: ConfigMap
metadata:
  name: contiv-agent-cfg
  namespace: kube-system
data:
  contiv.yaml: |-
    TCPstackDisabled: true
    UseTAPInterfaces: true
    TAPInterfaceVersion: 1
    NatExternalTraffic: true
    MTUSize: 1500
    IPAMConfig:
      PodSubnetCIDR: 10.1.0.0/16
      PodNetworkPrefixLen: 24
      PodIfIPCIDR: 10.2.1.0/24
      VPPHostSubnetCIDR: 172.30.0.0/16
      VPPHostNetworkPrefixLen: 24
      NodeInterconnectCIDR: 192.168.16.0/24
      VxlanCIDR: 192.168.30.0/24
      NodeInterconnectDHCP: False
  logs.conf: |
    defaultlevel: debug

---

apiVersion: v1
kind: ConfigMap
metadata:
  name: govpp-cfg
  namespace: kube-system
data:
  govpp.conf: |
    health-check-probe-interval: 1000000000
    health-check-reply-timeout: 500000000
    health-check-threshold: 3

---

###########################################################
#
# !!! DO NOT EDIT THINGS BELOW THIS LINE !!!
#
###########################################################


###########################################################
#  Components and other resources
###########################################################

# This installs the contiv-etcd (ETCD server to be used by Contiv) on the master node in a Kubernetes cluster.
# In odrer to dump the content of ETCD, you can use the kubectl exec command similar to this:
#   kubectl exec contiv-etcd-cxqhr -n kube-system etcdctl -- get --endpoints=[127.0.0.1:12379] --prefix="true" ""
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: contiv-etcd
  namespace: kube-system
  labels:
    k8s-app: contiv-etcd
spec:
  template:
    metadata:
      labels:
        k8s-app: contiv-etcd
      annotations:
        # Marks this pod as a critical add-on.
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      # Allow this pod to be rescheduled while the node is in "critical add-ons only" mode.
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
      - key: CriticalAddonsOnly
        operator: Exists
      # Only run this pod on the master.
      nodeSelector:
        node-role.kubernetes.io/master: ""
      hostNetwork: true

      containers:
        - name: contiv-etcd
          image: quay.io/coreos/etcd:v3.1.10
          imagePullPolicy: IfNotPresent
          env:
            - name: CONTIV_ETCD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            - name: ETCDCTL_API
              value: "3"
          command: ["/bin/sh","-c"]
          args: ["/usr/local/bin/etcd --name=contiv-etcd --data-dir=/var/etcd/contiv-data --advertise-client-urls=http://0.0.0.0:12379 --listen-client-urls=http://0.0.0.0:12379 --listen-peer-urls=http://0.0.0.0:12380"]

---

apiVersion: v1
kind: Service
metadata:
  name: contiv-etcd
  namespace: kube-system
spec:
  type: NodePort
  # Match contiv-etcd DaemonSet.
  selector:
    k8s-app: contiv-etcd
  ports:
  - port: 12379
    nodePort: 32379

---

# This config map contains ETCD configuration for connecting to the contiv-etcd defined above.
apiVersion: v1
kind: ConfigMap
metadata:
  name: contiv-etcd-cfg
  namespace: kube-system
data:
  etcd.conf: |
    insecure-transport: true
    dial-timeout: 5000000000
    endpoints:
      - "127.0.0.1:32379"

---

# This installs contiv-vswitch on each master and worker node in a Kubernetes cluster.
# It consists of the following containers:
#   - contiv-vswitch container: contains VPP and its management agent
#   - contiv-cni container: installs CNI on the host
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: contiv-vswitch
  namespace: kube-system
  labels:
    k8s-app: contiv-vswitch
spec:
  selector:
    matchLabels:
      k8s-app: contiv-vswitch
  template:
    metadata:
      labels:
        k8s-app: contiv-vswitch
      annotations:
        # Marks this pod as a critical add-on.
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      # Allow this pod to be rescheduled while the node is in "critical add-ons only" mode.
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
      - key: CriticalAddonsOnly
        operator: Exists
      hostNetwork: true
      hostPID: true

      # Init containers are executed before regular containers, must finish successfully before regular ones are started.
      initContainers:
      # This init container extracts/copies VPP LD_PRELOAD libs and default VPP config to the host.
      - name: vpp-init
        image: contivvpp/vswitch:latest
        imagePullPolicy: IfNotPresent
        command:
        - /bin/sh
        args:
        - -c
        - |
          set -eu
          chmod 700 /run/vpp
          rm -rf /dev/shm/db /dev/shm/global_vm /dev/shm/vpe-api /vpp-lib64/*
          cp -r "$LD_PRELOAD_LIB_DIR"/* /vpp-lib64/
          if [ ! -e /host/etc/vpp/contiv-vswitch.conf ]
          then
              cp /etc/vpp/contiv-vswitch.conf /host/etc/vpp/
          fi
          if ip link show vpp1 >/dev/null 2>&1
          then
               ip link del vpp1
          fi
          cp -f /usr/local/bin/vppctl /host/usr/local/bin/vppctl
        resources: {}
        securityContext:
          privileged: true
        volumeMounts:
          - name: usr-local-bin
            mountPath: /host/usr/local/bin
          - name: vpp-lib64
            mountPath: /vpp-lib64/
          - name: vpp-cfg
            mountPath: /host/etc/vpp
          - name: shm
            mountPath: /dev/shm
          - name: vpp-run
            mountPath: /run/vpp

      containers:
        # Runs contiv-vswitch container on each Kubernetes node.
        # It contains the vSwitch VPP and its management agent.
        - name: contiv-vswitch
          image: contivvpp/vswitch:latest
          imagePullPolicy: IfNotPresent
          securityContext:
            privileged: true
          ports:
            # readiness + liveness probe
            - containerPort: 9999
          readinessProbe:
            httpGet:
              path: /readiness
              port: 9999
            periodSeconds: 1
            initialDelaySeconds: 15
          livenessProbe:
            httpGet:
              path: /liveness
              port: 9999
            periodSeconds: 1
            initialDelaySeconds: 60
          env:
            - name: MICROSERVICE_LABEL
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: ETCDV3_CONFIG
              value: "/etc/etcd/etcd.conf"
          volumeMounts:
            - name: etcd-cfg
              mountPath: /etc/etcd
            - name: vpp-cfg
              mountPath: /etc/vpp
            - name: shm
              mountPath: /dev/shm
            - name: dev
              mountPath: /dev
            - name: vpp-run
              mountPath: /run/vpp
            - name: contiv-plugin-cfg
              mountPath: /etc/agent
            - name: govpp-plugin-cfg
              mountPath: /etc/govpp

        # This container installs the Contiv CNI binaries
        # and CNI network config file on each node.
        - name: contiv-cni
          image: contivvpp/cni:latest
          imagePullPolicy: IfNotPresent
          volumeMounts:
            - mountPath: /opt/cni/bin
              name: cni-bin-dir
            - mountPath: /etc/cni/net.d
              name: cni-net-dir

      volumes:
        # Used to connect to contiv-etcd.
        - name: etcd-cfg
          configMap:
            name: contiv-etcd-cfg
        # Used to install CNI.
        - name: cni-bin-dir
          hostPath:
            path: /opt/cni/bin
        - name: cni-net-dir
          hostPath:
            path: /etc/cni/net.d
        # VPP startup config folder.
        - name: vpp-cfg
          hostPath:
            path: /etc/vpp
        # To install vppctl.
        - name: usr-local-bin
          hostPath:
            path: /usr/local/bin
        # LD_PRELOAD library.
        - name: vpp-lib64
          hostPath:
            path: /tmp/ldpreload/vpp-lib64
        # /dev mount is required for DPDK-managed NICs on VPP (/dev/uio0) and for shared memory communication with VPP (/dev/shm)
        - name: dev
          hostPath:
            path: /dev
        - name: shm
          hostPath:
            path: /dev/shm
        # For CLI unix socket.
        - name: vpp-run
          hostPath:
            path: /run/vpp
        # Used to configure contiv plugin.
        - name: contiv-plugin-cfg
          configMap:
            name: contiv-agent-cfg
        # Used to configure govpp plugin.
        - name: govpp-plugin-cfg
          configMap:
            name: govpp-cfg

---

# This installs the contiv-ksr (Kubernetes State Reflector) on the master node in a Kubernetes cluster.
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: contiv-ksr
  namespace: kube-system
  labels:
    k8s-app: contiv-ksr
spec:
  template:
    metadata:
      labels:
        k8s-app: contiv-ksr
      annotations:
        # Marks this pod as a critical add-on.
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      # Allow this pod to be rescheduled while the node is in "critical add-ons only" mode.
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
      - key: CriticalAddonsOnly
        operator: Exists
      # Only run this pod on the master.
      nodeSelector:
        node-role.kubernetes.io/master: ""
      hostNetwork: true
      # This grants the required permissions to contiv-ksr.
      serviceAccountName: contiv-ksr

      containers:
        - name: contiv-ksr
          image: contivvpp/ksr:latest
          imagePullPolicy: IfNotPresent
          env:
            - name: ETCDV3_CONFIG
              value: "/etc/etcd/etcd.conf"
          volumeMounts:
            - name: etcd-cfg
              mountPath: /etc/etcd
          readinessProbe:
            httpGet:
              path: /readiness
              port: 9191
            periodSeconds: 1
            initialDelaySeconds: 10
          livenessProbe:
            httpGet:
              path: /liveness
              port: 9191
            periodSeconds: 1
            initialDelaySeconds: 30

      volumes:
        # Used to connect to contiv-etcd.
        - name: etcd-cfg
          configMap:
            name: contiv-etcd-cfg

---

# This cluster role defines a set of permissions required for contiv-ksr.
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: contiv-ksr
  namespace: kube-system
rules:
  - apiGroups:
    - ""
    - extensions
    resources:
      - pods
      - namespaces
      - networkpolicies
      - services
      - endpoints
      - nodes
    verbs:
      - watch
      - list

---

# This defines a service account for contiv-ksr.
apiVersion: v1
kind: ServiceAccount
metadata:
  name: contiv-ksr
  namespace: kube-system

---

# This binds the contiv-ksr cluster role with contiv-ksr service account.
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: contiv-ksr
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: contiv-ksr
subjects:
- kind: ServiceAccount
  name: contiv-ksr
  namespace: kube-system