Contiv multi-master and other fixes

Contiv's etcd was not being deployed correctly when using more than
one master.  To make it easier to manage, it has been moved into a
k8s container.

The api proxy was hardcoded to an old version (1.1.1), and in some
environments would run into a docker error.  This has been moved into
a k8s container for easier management.

The firewall was too permissive on several ports.  Many were open to
the world when they should have only been accessible inside the
cluster.

Many of the contiv role variables were not prefixed with 'contiv',
which may end up clobbering variables from another role.  Now all the
contiv specific role variables start with 'contiv_'.

The api proxy's default self-signed certificate was bundled with the
role.  This means someone with read-only MITM access and this key
could decrypt traffic.  Granted a user defined certificate from a
trusted CA should be used in a production environment, it is still
better to generate one in each environment when one is not provided.

13 files changed, 265 insertions, 19 deletions

diff --git a/roles/contiv/templates/aci-gw.service b/roles/contiv/templates/aci-gw.service
index 9b3f12567..e2813c99d 100644
--- a/roles/contiv/templates/aci-gw.service
+++ b/roles/contiv/templates/aci-gw.service
@@ -1,10 +1,10 @@
 [Unit]
 Description=Contiv ACI gw
-After=auditd.service systemd-user-sessions.service time-sync.target {{ openshift_docker_service_name }}.service
+After=auditd.service systemd-user-sessions.service time-sync.target {{ contiv_openshift_docker_service_name }}.service
 
 [Service]
-ExecStart={{ bin_dir }}/aci_gw.sh start
-ExecStop={{ bin_dir }}/aci_gw.sh stop
+ExecStart={{ contiv_bin_dir }}/aci_gw.sh start
+ExecStop={{ contiv_bin_dir }}/aci_gw.sh stop
 KillMode=control-group
 Restart=always
 RestartSec=10
diff --git a/roles/contiv/templates/aci_gw.j2 b/roles/contiv/templates/aci_gw.j2
index ab4ad46a6..5ff349945 100644
--- a/roles/contiv/templates/aci_gw.j2
+++ b/roles/contiv/templates/aci_gw.j2
@@ -11,13 +11,13 @@ start)
     set -e
 
     docker run --net=host \
-    -e "APIC_URL={{ apic_url }}" \
-    -e "APIC_USERNAME={{ apic_username }}" \
-    -e "APIC_PASSWORD={{ apic_password }}" \
-    -e "APIC_LEAF_NODE={{ apic_leaf_nodes }}" \
-    -e "APIC_PHYS_DOMAIN={{ apic_phys_dom }}" \
-    -e "APIC_EPG_BRIDGE_DOMAIN={{ apic_epg_bridge_domain }}" \
-    -e "APIC_CONTRACTS_UNRESTRICTED_MODE={{ apic_contracts_unrestricted_mode }}" \
+    -e "APIC_URL={{ contiv_apic_url }}" \
+    -e "APIC_USERNAME={{ contiv_apic_username }}" \
+    -e "APIC_PASSWORD={{ contiv_apic_password }}" \
+    -e "APIC_LEAF_NODE={{ contiv_apic_leaf_nodes }}" \
+    -e "APIC_PHYS_DOMAIN={{ contiv_apic_phys_dom }}" \
+    -e "APIC_EPG_BRIDGE_DOMAIN={{ contiv_apic_epg_bridge_domain }}" \
+    -e "APIC_CONTRACTS_UNRESTRICTED_MODE={{ contiv_apic_contracts_unrestricted_mode }}" \
     --name=contiv-aci-gw \
     contiv/aci-gw
     ;;
diff --git a/roles/contiv/templates/api-proxy-daemonset.yml.j2 b/roles/contiv/templates/api-proxy-daemonset.yml.j2
new file mode 100644
index 000000000..4d4388706
--- /dev/null
+++ b/roles/contiv/templates/api-proxy-daemonset.yml.j2
@@ -0,0 +1,56 @@
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: contiv-api-proxy
+  namespace: kube-system
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      name: contiv-api-proxy
+  template:
+    metadata:
+      namespace: kube-system
+      labels:
+        name: contiv-api-proxy
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+    spec:
+      serviceAccountName: contiv-api-proxy
+      hostNetwork: true
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: kubernetes.io/hostname
+                operator: In
+                values:
+{% for node in groups.oo_masters_to_config %}
+                  - "{{ node }}"
+{% endfor %}
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      containers:
+        - name: contiv-api-proxy
+          image: "{{ contiv_api_proxy_image_repo }}:{{ contiv_version }}"
+          args:
+            - "--listen-address=0.0.0.0:{{ contiv_api_proxy_port }}"
+            - --tls-key-file=/var/contiv/api_proxy_key.pem
+            - --tls-certificate=/var/contiv/api_proxy_cert.pem
+            - "--data-store-address={{ etcd_host }}"
+            - "--netmaster-address=127.0.0.1:{{ contiv_netmaster_port }}"
+          ports:
+            - containerPort: "{{ contiv_api_proxy_port }}"
+              hostPort: "{{ contiv_api_proxy_port }}"
+          volumeMounts:
+            - name: secret-volume
+              mountPath: /var/contiv
+              readOnly: true
+      volumes:
+        - name: secret-volume
+          secret:
+            secretName: contiv-api-proxy-secret
diff --git a/roles/contiv/templates/api-proxy-secrets.yml.j2 b/roles/contiv/templates/api-proxy-secrets.yml.j2
new file mode 100644
index 000000000..cd800c97d
--- /dev/null
+++ b/roles/contiv/templates/api-proxy-secrets.yml.j2
@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+    name: contiv-api-proxy-secret
+    namespace: kube-system
+    labels:
+      name: contiv-api-proxy-secret
+# Use data+b64encode, because stringData doesn't preserve newlines.
+data:
+    api_proxy_key.pem: "{{ key | b64encode }}"
+    api_proxy_cert.pem: "{{ cert | b64encode }}"
diff --git a/roles/contiv/templates/contiv.cfg.j2 b/roles/contiv/templates/contiv.cfg.j2
index f0e99c556..1dce9fcc2 100644
--- a/roles/contiv/templates/contiv.cfg.j2
+++ b/roles/contiv/templates/contiv.cfg.j2
@@ -1,5 +1,5 @@
 {
-  "K8S_API_SERVER": "https://{{ hostvars[groups['masters'][0]]['ansible_' + netmaster_interface].ipv4.address }}:{{ kube_master_api_port }}",
+  "K8S_API_SERVER": "https://{{ hostvars[groups['masters'][0]]['ansible_' + contiv_netmaster_interface].ipv4.address }}:{{ contiv_kube_master_api_port }}",
   "K8S_CA": "{{ openshift.common.config_base }}/node/ca.crt",
   "K8S_KEY": "{{ openshift.common.config_base }}/node/system:node:{{ openshift.common.hostname }}.key",
   "K8S_CERT": "{{ openshift.common.config_base }}/node/system:node:{{ openshift.common.hostname }}.crt",
diff --git a/roles/contiv/templates/contiv.cfg.master.j2 b/roles/contiv/templates/contiv.cfg.master.j2
index fac8e3c4c..ca29b8001 100644
--- a/roles/contiv/templates/contiv.cfg.master.j2
+++ b/roles/contiv/templates/contiv.cfg.master.j2
@@ -1,5 +1,5 @@
 {
-  "K8S_API_SERVER": "https://{{ hostvars[groups['masters'][0]]['ansible_' + netmaster_interface].ipv4.address }}:{{ kube_master_api_port }}",
+  "K8S_API_SERVER": "https://{{ hostvars[groups['masters'][0]]['ansible_' + contiv_netmaster_interface].ipv4.address }}:{{ contiv_kube_master_api_port }}",
   "K8S_CA": "{{ openshift.common.config_base }}/master/ca.crt",
   "K8S_KEY": "{{ openshift.common.config_base }}/master/system:node:{{ openshift.common.hostname }}.key",
   "K8S_CERT": "{{ openshift.common.config_base }}/master/system:node:{{ openshift.common.hostname }}.crt",
diff --git a/roles/contiv/templates/etcd-daemonset.yml.j2 b/roles/contiv/templates/etcd-daemonset.yml.j2
new file mode 100644
index 000000000..76937e670
--- /dev/null
+++ b/roles/contiv/templates/etcd-daemonset.yml.j2
@@ -0,0 +1,83 @@
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: contiv-etcd
+  namespace: kube-system
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      name: contiv-etcd
+  template:
+    metadata:
+      namespace: kube-system
+      labels:
+        name: contiv-etcd
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+    spec:
+      serviceAccountName: contiv-etcd
+      hostNetwork: true
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: kubernetes.io/hostname
+                operator: In
+                values:
+{% for node in groups.oo_masters_to_config %}
+                  - "{{ node }}"
+{% endfor %}
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      initContainers:
+        - name: contiv-etcd-init
+          image: "{{ contiv_etcd_init_image_repo }}:{{ contiv_etcd_init_image_tag }}"
+          env:
+            - name: ETCD_INIT_ARGSFILE
+              value: "{{ contiv_etcd_conf_dir }}/contiv-etcd-args"
+            - name: ETCD_INIT_LISTEN_PORT
+              value: "{{ contiv_etcd_port }}"
+            - name: ETCD_INIT_PEER_PORT
+              value: "{{ contiv_etcd_peer_port }}"
+            - name: ETCD_INIT_CLUSTER
+              value: "{{ contiv_etcd_peers }}"
+            - name: ETCD_INIT_DATA_DIR
+              value: "{{ contiv_etcd_data_dir }}"
+          volumeMounts:
+            - name: contiv-etcd-conf-dir
+              mountPath: "{{ contiv_etcd_conf_dir }}"
+          securityContext:
+            runAsUser: "{{ contiv_etcd_system_uid }}"
+            fsGroup: "{{ contiv_etcd_system_gid }}"
+      containers:
+        - name: contiv-etcd
+          image: "{{ contiv_etcd_image_repo }}:{{ contiv_etcd_image_tag }}"
+          command:
+            - sh
+            - -c
+            - 'exec etcd $(cat "$ETCD_INIT_ARGSFILE")'
+          env:
+            - name: ETCD_INIT_ARGSFILE
+              value: "{{ contiv_etcd_conf_dir }}/contiv-etcd-args"
+          volumeMounts:
+            - name: contiv-etcd-conf-dir
+              mountPath: "{{ contiv_etcd_conf_dir }}"
+            - name: contiv-etcd-data-dir
+              mountPath: "{{ contiv_etcd_data_dir }}"
+          securityContext:
+            runAsUser: "{{ contiv_etcd_system_uid }}"
+            fsGroup: "{{ contiv_etcd_system_gid }}"
+      volumes:
+        - name: contiv-etcd-data-dir
+          hostPath:
+            type: DirectoryOrCreate
+            path: "{{ contiv_etcd_data_dir }}"
+        - name: contiv-etcd-conf-dir
+          hostPath:
+            type: DirectoryOrCreate
+            path: "{{ contiv_etcd_conf_dir }}"
diff --git a/roles/contiv/templates/etcd-proxy-daemonset.yml.j2 b/roles/contiv/templates/etcd-proxy-daemonset.yml.j2
new file mode 100644
index 000000000..4ec6cfd76
--- /dev/null
+++ b/roles/contiv/templates/etcd-proxy-daemonset.yml.j2
@@ -0,0 +1,55 @@
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: contiv-etcd-proxy
+  namespace: kube-system
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      name: contiv-etcd-proxy
+  template:
+    metadata:
+      namespace: kube-system
+      labels:
+        name: contiv-etcd-proxy
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+    spec:
+      serviceAccountName: contiv-etcd
+      hostNetwork: true
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: kubernetes.io/hostname
+                operator: NotIn
+                values:
+{% for node in groups.oo_masters_to_config %}
+                  - "{{ node }}"
+{% endfor %}
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      containers:
+        - name: contiv-etcd-proxy
+          image: "{{ contiv_etcd_image_repo }}:{{ contiv_etcd_image_tag }}"
+          command:
+            - etcd
+            - "--proxy=on"
+            - "--listen-client-urls=http://127.0.0.1:{{ contiv_etcd_port }}"
+            - "--advertise-client-urls=http://127.0.0.1:{{ contiv_etcd_port }}"
+            - "--initial-cluster={{ contiv_etcd_peers }}"
+            - "--data-dir={{ contiv_etcd_data_dir }}"
+          volumeMounts:
+            - name: contiv-etcd-data-dir
+              mountPath: "{{ contiv_etcd_data_dir }}"
+          securityContext:
+            runAsUser: "{{ contiv_etcd_system_uid }}"
+            fsGroup: "{{ contiv_etcd_system_gid }}"
+      volumes:
+        - name: contiv-etcd-data-dir
+          emptyDir: {}
diff --git a/roles/contiv/templates/etcd-scc.yml.j2 b/roles/contiv/templates/etcd-scc.yml.j2
new file mode 100644
index 000000000..6c4bb1d1e
--- /dev/null
+++ b/roles/contiv/templates/etcd-scc.yml.j2
@@ -0,0 +1,42 @@
+allowHostDirVolumePlugin: true
+allowHostIPC: false
+allowHostNetwork: true
+allowHostPID: false
+allowHostPorts: false
+allowPrivilegedContainer: false
+allowedCapabilities: []
+allowedFlexVolumes: []
+apiVersion: v1
+defaultAddCapabilities: []
+fsGroup:
+  ranges:
+  - max: "{{ contiv_etcd_system_gid }}"
+    min: "{{ contiv_etcd_system_gid }}"
+  type: MustRunAs
+groups: []
+kind: SecurityContextConstraints
+metadata:
+  annotations:
+    kubernetes.io/description: 'For contiv-etcd only.'
+  creationTimestamp: null
+  name: contiv-etcd
+priority: null
+readOnlyRootFilesystem: true
+requiredDropCapabilities:
+- KILL
+- MKNOD
+- SETUID
+- SETGID
+runAsUser:
+  type: MustRunAs
+  uid: "{{ contiv_etcd_system_uid }}"
+seLinuxContext:
+  type: MustRunAs
+supplementalGroups:
+  type: MustRunAs
+users:
+- system:serviceaccount:kube-system:contiv-etcd
+volumes:
+- emptyDir
+- hostPath
+- secret
diff --git a/roles/contiv/templates/netmaster.env.j2 b/roles/contiv/templates/netmaster.env.j2
index 5b5c84a2e..9a38f32ea 100644
--- a/roles/contiv/templates/netmaster.env.j2
+++ b/roles/contiv/templates/netmaster.env.j2
@@ -1,2 +1 @@
-NETMASTER_ARGS='--cluster-store etcd://{{ etcd_url }}  --cluster-mode=kubernetes'
-
+NETMASTER_ARGS='--cluster-store {{ contiv_etcd_url }} --cluster-mode=kubernetes'
diff --git a/roles/contiv/templates/netmaster.service b/roles/contiv/templates/netmaster.service
index ce7d0c75e..b7289bc38 100644
--- a/roles/contiv/templates/netmaster.service
+++ b/roles/contiv/templates/netmaster.service
@@ -4,7 +4,7 @@ After=auditd.service systemd-user-sessions.service contiv-etcd.service
 
 [Service]
 EnvironmentFile=/etc/default/netmaster
-ExecStart={{ bin_dir }}/netmaster $NETMASTER_ARGS
+ExecStart={{ contiv_bin_dir }}/netmaster $NETMASTER_ARGS
 KillMode=control-group
 Restart=always
 RestartSec=10
diff --git a/roles/contiv/templates/netplugin.j2 b/roles/contiv/templates/netplugin.j2
index a4928cc3d..9ce44e2dc 100644
--- a/roles/contiv/templates/netplugin.j2
+++ b/roles/contiv/templates/netplugin.j2
@@ -1,7 +1,6 @@
 {% if contiv_encap_mode == "vlan" %}
-NETPLUGIN_ARGS='-vlan-if {{ netplugin_interface }} -ctrl-ip {{ netplugin_ctrl_ip }} -plugin-mode kubernetes -cluster-store etcd://{{ etcd_url }}'
+NETPLUGIN_ARGS='-vlan-if {{ contiv_netplugin_interface }} -ctrl-ip {{ contiv_netplugin_ctrl_ip }} -plugin-mode kubernetes -cluster-store {{ contiv_etcd_url }}'
 {% endif %}
 {% if contiv_encap_mode == "vxlan" %}
-NETPLUGIN_ARGS='-vtep-ip {{ netplugin_ctrl_ip }} -ctrl-ip {{ netplugin_ctrl_ip }} -plugin-mode kubernetes -cluster-store etcd://{{ etcd_url }}'
+NETPLUGIN_ARGS='-vtep-ip {{ contiv_netplugin_ctrl_ip }} -ctrl-ip {{ contiv_netplugin_ctrl_ip }} -plugin-mode kubernetes -cluster-store {{ contiv_etcd_url }}'
 {% endif %}
-
diff --git a/roles/contiv/templates/netplugin.service b/roles/contiv/templates/netplugin.service
index 6358d89ec..2e1ca1bdf 100644
--- a/roles/contiv/templates/netplugin.service
+++ b/roles/contiv/templates/netplugin.service
@@ -4,7 +4,7 @@ After=auditd.service systemd-user-sessions.service contiv-etcd.service
 
 [Service]
 EnvironmentFile=/etc/default/netplugin
-ExecStart={{ bin_dir }}/netplugin $NETPLUGIN_ARGS
+ExecStart={{ contiv_bin_dir }}/netplugin $NETPLUGIN_ARGS
 KillMode=control-group
 Restart=always
 RestartSec=10