Merge pull request #1175 from detiber/ha_port_fixes

Ha port fixes

16 files changed, 146 insertions, 114 deletions

diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 0df03f194..296176e49 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -43,6 +43,7 @@
           api_port: "{{ openshift_master_api_port | default(None) }}"
           api_url: "{{ openshift_master_api_url | default(None) }}"
           api_use_ssl: "{{ openshift_master_api_use_ssl | default(None) }}"
+          controllers_port: "{{ openshift_master_controllers_port | default(None) }}"
           public_api_url: "{{ openshift_master_public_api_url | default(None) }}"
           cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}"
           cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}"
diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py
index 831d78b83..0820989af 100755
--- a/roles/openshift_facts/library/openshift_facts.py
+++ b/roles/openshift_facts/library/openshift_facts.py
@@ -458,52 +458,68 @@ def set_url_facts_if_unset(facts):
                   were not already present
     """
     if 'master' in facts:
-        api_use_ssl = facts['master']['api_use_ssl']
-        api_port = facts['master']['api_port']
-        console_use_ssl = facts['master']['console_use_ssl']
-        console_port = facts['master']['console_port']
-        console_path = facts['master']['console_path']
-        etcd_use_ssl = facts['master']['etcd_use_ssl']
-        etcd_hosts = facts['master']['etcd_hosts']
-        etcd_port = facts['master']['etcd_port']
         hostname = facts['common']['hostname']
-        public_hostname = facts['common']['public_hostname']
         cluster_hostname = facts['master'].get('cluster_hostname')
         cluster_public_hostname = facts['master'].get('cluster_public_hostname')
+        public_hostname = facts['common']['public_hostname']
+        api_hostname = cluster_hostname if cluster_hostname else hostname
+        api_public_hostname = cluster_public_hostname if cluster_public_hostname else public_hostname
+        console_path = facts['master']['console_path']
+        etcd_hosts = facts['master']['etcd_hosts']
+
+        use_ssl = dict(
+            api=facts['master']['api_use_ssl'],
+            public_api=facts['master']['api_use_ssl'],
+            loopback_api=facts['master']['api_use_ssl'],
+            console=facts['master']['console_use_ssl'],
+            public_console=facts['master']['console_use_ssl'],
+            etcd=facts['master']['etcd_use_ssl']
+        )
+
+        ports = dict(
+            api=facts['master']['api_port'],
+            public_api=facts['master']['api_port'],
+            loopback_api=facts['master']['api_port'],
+            console=facts['master']['console_port'],
+            public_console=facts['master']['console_port'],
+            etcd=facts['master']['etcd_port'],
+        )
+
+        etcd_urls = []
+        if etcd_hosts != '':
+            facts['master']['etcd_port'] = ports['etcd']
+            facts['master']['embedded_etcd'] = False
+            for host in etcd_hosts:
+                etcd_urls.append(format_url(use_ssl['etcd'], host,
+                                            ports['etcd']))
+        else:
+            etcd_urls = [format_url(use_ssl['etcd'], hostname,
+                                    ports['etcd'])]
+
+        facts['master'].setdefault('etcd_urls', etcd_urls)
+
+        prefix_hosts = [('api', api_hostname),
+                        ('public_api', api_public_hostname),
+                        ('loopback_api', hostname)]
+
+        for prefix, host in prefix_hosts:
+            facts['master'].setdefault(prefix + '_url', format_url(use_ssl[prefix],
+                                                                   host,
+                                                                   ports[prefix]))
+
+
+        r_lhn = "{0}:{1}".format(api_hostname, ports['api']).replace('.', '-')
+        facts['master'].setdefault('loopback_cluster_name', r_lhn)
+        facts['master'].setdefault('loopback_context_name', "default/{0}/system:openshift-master".format(r_lhn))
+        facts['master'].setdefault('loopback_user', "system:openshift-master/{0}".format(r_lhn))
+
+        prefix_hosts = [('console', api_hostname), ('public_console', api_public_hostname)]
+        for prefix, host in prefix_hosts:
+            facts['master'].setdefault(prefix + '_url', format_url(use_ssl[prefix],
+                                                                   host,
+                                                                   ports[prefix],
+                                                                   console_path))
 
-        if 'etcd_urls' not in facts['master']:
-            etcd_urls = []
-            if etcd_hosts != '':
-                facts['master']['etcd_port'] = etcd_port
-                facts['master']['embedded_etcd'] = False
-                for host in etcd_hosts:
-                    etcd_urls.append(format_url(etcd_use_ssl, host,
-                                                etcd_port))
-            else:
-                etcd_urls = [format_url(etcd_use_ssl, hostname,
-                                        etcd_port)]
-            facts['master']['etcd_urls'] = etcd_urls
-        if 'api_url' not in facts['master']:
-            api_hostname = cluster_hostname if cluster_hostname else hostname
-            facts['master']['api_url'] = format_url(api_use_ssl, api_hostname,
-                                                    api_port)
-        if 'public_api_url' not in facts['master']:
-            api_public_hostname = cluster_public_hostname if cluster_public_hostname else public_hostname
-            facts['master']['public_api_url'] = format_url(api_use_ssl,
-                                                           api_public_hostname,
-                                                           api_port)
-        if 'console_url' not in facts['master']:
-            console_hostname = cluster_hostname if cluster_hostname else hostname
-            facts['master']['console_url'] = format_url(console_use_ssl,
-                                                        console_hostname,
-                                                        console_port,
-                                                        console_path)
-        if 'public_console_url' not in facts['master']:
-            console_public_hostname = cluster_public_hostname if cluster_public_hostname else public_hostname
-            facts['master']['public_console_url'] = format_url(console_use_ssl,
-                                                               console_public_hostname,
-                                                               console_port,
-                                                               console_path)
     return facts
 
 def set_aggregate_facts(facts):
@@ -1149,7 +1165,7 @@ class OpenShiftFacts(object):
         defaults['common'] = common
 
         if 'master' in roles:
-            master = dict(api_use_ssl=True, api_port='8443',
+            master = dict(api_use_ssl=True, api_port='8443', controllers_port='8444',
                           console_use_ssl=True, console_path='/console',
                           console_port='8443', etcd_use_ssl=True, etcd_hosts='',
                           etcd_port='4001', portal_net='172.30.0.0/16',
diff --git a/roles/openshift_master/defaults/main.yml b/roles/openshift_master/defaults/main.yml
index 9766d01ae..1f74d851a 100644
--- a/roles/openshift_master/defaults/main.yml
+++ b/roles/openshift_master/defaults/main.yml
@@ -6,7 +6,9 @@ os_firewall_allow:
 - service: etcd embedded
   port: 4001/tcp
 - service: api server https
-  port: 8443/tcp
+  port: "{{ openshift.master.api_port }}/tcp"
+- service: api controllers https
+  port: "{{ openshift.master.controllers_port }}/tcp"
 - service: dns tcp
   port: 53/tcp
 - service: dns udp
@@ -24,7 +26,5 @@ os_firewall_allow:
 os_firewall_deny:
 - service: api server http
   port: 8080/tcp
-- service: former web console port
-  port: 8444/tcp
 - service: former etcd peer port
   port: 7001/tcp
diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml
index 150b76fc8..44e7e3178 100644
--- a/roles/openshift_master/tasks/main.yml
+++ b/roles/openshift_master/tasks/main.yml
@@ -94,12 +94,12 @@
 - name: Install Master docker service file
   template:
     dest: "/etc/systemd/system/{{ openshift.common.service_type }}-master.service"
-    src: master.docker.service.j2
+    src: docker/master.docker.service.j2
   register: install_result
   when: openshift.common.is_containerized | bool and not openshift_master_ha | bool
-  
+
 - name: Create openshift.common.data_dir
-  file: 
+  file:
     path: "{{ openshift.common.data_dir }}"
     state: directory
     mode: 0755
@@ -174,31 +174,42 @@
   when: openshift.common.is_containerized | bool
 
 # workaround for missing systemd unit files for controllers/api
-- name: Create the api service file
-  template:
-    src: atomic-openshift-master-api{{ ha_suffix }}.service.j2
-    dest: "{{ ha_svcdir }}/{{ openshift.common.service_type }}-master-api.service"
-  when: openshift_master_ha | bool and openshift_master_cluster_method == "native"
-- name: Create the controllers service file
+- name: Create the systemd unit files
   template:
-    src: atomic-openshift-master-controllers{{ ha_suffix }}.service.j2
-    dest: "{{ ha_svcdir }}/{{ openshift.common.service_type }}-master-controllers.service"
+    src: "{{ ha_svc_template_path }}/atomic-openshift-master-{{ item }}.service.j2"
+    dest: "{{ ha_svcdir }}/{{ openshift.common.service_type }}-master-{{ item }}.service"
   when: openshift_master_ha | bool and openshift_master_cluster_method == "native"
-- name: Create the api env file
+  with_items:
+  - api
+  - controllers
+  register: create_unit_files
+
+- command: systemctl daemon-reload
+  when: create_unit_files | changed
+# end workaround for missing systemd unit files
+
+- name: Create the master api service env file
   template:
-    src: atomic-openshift-master-api.j2
+    src: "{{ ha_svc_template_path }}/atomic-openshift-master-api.j2"
     dest: /etc/sysconfig/{{ openshift.common.service_type }}-master-api
-    force: no
   when: openshift_master_ha | bool and openshift_master_cluster_method == "native"
-- name: Create the controllers env file
+  notify:
+  - restart master api
+
+- name: Create the master controllers service env file
   template:
-    src: atomic-openshift-master-controllers.j2
+    src: "{{ ha_svc_template_path }}/atomic-openshift-master-controllers.j2"
     dest: /etc/sysconfig/{{ openshift.common.service_type }}-master-controllers
-    force: no
-  when: openshift_master_ha | bool and openshift_master_cluster_method == "native"
-- command: systemctl daemon-reload
   when: openshift_master_ha | bool and openshift_master_cluster_method == "native"
-# end workaround for missing systemd unit files
+  notify:
+  - restart master controllers
+
+- name: Create the master service env file
+  template:
+    src: "atomic-openshift-master.j2"
+    dest: /etc/sysconfig/{{ openshift.common.service_type }}-master
+  notify:
+  - restart master
 
 - name: Create session secrets file
   template:
@@ -223,47 +234,36 @@
   - restart master api
   - restart master controllers
 
-- name: Configure master settings
-  lineinfile:
-    dest: /etc/sysconfig/{{ openshift.common.service_type }}-master
-    regexp: "{{ item.regex }}"
-    line: "{{ item.line }}"
-    create: yes
-  with_items:
-    - regex: '^OPTIONS='
-      line: "OPTIONS=--loglevel={{ openshift.master.debug_level }}"
-    - regex: '^CONFIG_FILE='
-      line: "CONFIG_FILE={{ openshift_master_config_file }}"
-  notify:
-  - restart master
-
-- name: Configure master api settings
-  lineinfile:
-    dest: /etc/sysconfig/{{ openshift.common.service_type }}-master-api
-    regexp: "{{ item.regex }}"
-    line: "{{ item.line }}"
-  with_items:
-    - regex: '^OPTIONS='
-      line: "OPTIONS=--loglevel={{ openshift.master.debug_level }} --listen=https://0.0.0.0:8443 --master=https://{{ openshift.common.ip }}:8443"
-    - regex: '^CONFIG_FILE='
-      line: "CONFIG_FILE={{ openshift_master_config_file }}"
-  when: openshift_master_ha | bool and openshift_master_cluster_method == "native"
-  notify:
-  - restart master api
-
-- name: Configure master controller settings
-  lineinfile:
-    dest: /etc/sysconfig/{{ openshift.common.service_type }}-master-controllers
-    regexp: "{{ item.regex }}"
-    line: "{{ item.line }}"
-  with_items:
-    - regex: '^OPTIONS='
-      line: "OPTIONS=--loglevel={{ openshift.master.debug_level }} --listen=https://0.0.0.0:8444"
-    - regex: '^CONFIG_FILE='
-      line: "CONFIG_FILE={{ openshift_master_config_file }}"
-  when: openshift_master_ha | bool and openshift_master_cluster_method == "native"
-  notify:
-  - restart master controllers
+- name: Test local loopback context
+  command: >
+    {{ openshift.common.client_binary }} config view
+    --config={{ openshift_master_loopback_config }}
+  changed_when: false
+  register: loopback_config
+
+- command: >
+    {{ openshift.common.client_binary }} config set-cluster
+    --certificate-authority={{ openshift_master_config_dir }}/ca.crt
+    --embed-certs=true --server={{ openshift.master.loopback_api_url }}
+    {{ openshift.master.loopback_cluster_name }}
+    --config={{ openshift_master_loopback_config }}
+  when: loopback_context_string not in loopback_config.stdout
+  register: set_loopback_cluster
+
+- command: >
+    {{ openshift.common.client_binary }} config set-context
+    --cluster={{ openshift.master.loopback_cluster_name }}
+    --namespace=default --user={{ openshift.master.loopback_user }}
+    {{ openshift.master.loopback_context_name }}
+    --config={{ openshift_master_loopback_config }}
+  when: set_loopback_cluster | changed
+  register: set_loopback_context
+
+- command: >
+    {{ openshift.common.client_binary }} config use-context {{ openshift.master.loopback_context_name }}
+    --config={{ openshift_master_loopback_config }}
+  when: set_loopback_context | changed
+  register: set_current_context
 
 - name: Start and enable master
   service: name={{ openshift.common.service_type }}-master enabled=yes state=started
diff --git a/roles/openshift_master/templates/atomic-openshift-master-controllers.j2 b/roles/openshift_master/templates/atomic-openshift-master.j2
index 205934248..81bae5470 100644
--- a/roles/openshift_master/templates/atomic-openshift-master-controllers.j2
+++ b/roles/openshift_master/templates/atomic-openshift-master.j2
@@ -1,5 +1,5 @@
-OPTIONS=
-CONFIG_FILE={{ openshift_master_config_dir }}/master-config.yaml
+OPTIONS=--loglevel={{ openshift.master.debug_level }}
+CONFIG_FILE={{ openshift_master_config_file }}
 
 # Proxy configuration
 # Origin uses standard HTTP_PROXY environment variables. Be sure to set
diff --git a/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-api.j2 b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-api.j2
new file mode 120000
index 000000000..4bb7095ee
--- /dev/null
+++ b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-api.j2
@@ -0,0 +1 @@
+../native-cluster/atomic-openshift-master-api.j2
\ No newline at end of file
diff --git a/roles/openshift_master/templates/atomic-openshift-master-api.docker.service.j2 b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-api.service.j2
index 936c39edf..a935b82f6 100644
--- a/roles/openshift_master/templates/atomic-openshift-master-api.docker.service.j2
+++ b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-api.service.j2
@@ -23,4 +23,4 @@ Restart=always
 
 [Install]
 WantedBy=multi-user.target
-WantedBy={{ openshift.common.service_type }}-node.service
\ No newline at end of file
+WantedBy={{ openshift.common.service_type }}-node.service
diff --git a/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-controllers.j2 b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-controllers.j2
new file mode 120000
index 000000000..8714ebbae
--- /dev/null
+++ b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-controllers.j2
@@ -0,0 +1 @@
+../native-cluster/atomic-openshift-master-controllers.j2
\ No newline at end of file
diff --git a/roles/openshift_master/templates/atomic-openshift-master-controllers.docker.service.j2 b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-controllers.service.j2
index 6ba7d6e2a..6ba7d6e2a 100644
--- a/roles/openshift_master/templates/atomic-openshift-master-controllers.docker.service.j2
+++ b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-controllers.service.j2
diff --git a/roles/openshift_master/templates/master.docker.service.j2 b/roles/openshift_master/templates/docker/master.docker.service.j2
index 23781a313..23781a313 100644
--- a/roles/openshift_master/templates/master.docker.service.j2
+++ b/roles/openshift_master/templates/docker/master.docker.service.j2
diff --git a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2
new file mode 100644
index 000000000..c9aa15b41
--- /dev/null
+++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2
@@ -0,0 +1,9 @@
+OPTIONS=--loglevel={{ openshift.master.debug_level }} --listen={{ 'https' if openshift.master.api_use_ssl else 'http' }}://{{ openshift.master.bind_addr }}:{{ openshift.master.api_port }} --master={{ openshift.master.loopback_api_url }}:{{ openshift.master.api_port }}
+CONFIG_FILE={{ openshift_master_config_file }}
+
+# Proxy configuration
+# Origin uses standard HTTP_PROXY environment variables. Be sure to set
+# NO_PROXY for your master
+#NO_PROXY=master.example.com
+#HTTP_PROXY=http://USER:PASSWORD@IPADDR:PORT
+#HTTPS_PROXY=https://USER:PASSWORD@IPADDR:PORT
diff --git a/roles/openshift_master/templates/atomic-openshift-master-api.service.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.service.j2
index ba19fb348..ba19fb348 100644
--- a/roles/openshift_master/templates/atomic-openshift-master-api.service.j2
+++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.service.j2
diff --git a/roles/openshift_master/templates/atomic-openshift-master-api.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2
index 205934248..cdc56eece 100644
--- a/roles/openshift_master/templates/atomic-openshift-master-api.j2
+++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2
@@ -1,5 +1,5 @@
-OPTIONS=
-CONFIG_FILE={{ openshift_master_config_dir }}/master-config.yaml
+OPTIONS=--loglevel={{ openshift.master.debug_level }} --listen={{ 'https' if openshift.master.api_use_ssl else 'http' }}://{{ openshift.master.bind_addr }}:{{ openshift.master.controllers_port }}
+CONFIG_FILE={{ openshift_master_config_file }}
 
 # Proxy configuration
 # Origin uses standard HTTP_PROXY environment variables. Be sure to set
diff --git a/roles/openshift_master/templates/atomic-openshift-master-controllers.service.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.service.j2
index e6e97b24f..e6e97b24f 100644
--- a/roles/openshift_master/templates/atomic-openshift-master-controllers.service.j2
+++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.service.j2
diff --git a/roles/openshift_master/vars/main.yml b/roles/openshift_master/vars/main.yml
index 534465451..fe88c3c16 100644
--- a/roles/openshift_master/vars/main.yml
+++ b/roles/openshift_master/vars/main.yml
@@ -1,11 +1,16 @@
 ---
 openshift_master_config_dir: "{{ openshift.common.config_base }}/master"
 openshift_master_config_file: "{{ openshift_master_config_dir }}/master-config.yaml"
+openshift_master_loopback_config: "{{ openshift_master_config_dir }}/openshift-master.kubeconfig"
+loopback_context_string: "current-context: {{ openshift.master.loopback_context_name }}"
 openshift_master_scheduler_conf: "{{ openshift_master_config_dir }}/scheduler.json"
 openshift_master_session_secrets_file: "{{ openshift_master_config_dir }}/session-secrets.yaml"
 openshift_master_policy: "{{ openshift_master_config_dir }}/policy.json"
 openshift_version: "{{ openshift_pkg_version | default('') }}"
 
+ha_svc_template_path: "{{ 'docker-cluster' if openshift.common.is_containerized | bool else 'native-cluster' }}"
+ha_svc_svc_dir: "{{ '/etc/systemd/system' if openshift.common.is_containerized | bool else '/usr/lib/systemd/system' }}"
+
 openshift_master_valid_grant_methods:
 - auto
 - prompt
diff --git a/roles/openshift_master_cluster/tasks/configure.yml b/roles/openshift_master_cluster/tasks/configure.yml
index 7ab9afb51..1b94598dd 100644
--- a/roles/openshift_master_cluster/tasks/configure.yml
+++ b/roles/openshift_master_cluster/tasks/configure.yml
@@ -34,11 +34,10 @@
 - name: Disable stonith
   command: pcs property set stonith-enabled=false
 
-# TODO: handle case where api port is not 8443
 - name: Wait for the clustered master service to be available
   wait_for:
     host: "{{ openshift_master_cluster_vip }}"
-    port: 8443
+    port: "{{ openshift.master.api_port }}"
     state: started
     timeout: 180
     delay: 90