Add support for external glusterfs as registry backend

14 files changed, 128 insertions, 48 deletions

diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py
index 99ebb7e36..0cb5f40c4 100755
--- a/roles/openshift_facts/library/openshift_facts.py
+++ b/roles/openshift_facts/library/openshift_facts.py
@@ -1972,6 +1972,7 @@ class OpenShiftFacts(object):
                         glusterfs=dict(
                             endpoints='glusterfs-registry-endpoints',
                             path='glusterfs-registry-volume',
+                            ips=[],
                             readOnly=False,
                             swap=False,
                             swapcopy=True),
diff --git a/roles/openshift_hosted/README.md b/roles/openshift_hosted/README.md
index d6f6e3e09..a1c2c3956 100644
--- a/roles/openshift_hosted/README.md
+++ b/roles/openshift_hosted/README.md
@@ -34,13 +34,27 @@ variables also control configuration behavior:
 
 | Name                                         | Default value | Description                                                                  |
 |----------------------------------------------|---------------|------------------------------------------------------------------------------|
-| openshift_hosted_registry_glusterfs_swap     | False         | Whether to swap an existing registry's storage volume for a GlusterFS volume |
-| openshift_hosted_registry_glusterfs_swapcopy | True          | If swapping, also copy the current contents of the registry volume           |
+| openshift_hosted_registry_storage_glusterfs_endpoints | glusterfs-registry-endpoints | The name for the Endpoints resource that will point the registry to the GlusterFS nodes
+| openshift_hosted_registry_storage_glusterfs_path      | glusterfs-registry-volume    | The name for the GlusterFS volume that will provide registry storage
+| openshift_hosted_registry_storage_glusterfs_readonly  | False                        | Whether the GlusterFS volume should be read-only
+| openshift_hosted_registry_storage_glusterfs_swap      | False                        | Whether to swap an existing registry's storage volume for a GlusterFS volume
+| openshift_hosted_registry_storage_glusterfs_swapcopy  | True                         | If swapping, copy the contents of the pre-existing registry storage to the new GlusterFS volume
+| openshift_hosted_registry_storage_glusterfs_ips       | `[]`                         | A list of IP addresses of the nodes of the GlusterFS cluster to use for hosted registry storage
+
+**NOTE:** Configuring a value for
+`openshift_hosted_registry_storage_glusterfs_ips` with a `glusterfs_registry`
+host group is not allowed. Specifying a `glusterfs_registry` host group 
+indicates that a new GlusterFS cluster should be configured, whereas 
+specifying `openshift_hosted_registry_storage_glusterfs_ips` indicates wanting 
+to use a pre-configured GlusterFS cluster for the registry storage.
+
+_
 
 Dependencies
 ------------
 
 * openshift_hosted_facts
+* openshift_persistent_volumes
 
 Example Playbook
 ----------------
@@ -56,6 +70,10 @@ Example Playbook
       cafile: /path/to/my-router-ca.crt
     openshift_hosted_router_registryurl: 'registry.access.redhat.com/openshift3/ose-haproxy-router:v3.0.2.0'
     openshift_hosted_router_selector: 'type=infra'
+    openshift_hosted_registry_storage_kind=glusterfs
+    openshift_hosted_registry_storage_glusterfs_path=external_glusterfs_volume_name
+    openshift_hosted_registry_storage_glusterfs_ips=['192.168.20.239','192.168.20.96','192.168.20.114']
+
 ```
 
 License
diff --git a/roles/openshift_hosted/defaults/main.yml b/roles/openshift_hosted/defaults/main.yml
index 2af42fba4..e70c0c420 100644
--- a/roles/openshift_hosted/defaults/main.yml
+++ b/roles/openshift_hosted/defaults/main.yml
@@ -70,7 +70,6 @@ r_openshift_hosted_registry_use_firewalld: "{{ os_firewall_use_firewalld | defau
 openshift_hosted_registry_name: docker-registry
 openshift_hosted_registry_wait: "{{ not (openshift_master_bootstrap_enabled | default(False)) }}"
 openshift_hosted_registry_cert_expire_days: 730
-
 r_openshift_hosted_registry_os_firewall_deny: []
 r_openshift_hosted_registry_os_firewall_allow:
 - service: Docker Registry Port
diff --git a/roles/openshift_hosted/tasks/registry.yml b/roles/openshift_hosted/tasks/registry.yml
index eaaac9da2..9f2ef4e40 100644
--- a/roles/openshift_hosted/tasks/registry.yml
+++ b/roles/openshift_hosted/tasks/registry.yml
@@ -1,4 +1,10 @@
 ---
+- name: Create temp directory for doing work in
+  command: mktemp -d /tmp/openshift-hosted-ansible-XXXXXX
+  register: mktempHosted
+  changed_when: False
+  check_mode: no
+
 - name: setup firewall
   include: firewall.yml
   vars:
@@ -36,13 +42,13 @@
       l_default_replicas: "{{ l_node_count if openshift.hosted.registry.storage.kind | default(none) is not none else 1 }}"
     when: l_node_count | int > 0
 
-
 - name: set openshift_hosted facts
   set_fact:
     openshift_hosted_registry_replicas: "{{ openshift.hosted.registry.replicas | default(l_default_replicas) }}"
     openshift_hosted_registry_namespace: "{{ openshift.hosted.registry.namespace | default('default') }}"
     openshift_hosted_registry_selector: "{{ openshift.hosted.registry.selector }}"
     openshift_hosted_registry_images: "{{ openshift.hosted.registry.registryurl | default('openshift3/ose-${component}:${version}')}}"
+    openshift_hosted_registry_storage_glusterfs_ips: "{%- set gluster_ips = [] %}{% if groups.glusterfs_registry is defined %}{% for node in groups.glusterfs_registry %}{%- set _ = gluster_ips.append(hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip)) %}{% endfor %}{{ gluster_ips }}{% elif groups.glusterfs is defined %}{% for node in groups.glusterfs %}{%- set _ = gluster_ips.append(hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip)) %}{% endfor %}{{ gluster_ips }}{% else %}{{ openshift.hosted.registry.storage.glusterfs.ips }}{% endif %}"
 
 - name: Update registry environment variables when pushing via dns
   set_fact:
@@ -113,6 +119,11 @@
   when:
   - openshift.hosted.registry.storage.kind | default(none) in ['nfs', 'openstack', 'glusterfs']
 
+- include: storage/glusterfs_endpoints.yml
+  when:
+  - openshift_hosted_registry_storage_glusterfs_ips|length > 0
+  - openshift.hosted.registry.storage.kind | default(none) in ['glusterfs']
+
 - name: Create OpenShift registry
   oc_adm_registry:
     name: "{{ openshift_hosted_registry_name }}"
@@ -141,3 +152,10 @@
 - include: storage/glusterfs.yml
   when:
   - openshift.hosted.registry.storage.kind | default(none) == 'glusterfs' or openshift.hosted.registry.storage.glusterfs.swap
+
+- name: Delete temp directory
+  file:
+    name: "{{ mktempHosted.stdout }}"
+    state: absent
+  changed_when: False
+  check_mode: no
diff --git a/roles/openshift_hosted/tasks/storage/glusterfs.yml b/roles/openshift_hosted/tasks/storage/glusterfs.yml
index c2954fde1..06c22a5a2 100644
--- a/roles/openshift_hosted/tasks/storage/glusterfs.yml
+++ b/roles/openshift_hosted/tasks/storage/glusterfs.yml
@@ -35,7 +35,7 @@
   mount:
     state: mounted
     fstype: glusterfs
-    src: "{% if 'glusterfs_registry' in groups %}{% set node = groups.glusterfs_registry[0] %}{% else %}{% set node = groups.glusterfs[0] %}{% endif %}{% if 'glusterfs_hostname' in hostvars[node] %}{{ hostvars[node].glusterfs_hostname }}{% elif 'openshift' in hostvars[node] %}{{ hostvars[node].openshift.node.nodename }}{% else %}{{ node }}{% endif %}:/{{ openshift.hosted.registry.storage.glusterfs.path }}"
+    src: "{% if 'glusterfs_registry' in groups %}{% set node = groups.glusterfs_registry[0] %}{% elif 'glusterfs' in groups %}{% set node = groups.glusterfs[0] %}{% endif %}{% if openshift_hosted_registry_storage_glusterfs_ips is defined and openshift_hosted_registry_storage_glusterfs_ips|length > 0 %}{{ openshift_hosted_registry_storage_glusterfs_ips[0] }}{% elif 'glusterfs_hostname' in hostvars[node] %}{{ hostvars[node].glusterfs_hostname }}{% elif 'openshift' in hostvars[node] %}{{ hostvars[node].openshift.node.nodename }}{% else %}{{ node }}{% endif %}:/{{ openshift.hosted.registry.storage.glusterfs.path }}"
     name: "{{ mktemp.stdout }}"
 
 - name: Set registry volume permissions
diff --git a/roles/openshift_hosted/tasks/storage/glusterfs_endpoints.yml b/roles/openshift_hosted/tasks/storage/glusterfs_endpoints.yml
new file mode 100644
index 000000000..0f4381748
--- /dev/null
+++ b/roles/openshift_hosted/tasks/storage/glusterfs_endpoints.yml
@@ -0,0 +1,16 @@
+---
+- name: Generate GlusterFS registry endpoints
+  template:
+    src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-endpoints.yml.j2"
+    dest: "{{ mktempHosted.stdout }}/glusterfs-registry-endpoints.yml"
+
+- name: Generate GlusterFS registry service
+  template:
+    src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-service.yml.j2"
+    dest: "{{ mktempHosted.stdout }}/glusterfs-registry-service.yml"
+
+- name: Create GlusterFS registry service and endpoint
+  command: "{{ openshift.common.client_binary }} apply -f {{ item }} -n {{ openshift.hosted.registry.namespace | default('default') }}"
+  with_items:
+  - "{{ mktempHosted.stdout }}/glusterfs-registry-service.yml"
+  - "{{ mktempHosted.stdout }}/glusterfs-registry-endpoints.yml"
diff --git a/roles/openshift_hosted/templates/v3.6/glusterfs-registry-endpoints.yml.j2 b/roles/openshift_hosted/templates/v3.6/glusterfs-registry-endpoints.yml.j2
new file mode 100644
index 000000000..607d25533
--- /dev/null
+++ b/roles/openshift_hosted/templates/v3.6/glusterfs-registry-endpoints.yml.j2
@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: {{ openshift.hosted.registry.storage.glusterfs.endpoints }}
+subsets:
+- addresses:
+{% for ip in openshift_hosted_registry_storage_glusterfs_ips %}
+  - ip: {{ ip }}
+{% endfor %}
+  ports:
+  - port: 1
diff --git a/roles/openshift_hosted/templates/v3.6/glusterfs-registry-service.yml.j2 b/roles/openshift_hosted/templates/v3.6/glusterfs-registry-service.yml.j2
new file mode 100644
index 000000000..452c7c3e1
--- /dev/null
+++ b/roles/openshift_hosted/templates/v3.6/glusterfs-registry-service.yml.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ openshift.hosted.registry.storage.glusterfs.endpoints }}
+spec:
+  ports:
+  - port: 1
+status:
+  loadBalancer: {}
diff --git a/roles/openshift_hosted/templates/v3.7/glusterfs-registry-endpoints.yml.j2 b/roles/openshift_hosted/templates/v3.7/glusterfs-registry-endpoints.yml.j2
new file mode 100644
index 000000000..607d25533
--- /dev/null
+++ b/roles/openshift_hosted/templates/v3.7/glusterfs-registry-endpoints.yml.j2
@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: {{ openshift.hosted.registry.storage.glusterfs.endpoints }}
+subsets:
+- addresses:
+{% for ip in openshift_hosted_registry_storage_glusterfs_ips %}
+  - ip: {{ ip }}
+{% endfor %}
+  ports:
+  - port: 1
diff --git a/roles/openshift_hosted/templates/v3.7/glusterfs-registry-service.yml.j2 b/roles/openshift_hosted/templates/v3.7/glusterfs-registry-service.yml.j2
new file mode 100644
index 000000000..452c7c3e1
--- /dev/null
+++ b/roles/openshift_hosted/templates/v3.7/glusterfs-registry-service.yml.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ openshift.hosted.registry.storage.glusterfs.endpoints }}
+spec:
+  ports:
+  - port: 1
+status:
+  loadBalancer: {}
diff --git a/roles/openshift_sanitize_inventory/tasks/unsupported.yml b/roles/openshift_sanitize_inventory/tasks/unsupported.yml
index b70ab90a1..1c4984467 100644
--- a/roles/openshift_sanitize_inventory/tasks/unsupported.yml
+++ b/roles/openshift_sanitize_inventory/tasks/unsupported.yml
@@ -40,3 +40,27 @@
       openshift_master_dynamic_provisioning_enabled to True and set an
       openshift_cloudprovider_kind. You can disable this check with
       'dynamic_volumes_check=False'.
+
+#if we have registry backend as glusterfs, and we have clashing configuration.
+- name: Ensure the hosted registry's GlusterFS storage is configured correctly
+  when:
+  - openshift_hosted_registry_storage_kind | default(none) in ['glusterfs']
+  - openshift_hosted_registry_storage_glusterfs_ips is defined and openshift_hosted_registry_storage_glusterfs_ips != ''
+  - "'glusterfs_registry' in groups | default([])"
+  fail:
+    msg: |-
+      Configuring a value for openshift_hosted_registry_storage_glusterfs_ips and with a glusterfs_registry host group is not allowed.
+      Specifying a glusterfs_registry host group indicates that a new GlusterFS cluster should be configured, whereas
+      specifying openshift_hosted_registry_storage_glusterfs_ips indicates wanting to use a pre-configured GlusterFS cluster for the registry storage.
+
+#if we have registry backend as glusterfs and no gluster specified.
+- name: Ensure the hosted registry's GlusterFS storage is configured correctly
+  when:
+  - openshift_hosted_registry_storage_kind | default(none) in ['glusterfs']
+  - not openshift_hosted_registry_storage_glusterfs_ips is defined
+  - not 'glusterfs_registry' in groups | default([])
+  - not 'glusterfs' in groups | default([])
+  fail:
+    msg: |-
+      Configuring a value for openshift_hosted_registry_storage_kind=glusterfs without a any glusterfs option is not allowed.
+      Specify either openshift_hosted_registry_storage_glusterfs_ips variable or glusterfs, glusterfs_registry host groups.
diff --git a/roles/openshift_storage_glusterfs/README.md b/roles/openshift_storage_glusterfs/README.md
index 54adcf78d..be749a2e1 100644
--- a/roles/openshift_storage_glusterfs/README.md
+++ b/roles/openshift_storage_glusterfs/README.md
@@ -133,8 +133,7 @@ are an exception:
 | openshift_storage_glusterfs_registry_heketi_admin_key | auto-generated        | Separate from the above
 | openshift_storage_glusterfs_registry_heketi_user_key  | auto-generated        | Separate from the above
 
-Additionally, this role's behavior responds to the following registry-specific
-variables:
+Additionally, this role's behavior responds to several registry-specific variables in the [openshift_hosted role](../openshift_hosted/README.md):
 
 | Name                                                  | Default value                | Description                             |
 |-------------------------------------------------------|------------------------------|-----------------------------------------|
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml
index 1ede0ae94..b8fd7979f 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml
@@ -44,6 +44,6 @@
     glusterfs_heketi_ssh_sudo: "{{ openshift_storage_glusterfs_heketi_ssh_sudo | bool }}"
     glusterfs_heketi_ssh_keyfile: "{{ openshift_storage_glusterfs_heketi_ssh_keyfile }}"
     glusterfs_heketi_fstab: "{{ openshift_storage_glusterfs_heketi_fstab }}"
-    glusterfs_nodes: "{{ groups.glusterfs }}"
+    glusterfs_nodes: "{{ groups.glusterfs | default([]]) }}"
 
 - include: glusterfs_common.yml
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
index 1fa42efa7..d3cba61cf 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
@@ -44,52 +44,13 @@
     glusterfs_heketi_ssh_sudo: "{{ openshift_storage_glusterfs_registry_heketi_ssh_sudo | bool }}"
     glusterfs_heketi_ssh_keyfile: "{{ openshift_storage_glusterfs_registry_heketi_ssh_keyfile }}"
     glusterfs_heketi_fstab: "{{ openshift_storage_glusterfs_registry_heketi_fstab }}"
-    glusterfs_nodes: "{{ groups.glusterfs_registry | default(groups.glusterfs) }}"
+    glusterfs_nodes: "{% if groups.glusterfs_registry is defined %}{% set nodes = groups.glusterfs_registry %}{% elif 'groups.glusterfs' is defined %}{% set nodes = groups.glusterfs %}{% else %}{% set nodes = '[]' %}{% endif %}{{ nodes }}"
 
 - include: glusterfs_common.yml
   when:
   - glusterfs_nodes | default([]) | count > 0
   - "'glusterfs' not in groups or glusterfs_nodes != groups.glusterfs"
 
-- name: Delete pre-existing GlusterFS registry resources
-  oc_obj:
-    namespace: "{{ glusterfs_namespace }}"
-    kind: "{{ item.kind }}"
-    name: "{{ item.name }}"
-    state: absent
-  with_items:
-  - kind: "svc"
-    name: "glusterfs-{{ glusterfs_name | default }}-endpoints"
-  failed_when: False
-
-- name: Generate GlusterFS registry endpoints
-  template:
-    src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-endpoints.yml.j2"
-    dest: "{{ mktemp.stdout }}/glusterfs-registry-endpoints.yml"
-
-- name: Copy GlusterFS registry service
-  template:
-    src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-service.yml.j2"
-    dest: "{{ mktemp.stdout }}/glusterfs-registry-service.yml"
-
-- name: Create GlusterFS registry endpoints
-  oc_obj:
-    namespace: "{{ openshift.hosted.registry.namespace | default('default') }}"
-    state: present
-    kind: endpoints
-    name: "glusterfs-{{ glusterfs_name }}-endpoints"
-    files:
-    - "{{ mktemp.stdout }}/glusterfs-registry-endpoints.yml"
-
-- name: Create GlusterFS registry service
-  oc_obj:
-    namespace: "{{ openshift.hosted.registry.namespace | default('default') }}"
-    state: present
-    kind: service
-    name: "glusterfs-{{ glusterfs_name }}-endpoints"
-    files:
-    - "{{ mktemp.stdout }}/glusterfs-registry-service.yml"
-
 - name: Check if GlusterFS registry volume exists
   command: "{{ glusterfs_heketi_client }} volume list"
   register: registry_volume