Merge pull request #1173 from abutcher/nfs-group

Refactor storage options

15 files changed, 267 insertions, 63 deletions

diff --git a/roles/openshift_common/tasks/main.yml b/roles/openshift_common/tasks/main.yml
index ff8c3b50f..98b0b11ea 100644
--- a/roles/openshift_common/tasks/main.yml
+++ b/roles/openshift_common/tasks/main.yml
@@ -39,11 +39,15 @@
   action: "{{ ansible_pkg_mgr }} name={{ openshift.common.service_type }}{{ openshift_version | default('') }} state=present"
   when: not openshift.common.is_containerized | bool
 
-- name: Set version facts
+# This invocation also updates the version facts which are necessary
+# for setting the hostname below.
+- name: openshift_facts
   openshift_facts:
+    role: hosted
+    openshift_env: "{{ hostvars[inventory_hostname] | oo_openshift_env }}"
 
-  # For enterprise versions < 3.1 and origin versions < 1.1 we want to set the
-  # hostname by default.
+# For enterprise versions < 3.1 and origin versions < 1.1 we want to set the
+# hostname by default.
 - set_fact:
     set_hostname_default: "{{ not openshift.common.version_greater_than_3_1_or_1_1 }}"
 
diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py
index 40e54d706..85c8abdf0 100755
--- a/roles/openshift_facts/library/openshift_facts.py
+++ b/roles/openshift_facts/library/openshift_facts.py
@@ -957,12 +957,12 @@ def merge_facts(orig, new, additive_facts_to_overwrite):
                 # Fact is additive so we'll combine orig and new.
                 if isinstance(value, list) and isinstance(new[key], list):
                     new_fact = []
-                    for item in copy.deepcopy(value) + copy.copy(new[key]):
+                    for item in copy.deepcopy(value) + copy.deepcopy(new[key]):
                         if item not in new_fact:
                             new_fact.append(item)
                     facts[key] = new_fact
             else:
-                facts[key] = copy.copy(new[key])
+                facts[key] = copy.deepcopy(new[key])
         else:
             facts[key] = copy.deepcopy(value)
     new_keys = set(new.keys()) - set(orig.keys())
@@ -1108,9 +1108,11 @@ class OpenShiftFacts(object):
         Raises:
             OpenShiftFactsUnsupportedRoleError:
     """
-    known_roles = ['common', 'master', 'node', 'etcd', 'nfs']
+    known_roles = ['common', 'master', 'node', 'etcd', 'hosted']
 
-    def __init__(self, role, filename, local_facts, additive_facts_to_overwrite=False):
+    # Disabling too-many-arguments, this should be cleaned up as a TODO item.
+    # pylint: disable=too-many-arguments
+    def __init__(self, role, filename, local_facts, additive_facts_to_overwrite=False, openshift_env=None):
         self.changed = False
         self.filename = filename
         if role not in self.known_roles:
@@ -1119,9 +1121,9 @@ class OpenShiftFacts(object):
             )
         self.role = role
         self.system_facts = ansible_facts(module)
-        self.facts = self.generate_facts(local_facts, additive_facts_to_overwrite)
+        self.facts = self.generate_facts(local_facts, additive_facts_to_overwrite, openshift_env)
 
-    def generate_facts(self, local_facts, additive_facts_to_overwrite):
+    def generate_facts(self, local_facts, additive_facts_to_overwrite, openshift_env):
         """ Generate facts
 
             Args:
@@ -1133,7 +1135,7 @@ class OpenShiftFacts(object):
             Returns:
                 dict: The generated facts
         """
-        local_facts = self.init_local_facts(local_facts, additive_facts_to_overwrite)
+        local_facts = self.init_local_facts(local_facts, additive_facts_to_overwrite, openshift_env)
         roles = local_facts.keys()
 
         defaults = self.get_defaults(roles)
@@ -1205,10 +1207,23 @@ class OpenShiftFacts(object):
                         iptables_sync_period='5s', set_node_ip=False)
             defaults['node'] = node
 
-        if 'nfs' in roles:
-            nfs = dict(exports_dir='/var/export', registry_volume='regvol',
-                       export_options='*(rw,sync,all_squash)')
-            defaults['nfs'] = nfs
+        defaults['hosted'] = dict(
+            registry=dict(
+                storage=dict(
+                    kind=None,
+                    volume=dict(
+                        name='registry',
+                        size='5Gi'
+                    ),
+                    nfs=dict(
+                        directory='/exports',
+                        options='*(rw,root_squash)'),
+                    host=None,
+                    access_modes=['ReadWriteMany'],
+                    create_pv=True
+                )
+            )
+        )
 
         return defaults
 
@@ -1287,7 +1302,9 @@ class OpenShiftFacts(object):
         )
         return provider_facts
 
-    def init_local_facts(self, facts=None, additive_facts_to_overwrite=False):
+    # Disabling too-many-branches. This should be cleaned up as a TODO item.
+    #pylint: disable=too-many-branches
+    def init_local_facts(self, facts=None, additive_facts_to_overwrite=False, openshift_env=None):
         """ Initialize the provider facts
 
             Args:
@@ -1300,10 +1317,27 @@ class OpenShiftFacts(object):
                       local facts
         """
         changed = False
-        facts_to_set = {self.role: dict()}
+
+        facts_to_set = dict()
+
         if facts is not None:
             facts_to_set[self.role] = facts
 
+        if openshift_env != {} and openshift_env != None:
+            for fact, value in openshift_env.iteritems():
+                oo_env_facts = dict()
+                current_level = oo_env_facts
+                keys = fact.split('_')[1:]
+                if keys[0] != self.role:
+                    continue
+                for key in keys:
+                    if key == keys[-1]:
+                        current_level[key] = value
+                    elif key not in current_level:
+                        current_level[key] = dict()
+                        current_level = current_level[key]
+                facts_to_set = merge_facts(facts_to_set, oo_env_facts, [])
+
         local_facts = get_local_facts_from_file(self.filename)
 
         for arg in ['labels', 'annotations']:
@@ -1314,11 +1348,12 @@ class OpenShiftFacts(object):
         new_local_facts = merge_facts(local_facts, facts_to_set, additive_facts_to_overwrite)
         for facts in new_local_facts.values():
             keys_to_delete = []
-            for fact, value in facts.iteritems():
-                if value == "" or value is None:
-                    keys_to_delete.append(fact)
-            for key in keys_to_delete:
-                del facts[key]
+            if isinstance(facts, dict):
+                for fact, value in facts.iteritems():
+                    if value == "" or value is None:
+                        keys_to_delete.append(fact)
+                for key in keys_to_delete:
+                    del facts[key]
 
         if new_local_facts != local_facts:
             self.validate_local_facts(new_local_facts)
@@ -1406,6 +1441,7 @@ def main():
                       choices=OpenShiftFacts.known_roles),
             local_facts=dict(default=None, type='dict', required=False),
             additive_facts_to_overwrite=dict(default=[], type='list', required=False),
+            openshift_env=dict(default={}, type='dict', required=False)
         ),
         supports_check_mode=True,
         add_file_common_args=True,
@@ -1414,9 +1450,15 @@ def main():
     role = module.params['role']
     local_facts = module.params['local_facts']
     additive_facts_to_overwrite = module.params['additive_facts_to_overwrite']
+    openshift_env = module.params['openshift_env']
+
     fact_file = '/etc/ansible/facts.d/openshift.fact'
 
-    openshift_facts = OpenShiftFacts(role, fact_file, local_facts, additive_facts_to_overwrite)
+    openshift_facts = OpenShiftFacts(role,
+                                     fact_file,
+                                     local_facts,
+                                     additive_facts_to_overwrite,
+                                     openshift_env)
 
     file_params = module.params.copy()
     file_params['path'] = fact_file
diff --git a/roles/openshift_persistent_volumes/README.md b/roles/openshift_persistent_volumes/README.md
new file mode 100644
index 000000000..34ae89536
--- /dev/null
+++ b/roles/openshift_persistent_volumes/README.md
@@ -0,0 +1,60 @@
+OpenShift NFS Server
+====================
+
+OpenShift Persistent Volumes
+
+Requirements
+------------
+
+Role Variables
+--------------
+
+From this role:
+| Name                     | Default value |                                                                                     |
+|--------------------------|---------------|-------------------------------------------------------------------------------------|
+| persistent_volumes       | []            | List of persistent volume dictionaries, keys: name, capacity, access_modes, storage |
+| persistent_volume_claims | []            | List of persistent volume claim dictionaries, keys: name, capacity, access_modes    |
+
+
+From openshift_common:
+| Name                          | Default Value  |                                        |
+|-------------------------------|----------------|----------------------------------------|
+| openshift_debug_level         | 2              | Global openshift debug log verbosity   |
+
+
+Dependencies
+------------
+
+
+Example Playbook
+----------------
+
+- name: Create persistent volumes/claims
+  hosts: oo_first_master
+  vars:
+    persistent_volumes:
+    - name: "registry-volume"
+      capacity: "5Gi"
+      access_modes:
+      - "ReadWriteMany"
+      storage:
+        nfs:
+          server: "nfs.example.com"
+          path: "/var/exports/registry"
+    persistent_volume_claims:
+    - name: "registry-claim"
+      capacity: "5Gi"
+      access_modes:
+      - "ReadWriteMany"
+  roles:
+  - role: openshift_persistent_volumes
+
+License
+-------
+
+Apache License, Version 2.0
+
+Author Information
+------------------
+
+Andrew Butcher (abutcher@redhat.com)
diff --git a/roles/openshift_persistent_volumes/meta/main.yml b/roles/openshift_persistent_volumes/meta/main.yml
new file mode 100644
index 000000000..d9f6fc01a
--- /dev/null
+++ b/roles/openshift_persistent_volumes/meta/main.yml
@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Andrew Butcher
+  description: OpenShift Persistent Volumes
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.9
+  platforms:
+  - name: EL
+    versions:
+    - 7
+dependencies:
+- { role: openshift_common }
diff --git a/roles/openshift_persistent_volumes/tasks/main.yml b/roles/openshift_persistent_volumes/tasks/main.yml
new file mode 100644
index 000000000..2455fc792
--- /dev/null
+++ b/roles/openshift_persistent_volumes/tasks/main.yml
@@ -0,0 +1,50 @@
+---
+- name: Create temp directory for volume definitions
+  command: mktemp -d /tmp/openshift-ansible-XXXXXXX
+  register: mktemp
+  changed_when: False
+
+- name: Copy the admin client config(s)
+  command: >
+    cp {{ openshift_master_config_dir }}/admin.kubeconfig {{ mktemp.stdout }}/admin.kubeconfig
+  changed_when: False
+
+- name: Deploy PersistentVolume definitions
+  template:
+    dest: "{{ mktemp.stdout }}/persistent-volumes.yml"
+    src: persistent-volume.yml.j2
+  when: persistent_volumes | length > 0
+  changed_when: False
+
+- name: Create PersistentVolumes
+  command: >
+    {{ openshift.common.client_binary }} create
+    -f {{ mktemp.stdout }}/persistent-volumes.yml
+    --config={{ mktemp.stdout }}/admin.kubeconfig
+  register: pv_create_output
+  when: persistent_volumes | length > 0
+  failed_when: ('already exists' not in pv_create_output.stderr if pv_create_output.stderr else False) or ('created' not in pv_create_output.stdout if pv_create_output.stdout else False)
+  changed_when: ('created' in pv_create_output.stdout)
+
+- name: Deploy PersistentVolumeClaim definitions
+  template:
+    dest: "{{ mktemp.stdout }}/persistent-volume-claims.yml"
+    src: persistent-volume-claim.yml.j2
+  when: persistent_volume_claims | length > 0
+  changed_when: False
+
+- name: Create PersistentVolumeClaims
+  command: >
+    {{ openshift.common.client_binary }} create
+    -f {{ mktemp.stdout }}/persistent-volume-claims.yml
+    --config={{ mktemp.stdout }}/admin.kubeconfig
+  register: pvc_create_output
+  when: persistent_volume_claims | length > 0
+  failed_when: ('already exists' not in pvc_create_output.stderr if pvc_create_output.stderr else False) or ('created' not in pvc_create_output.stdout if pvc_create_output.stdout else False)
+  changed_when: ('created' in pvc_create_output.stdout)
+
+- name: Delete temp directory
+  file:
+    name: "{{ mktemp.stdout }}"
+    state: absent
+  changed_when: False
diff --git a/roles/openshift_persistent_volumes/templates/persistent-volume-claim.yml.j2 b/roles/openshift_persistent_volumes/templates/persistent-volume-claim.yml.j2
new file mode 100644
index 000000000..58b3e1c67
--- /dev/null
+++ b/roles/openshift_persistent_volumes/templates/persistent-volume-claim.yml.j2
@@ -0,0 +1,14 @@
+---
+apiVersion: "v1"
+kind: "List"
+items:
+{% for claim in persistent_volume_claims %}
+- kind: "PersistentVolumeClaim"
+  metadata:
+    name: "{{ claim.name }}"
+  spec:
+    accessModes: {{ claim.access_modes | to_padded_yaml(2, 2) }}
+    resources:
+      requests:
+        storage: "{{ claim.capacity }}"
+{% endfor %}
diff --git a/roles/openshift_persistent_volumes/templates/persistent-volume.yml.j2 b/roles/openshift_persistent_volumes/templates/persistent-volume.yml.j2
new file mode 100644
index 000000000..5714b6b0d
--- /dev/null
+++ b/roles/openshift_persistent_volumes/templates/persistent-volume.yml.j2
@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: List
+items:
+{% for volume in persistent_volumes %}
+- kind: PersistentVolume
+  metadata:
+    name: "{{ volume.name }}"
+  spec:
+    capacity:
+      storage: "{{ volume.capacity }}"
+    accessModes: {{ volume.access_modes | to_padded_yaml(2, 2) }}
+    {{ volume.storage.keys()[0] }}: {{ volume.storage[volume.storage.keys()[0]] | to_padded_yaml(3, 2) }}
+{% endfor %}
diff --git a/roles/openshift_persistent_volumes/vars/main.yml b/roles/openshift_persistent_volumes/vars/main.yml
new file mode 100644
index 000000000..9967e26f4
--- /dev/null
+++ b/roles/openshift_persistent_volumes/vars/main.yml
@@ -0,0 +1,2 @@
+---
+openshift_master_config_dir: "{{ openshift.common.config_base }}/master"
diff --git a/roles/openshift_registry/defaults/main.yml b/roles/openshift_registry/defaults/main.yml
new file mode 100644
index 000000000..17a0d5301
--- /dev/null
+++ b/roles/openshift_registry/defaults/main.yml
@@ -0,0 +1,2 @@
+---
+registry_volume_claim: 'registry-claim'
diff --git a/roles/openshift_registry/tasks/main.yml b/roles/openshift_registry/tasks/main.yml
index 2804e8f2e..1eeec2fbb 100644
--- a/roles/openshift_registry/tasks/main.yml
+++ b/roles/openshift_registry/tasks/main.yml
@@ -1,28 +1,24 @@
 ---
-- set_fact: _oreg_images="--images='{{ openshift.master.registry_url }}'"
-
-- set_fact: _oreg_selector="--selector='{{ openshift.master.registry_selector }}'"
-
 - name: Deploy OpenShift Registry
   command: >
     {{ openshift.common.admin_binary }} registry
-    --create --service-account=registry {{ _oreg_selector }}
-    --credentials={{ openshift_master_config_dir }}/openshift-registry.kubeconfig {{ _oreg_images }}
-  register: _oreg_results
-  changed_when: "'service exists' not in _oreg_results.stdout"
+    --create --replicas={{ openshift.master.infra_nodes | length }}
+    --service-account=registry {{ oreg_selector }}
+    --credentials={{ openshift_master_config_dir }}/openshift-registry.kubeconfig {{ oreg_images }}
+  register: oreg_results
+  changed_when: "'service exists' not in oreg_results.stdout"
 
-- name: Determine if nfs volume is already attached
+- name: Determine if volume is already attached to dc/docker-registry
   command: "{{ openshift.common.client_binary }} get -o template dc/docker-registry --template=\\{\\{.spec.template.spec.volumes\\}\\}"
+  changed_when: false
   register: registry_volumes_output
-  when: attach_registry_volume | bool
 
 - set_fact:
-    volume_already_attached: "{{ 'server:' + nfs_host in registry_volumes_output.stdout and 'path:' + registry_volume_path in registry_volumes_output.stdout }}"
-  when: attach_registry_volume | bool
+    volume_attached: "{{ registry_volume_claim in registry_volumes_output.stdout }}"
 
-- name: Add nfs volume to dc/docker-registry
+- name: Add volume to dc/docker-registry
   command: >
     {{ openshift.common.client_binary }} volume dc/docker-registry
-    --add --overwrite --name=registry-storage --mount-path=/registry
-    --source='{"nfs": {"server": "{{ nfs_host }}", "path": "{{ registry_volume_path }}"}}'
-  when: attach_registry_volume | bool and not volume_already_attached | bool
+    --add --overwrite -t persistentVolumeClaim --claim-name={{ registry_volume_claim }}
+    --name=registry-storage
+  when: not volume_attached | bool
diff --git a/roles/openshift_registry/vars/main.yml b/roles/openshift_registry/vars/main.yml
index 9967e26f4..306350a5a 100644
--- a/roles/openshift_registry/vars/main.yml
+++ b/roles/openshift_registry/vars/main.yml
@@ -1,2 +1,4 @@
 ---
 openshift_master_config_dir: "{{ openshift.common.config_base }}/master"
+oreg_images: "--images='{{ openshift.master.registry_url }}'"
+oreg_selector: "--selector='{{ openshift.master.registry_selector }}'"
diff --git a/roles/openshift_storage_nfs/README.md b/roles/openshift_storage_nfs/README.md
index 548e146cb..dd988b849 100644
--- a/roles/openshift_storage_nfs/README.md
+++ b/roles/openshift_storage_nfs/README.md
@@ -15,11 +15,11 @@ Role Variables
 --------------
 
 From this role:
-| Name                          | Default value         |                                                  |
-|-------------------------------|-----------------------|--------------------------------------------------|
-| openshift_nfs_exports_dir     | /var/export           | Root export directory.                           |
-| openshift_nfs_registry_volume | regvol                | Registry volume within openshift_nfs_exports_dir |
-| openshift_nfs_export_options  | *(rw,sync,all_squash) | NFS options for configured exports.              |
+| Name                                            | Default value         |                                                             |
+|-------------------------------------------------|-----------------------|-------------------------------------------------------------|
+| openshift_hosted_registry_storage_nfs_directory | /exports              | Root export directory.                                      |
+| openshift_hosted_registry_storage_volume_name   | registry              | Registry volume within openshift_hosted_registry_volume_dir |
+| openshift_hosted_registry_storage_nfs_options   | *(rw,root_squash)     | NFS options for configured exports.                         |
 
 
 From openshift_common:
@@ -31,8 +31,6 @@ From openshift_common:
 Dependencies
 ------------
 
-
-
 Example Playbook
 ----------------
 
diff --git a/roles/openshift_storage_nfs/defaults/main.yml b/roles/openshift_storage_nfs/defaults/main.yml
index e25062c00..5f6893129 100644
--- a/roles/openshift_storage_nfs/defaults/main.yml
+++ b/roles/openshift_storage_nfs/defaults/main.yml
@@ -1,7 +1,13 @@
 ---
-exports_dir: /var/export
-registry_volume: regvol
-export_options: '*(rw,sync,all_squash)'
+openshift:
+  hosted:
+    registry:
+      storage:
+        nfs:
+          directory: "/exports"
+          options: "*(rw,root_squash)"
+        volume:
+          name: "registry"
 os_firewall_use_firewalld: False
 os_firewall_allow:
 - service: nfs
diff --git a/roles/openshift_storage_nfs/tasks/main.yml b/roles/openshift_storage_nfs/tasks/main.yml
index 64b121ade..fdd7bd3f1 100644
--- a/roles/openshift_storage_nfs/tasks/main.yml
+++ b/roles/openshift_storage_nfs/tasks/main.yml
@@ -1,31 +1,34 @@
 ---
-- name: Set nfs facts
-  openshift_facts:
-    role: nfs
-    local_facts:
-      exports_dir: "{{ openshift_nfs_exports_dir | default(None) }}"
-      export_options: "{{ openshift_nfs_export_options | default(None) }}"
-      registry_volume: "{{ openshift_nfs_registry_volume | default(None) }}"
-
 - name: Install nfs-utils
   yum:
     pkg: nfs-utils
     state: present
 
+- name: Configure NFS
+  lineinfile:
+    dest: /etc/sysconfig/nfs
+    regexp: '^RPCNFSDARGS=.*$'
+    line: 'RPCNFSDARGS="-N 2 -N 3"'
+  register: nfs_config
+
+- name: Restart nfs-config
+  service: name=nfs-config state=restarted
+  when: nfs_config | changed
+
 - name: Ensure exports directory exists
   file:
-    path: "{{ openshift.nfs.exports_dir }}"
+    path: "{{ openshift.hosted.registry.storage.nfs.directory }}"
     state: directory
 
 - name: Ensure export directories exist
   file:
-    path: "{{ openshift.nfs.exports_dir }}/{{ item }}"
+    path: "{{ openshift.hosted.registry.storage.nfs.directory }}/{{ item }}"
     state: directory
     mode: 0777
     owner: nfsnobody
     group: nfsnobody
   with_items:
-  - "{{ openshift.nfs.registry_volume }}"
+  - "{{ openshift.hosted.registry.storage.volume.name }}"
 
 - name: Configure exports
   template:
@@ -44,6 +47,4 @@
   - nfs-server
 
 - set_fact:
-    nfs_service_status_changed: "{{ True in (start_result.results
-                                    | map(attribute='changed')
-                                    | list) }}"
+    nfs_service_status_changed: "{{ start_result | changed }}"
diff --git a/roles/openshift_storage_nfs/templates/exports.j2 b/roles/openshift_storage_nfs/templates/exports.j2
index 702473040..c1e1994b0 100644
--- a/roles/openshift_storage_nfs/templates/exports.j2
+++ b/roles/openshift_storage_nfs/templates/exports.j2
@@ -1 +1 @@
-{{ openshift.nfs.exports_dir }}/{{ openshift.nfs.registry_volume }} {{ openshift.nfs.export_options }}
+{{ openshift.hosted.registry.storage.nfs.directory }}/{{ openshift.hosted.registry.storage.volume.name }} {{ openshift.hosted.registry.storage.nfs.options }}