Merge pull request #6365 from mtnbikenc/consolidate-etcd-upgrade

Playbook Consolidation - etcd Upgrade

16 files changed, 199 insertions, 39 deletions

diff --git a/playbooks/openshift-etcd/private/ca.yml b/playbooks/openshift-etcd/private/ca.yml
index c9f186e72..f3bb3c2d1 100644
--- a/playbooks/openshift-etcd/private/ca.yml
+++ b/playbooks/openshift-etcd/private/ca.yml
@@ -7,7 +7,7 @@
   tasks:
   - include_role:
       name: etcd
-      tasks_from: ca
+      tasks_from: ca.yml
     vars:
       etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
       etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
diff --git a/playbooks/openshift-etcd/private/certificates-backup.yml b/playbooks/openshift-etcd/private/certificates-backup.yml
index d738c8207..ce21a1f96 100644
--- a/playbooks/openshift-etcd/private/certificates-backup.yml
+++ b/playbooks/openshift-etcd/private/certificates-backup.yml
@@ -5,10 +5,10 @@
   tasks:
   - include_role:
       name: etcd
-      tasks_from: backup_generated_certificates
+      tasks_from: backup_generated_certificates.yml
   - include_role:
       name: etcd
-      tasks_from: remove_generated_certificates
+      tasks_from: remove_generated_certificates.yml
 
 - name: Backup deployed etcd certificates
   hosts: oo_etcd_to_config
@@ -16,4 +16,4 @@
   tasks:
   - include_role:
       name: etcd
-      tasks_from: backup_server_certificates
+      tasks_from: backup_server_certificates.yml
diff --git a/playbooks/openshift-etcd/private/embedded2external.yml b/playbooks/openshift-etcd/private/embedded2external.yml
index 514319b88..c7a532622 100644
--- a/playbooks/openshift-etcd/private/embedded2external.yml
+++ b/playbooks/openshift-etcd/private/embedded2external.yml
@@ -20,7 +20,7 @@
   - name: Check the master API is ready
     include_role:
       name: openshift_master
-      tasks_from: check_master_api_is_ready
+      tasks_from: check_master_api_is_ready.yml
   - set_fact:
       master_service: "{{ openshift.common.service_type + '-master' }}"
       embedded_etcd_backup_suffix: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
@@ -34,7 +34,7 @@
   # Can't use with_items with include_role: https://github.com/ansible/ansible/issues/21285
   - include_role:
       name: etcd
-      tasks_from: backup
+      tasks_from: backup.yml
     vars:
       r_etcd_common_backup_tag: pre-migrate
       r_etcd_common_embedded_etcd: "{{ true }}"
@@ -42,7 +42,7 @@
 
   - include_role:
       name: etcd
-      tasks_from: backup.archive
+      tasks_from: backup.archive.yml
     vars:
       r_etcd_common_backup_tag: pre-migrate
       r_etcd_common_embedded_etcd: "{{ true }}"
@@ -58,7 +58,7 @@
   tasks:
   - include_role:
       name: etcd
-      tasks_from: backup_master_etcd_certificates
+      tasks_from: backup_master_etcd_certificates.yml
 
 - name: Redeploy master etcd certificates
   import_playbook: master_etcd_certificates.yml
@@ -75,10 +75,10 @@
   pre_tasks:
   - include_role:
       name: etcd
-      tasks_from: disable_etcd
+      tasks_from: disable_etcd.yml
   - include_role:
       name: etcd
-      tasks_from: clean_data
+      tasks_from: clean_data.yml
 
 # 6. copy the embedded etcd backup to the external host
 # TODO(jchaloup): if the etcd and first master are on the same host, just copy the directory
@@ -93,7 +93,7 @@
 
   - include_role:
       name: etcd
-      tasks_from: backup.fetch
+      tasks_from: backup.fetch.yml
     vars:
       etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}"
       r_etcd_common_backup_tag: pre-migrate
@@ -103,7 +103,7 @@
 
   - include_role:
       name: etcd
-      tasks_from: backup.copy
+      tasks_from: backup.copy.yml
     vars:
       etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}"
       r_etcd_common_backup_tag: pre-migrate
@@ -124,14 +124,14 @@
   tasks:
   - include_role:
       name: etcd
-      tasks_from: backup.unarchive
+      tasks_from: backup.unarchive.yml
     vars:
       r_etcd_common_backup_tag: pre-migrate
       r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}"
 
   - include_role:
       name: etcd
-      tasks_from: backup.force_new_cluster
+      tasks_from: backup.force_new_cluster.yml
     vars:
       r_etcd_common_backup_tag: pre-migrate
       r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}"
@@ -145,7 +145,7 @@
   tasks:
   - include_role:
       name: openshift_master
-      tasks_from: configure_external_etcd
+      tasks_from: configure_external_etcd.yml
     vars:
       etcd_peer_url_scheme: "https"
       etcd_ip: "{{ hostvars[groups.oo_etcd_to_config.0].openshift.common.ip }}"
diff --git a/playbooks/openshift-etcd/private/migrate.yml b/playbooks/openshift-etcd/private/migrate.yml
index 4269918c2..834bd242d 100644
--- a/playbooks/openshift-etcd/private/migrate.yml
+++ b/playbooks/openshift-etcd/private/migrate.yml
@@ -17,7 +17,7 @@
   tasks:
   - include_role:
       name: etcd
-      tasks_from: migrate.pre_check
+      tasks_from: migrate.pre_check.yml
     vars:
       r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
       etcd_peer: "{{ ansible_default_ipv4.address }}"
@@ -46,7 +46,7 @@
   post_tasks:
   - include_role:
       name: etcd
-      tasks_from: backup
+      tasks_from: backup.yml
     vars:
       r_etcd_common_backup_tag: pre-migration
       r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
@@ -74,7 +74,7 @@
   pre_tasks:
   - include_role:
       name: etcd
-      tasks_from: disable_etcd
+      tasks_from: disable_etcd.yml
 
 - name: Migrate data on first etcd
   hosts: oo_etcd_to_migrate[0]
@@ -82,7 +82,7 @@
   tasks:
   - include_role:
       name: etcd
-      tasks_from: migrate
+      tasks_from: migrate.yml
     vars:
       r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
       etcd_peer: "{{ openshift.common.ip }}"
@@ -95,7 +95,7 @@
   tasks:
   - include_role:
       name: etcd
-      tasks_from: clean_data
+      tasks_from: clean_data.yml
     vars:
       r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
       etcd_peer: "{{ openshift.common.ip }}"
@@ -132,7 +132,7 @@
   tasks:
   - include_role:
       name: etcd
-      tasks_from: migrate.add_ttls
+      tasks_from: migrate.add_ttls.yml
     vars:
       etcd_peer: "{{ hostvars[groups.oo_etcd_to_migrate.0].openshift.common.ip }}"
       etcd_url_scheme: "https"
@@ -144,7 +144,7 @@
   tasks:
   - include_role:
       name: etcd
-      tasks_from: migrate.configure_master
+      tasks_from: migrate.configure_master.yml
     when: etcd_migration_failed | length == 0
   - debug:
       msg: "Skipping master re-configuration since migration failed."
diff --git a/playbooks/openshift-etcd/private/redeploy-ca.yml b/playbooks/openshift-etcd/private/redeploy-ca.yml
index cc5d57031..158bcb849 100644
--- a/playbooks/openshift-etcd/private/redeploy-ca.yml
+++ b/playbooks/openshift-etcd/private/redeploy-ca.yml
@@ -16,12 +16,12 @@
   tasks:
   - include_role:
       name: etcd
-      tasks_from: backup_ca_certificates
+      tasks_from: backup_ca_certificates.yml
   - include_role:
       name: etcd
-      tasks_from: remove_ca_certificates
+      tasks_from: remove_ca_certificates.yml
 
-- include: ca.yml
+- import_playbook: ca.yml
 
 - name: Create temp directory for syncing certs
   hosts: localhost
@@ -44,7 +44,7 @@
       etcd_sync_cert_dir: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}"
       etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
 
-- include: restart.yml
+- import_playbook: restart.yml
   # Do not restart etcd when etcd certificates were previously expired.
   when: ('expired' not in (hostvars
                            | oo_select_keys(groups['etcd'])
@@ -56,7 +56,7 @@
   tasks:
   - include_role:
       name: etcd
-      tasks_from: retrieve_ca_certificates
+      tasks_from: retrieve_ca_certificates.yml
     vars:
       etcd_sync_cert_dir: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}"
 
@@ -82,7 +82,7 @@
       state: absent
     changed_when: false
 
-- include: ../../openshift-master/private/restart.yml
+- import_playbook: ../../openshift-master/private/restart.yml
   # Do not restart masters when master or etcd certificates were previously expired.
   when:
   # masters
diff --git a/playbooks/openshift-etcd/private/redeploy-certificates.yml b/playbooks/openshift-etcd/private/redeploy-certificates.yml
index cc1e6adf5..1c8eb27ac 100644
--- a/playbooks/openshift-etcd/private/redeploy-certificates.yml
+++ b/playbooks/openshift-etcd/private/redeploy-certificates.yml
@@ -11,8 +11,8 @@
   # certificates were previously expired.
   - role: openshift_certificate_expiry
 
-- include: certificates-backup.yml
+- import_playbook: certificates-backup.yml
 
-- include: certificates.yml
+- import_playbook: certificates.yml
   vars:
     etcd_certificates_redeploy: true
diff --git a/playbooks/openshift-etcd/private/scaleup.yml b/playbooks/openshift-etcd/private/scaleup.yml
index fac8e3f02..3ef043ec8 100644
--- a/playbooks/openshift-etcd/private/scaleup.yml
+++ b/playbooks/openshift-etcd/private/scaleup.yml
@@ -32,7 +32,7 @@
     until: etcd_add_check.rc == 0
   - include_role:
       name: etcd
-      tasks_from: server_certificates
+      tasks_from: server_certificates.yml
     vars:
       etcd_peers: "{{ groups.oo_new_etcd_to_config | default([], true) }}"
       etcd_certificates_etcd_hosts: "{{ groups.oo_new_etcd_to_config | default([], true) }}"
@@ -78,4 +78,4 @@
   post_tasks:
   - include_role:
       name: openshift_master
-      tasks_from: update_etcd_client_urls
+      tasks_from: update_etcd_client_urls.yml
diff --git a/playbooks/openshift-etcd/private/server_certificates.yml b/playbooks/openshift-etcd/private/server_certificates.yml
index 14c74baf3..695b53990 100644
--- a/playbooks/openshift-etcd/private/server_certificates.yml
+++ b/playbooks/openshift-etcd/private/server_certificates.yml
@@ -7,7 +7,7 @@
   post_tasks:
     - include_role:
         name: etcd
-        tasks_from: server_certificates
+        tasks_from: server_certificates.yml
       vars:
         etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
         etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
diff --git a/playbooks/openshift-etcd/private/upgrade_backup.yml b/playbooks/openshift-etcd/private/upgrade_backup.yml
new file mode 100644
index 000000000..22ed7e610
--- /dev/null
+++ b/playbooks/openshift-etcd/private/upgrade_backup.yml
@@ -0,0 +1,28 @@
+---
+- name: Backup etcd
+  hosts: oo_etcd_hosts_to_backup
+  roles:
+  - role: openshift_etcd_facts
+  post_tasks:
+  - include_role:
+      name: etcd
+      tasks_from: backup.yml
+    vars:
+      r_etcd_common_backup_tag: "{{ etcd_backup_tag }}"
+      r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
+      r_etcd_common_backup_sufix_name: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
+
+- name: Gate on etcd backup
+  hosts: localhost
+  connection: local
+  become: no
+  tasks:
+  - set_fact:
+      etcd_backup_completed: "{{ hostvars
+                                 | oo_select_keys(groups.oo_etcd_hosts_to_backup)
+                                 | oo_collect('inventory_hostname', {'r_etcd_common_backup_complete': true}) }}"
+  - set_fact:
+      etcd_backup_failed: "{{ groups.oo_etcd_hosts_to_backup | difference(etcd_backup_completed) | list }}"
+  - fail:
+      msg: "Upgrade cannot continue. The following hosts did not complete etcd backup: {{ etcd_backup_failed | join(',') }}"
+    when: etcd_backup_failed | length > 0
diff --git a/playbooks/openshift-etcd/private/upgrade_image_members.yml b/playbooks/openshift-etcd/private/upgrade_image_members.yml
new file mode 100644
index 000000000..c133c0201
--- /dev/null
+++ b/playbooks/openshift-etcd/private/upgrade_image_members.yml
@@ -0,0 +1,17 @@
+---
+# INPUT etcd_upgrade_version
+# INPUT etcd_container_version
+# INPUT openshift.common.is_containerized
+- name: Upgrade containerized hosts to {{ etcd_upgrade_version }}
+  hosts: oo_etcd_hosts_to_upgrade
+  serial: 1
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: upgrade_image.yml
+    vars:
+      r_etcd_upgrade_version: "{{ etcd_upgrade_version }}"
+      etcd_peer: "{{ openshift.common.hostname }}"
+    when:
+    - etcd_container_version | default('99') | version_compare(etcd_upgrade_version,'<')
+    - openshift.common.is_containerized | bool
diff --git a/playbooks/openshift-etcd/private/upgrade_main.yml b/playbooks/openshift-etcd/private/upgrade_main.yml
new file mode 100644
index 000000000..e373a4a4c
--- /dev/null
+++ b/playbooks/openshift-etcd/private/upgrade_main.yml
@@ -0,0 +1,29 @@
+---
+# For 1.4/3.4 we want to upgrade everyone to etcd-3.0. etcd docs say to
+# upgrade from 2.0.x to 2.1.x to 2.2.x to 2.3.x to 3.0.x. While this is a tedius
+# task for RHEL and CENTOS it's simply not possible in Fedora unless you've
+# mirrored packages on your own because only the GA and latest versions are
+# available in the repos. So for Fedora we'll simply skip this, sorry.
+
+- name: Backup etcd before upgrading anything
+  import_playbook: upgrade_backup.yml
+  vars:
+    etcd_backup_tag: "pre-upgrade-"
+  when: openshift_etcd_backup | default(true) | bool
+
+- name: Drop etcdctl profiles
+  hosts: oo_etcd_hosts_to_upgrade
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: drop_etcdctl.yml
+
+- name: Perform etcd upgrade
+  import_playbook: upgrade_step.yml
+  when: openshift_etcd_upgrade | default(true) | bool
+
+- name: Backup etcd
+  import_playbook: upgrade_backup.yml
+  vars:
+    etcd_backup_tag: "post-3.0-"
+  when: openshift_etcd_backup | default(true) | bool
diff --git a/playbooks/openshift-etcd/private/upgrade_rpm_members.yml b/playbooks/openshift-etcd/private/upgrade_rpm_members.yml
new file mode 100644
index 000000000..902c39d9c
--- /dev/null
+++ b/playbooks/openshift-etcd/private/upgrade_rpm_members.yml
@@ -0,0 +1,18 @@
+---
+# INPUT etcd_upgrade_version
+# INPUT etcd_rpm_version
+# INPUT openshift.common.is_containerized
+- name: Upgrade to {{ etcd_upgrade_version }}
+  hosts: oo_etcd_hosts_to_upgrade
+  serial: 1
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: upgrade_rpm.yml
+    vars:
+      r_etcd_upgrade_version: "{{ etcd_upgrade_version }}"
+      etcd_peer: "{{ openshift.common.hostname }}"
+    when:
+    - etcd_rpm_version.stdout | default('99') | version_compare(etcd_upgrade_version, '<')
+    - ansible_distribution == 'RedHat'
+    - not openshift.common.is_containerized | bool
diff --git a/playbooks/openshift-etcd/private/upgrade_step.yml b/playbooks/openshift-etcd/private/upgrade_step.yml
new file mode 100644
index 000000000..60127fc68
--- /dev/null
+++ b/playbooks/openshift-etcd/private/upgrade_step.yml
@@ -0,0 +1,64 @@
+---
+- name: Determine etcd version
+  hosts: oo_etcd_hosts_to_upgrade
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: version_detect.yml
+
+- import_playbook: upgrade_rpm_members.yml
+  vars:
+    etcd_upgrade_version: '2.1'
+
+- import_playbook: upgrade_rpm_members.yml
+  vars:
+    etcd_upgrade_version: '2.2'
+
+- import_playbook: upgrade_image_members.yml
+  vars:
+    etcd_upgrade_version: '2.2.5'
+
+- import_playbook: upgrade_rpm_members.yml
+  vars:
+    etcd_upgrade_version: '2.3'
+
+- import_playbook: upgrade_image_members.yml
+  vars:
+    etcd_upgrade_version: '2.3.7'
+
+- import_playbook: upgrade_rpm_members.yml
+  vars:
+    etcd_upgrade_version: '3.0'
+
+- import_playbook: upgrade_image_members.yml
+  vars:
+    etcd_upgrade_version: '3.0.15'
+
+- import_playbook: upgrade_rpm_members.yml
+  vars:
+    etcd_upgrade_version: '3.1'
+
+- import_playbook: upgrade_image_members.yml
+  vars:
+    etcd_upgrade_version: '3.1.3'
+
+- import_playbook: upgrade_rpm_members.yml
+  vars:
+    etcd_upgrade_version: '3.2'
+
+- import_playbook: upgrade_image_members.yml
+  vars:
+    etcd_upgrade_version: '3.2.7'
+
+- name: Upgrade fedora to latest
+  hosts: oo_etcd_hosts_to_upgrade
+  serial: 1
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: upgrade_image.yml
+    vars:
+      etcd_peer: "{{ openshift.common.hostname }}"
+    when:
+    - ansible_distribution == 'Fedora'
+    - not openshift.common.is_containerized | bool
diff --git a/playbooks/openshift-etcd/redeploy-ca.yml b/playbooks/openshift-etcd/redeploy-ca.yml
index b1d23675d..769d694ba 100644
--- a/playbooks/openshift-etcd/redeploy-ca.yml
+++ b/playbooks/openshift-etcd/redeploy-ca.yml
@@ -1,4 +1,4 @@
 ---
-- include: ../init/main.yml
+- import_playbook: ../init/main.yml
 
-- include: private/redeploy-ca.yml
+- import_playbook: private/redeploy-ca.yml
diff --git a/playbooks/openshift-etcd/redeploy-certificates.yml b/playbooks/openshift-etcd/redeploy-certificates.yml
index 1bd302c03..753878d70 100644
--- a/playbooks/openshift-etcd/redeploy-certificates.yml
+++ b/playbooks/openshift-etcd/redeploy-certificates.yml
@@ -1,10 +1,10 @@
 ---
-- include: ../init/main.yml
+- import_playbook: ../init/main.yml
 
-- include: private/redeploy-certificates.yml
+- import_playbook: private/redeploy-certificates.yml
 
-- include: private/restart.yml
+- import_playbook: private/restart.yml
   vars:
     g_etcd_certificates_expired: "{{ ('expired' in (hostvars | oo_select_keys(groups['etcd']) | oo_collect('check_results.check_results.etcd') | oo_collect('health'))) | bool }}"
 
-- include: ../openshift-master/private/restart.yml
+- import_playbook: ../openshift-master/private/restart.yml
diff --git a/playbooks/openshift-etcd/upgrade.yml b/playbooks/openshift-etcd/upgrade.yml
new file mode 100644
index 000000000..ccc797527
--- /dev/null
+++ b/playbooks/openshift-etcd/upgrade.yml
@@ -0,0 +1,4 @@
+---
+- import_playbook: ../init/evaluate_groups.yml
+
+- import_playbook: private/upgrade_main.yml