From 483e58decb4daae9c355a5be8f090d1778133143 Mon Sep 17 00:00:00 2001
From: Scott Dodson <sdodson@redhat.com>
Date: Wed, 5 Apr 2017 15:43:39 -0400
Subject: Copy v3 data dir when performing backup

Fixes https://bugzilla.redhat.com/show_bug.cgi?id=1433272
---
 .../common/openshift-cluster/upgrades/etcd/backup.yml    | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

(limited to 'playbooks/common/openshift-cluster')

diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
index 7ef79afa9..347621a14 100644
--- a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
@@ -3,7 +3,6 @@
   hosts: etcd_hosts_to_backup
   vars:
     embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
-    timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
     etcdctl_command: "{{ 'etcdctl' if not openshift.common.is_containerized or embedded_etcd else 'docker exec etcd_container etcdctl' if not openshift.common.is_etcd_system_container else 'runc exec etcd etcdctl' }}"
   roles:
   - openshift_facts
@@ -13,6 +12,8 @@
       role: etcd
       local_facts: {}
     when: "'etcd' not in openshift"
+  - set_fact:
+      timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
 
   - stat: path=/var/lib/openshift
     register: var_lib_openshift
@@ -77,6 +78,19 @@
       {{ etcdctl_command }} backup --data-dir={{ openshift.etcd.etcd_data_dir }}
       --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ backup_tag | default('') }}{{ timestamp }}
 
+  # According to the docs change you can simply copy snap/db
+  # https://github.com/openshift/openshift-docs/commit/b38042de02d9780842dce95cfa0ef45d53b58bc6
+  - name: Check for v3 data store
+    stat:
+      path: "{{ openshift.etcd.etcd_data_dir }}/member/snap/db"
+    register: v3_db
+
+  - name: Copy etcd v3 data store
+    command: >
+      cp -a {{ openshift.etcd.etcd_data_dir }}/member/snap
+      {{ openshift.common.data_dir }}/etcd-backup-{{ backup_tag | default('') }}{{ timestamp }}/member/
+    when: v3_db.stat.exists
+
   - set_fact:
       etcd_backup_complete: True
 
-- 
cgit v1.2.3


From da3c31cae2870a66a0524ae06c37a3fb44e1b312 Mon Sep 17 00:00:00 2001
From: Scott Dodson <sdodson@redhat.com>
Date: Fri, 7 Apr 2017 09:13:43 -0400
Subject: Create member/snap directory encase it doesn't exist

Fixes

TASK [Copy etcd v3 data store]
*************************************************
fatal: [host.redhat.com]: FAILED! => {
    "changed": true,
    "cmd": [
        "cp",
        "-a",
        "/var/lib/etcd//member/snap",
        "/var/lib/origin/etcd-backup-pre-upgrade-20170407055413/member/"
    ],
    "delta": "0:00:00.003152",
    "end": "2017-04-07 01:54:17.584685",
    "failed": true,
    "rc": 1,
    "start": "2017-04-07 01:54:17.581533",
    "warnings": []
}

STDERR:

cp: cannot create directory
?/var/lib/origin/etcd-backup-pre-upgrade-20170407055413/member/?: No
such file or directory
---
 playbooks/common/openshift-cluster/upgrades/etcd/backup.yml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

(limited to 'playbooks/common/openshift-cluster')

diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
index 347621a14..31a6c6c66 100644
--- a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
@@ -85,10 +85,15 @@
       path: "{{ openshift.etcd.etcd_data_dir }}/member/snap/db"
     register: v3_db
 
+  - name: Ensure v3 backup directory exists
+    file:
+      path: "{{ openshift.common.data_dir }}/etcd-backup-{{ backup_tag | default('') }}{{ timestamp }}/member/snap"
+      state: directory
+
   - name: Copy etcd v3 data store
     command: >
-      cp -a {{ openshift.etcd.etcd_data_dir }}/member/snap
-      {{ openshift.common.data_dir }}/etcd-backup-{{ backup_tag | default('') }}{{ timestamp }}/member/
+      cp -a {{ openshift.etcd.etcd_data_dir }}/member/snap/db
+      {{ openshift.common.data_dir }}/etcd-backup-{{ backup_tag | default('') }}{{ timestamp }}/member/snap/
     when: v3_db.stat.exists
 
   - set_fact:
-- 
cgit v1.2.3


From 1ce31d156ef800da73544622558da9aa0fc647e4 Mon Sep 17 00:00:00 2001
From: Scott Dodson <sdodson@redhat.com>
Date: Mon, 10 Apr 2017 13:13:32 -0400
Subject: Store backups in /var/lib/etcd/openshift-backup

Because containerized installs don't mount /var/lib/origin and we
switched to running the backup inside the container that meant that we
were backing up the etcd data into a directory inside the container
filesystem. Since we have no other volume mounted we need to backup into
/var/lib/etcd.
---
 .../openshift-cluster/upgrades/etcd/backup.yml     | 31 +++++++++-------------
 1 file changed, 12 insertions(+), 19 deletions(-)

(limited to 'playbooks/common/openshift-cluster')

diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
index 31a6c6c66..6c763ce46 100644
--- a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
@@ -4,6 +4,7 @@
   vars:
     embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
     etcdctl_command: "{{ 'etcdctl' if not openshift.common.is_containerized or embedded_etcd else 'docker exec etcd_container etcdctl' if not openshift.common.is_etcd_system_container else 'runc exec etcd etcdctl' }}"
+    timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
   roles:
   - openshift_facts
   tasks:
@@ -13,30 +14,20 @@
       local_facts: {}
     when: "'etcd' not in openshift"
   - set_fact:
-      timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
-
-  - stat: path=/var/lib/openshift
-    register: var_lib_openshift
-
-  - stat: path=/var/lib/origin
-    register: var_lib_origin
-
-  - name: Create origin symlink if necessary
-    file: src=/var/lib/openshift/ dest=/var/lib/origin state=link
-    when: var_lib_openshift.stat.exists == True and var_lib_origin.stat.exists == False
+      etcd_backup_dir: "{{ openshift.etcd.etcd_data_dir }}/openshift-backup-{{ backup_tag | default('') }}{{ timestamp }}"
 
   # TODO: replace shell module with command and update later checks
   # We assume to be using the data dir for all backups.
   - name: Check available disk space for etcd backup
-    shell: df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1
+    shell: df --output=avail -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1
     register: avail_disk
     # AUDIT:changed_when: `false` because we are only inspecting
     # state, not manipulating anything
     changed_when: false
 
   # TODO: replace shell module with command and update later checks
-  - name: Check current embedded etcd disk usage
-    shell: du -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1 | cut -f1
+  - name: Check current etcd disk usage
+    shell: du --exclude='*openshift-backup*' -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1 | cut -f1
     register: etcd_disk_usage
     when: embedded_etcd | bool
     # AUDIT:changed_when: `false` because we are only inspecting
@@ -71,12 +62,12 @@
     package:
       name: etcd
       state: latest
-    when: ( embedded_etcd | bool or openshift.common.is_containerized ) and not openshift.common.is_atomic
+    when: embedded_etcd | bool
 
   - name: Generate etcd backup
     command: >
       {{ etcdctl_command }} backup --data-dir={{ openshift.etcd.etcd_data_dir }}
-      --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ backup_tag | default('') }}{{ timestamp }}
+      --backup-dir={{ etcd_backup_dir }}
 
   # According to the docs change you can simply copy snap/db
   # https://github.com/openshift/openshift-docs/commit/b38042de02d9780842dce95cfa0ef45d53b58bc6
@@ -85,15 +76,17 @@
       path: "{{ openshift.etcd.etcd_data_dir }}/member/snap/db"
     register: v3_db
 
+  # TODO: this should be safe to remove now that we've figured out that we were dumping the data
+  # inside the container
   - name: Ensure v3 backup directory exists
     file:
-      path: "{{ openshift.common.data_dir }}/etcd-backup-{{ backup_tag | default('') }}{{ timestamp }}/member/snap"
+      path: "{{ etcd_backup_dir }}/member/snap"
       state: directory
 
   - name: Copy etcd v3 data store
     command: >
       cp -a {{ openshift.etcd.etcd_data_dir }}/member/snap/db
-      {{ openshift.common.data_dir }}/etcd-backup-{{ backup_tag | default('') }}{{ timestamp }}/member/snap/
+      {{ etcd_backup_dir }}/member/snap/
     when: v3_db.stat.exists
 
   - set_fact:
@@ -101,7 +94,7 @@
 
   - name: Display location of etcd backup
     debug:
-      msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ backup_tag | default('') }}{{ timestamp }}"
+      msg: "Etcd backup created in {{ etcd_backup_dir }}"
 
 - name: Gate on etcd backup
   hosts: localhost
-- 
cgit v1.2.3


From 974f01c81d11027a7d944fc547df4ab557e99e58 Mon Sep 17 00:00:00 2001
From: Scott Dodson <sdodson@redhat.com>
Date: Mon, 10 Apr 2017 13:27:57 -0400
Subject: Cleanup comments and remove extraneous tasks

---
 .../openshift-cluster/upgrades/etcd/backup.yml     | 30 +++++-----------------
 1 file changed, 7 insertions(+), 23 deletions(-)

(limited to 'playbooks/common/openshift-cluster')

diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
index 6c763ce46..fb51a0061 100644
--- a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
@@ -17,7 +17,6 @@
       etcd_backup_dir: "{{ openshift.etcd.etcd_data_dir }}/openshift-backup-{{ backup_tag | default('') }}{{ timestamp }}"
 
   # TODO: replace shell module with command and update later checks
-  # We assume to be using the data dir for all backups.
   - name: Check available disk space for etcd backup
     shell: df --output=avail -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1
     register: avail_disk
@@ -44,25 +43,17 @@
   # For non containerized and non embedded we should have the correct version of
   # etcd installed already. So don't do anything.
   #
-  # For embedded or containerized we need to use the latest because OCP 3.3 uses
-  # a version of etcd that can only be backed up with etcd-3.x and if it's
-  # containerized then etcd version may be newer than that on the host so
-  # upgrade it.
+  # For containerized installs we now exec into etcd_container
   #
-  # On atomic we have neither yum nor dnf so ansible throws a hard to debug error
-  # if you use package there, like this: "Could not find a module for unknown."
-  # see https://bugzilla.redhat.com/show_bug.cgi?id=1408668
-  #
-  # TODO - We should refactor all containerized backups to use the containerized
-  # version of etcd to perform the backup rather than relying on the host's
-  # binaries. Until we do that we'll continue to have problems backing up etcd
-  # when atomic host has an older version than the version that's running in the
-  # container whether that's embedded or not
-  - name: Install latest etcd for containerized or embedded
+  # For embedded non containerized we need to ensure we have the latest version
+  # etcd on the host.
+  - name: Install latest etcd for embedded
     package:
       name: etcd
       state: latest
-    when: embedded_etcd | bool
+    when:
+    - embedded_etcd | bool
+    - not openshift.common.is_atomic | bool
 
   - name: Generate etcd backup
     command: >
@@ -76,13 +67,6 @@
       path: "{{ openshift.etcd.etcd_data_dir }}/member/snap/db"
     register: v3_db
 
-  # TODO: this should be safe to remove now that we've figured out that we were dumping the data
-  # inside the container
-  - name: Ensure v3 backup directory exists
-    file:
-      path: "{{ etcd_backup_dir }}/member/snap"
-      state: directory
-
   - name: Copy etcd v3 data store
     command: >
       cp -a {{ openshift.etcd.etcd_data_dir }}/member/snap/db
-- 
cgit v1.2.3