retry package operations

When a package install/update fails due to network blips or other spotty
availability, retry it. If the failure is a real failure (e.g. package
is really not there) it still fails after 3 tries (Ansible default).

50 files changed, 142 insertions, 0 deletions

diff --git a/playbooks/adhoc/uninstall.yml b/playbooks/adhoc/uninstall.yml
index 5ed55a817..0c2a2c7e8 100644
--- a/playbooks/adhoc/uninstall.yml
+++ b/playbooks/adhoc/uninstall.yml
@@ -126,10 +126,14 @@
         - origin-sdn-ovs
         - tuned-profiles-openshift-node
         - tuned-profiles-origin-node
+        register: result
+        until: result | success
 
       - name: Remove flannel package
         package: name=flannel state=absent
         when: openshift_use_flannel | default(false) | bool
+        register: result
+        until: result | success
       when: not is_atomic | bool
 
     - shell: systemctl reset-failed
@@ -382,6 +386,8 @@
     - origin-master
     - pacemaker
     - pcs
+    register: result
+    until: result | success
 
   - shell: systemctl reset-failed
     changed_when: False
@@ -497,6 +503,8 @@
     with_items:
     - etcd
     - etcd3
+    register: result
+    until: result | success
 
   - shell: systemctl reset-failed
     changed_when: False
@@ -554,6 +562,8 @@
     when: not is_atomic | bool and openshift_remove_all | default(True) | bool
     with_items:
     - haproxy
+    register: result
+    until: result | success
 
   - shell: systemctl reset-failed
     changed_when: False
diff --git a/playbooks/common/openshift-cluster/upgrades/docker/tasks/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/docker/tasks/upgrade.yml
index 808cc562c..83be290e6 100644
--- a/playbooks/common/openshift-cluster/upgrades/docker/tasks/upgrade.yml
+++ b/playbooks/common/openshift-cluster/upgrades/docker/tasks/upgrade.yml
@@ -41,6 +41,8 @@
 
 - name: Upgrade Docker
   package: name=docker{{ '-' + docker_version }} state=present
+  register: result
+  until: result | success
 
 - include: restart.yml
   when: not skip_docker_restart | default(False) | bool
diff --git a/playbooks/init/facts.yml b/playbooks/init/facts.yml
index 05142f9b6..1166ac538 100644
--- a/playbooks/init/facts.yml
+++ b/playbooks/init/facts.yml
@@ -84,6 +84,8 @@
       - "{{ 'python3-dbus' if ansible_distribution == 'Fedora' else 'dbus-python' }}"
       - "{{ 'python3-PyYAML' if ansible_distribution == 'Fedora' else 'PyYAML' }}"
       - yum-utils
+      register: result
+      until: result | success
 
     - name: Ensure various deps for running system containers are installed
       package:
@@ -100,6 +102,8 @@
         or (openshift_use_openvswitch_system_container | default(False)) | bool
         or (openshift_use_node_system_container | default(False)) | bool
         or (openshift_use_master_system_container | default(False)) | bool
+      register: result
+      until: result | success
 
   - name: Gather Cluster facts and set is_containerized if needed
     openshift_facts:
diff --git a/playbooks/openshift-node/private/network_manager.yml b/playbooks/openshift-node/private/network_manager.yml
index fffed4dfb..7211787be 100644
--- a/playbooks/openshift-node/private/network_manager.yml
+++ b/playbooks/openshift-node/private/network_manager.yml
@@ -9,6 +9,8 @@
     package:
       name: 'NetworkManager'
       state: present
+    register: result
+    until: result | success
 
   - name: configure NetworkManager
     lineinfile:
diff --git a/roles/cockpit/tasks/main.yml b/roles/cockpit/tasks/main.yml
index 34754502a..f63b3e49b 100644
--- a/roles/cockpit/tasks/main.yml
+++ b/roles/cockpit/tasks/main.yml
@@ -11,6 +11,8 @@
     - cockpit-docker
     - "{{ cockpit_plugins }}"
   when: not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
 
 - name: Enable cockpit-ws
   systemd:
diff --git a/roles/contiv/tasks/download_bins.yml b/roles/contiv/tasks/download_bins.yml
index 319fce46c..741c1d1da 100644
--- a/roles/contiv/tasks/download_bins.yml
+++ b/roles/contiv/tasks/download_bins.yml
@@ -8,6 +8,8 @@
   yum:
     name: bzip2
     state: installed
+  register: result
+  until: result | success
 
 - name: Download Bins | Download Contiv tar file
   get_url:
diff --git a/roles/contiv/tasks/pkgMgrInstallers/centos-install.yml b/roles/contiv/tasks/pkgMgrInstallers/centos-install.yml
index 91e6aadf3..62b4716a3 100644
--- a/roles/contiv/tasks/pkgMgrInstallers/centos-install.yml
+++ b/roles/contiv/tasks/pkgMgrInstallers/centos-install.yml
@@ -3,6 +3,8 @@
   yum:
     pkg=net-tools
     state=latest
+  register: result
+  until: result | success
 
 - name: PkgMgr RHEL/CentOS | Get openstack ocata rpm
   get_url:
@@ -20,6 +22,8 @@
   yum: name=/tmp/rdo-release-ocata-2.noarch.rpm state=present
   tags:
     - ovs_install
+  register: result
+  until: result | success
 
 - name: PkgMgr RHEL/CentOS | Install ovs
   yum:
@@ -31,3 +35,5 @@
     no_proxy: "{{ no_proxy|default('') }}"
   tags:
     - ovs_install
+  register: result
+  until: result | success
diff --git a/roles/contiv_facts/tasks/fedora-install.yml b/roles/contiv_facts/tasks/fedora-install.yml
index db56a18c0..a57f6eb19 100644
--- a/roles/contiv_facts/tasks/fedora-install.yml
+++ b/roles/contiv_facts/tasks/fedora-install.yml
@@ -3,6 +3,8 @@
   yum:
     name: dnf
     state: installed
+  register: result
+  until: result | success
 
 - name: Update repo cache
   command: dnf update -y
diff --git a/roles/docker/tasks/package_docker.yml b/roles/docker/tasks/package_docker.yml
index 5437275a2..044b04478 100644
--- a/roles/docker/tasks/package_docker.yml
+++ b/roles/docker/tasks/package_docker.yml
@@ -37,6 +37,8 @@
 - name: Install Docker
   package: name=docker{{ '-' + docker_version if docker_version is defined else '' }} state=present
   when: not openshift.common.is_atomic | bool and not curr_docker_version | skipped and not curr_docker_version.stdout != ''
+  register: result
+  until: result | success
 
 - block:
   # Extend the default Docker service unit file when using iptables-services
diff --git a/roles/docker/tasks/systemcontainer_crio.yml b/roles/docker/tasks/systemcontainer_crio.yml
index 17800d4e5..3439aa353 100644
--- a/roles/docker/tasks/systemcontainer_crio.yml
+++ b/roles/docker/tasks/systemcontainer_crio.yml
@@ -29,6 +29,8 @@
     name: container-selinux
     state: present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Check we are not using node as a Docker container with CRI-O
   fail: msg='Cannot use CRI-O with node configured as a Docker container'
@@ -42,6 +44,8 @@
     name: atomic
     state: present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 # At the time of writing the atomic command requires runc for it's own use. This
 # task is here in the even that the atomic package ever removes the dependency.
@@ -50,6 +54,8 @@
     name: runc
     state: present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 
 - name: Check that overlay is in the kernel
diff --git a/roles/docker/tasks/systemcontainer_docker.yml b/roles/docker/tasks/systemcontainer_docker.yml
index f69acb9a5..881d83f50 100644
--- a/roles/docker/tasks/systemcontainer_docker.yml
+++ b/roles/docker/tasks/systemcontainer_docker.yml
@@ -34,6 +34,8 @@
     name: container-selinux
     state: present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 # Used to pull and install the system container
 - name: Ensure atomic is installed
@@ -41,6 +43,8 @@
     name: atomic
     state: present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 # At the time of writing the atomic command requires runc for it's own use. This
 # task is here in the even that the atomic package ever removes the dependency.
@@ -49,11 +53,15 @@
     name: runc
     state: present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 # Make sure Docker is installed so we are able to use the client
 - name: Install Docker so we can use the client
   package: name=docker{{ '-' + docker_version if docker_version is defined else '' }} state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 # Make sure docker is disabled. Errors are ignored.
 - name: Disable Docker
diff --git a/roles/etcd/tasks/auxiliary/drop_etcdctl.yml b/roles/etcd/tasks/auxiliary/drop_etcdctl.yml
index 11bd2310e..603f2531f 100644
--- a/roles/etcd/tasks/auxiliary/drop_etcdctl.yml
+++ b/roles/etcd/tasks/auxiliary/drop_etcdctl.yml
@@ -2,6 +2,8 @@
 - name: Install etcd for etcdctl
   package: name=etcd{{ '-' + etcd_version if etcd_version is defined else '' }} state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Configure etcd profile.d aliases
   template:
diff --git a/roles/etcd/tasks/backup/backup.yml b/roles/etcd/tasks/backup/backup.yml
index afb84eb58..9da023dbd 100644
--- a/roles/etcd/tasks/backup/backup.yml
+++ b/roles/etcd/tasks/backup/backup.yml
@@ -43,6 +43,8 @@
   when:
   - r_etcd_common_embedded_etcd | bool
   - not l_ostree_booted.stat.exists | bool
+  register: result
+  until: result | success
 
 - name: Check selinux label of '{{ etcd_data_dir }}'
   command: >
diff --git a/roles/etcd/tasks/certificates/deploy_ca.yml b/roles/etcd/tasks/certificates/deploy_ca.yml
index 3d32290a2..bd4dafafd 100644
--- a/roles/etcd/tasks/certificates/deploy_ca.yml
+++ b/roles/etcd/tasks/certificates/deploy_ca.yml
@@ -6,6 +6,8 @@
   when: not etcd_is_atomic | bool
   delegate_to: "{{ etcd_ca_host }}"
   run_once: true
+  register: result
+  until: result | success
 
 - file:
     path: "{{ item }}"
diff --git a/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml b/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml
index 00b8f4a0b..f4726940a 100644
--- a/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml
+++ b/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml
@@ -4,6 +4,8 @@
     name: "etcd{{ '-' + etcd_version if etcd_version is defined else '' }}"
     state: present
   when: not etcd_is_containerized | bool
+  register: result
+  until: result | success
 
 - name: Check status of etcd certificates
   stat:
diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml
index 5ee9335f5..b2100801f 100644
--- a/roles/etcd/tasks/main.yml
+++ b/roles/etcd/tasks/main.yml
@@ -12,6 +12,8 @@
 - name: Install etcd
   package: name=etcd{{ '-' + etcd_version if etcd_version is defined else '' }} state=present
   when: not etcd_is_containerized | bool
+  register: result
+  until: result | success
 
 - include_tasks: drop_etcdctl.yml
   when:
diff --git a/roles/etcd/tasks/upgrade/upgrade_image.yml b/roles/etcd/tasks/upgrade/upgrade_image.yml
index 24071f9ad..6e712ba74 100644
--- a/roles/etcd/tasks/upgrade/upgrade_image.yml
+++ b/roles/etcd/tasks/upgrade/upgrade_image.yml
@@ -44,6 +44,8 @@
     name: etcd
     state: latest
   when: not l_ostree_booted.stat.exists | bool
+  register: result
+  until: result | success
 
 - name: Verify cluster is healthy
   command: "{{ etcdctlv2 }} cluster-health"
diff --git a/roles/etcd/tasks/upgrade/upgrade_rpm.yml b/roles/etcd/tasks/upgrade/upgrade_rpm.yml
index 505e28afb..e98def46e 100644
--- a/roles/etcd/tasks/upgrade/upgrade_rpm.yml
+++ b/roles/etcd/tasks/upgrade/upgrade_rpm.yml
@@ -18,6 +18,8 @@
   package:
     name: "{{ l_etcd_target_package }}"
     state: latest
+  register: result
+  until: result | success
 
 - lineinfile:
     destfile: "{{ etcd_conf_file }}"
diff --git a/roles/flannel/tasks/main.yml b/roles/flannel/tasks/main.yml
index 3a8945a82..befe1b2e6 100644
--- a/roles/flannel/tasks/main.yml
+++ b/roles/flannel/tasks/main.yml
@@ -3,6 +3,8 @@
   become: yes
   package: name=flannel state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Set flannel etcd options
   become: yes
diff --git a/roles/nickhammond.logrotate/tasks/main.yml b/roles/nickhammond.logrotate/tasks/main.yml
index 657cb10ec..32d3acb86 100644
--- a/roles/nickhammond.logrotate/tasks/main.yml
+++ b/roles/nickhammond.logrotate/tasks/main.yml
@@ -2,6 +2,8 @@
 - name: nickhammond.logrotate | Install logrotate
   package: name=logrotate state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: nickhammond.logrotate | Setup logrotate.d scripts
   template:
diff --git a/roles/nuage_ca/tasks/main.yaml b/roles/nuage_ca/tasks/main.yaml
index 8d73e6840..46929fa1f 100644
--- a/roles/nuage_ca/tasks/main.yaml
+++ b/roles/nuage_ca/tasks/main.yaml
@@ -2,6 +2,8 @@
 - name: Install openssl
   package: name=openssl state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Create CA directory
   file: path="{{ nuage_ca_dir }}" state=directory
diff --git a/roles/openshift_ca/tasks/main.yml b/roles/openshift_ca/tasks/main.yml
index 31f0f8e7a..05e0a1352 100644
--- a/roles/openshift_ca/tasks/main.yml
+++ b/roles/openshift_ca/tasks/main.yml
@@ -13,6 +13,7 @@
     state: present
   when: not openshift.common.is_containerized | bool
   register: install_result
+  until: install_result | success
   delegate_to: "{{ openshift_ca_host }}"
   run_once: true
 
diff --git a/roles/openshift_cli/tasks/main.yml b/roles/openshift_cli/tasks/main.yml
index 6aa15d568..7b046b2c4 100644
--- a/roles/openshift_cli/tasks/main.yml
+++ b/roles/openshift_cli/tasks/main.yml
@@ -8,6 +8,8 @@
 - name: Install clients
   package: name={{ openshift.common.service_type }}-clients state=present
   when: not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
 
 - block:
   - name: Pull CLI Image
@@ -47,3 +49,5 @@
 - name: Install bash completion for oc tools
   package: name=bash-completion state=present
   when: not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
diff --git a/roles/openshift_clock/tasks/main.yaml b/roles/openshift_clock/tasks/main.yaml
index f8b02524a..82c73b583 100644
--- a/roles/openshift_clock/tasks/main.yaml
+++ b/roles/openshift_clock/tasks/main.yaml
@@ -9,6 +9,8 @@
   when:
     - openshift_clock_enabled | bool
     - chrony_installed.rc != 0
+  register: result
+  until: result | success
 
 - name: Start and enable ntpd/chronyd
   command: timedatectl set-ntp true
diff --git a/roles/openshift_excluder/tasks/install.yml b/roles/openshift_excluder/tasks/install.yml
index 7a5bebf6f..3ac55894f 100644
--- a/roles/openshift_excluder/tasks/install.yml
+++ b/roles/openshift_excluder/tasks/install.yml
@@ -13,6 +13,8 @@
     when:
     - r_openshift_excluder_enable_docker_excluder | bool
     - ansible_pkg_mgr == "yum"
+    register: result
+    until: result | success
 
 
   # For DNF we do not need the "*" and if we add it, it causes an error because
@@ -26,6 +28,8 @@
     when:
     - r_openshift_excluder_enable_docker_excluder | bool
     - ansible_pkg_mgr == "dnf"
+    register: result
+    until: result | success
 
   - name: Install openshift excluder - yum
     package:
@@ -34,6 +38,8 @@
     when:
     - r_openshift_excluder_enable_openshift_excluder | bool
     - ansible_pkg_mgr == "yum"
+    register: result
+    until: result | success
 
   # For DNF we do not need the "*" and if we add it, it causes an error because
   # it's not a valid pkg_spec
@@ -46,6 +52,8 @@
     when:
     - r_openshift_excluder_enable_openshift_excluder | bool
     - ansible_pkg_mgr == "dnf"
+    register: result
+    until: result | success
 
   - set_fact:
       r_openshift_excluder_install_ran: True
diff --git a/roles/openshift_expand_partition/tasks/main.yml b/roles/openshift_expand_partition/tasks/main.yml
index 4cb5418c6..b7acb0c5a 100644
--- a/roles/openshift_expand_partition/tasks/main.yml
+++ b/roles/openshift_expand_partition/tasks/main.yml
@@ -2,6 +2,8 @@
 - name: Ensure growpart is installed
   package: name=cloud-utils-growpart state=present
   when: not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
 
 - name: Determine if growpart is installed
   command: "rpm -q cloud-utils-growpart"
diff --git a/roles/openshift_loadbalancer/tasks/main.yml b/roles/openshift_loadbalancer/tasks/main.yml
index c87a327a4..79c5793d9 100644
--- a/roles/openshift_loadbalancer/tasks/main.yml
+++ b/roles/openshift_loadbalancer/tasks/main.yml
@@ -5,6 +5,8 @@
 - name: Install haproxy
   package: name=haproxy state=present
   when: not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
 
 - name: Pull haproxy image
   command: >
diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml
index d570a1c7f..e52cd6231 100644
--- a/roles/openshift_master/tasks/main.yml
+++ b/roles/openshift_master/tasks/main.yml
@@ -39,6 +39,8 @@
     state: present
   when:
   - not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
 
 - name: Create r_openshift_master_data_dir
   file:
@@ -88,6 +90,8 @@
   - item.kind == 'HTPasswdPasswordIdentityProvider'
   - not openshift.common.is_atomic | bool
   with_items: "{{ openshift.master.identity_providers }}"
+  register: result
+  until: result | success
 
 - name: Ensure htpasswd directory exists
   file:
@@ -306,6 +310,7 @@
   - openshift.master.cluster_method == 'pacemaker'
   - not openshift.common.is_containerized | bool
   register: l_install_result
+  until: l_install_result | success
 
 - name: Start and enable cluster service
   systemd:
diff --git a/roles/openshift_master/tasks/upgrade/rpm_upgrade.yml b/roles/openshift_master/tasks/upgrade/rpm_upgrade.yml
index f914a9978..caab3045a 100644
--- a/roles/openshift_master/tasks/upgrade/rpm_upgrade.yml
+++ b/roles/openshift_master/tasks/upgrade/rpm_upgrade.yml
@@ -18,3 +18,5 @@
       - "{{ openshift.common.service_type }}-sdn-ovs{{ openshift_pkg_version }}"
       - "{{ openshift.common.service_type }}-clients{{ openshift_pkg_version }}"
       - "tuned-profiles-{{ openshift.common.service_type }}-node{{ openshift_pkg_version }}"
+  register: result
+  until: result | success
diff --git a/roles/openshift_nfs/tasks/setup.yml b/roles/openshift_nfs/tasks/setup.yml
index 3070de495..edb854467 100644
--- a/roles/openshift_nfs/tasks/setup.yml
+++ b/roles/openshift_nfs/tasks/setup.yml
@@ -5,6 +5,8 @@
 
 - name: Install nfs-utils
   package: name=nfs-utils state=present
+  register: result
+  until: result | success
 
 - name: Configure NFS
   lineinfile:
diff --git a/roles/openshift_node/tasks/bootstrap.yml b/roles/openshift_node/tasks/bootstrap.yml
index ac43ef039..a042bc01b 100644
--- a/roles/openshift_node/tasks/bootstrap.yml
+++ b/roles/openshift_node/tasks/bootstrap.yml
@@ -4,6 +4,8 @@
     name: "{{ item }}"
     state: present
   with_items: "{{ r_openshift_node_image_prep_packages }}"
+  register: result
+  until: result | success
 
 - name: create the directory for node
   file:
diff --git a/roles/openshift_node/tasks/dnsmasq.yml b/roles/openshift_node/tasks/dnsmasq.yml
index 22bdce6c6..f210a3a21 100644
--- a/roles/openshift_node/tasks/dnsmasq.yml
+++ b/roles/openshift_node/tasks/dnsmasq.yml
@@ -13,6 +13,8 @@
 - name: Install dnsmasq
   package: name=dnsmasq state=installed
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: ensure origin/node directory exists
   file:
diff --git a/roles/openshift_node/tasks/dnsmasq/no-network-manager.yml b/roles/openshift_node/tasks/dnsmasq/no-network-manager.yml
index dede2fb8f..541c8115a 100644
--- a/roles/openshift_node/tasks/dnsmasq/no-network-manager.yml
+++ b/roles/openshift_node/tasks/dnsmasq/no-network-manager.yml
@@ -7,5 +7,7 @@
     name: NetworkManager
     state: present
   notify: restart NetworkManager
+  register: result
+  until: result | success
 
 - include_tasks: network-manager.yml
diff --git a/roles/openshift_node/tasks/docker/upgrade.yml b/roles/openshift_node/tasks/docker/upgrade.yml
index ebe87d6fd..d743d2188 100644
--- a/roles/openshift_node/tasks/docker/upgrade.yml
+++ b/roles/openshift_node/tasks/docker/upgrade.yml
@@ -36,5 +36,7 @@
 
 - name: Upgrade Docker
   package: name=docker{{ '-' + docker_version }} state=present
+  register: result
+  until: result | success
 
 # starting docker happens back in ../main.yml where it calls ../restart.yml
diff --git a/roles/openshift_node/tasks/install.yml b/roles/openshift_node/tasks/install.yml
index 9a91e2fb6..1ed4a05c1 100644
--- a/roles/openshift_node/tasks/install.yml
+++ b/roles/openshift_node/tasks/install.yml
@@ -5,6 +5,8 @@
     package:
       name: "{{ openshift.common.service_type }}-node{{ (openshift_pkg_version | default('')) | oo_image_tag_to_rpm_version(include_dash=True) }}"
       state: present
+    register: result
+    until: result | success
 
   - name: Install sdn-ovs package
     package:
@@ -12,11 +14,15 @@
       state: present
     when:
     - openshift_node_use_openshift_sdn | bool
+    register: result
+    until: result | success
 
   - name: Install conntrack-tools package
     package:
       name: "conntrack-tools"
       state: present
+    register: result
+    until: result | success
 
 - when:
   - openshift.common.is_containerized | bool
diff --git a/roles/openshift_node/tasks/storage_plugins/ceph.yml b/roles/openshift_node/tasks/storage_plugins/ceph.yml
index 037efe81a..72a3b837f 100644
--- a/roles/openshift_node/tasks/storage_plugins/ceph.yml
+++ b/roles/openshift_node/tasks/storage_plugins/ceph.yml
@@ -2,3 +2,5 @@
 - name: Install Ceph storage plugin dependencies
   package: name=ceph-common state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
diff --git a/roles/openshift_node/tasks/storage_plugins/glusterfs.yml b/roles/openshift_node/tasks/storage_plugins/glusterfs.yml
index 1b8a7ad50..08ea71a0c 100644
--- a/roles/openshift_node/tasks/storage_plugins/glusterfs.yml
+++ b/roles/openshift_node/tasks/storage_plugins/glusterfs.yml
@@ -2,6 +2,8 @@
 - name: Install GlusterFS storage plugin dependencies
   package: name=glusterfs-fuse state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Check for existence of fusefs sebooleans
   command: getsebool {{ item }}
diff --git a/roles/openshift_node/tasks/storage_plugins/iscsi.yml b/roles/openshift_node/tasks/storage_plugins/iscsi.yml
index 1c5478c55..ece68dc71 100644
--- a/roles/openshift_node/tasks/storage_plugins/iscsi.yml
+++ b/roles/openshift_node/tasks/storage_plugins/iscsi.yml
@@ -2,3 +2,5 @@
 - name: Install iSCSI storage plugin dependencies
   package: name=iscsi-initiator-utils state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
diff --git a/roles/openshift_node/tasks/storage_plugins/nfs.yml b/roles/openshift_node/tasks/storage_plugins/nfs.yml
index 7e1035893..5eacf42e8 100644
--- a/roles/openshift_node/tasks/storage_plugins/nfs.yml
+++ b/roles/openshift_node/tasks/storage_plugins/nfs.yml
@@ -2,6 +2,8 @@
 - name: Install NFS storage plugin dependencies
   package: name=nfs-utils state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Check for existence of nfs sebooleans
   command: getsebool {{ item }}
diff --git a/roles/openshift_node/tasks/upgrade.yml b/roles/openshift_node/tasks/upgrade.yml
index ff3478800..561b56918 100644
--- a/roles/openshift_node/tasks/upgrade.yml
+++ b/roles/openshift_node/tasks/upgrade.yml
@@ -107,6 +107,8 @@
     name: openvswitch
     state: latest
   when: not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
 
 - name: Update oreg value
   yedit:
diff --git a/roles/openshift_node/tasks/upgrade/rpm_upgrade.yml b/roles/openshift_node/tasks/upgrade/rpm_upgrade.yml
index a998acf21..fcbe1a598 100644
--- a/roles/openshift_node/tasks/upgrade/rpm_upgrade.yml
+++ b/roles/openshift_node/tasks/upgrade/rpm_upgrade.yml
@@ -8,10 +8,14 @@
 # We verified latest rpm available is suitable, so just yum update.
 - name: Upgrade packages
   package: "name={{ openshift.common.service_type }}-{{ component }}{{ openshift_pkg_version }} state=present"
+  register: result
+  until: result | success
 
 - name: Ensure python-yaml present for config upgrade
   package: name=PyYAML state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Install Node service file
   template:
diff --git a/roles/openshift_openstack/tasks/node-packages.yml b/roles/openshift_openstack/tasks/node-packages.yml
index 7864f5269..e41104af1 100644
--- a/roles/openshift_openstack/tasks/node-packages.yml
+++ b/roles/openshift_openstack/tasks/node-packages.yml
@@ -6,6 +6,8 @@
     name: "{{ item }}"
     state: latest
   with_items: "{{ openshift_openstack_required_packages }}"
+  register: result
+  until: result | success
 
 - name: Install debug packages (optional)
   yum:
@@ -13,3 +15,5 @@
     state: latest
   with_items: "{{ openshift_openstack_debug_packages }}"
   when: openshift_openstack_install_debug_packages|bool
+  register: result
+  until: result | success
diff --git a/roles/openshift_repos/tasks/main.yaml b/roles/openshift_repos/tasks/main.yaml
index 552a22a0f..5e7bde1e1 100644
--- a/roles/openshift_repos/tasks/main.yaml
+++ b/roles/openshift_repos/tasks/main.yaml
@@ -9,6 +9,8 @@
   # TODO: This needs to be removed and placed into a role
   - name: Ensure libselinux-python is installed
     package: name=libselinux-python state=present
+    register: result
+    until: result | success
 
   - name: Remove openshift_additional.repo file
     file:
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
index 2a678af57..4b33e91b4 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
@@ -4,6 +4,8 @@
   when:
   - not openshift.common.is_atomic | bool
   - not glusterfs_heketi_is_native | bool
+  register: result
+  until: result | success
 
 - name: Verify heketi-cli is installed
   shell: "command -v {{ glusterfs_heketi_cli }} >/dev/null 2>&1 || { echo >&2 'ERROR: Make sure heketi-cli is available, then re-run the installer'; exit 1; }"
diff --git a/roles/openshift_storage_nfs/tasks/main.yml b/roles/openshift_storage_nfs/tasks/main.yml
index 24264fa43..c25cad74c 100644
--- a/roles/openshift_storage_nfs/tasks/main.yml
+++ b/roles/openshift_storage_nfs/tasks/main.yml
@@ -4,6 +4,8 @@
 
 - name: Install nfs-utils
   package: name=nfs-utils state=present
+  register: result
+  until: result | success
 
 - name: Configure NFS
   lineinfile:
diff --git a/roles/openshift_storage_nfs_lvm/tasks/nfs.yml b/roles/openshift_storage_nfs_lvm/tasks/nfs.yml
index 03f4fcec0..bee786a90 100644
--- a/roles/openshift_storage_nfs_lvm/tasks/nfs.yml
+++ b/roles/openshift_storage_nfs_lvm/tasks/nfs.yml
@@ -2,6 +2,8 @@
 - name: Install NFS server
   package: name=nfs-utils state=present
   when: not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
 
 - name: Start rpcbind
   systemd:
diff --git a/roles/os_firewall/tasks/firewalld.yml b/roles/os_firewall/tasks/firewalld.yml
index 54430f402..1e27ebaf9 100644
--- a/roles/os_firewall/tasks/firewalld.yml
+++ b/roles/os_firewall/tasks/firewalld.yml
@@ -8,6 +8,8 @@
   package:
     name: firewalld
     state: present
+  register: result
+  until: result | success
 
 - name: Ensure iptables services are not enabled
   systemd:
diff --git a/roles/os_firewall/tasks/iptables.yml b/roles/os_firewall/tasks/iptables.yml
index 2d74f2e48..a7c13e487 100644
--- a/roles/os_firewall/tasks/iptables.yml
+++ b/roles/os_firewall/tasks/iptables.yml
@@ -22,6 +22,8 @@
     - iptables
     - iptables-services
   when: not r_os_firewall_is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Start and enable iptables service
   systemd:
diff --git a/roles/os_update_latest/tasks/main.yml b/roles/os_update_latest/tasks/main.yml
index 6b5fd0106..60d665587 100644
--- a/roles/os_update_latest/tasks/main.yml
+++ b/roles/os_update_latest/tasks/main.yml
@@ -1,3 +1,5 @@
 ---
 - name: Update all packages
   package: name=* state=latest
+  register: result
+  until: result | success
diff --git a/roles/rhel_subscribe/tasks/main.yml b/roles/rhel_subscribe/tasks/main.yml
index b06f51908..9ca49b569 100644
--- a/roles/rhel_subscribe/tasks/main.yml
+++ b/roles/rhel_subscribe/tasks/main.yml
@@ -36,6 +36,8 @@
   yum:
     name: subscription-manager
     state: present
+  register: result
+  until: result | success
 
 - name: RedHat subscriptions
   redhat_subscription: