Merge pull request #5948 from mtnbikenc/checkpoint-times

Automatic merge from submit-queue.

Add execution times to checkpoint status

This PR adds execution times to the checkpoint status report at the end of an installer playbook run.

- Checkpoint plays now use `all` host group to avoid using an undefined host group during the Initialization phase as well as maintaining the need to collect inventory vars for host group conditionals
- `run_once: true` is used to limit the execution to only one host
- `installer_phase_*` data is now stored as a dict so multiple data points can be stored for that phase
- Documentation updated


```
INSTALLER STATUS ***************************************************************
Initialization             : Complete (0:02:14)
Health Check               : Complete (0:01:10)
etcd Install               : Complete (0:02:01)
Master Install             : Complete (0:11:43)
Master Additional Install  : Complete (0:00:54)
Node Install               : Complete (0:14:11)
Hosted Install             : Complete (0:03:28)
```

2 files changed, 41 insertions, 85 deletions

diff --git a/roles/installer_checkpoint/README.md b/roles/installer_checkpoint/README.md
index 83e00e504..18d182ec9 100644
--- a/roles/installer_checkpoint/README.md
+++ b/roles/installer_checkpoint/README.md
@@ -92,31 +92,34 @@ phase/component and then a final play for setting `installer_hase_initialize` to
 # common/openshift-cluster/std_include.yml
 ---
 - name: Initialization Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   roles:
   - installer_checkpoint
   tasks:
   - name: Set install initialization 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_initialize: "In Progress"
-      aggregate: false
+        installer_phase_initialize:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 #...
 # Various plays here
 #...
 
 - name: Initialization Checkpoint End
-  hosts: localhost
-  connection: local
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set install initialization 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_initialize: "Complete"
-      aggregate: false
+        installer_phase_initialize:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 ``` 
 
 Each phase or component of the installer will follow a similar pattern, with the
@@ -139,37 +142,25 @@ localhost            : ok=24   changed=0    unreachable=0    failed=0
 
 
 INSTALLER STATUS ***************************************************************
-Initialization             : Complete
-etcd Install               : Complete
-NFS Install                : Not Started
-Load balancer Install      : Not Started
-Master Install             : Complete
-Master Additional Install  : Complete
-Node Install               : Complete
-GlusterFS Install          : Not Started
-Hosted Install             : Complete
-Metrics Install            : Not Started
-Logging Install            : Not Started
-Service Catalog Install    : Not Started
+Initialization             : Complete (0:02:14)
+Health Check               : Complete (0:01:10)
+etcd Install               : Complete (0:02:01)
+Master Install             : Complete (0:11:43)
+Master Additional Install  : Complete (0:00:54)
+Node Install               : Complete (0:14:11)
+Hosted Install             : Complete (0:03:28)
 ```
 
 Example display if a failure occurs during execution:
 
 ```
 INSTALLER STATUS ***************************************************************
-Initialization             : Complete
-etcd Install               : Complete
-NFS Install                : Not Started
-Load balancer Install      : Not Started
-Master Install             : In Progress
-     This phase can be restarted by running: playbooks/byo/openshift-master/config.yml
-Master Additional Install  : Not Started
-Node Install               : Not Started
-GlusterFS Install          : Not Started
-Hosted Install             : Not Started
-Metrics Install            : Not Started
-Logging Install            : Not Started
-Service Catalog Install    : Not Started
+Initialization             : Complete (0:02:14)
+Health Check               : Complete (0:01:10)
+etcd Install               : Complete (0:02:58)
+Master Install             : Complete (0:09:20)
+Master Additional Install  : In Progress (0:20:04)
+    This phase can be restarted by running: playbooks/byo/openshift-master/additional_config.yml
 ```
 
 [set_stats]: http://docs.ansible.com/ansible/latest/set_stats_module.html
diff --git a/roles/installer_checkpoint/callback_plugins/installer_checkpoint.py b/roles/installer_checkpoint/callback_plugins/installer_checkpoint.py
index b17358882..556e9127f 100644
--- a/roles/installer_checkpoint/callback_plugins/installer_checkpoint.py
+++ b/roles/installer_checkpoint/callback_plugins/installer_checkpoint.py
@@ -1,60 +1,10 @@
 """Ansible callback plugin to print a summary completion status of installation
 phases.
 """
+from datetime import datetime
 from ansible.plugins.callback import CallbackBase
 from ansible import constants as C
 
-DOCUMENTATION = '''
-
-'''
-
-EXAMPLES = '''
----------------------------------------------
-Example display of a successful playbook run:
-
-PLAY RECAP *********************************************************************
-master01.example.com : ok=158  changed=16   unreachable=0    failed=0
-node01.example.com   : ok=469  changed=74   unreachable=0    failed=0
-node02.example.com   : ok=157  changed=17   unreachable=0    failed=0
-localhost            : ok=24   changed=0    unreachable=0    failed=0
-
-
-INSTALLER STATUS ***************************************************************
-Initialization             : Complete
-etcd Install               : Complete
-NFS Install                : Not Started
-Load balancer Install      : Not Started
-Master Install             : Complete
-Master Additional Install  : Complete
-Node Install               : Complete
-GlusterFS Install          : Not Started
-Hosted Install             : Complete
-Metrics Install            : Not Started
-Logging Install            : Not Started
-Prometheus Install         : Not Started
-Service Catalog Install    : Not Started
-
------------------------------------------------------
-Example display if a failure occurs during execution:
-
-INSTALLER STATUS ***************************************************************
-Initialization             : Complete
-etcd Install               : Complete
-NFS Install                : Not Started
-Load balancer Install      : Not Started
-Master Install             : In Progress
-     This phase can be restarted by running: playbooks/byo/openshift-master/config.yml
-Master Additional Install  : Not Started
-Node Install               : Not Started
-GlusterFS Install          : Not Started
-Hosted Install             : Not Started
-Metrics Install            : Not Started
-Logging Install            : Not Started
-Prometheus Install         : Not Started
-Service Catalog Install    : Not Started
-
-'''
-
 
 class CallbackModule(CallbackBase):
     """This callback summarizes installation phase status."""
@@ -163,9 +113,10 @@ class CallbackModule(CallbackBase):
                 phase_title = phase_attributes[phase]['title']
                 padding = max_column - len(phase_title) + 2
                 if phase in stats.custom['_run']:
-                    phase_status = stats.custom['_run'][phase]
+                    phase_status = stats.custom['_run'][phase]['status']
+                    phase_time = phase_time_delta(stats.custom['_run'][phase])
                     self._display.display(
-                        '{}{}: {}'.format(phase_title, ' ' * padding, phase_status),
+                        '{}{}: {} ({})'.format(phase_title, ' ' * padding, phase_status, phase_time),
                         color=self.phase_color(phase_status))
                     if phase_status == 'In Progress' and phase != 'installer_phase_initialize':
                         self._display.display(
@@ -192,3 +143,17 @@ class CallbackModule(CallbackBase):
             phase_color = C.COLOR_WARN
 
         return phase_color
+
+
+def phase_time_delta(phase):
+    """ Calculate the difference between phase start and end times """
+    time_format = '%Y%m%d%H%M%SZ'
+    phase_start = datetime.strptime(phase['start'], time_format)
+    if 'end' not in phase:
+        # The phase failed so set the end time to now
+        phase_end = datetime.now()
+    else:
+        phase_end = datetime.strptime(phase['end'], time_format)
+    delta = str(phase_end - phase_start).split(".")[0]  # Trim microseconds
+
+    return delta