Make rollout status check best-effort, add poll

We cannot rely on the `watch.Until` call in the `rollout status`
subcommand for the time being, so we need to ignore the result of this
call. This will make the rollout status check best-effort, so we need to
follow it with a poll for the actual status of the rollout, which we can
extract from the `openshift.io/deployment.phase` annotation on the
ReplicationControllers. This annotation can have only three values --
`Running`, `Complete` and `Failed`. If we poll on this attribute until
we stop seeing `Running`, we can then inspect the last result for
`Failed`; if it's present, we have failed the deployment.

Signed-off-by: Steve Kuznetsov <skuznets@redhat.com>

2 files changed, 52 insertions, 2 deletions

diff --git a/roles/openshift_hosted/tasks/registry/registry.yml b/roles/openshift_hosted/tasks/registry/registry.yml
index 7945e16ca..d895e9a68 100644
--- a/roles/openshift_hosted/tasks/registry/registry.yml
+++ b/roles/openshift_hosted/tasks/registry/registry.yml
@@ -124,11 +124,34 @@
     edits: "{{ openshift_hosted_registry_edits }}"
     force: "{{ True|bool in openshift_hosted_registry_force }}"
 
-- name: Ensure OpenShift registry correctly rolls out
+- name: Ensure OpenShift registry correctly rolls out (best-effort today)
   command: |
     oc rollout status deploymentconfig {{ openshift_hosted_registry_name }} \
                       --namespace {{ openshift_hosted_registry_namespace }} \
                       --config {{ openshift.common.config_base }}/master/admin.kubeconfig
+  async: 600
+  poll: 15
+  failed_when: false
+
+- name: Determine the latest version of the OpenShift registry deployment
+  command: |
+    oc get deploymentconfig {{ openshift_hosted_registry_name }} \
+           --namespace {{ openshift_hosted_registry_namespace }} \
+           --config {{ openshift.common.config_base }}/master/admin.kubeconfig \
+           -o jsonpath='{ .status.latestVersion }'
+  register: openshift_hosted_registry_latest_version
+
+- name: Sanity-check that the OpenShift registry rolled out correctly
+  command: |
+    oc get replicationcontroller {{ openshift_hosted_registry_name }}-{{ openshift_hosted_registry_latest_version.stdout }} \
+           --namespace {{ openshift_hosted_registry_namespace }} \
+           --config {{ openshift.common.config_base }}/master/admin.kubeconfig \
+           -o jsonpath='{ .metadata.annotations.openshift\.io/deployment\.phase }'
+  register: openshift_hosted_registry_rc_phase
+  until: "'Running' not in openshift_hosted_registry_rc_phase.stdout"
+  delay: 15
+  retries: 40
+  failed_when: "'Failed' in openshift_hosted_registry_rc_phase.stdout"
 
 - include: storage/glusterfs.yml
   when:
diff --git a/roles/openshift_hosted/tasks/router/router.yml b/roles/openshift_hosted/tasks/router/router.yml
index 3382abd7b..160ae2f5e 100644
--- a/roles/openshift_hosted/tasks/router/router.yml
+++ b/roles/openshift_hosted/tasks/router/router.yml
@@ -90,9 +90,36 @@
     stats_port: "{{ item.stats_port }}"
   with_items: "{{ openshift_hosted_routers }}"
 
-- name: Ensure OpenShift router correctly rolls out
+- name: Ensure OpenShift router correctly rolls out (best-effort today)
   command: |
     oc rollout status deploymentconfig {{ item.name }} \
                       --namespace {{ item.namespace | default('default') }} \
                       --config {{ openshift.common.config_base }}/master/admin.kubeconfig
+  async: 600
+  poll: 15
   with_items: "{{ openshift_hosted_routers }}"
+  failed_when: false
+
+- name: Determine the latest version of the OpenShift router deployment
+  command: |
+    oc get deploymentconfig {{ item.name }} \
+           --namespace {{ item.namespace }} \
+           --config {{ openshift.common.config_base }}/master/admin.kubeconfig \
+           -o jsonpath='{ .status.latestVersion }'
+  register: openshift_hosted_routers_latest_version
+  with_items: "{{ openshift_hosted_routers }}"
+
+- name: Poll for OpenShift router deployment success
+  command: |
+    oc get replicationcontroller {{ item.0.name }}-{{ item.1.stdout }} \
+           --namespace {{ item.0.namespace }} \
+           --config {{ openshift.common.config_base }}/master/admin.kubeconfig \
+           -o jsonpath='{ .metadata.annotations.openshift\.io/deployment\.phase }'
+  register: openshift_hosted_router_rc_phase
+  until: "'Running' not in openshift_hosted_router_rc_phase.stdout"
+  delay: 15
+  retries: 40
+  failed_when: "'Failed' in openshift_hosted_router_rc_phase.stdout"
+  with_together:
+  - "{{ openshift_hosted_routers }}"
+  - "{{ openshift_hosted_routers_latest_version.results }}"