diff options
Diffstat (limited to 'playbooks')
7 files changed, 155 insertions, 105 deletions
diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml index 13e1da961..5d3280328 100644 --- a/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml +++ b/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml @@ -22,12 +22,24 @@ hosts: oo_masters_to_config:oo_nodes_to_upgrade:oo_etcd_to_config serial: 1 any_errors_fatal: true + + roles: + - lib_openshift + tasks: - - name: Prepare for Node draining - command: > - {{ openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename }} --schedulable=false + - name: Mark node unschedulable + oadm_manage_node: + node: "{{ openshift.node.nodename | lower }}" + schedulable: False delegate_to: "{{ groups.oo_first_master.0 }}" - when: l_docker_upgrade is defined and l_docker_upgrade | bool and inventory_hostname in groups.oo_nodes_to_upgrade + retries: 10 + delay: 5 + register: node_unschedulable + until: node_unschedulable|succeeded + when: + - l_docker_upgrade is defined + - l_docker_upgrade | bool + - inventory_hostname in groups.oo_nodes_to_upgrade - name: Drain Node for Kubelet upgrade command: > @@ -39,7 +51,12 @@ when: l_docker_upgrade is defined and l_docker_upgrade | bool - name: Set node schedulability - command: > - {{ openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename }} --schedulable=true + oadm_manage_node: + node: "{{ openshift.node.nodename | lower }}" + schedulable: True delegate_to: "{{ groups.oo_first_master.0 }}" - when: l_docker_upgrade is defined and l_docker_upgrade | bool and inventory_hostname in groups.oo_nodes_to_upgrade and openshift.node.schedulable | bool + retries: 10 + delay: 5 + register: node_schedulable + until: node_schedulable|succeeded + when: node_unschedulable|changed diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml index 45aabf3e4..7ef79afa9 100644 --- a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml +++ b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml @@ -29,12 +29,18 @@ - name: Check available disk space for etcd backup shell: df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1 register: avail_disk + # AUDIT:changed_when: `false` because we are only inspecting + # state, not manipulating anything + changed_when: false # TODO: replace shell module with command and update later checks - name: Check current embedded etcd disk usage shell: du -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1 | cut -f1 register: etcd_disk_usage when: embedded_etcd | bool + # AUDIT:changed_when: `false` because we are only inspecting + # state, not manipulating anything + changed_when: false - name: Abort if insufficient disk space for etcd backup fail: diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml index 690858c53..a9b5b94e6 100644 --- a/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml +++ b/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml @@ -9,21 +9,36 @@ register: etcd_rpm_version failed_when: false when: not openshift.common.is_containerized | bool + # AUDIT:changed_when: `false` because we are only inspecting + # state, not manipulating anything + changed_when: false + - name: Record containerized etcd version command: docker exec etcd_container rpm -qa --qf '%{version}' etcd\* register: etcd_container_version failed_when: false when: openshift.common.is_containerized | bool + # AUDIT:changed_when: `false` because we are only inspecting + # state, not manipulating anything + changed_when: false + - name: Record containerized etcd version command: docker exec etcd_container rpm -qa --qf '%{version}' etcd\* register: etcd_container_version failed_when: false when: openshift.common.is_containerized | bool and not openshift.common.is_etcd_system_container | bool + # AUDIT:changed_when: `false` because we are only inspecting + # state, not manipulating anything + changed_when: false + - name: Record containerized etcd version command: runc exec etcd_container rpm -qa --qf '%{version}' etcd\* register: etcd_container_version failed_when: false when: openshift.common.is_containerized | bool and openshift.common.is_etcd_system_container | bool + # AUDIT:changed_when: `false` because we are only inspecting + # state, not manipulating anything + changed_when: false # I really dislike this copy/pasta but I wasn't able to find a way to get it to loop # through hosts, then loop through tasks only when appropriate diff --git a/playbooks/common/openshift-cluster/upgrades/initialize_nodes_to_upgrade.yml b/playbooks/common/openshift-cluster/upgrades/initialize_nodes_to_upgrade.yml index 37c89374c..046535680 100644 --- a/playbooks/common/openshift-cluster/upgrades/initialize_nodes_to_upgrade.yml +++ b/playbooks/common/openshift-cluster/upgrades/initialize_nodes_to_upgrade.yml @@ -1,20 +1,17 @@ --- - name: Filter list of nodes to be upgraded if necessary hosts: oo_first_master + + roles: + - lib_openshift + tasks: - name: Retrieve list of openshift nodes matching upgrade label - command: > - {{ openshift.common.client_binary }} - get nodes - --config={{ openshift.common.config_base }}/master/admin.kubeconfig - --selector={{ openshift_upgrade_nodes_label }} - -o jsonpath='{.items[*].metadata.name}' - register: matching_nodes - changed_when: false - when: openshift_upgrade_nodes_label is defined - - - set_fact: - nodes_to_upgrade: "{{ matching_nodes.stdout.split(' ') }}" + oc_obj: + state: list + kind: node + selector: "{{ openshift_upgrade_nodes_label }}" + register: nodes_to_upgrade when: openshift_upgrade_nodes_label is defined # We got a list of nodes with the label, now we need to match these with inventory hosts @@ -26,7 +23,9 @@ ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" ansible_become: "{{ g_sudo | default(omit) }}" with_items: " {{ groups['oo_nodes_to_config'] }}" - when: openshift_upgrade_nodes_label is defined and hostvars[item].openshift.common.hostname in nodes_to_upgrade + when: + - openshift_upgrade_nodes_label is defined + - hostvars[item].openshift.common.hostname in nodes_to_upgrade.results.results[0]['items'] | map(attribute='metadata.name') | list changed_when: false # Build up the oo_nodes_to_upgrade group, use the list filtered by label if diff --git a/playbooks/common/openshift-cluster/upgrades/post_control_plane.yml b/playbooks/common/openshift-cluster/upgrades/post_control_plane.yml index 4135f7e94..f0191e380 100644 --- a/playbooks/common/openshift-cluster/upgrades/post_control_plane.yml +++ b/playbooks/common/openshift-cluster/upgrades/post_control_plane.yml @@ -9,77 +9,100 @@ registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', openshift_image_tag ) }}" router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', openshift_image_tag ) }}" oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig" - roles: - - openshift_manageiq - # Create the new templates shipped in 3.2, existing templates are left - # unmodified. This prevents the subsequent role definition for - # openshift_examples from failing when trying to replace templates that do - # not already exist. We could have potentially done a replace --force to - # create and update in one step. - - openshift_examples - - openshift_hosted_templates - # Update the existing templates - - role: openshift_examples - registry_url: "{{ openshift.master.registry_url }}" - openshift_examples_import_command: replace - - role: openshift_hosted_templates - registry_url: "{{ openshift.master.registry_url }}" - openshift_hosted_templates_import_command: replace - pre_tasks: + pre_tasks: + - name: Load lib_openshift modules + include_role: + name: lib_openshift # TODO: remove temp_skip_router_registry_upgrade variable. This is a short term hack # to allow ops to use this control plane upgrade, without triggering router/registry # upgrade which has not yet been synced with their process. - name: Collect all routers - command: > - {{ oc_cmd }} get pods --all-namespaces -l 'router' -o json + oc_obj: + state: list + kind: pods + all_namespaces: True + selector: 'router' register: all_routers - failed_when: false - changed_when: false when: temp_skip_router_registry_upgrade is not defined - - set_fact: haproxy_routers="{{ (all_routers.stdout | from_json)['items'] | oo_pods_match_component(openshift_deployment_type, 'haproxy-router') | oo_select_keys_from_list(['metadata']) }}" - when: all_routers.rc == 0 and temp_skip_router_registry_upgrade is not defined + - set_fact: haproxy_routers="{{ (all_routers.reults.results[0]['items'] | oo_pods_match_component(openshift_deployment_type, 'haproxy-router') | oo_select_keys_from_list(['metadata']) }}" + when: + - all_routers.results.returncode == 0 + - temp_skip_router_registry_upgrade is not defined - set_fact: haproxy_routers=[] - when: all_routers.rc != 0 and temp_skip_router_registry_upgrade is not defined + when: + - all_routers.results.returncode != 0 + - temp_skip_router_registry_upgrade is not defined - name: Update router image to current version - when: all_routers.rc == 0 and temp_skip_router_registry_upgrade is not defined + when: + - all_routers.results.returncode == 0 + - temp_skip_router_registry_upgrade is not defined command: > {{ oc_cmd }} patch dc/{{ item['labels']['deploymentconfig'] }} -n {{ item['namespace'] }} -p '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}","livenessProbe":{"tcpSocket":null,"httpGet":{"path": "/healthz", "port": 1936, "host": "localhost", "scheme": "HTTP"},"initialDelaySeconds":10,"timeoutSeconds":1}}]}}}}' --api-version=v1 with_items: "{{ haproxy_routers }}" + # AUDIT:changed_when_note: `false` not being set here. What we + # need to do is check the current router image version and see if + # this task needs to be ran. - name: Check for default registry - command: > - {{ oc_cmd }} get -n default dc/docker-registry + oc_obj: + state: list + kind: dc + name: docker-registry register: _default_registry - failed_when: false - changed_when: false when: temp_skip_router_registry_upgrade is not defined - name: Update registry image to current version - when: _default_registry.rc == 0 and temp_skip_router_registry_upgrade is not defined + when: + - _default_registry.results.results[0] != {} + - temp_skip_router_registry_upgrade is not defined command: > {{ oc_cmd }} patch dc/docker-registry -n default -p '{"spec":{"template":{"spec":{"containers":[{"name":"registry","image":"{{ registry_image }}"}]}}}}' --api-version=v1 + # AUDIT:changed_when_note: `false` not being set here. What we + # need to do is check the current registry image version and see + # if this task needs to be ran. + + roles: + - openshift_manageiq + # Create the new templates shipped in 3.2, existing templates are left + # unmodified. This prevents the subsequent role definition for + # openshift_examples from failing when trying to replace templates that do + # not already exist. We could have potentially done a replace --force to + # create and update in one step. + - openshift_examples + - openshift_hosted_templates + # Update the existing templates + - role: openshift_examples + registry_url: "{{ openshift.master.registry_url }}" + openshift_examples_import_command: replace + - role: openshift_hosted_templates + registry_url: "{{ openshift.master.registry_url }}" + openshift_hosted_templates_import_command: replace # Check for warnings to be printed at the end of the upgrade: - name: Check for warnings hosts: oo_masters_to_config tasks: # Check if any masters are using pluginOrderOverride and warn if so, only for 1.3/3.3 and beyond: - - command: > - grep pluginOrderOverride {{ openshift.common.config_base }}/master/master-config.yaml + - name: grep pluginOrderOverride + command: grep pluginOrderOverride {{ openshift.common.config_base }}/master/master-config.yaml register: grep_plugin_order_override when: openshift.common.version_gte_3_3_or_1_3 | bool - failed_when: false + changed_when: false + - name: Warn if pluginOrderOverride is in use in master-config.yaml - debug: msg="WARNING pluginOrderOverride is being deprecated in master-config.yaml, please see https://docs.openshift.com/enterprise/latest/architecture/additional_concepts/admission_controllers.html for more information." - when: not grep_plugin_order_override | skipped and grep_plugin_order_override.rc == 0 + debug: + msg: "WARNING pluginOrderOverride is being deprecated in master-config.yaml, please see https://docs.openshift.com/enterprise/latest/architecture/additional_concepts/admission_controllers.html for more information." + when: + - not grep_plugin_order_override | skipped + - grep_plugin_order_override.rc == 0 - include: ../reset_excluder.yml tags: diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml index db2c27919..a4aefcdac 100644 --- a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml +++ b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml @@ -238,29 +238,22 @@ any_errors_fatal: true pre_tasks: + - name: Load lib_openshift modules + include_role: + name: lib_openshift + # TODO: To better handle re-trying failed upgrades, it would be nice to check if the node # or docker actually needs an upgrade before proceeding. Perhaps best to save this until # we merge upgrade functionality into the base roles and a normal config.yml playbook run. - - name: Determine if node is currently scheduleable - command: > - {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} get node {{ openshift.node.nodename | lower }} -o json - register: node_output - delegate_to: "{{ groups.oo_first_master.0 }}" - changed_when: false - - - set_fact: - was_schedulable: "{{ 'unschedulable' not in (node_output.stdout | from_json).spec }}" - - name: Mark node unschedulable - command: > - {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=false + oadm_manage_node: + node: "{{ openshift.node.nodename | lower }}" + schedulable: False delegate_to: "{{ groups.oo_first_master.0 }}" - # NOTE: There is a transient "object has been modified" error here, allow a couple - # retries for a more reliable upgrade. - register: node_unsched - until: node_unsched.rc == 0 - retries: 3 - delay: 1 + retries: 10 + delay: 5 + register: node_unschedulable + until: node_unschedulable|succeeded - name: Drain Node for Kubelet upgrade command: > @@ -268,17 +261,19 @@ delegate_to: "{{ groups.oo_first_master.0 }}" roles: + - lib_openshift - openshift_facts - docker - openshift_node_upgrade post_tasks: - name: Set node schedulability - command: > - {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=true + oadm_manage_node: + node: "{{ openshift.node.nodename | lower }}" + schedulable: True delegate_to: "{{ groups.oo_first_master.0 }}" - when: was_schedulable | bool - register: node_sched - until: node_sched.rc == 0 - retries: 3 - delay: 1 + retries: 10 + delay: 5 + register: node_schedulable + until: node_schedulable|succeeded + when: node_unschedulable|changed diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml index e45b635f7..e3a98fd9b 100644 --- a/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml +++ b/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml @@ -7,29 +7,22 @@ any_errors_fatal: true pre_tasks: + - name: Load lib_openshift modules + include_role: + name: lib_openshift + # TODO: To better handle re-trying failed upgrades, it would be nice to check if the node # or docker actually needs an upgrade before proceeding. Perhaps best to save this until # we merge upgrade functionality into the base roles and a normal config.yml playbook run. - - name: Determine if node is currently scheduleable - command: > - {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} get node {{ openshift.node.nodename | lower }} -o json - register: node_output - delegate_to: "{{ groups.oo_first_master.0 }}" - changed_when: false - - - set_fact: - was_schedulable: "{{ 'unschedulable' not in (node_output.stdout | from_json).spec }}" - - name: Mark node unschedulable - command: > - {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=false + oadm_manage_node: + node: "{{ openshift.node.nodename | lower }}" + schedulable: False delegate_to: "{{ groups.oo_first_master.0 }}" - # NOTE: There is a transient "object has been modified" error here, allow a couple - # retries for a more reliable upgrade. - register: node_unsched - until: node_unsched.rc == 0 - retries: 3 - delay: 1 + retries: 10 + delay: 5 + register: node_unschedulable + until: node_unschedulable|succeeded - name: Drain Node for Kubelet upgrade command: > @@ -37,20 +30,22 @@ delegate_to: "{{ groups.oo_first_master.0 }}" roles: + - lib_openshift - openshift_facts - docker - openshift_node_upgrade post_tasks: - name: Set node schedulability - command: > - {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=true + oadm_manage_node: + node: "{{ openshift.node.nodename | lower }}" + schedulable: True delegate_to: "{{ groups.oo_first_master.0 }}" - when: was_schedulable | bool - register: node_sched - until: node_sched.rc == 0 - retries: 3 - delay: 1 + retries: 10 + delay: 5 + register: node_schedulable + until: node_schedulable|succeeded + when: node_unschedulable|changed - include: ../reset_excluder.yml tags: |