diff options
Diffstat (limited to 'playbooks')
27 files changed, 605 insertions, 415 deletions
diff --git a/playbooks/adhoc/upgrades/filter_plugins b/playbooks/adhoc/upgrades/filter_plugins deleted file mode 120000 index b0b7a3414..000000000 --- a/playbooks/adhoc/upgrades/filter_plugins +++ /dev/null @@ -1 +0,0 @@ -../../../filter_plugins/
\ No newline at end of file diff --git a/playbooks/adhoc/upgrades/lookup_plugins b/playbooks/adhoc/upgrades/lookup_plugins deleted file mode 120000 index 73cafffe5..000000000 --- a/playbooks/adhoc/upgrades/lookup_plugins +++ /dev/null @@ -1 +0,0 @@ -../../../lookup_plugins/
\ No newline at end of file diff --git a/playbooks/adhoc/upgrades/roles b/playbooks/adhoc/upgrades/roles deleted file mode 120000 index e2b799b9d..000000000 --- a/playbooks/adhoc/upgrades/roles +++ /dev/null @@ -1 +0,0 @@ -../../../roles/
\ No newline at end of file diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml deleted file mode 100644 index 324f5fba3..000000000 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ /dev/null @@ -1,407 +0,0 @@ ---- -- name: Load master facts - hosts: masters - roles: - - openshift_facts - -- name: Verify upgrade can proceed - hosts: masters[0] - vars: - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - gather_facts: no - tasks: - # Pacemaker is currently the only supported upgrade path for multiple masters - - fail: - msg: "openshift_master_cluster_method must be set to 'pacemaker'" - when: openshift_master_ha | bool and ((openshift_master_cluster_method is not defined) or (openshift_master_cluster_method is defined and openshift_master_cluster_method != "pacemaker")) - -- name: Run pre-upgrade checks on first master - hosts: masters[0] - tasks: - # If this script errors out ansible will show the default stdout/stderr - # which contains details for the user: - - script: files/pre-upgrade-check - -- name: Evaluate etcd_hosts - hosts: localhost - tasks: - - name: Evaluate etcd hosts - add_host: - name: "{{ groups.masters.0 }}" - groups: etcd_hosts - when: hostvars[groups.masters.0].openshift.master.embedded_etcd | bool - - name: Evaluate etcd hosts - add_host: - name: "{{ item }}" - groups: etcd_hosts - with_items: groups.etcd - when: not hostvars[groups.masters.0].openshift.master.embedded_etcd | bool - -- name: Backup etcd - hosts: etcd_hosts - vars: - embedded_etcd: "{{ openshift.master.embedded_etcd }}" - timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" - roles: - - openshift_facts - tasks: - - - stat: path=/var/lib/openshift - register: var_lib_openshift - - - stat: path=/var/lib/origin - register: var_lib_origin - - - name: Create origin symlink if necessary - file: src=/var/lib/openshift/ dest=/var/lib/origin state=link - when: var_lib_openshift.stat.exists == True and var_lib_origin.stat.exists == False - - - name: Check available disk space for etcd backup - # We assume to be using the data dir for all backups. - shell: > - df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1 - register: avail_disk - - - name: Check current embedded etcd disk usage - shell: > - du -k {{ openshift.master.etcd_data_dir }} | tail -n 1 | cut -f1 - register: etcd_disk_usage - when: embedded_etcd | bool - - - name: Abort if insufficient disk space for etcd backup - fail: - msg: > - {{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup, - {{ avail_disk.stdout }} Kb available. - when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int) - - - name: Install etcd (for etcdctl) - yum: - pkg: etcd - state: latest - - - name: Generate etcd backup - command: > - etcdctl backup --data-dir={{ openshift.master.etcd_data_dir }} - --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }} - - - name: Display location of etcd backup - debug: - msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}" - -- name: Update deployment type - hosts: OSEv3 - roles: - - openshift_facts - post_tasks: - - openshift_facts: - role: common - local_facts: - deployment_type: "{{ deployment_type }}" - - -- name: Perform upgrade version checking - hosts: masters[0] - tasks: - - name: Clean yum cache - command: yum clean all - - - name: Determine available versions - script: files/versions.sh {{ openshift.common.service_type }} openshift - register: g_versions_result - - - set_fact: - g_aos_versions: "{{ g_versions_result.stdout | from_yaml }}" - - - set_fact: - g_new_version: "{{ g_aos_versions.curr_version.split('-', 1).0 if g_aos_versions.avail_version is none else g_aos_versions.avail_version.split('-', 1).0 }}" - - - fail: - msg: This playbook requires Origin 1.0.6 or later - when: deployment_type == 'origin' and g_aos_versions.curr_version | version_compare('1.0.6','<') - - # TODO: This should be specific to the 3.1 upgrade playbook (coming in future refactor), otherwise we are blocking 3.0.1 to 3.0.2 here. - - fail: - msg: Atomic OpenShift 3.1 packages not found - when: deployment_type in ['openshift-enterprise', 'atomic-openshift'] and g_aos_versions.curr_version | version_compare('3.0.2.900','<') and (g_aos_versions.avail_version is none or g_aos_versions.avail_version | version_compare('3.0.2.900','<')) - # Deployment type 'enterprise' is no longer valid if we're upgrading to 3.1 or beyond. - # (still valid for 3.0.x to 3.0.y however) Using the global deployment_type here as - # we're checking what was requested by the upgrade, not the current type on the system. - - fail: - msg: "Deployment type enterprise not supported for upgrade" - when: deployment_type == "enterprise" and g_aos_versions.curr_version | version_compare('3.1', '>=') - - -- name: Upgrade masters - hosts: masters - vars: - openshift_version: "{{ openshift_pkg_version | default('') }}" - tasks: - - name: Upgrade to latest available kernel - yum: - pkg: kernel - state: latest - - - name: Upgrade master packages - command: yum update -y {{ openshift.common.service_type }}-master{{ openshift_version }} - - - name: Ensure python-yaml present for config upgrade - yum: - pkg: PyYAML - state: installed - - - name: Upgrade master configuration - openshift_upgrade_config: - from_version: '3.0' - to_version: '3.1' - role: master - config_base: "{{ hostvars[inventory_hostname].openshift.common.config_base }}" - when: deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and g_aos_versions.curr_version | version_compare('3.1', '>=') - - - set_fact: - master_certs_missing: True - master_cert_subdir: master-{{ openshift.common.hostname }} - master_cert_config_dir: "{{ openshift.common.config_base }}/master" - -- name: Create temp directory for syncing certs - hosts: localhost - gather_facts: no - tasks: - - name: Create local temp directory for syncing certs - local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX - register: g_master_mktemp - changed_when: False - -- name: Generate missing master certificates - hosts: masters[0] - vars: - master_hostnames: "{{ hostvars - | oo_select_keys(groups.masters) - | oo_collect('openshift.common.all_hostnames') - | oo_flatten | unique }}" - master_generated_certs_dir: "{{ openshift.common.config_base }}/generated-configs" - masters_needing_certs: "{{ hostvars - | oo_select_keys(groups.masters) - | difference([groups.masters.0]) }}" - sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" - openshift_deployment_type: "{{ deployment_type }}" - roles: - - openshift_master_certificates - post_tasks: - - name: Remove generated etcd client certs when using external etcd - file: - path: "{{ master_generated_certs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}" - state: absent - when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config - with_nested: - - masters_needing_certs - - - master.etcd-client.crt - - master.etcd-client.key - - - name: Create a tarball of the master certs - command: > - tar -czvf {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz - -C {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }} . - with_items: masters_needing_certs - - - name: Retrieve the master cert tarball from the master - fetch: - src: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz" - dest: "{{ sync_tmpdir }}/" - flat: yes - fail_on_missing: yes - validate_checksum: yes - with_items: masters_needing_certs - -- name: Sync certs and restart masters post configuration change - hosts: masters - vars: - sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - tasks: - - name: Unarchive the tarball on the master - unarchive: - src: "{{ sync_tmpdir }}/{{ master_cert_subdir }}.tgz" - dest: "{{ master_cert_config_dir }}" - when: inventory_hostname != groups.masters.0 - - - name: Restart master services - service: name="{{ openshift.common.service_type}}-master" state=restarted - when: not openshift_master_ha | bool - -- name: Destroy cluster - hosts: masters[0] - vars: - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - openshift_deployment_type: "{{ deployment_type }}" - pre_tasks: - - name: Check for configured cluster - stat: - path: /etc/corosync/corosync.conf - register: corosync_conf - when: openshift_master_ha | bool - - name: Destroy cluster - command: pcs cluster destroy --all - when: openshift_master_ha | bool and corosync_conf.stat.exists == true - -- name: Start pcsd on masters - hosts: masters - vars: - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - tasks: - - name: Start pcsd - service: name=pcsd enabled=yes state=started - when: openshift_master_ha | bool - -- name: Re-create cluster - hosts: masters[0] - vars: - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - openshift_deployment_type: "{{ deployment_type }}" - omc_cluster_hosts: "{{ groups.masters | join(' ') }}" - roles: - - role: openshift_master_cluster - when: openshift_master_ha | bool - -- name: Delete temporary directory on localhost - hosts: localhost - gather_facts: no - tasks: - - file: name={{ g_master_mktemp.stdout }} state=absent - changed_when: False - - -- name: Upgrade nodes - hosts: nodes - vars: - openshift_version: "{{ openshift_pkg_version | default('') }}" - roles: - - openshift_facts - tasks: - - name: Upgrade node packages - command: yum update -y {{ openshift.common.service_type }}-node{{ openshift_version }} - - name: Restart node services - service: name="{{ openshift.common.service_type }}-node" state=restarted - -- name: Update cluster policy and policy bindings - hosts: masters[0] - vars: - origin_reconcile_bindings: "{{ deployment_type == 'origin' and g_new_version | version_compare('1.0.6', '>') }}" - ent_reconcile_bindings: "{{ deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and g_new_version | version_compare('3.0.2','>') }}" - tasks: - - name: oadm policy reconcile-cluster-roles --confirm - command: > - {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig - policy reconcile-cluster-roles --confirm - - - name: oadm policy reconcile-cluster-role-bindings --confirm - command: > - {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig - policy reconcile-cluster-role-bindings - --exclude-groups=system:authenticated - --exclude-groups=system:unauthenticated - --exclude-users=system:anonymous - --additive-only=true --confirm - when: origin_reconcile_bindings | bool or ent_reconcile_bindings | bool - - -- name: Restart masters post reconcile - hosts: masters - vars: - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - tasks: - - name: Restart master services - service: name="{{ openshift.common.service_type}}-master" state=restarted - when: not openshift_master_ha | bool - -- name: Restart cluster post reconcile - hosts: masters[0] - vars: - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - tasks: - - name: Restart master cluster - command: pcs resource restart master - when: openshift_master_ha | bool - - name: Wait for the clustered master service to be available - wait_for: - host: "{{ openshift_master_cluster_vip }}" - port: 8443 - state: started - timeout: 180 - delay: 90 - when: openshift_master_ha | bool - -- name: Upgrade default router and registry - hosts: masters[0] - vars: - - registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', 'v' + g_new_version ) }}" - - router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + g_new_version ) }}" - - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig" - tasks: - - name: Check for default router - command: > - {{ oc_cmd }} get -n default dc/router - register: _default_router - failed_when: false - changed_when: false - - name: Check for allowHostNetwork and allowHostPorts - when: _default_router.rc == 0 - shell: > - {{ oc_cmd }} get -o yaml scc/privileged | /usr/bin/grep -e allowHostPorts -e allowHostNetwork - register: _scc - - name: Grant allowHostNetwork and allowHostPorts - when: - - _default_router.rc == 0 - - "'false' in _scc.stdout" - command: > - {{ oc_cmd }} patch scc/privileged -p '{"allowHostPorts":true,"allowHostNetwork":true}' --loglevel=9 - - name: Update deployment config to 1.0.4/3.0.1 spec - when: _default_router.rc == 0 - command: > - {{ oc_cmd }} patch dc/router -p - '{"spec":{"strategy":{"rollingParams":{"updatePercent":-10},"spec":{"serviceAccount":"router","serviceAccountName":"router"}}}}' - - name: Switch to hostNetwork=true - when: _default_router.rc == 0 - command: > - {{ oc_cmd }} patch dc/router -p '{"spec":{"template":{"spec":{"hostNetwork":true}}}}' - - name: Update router image to current version - when: _default_router.rc == 0 - command: > - {{ oc_cmd }} patch dc/router -p - '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}"}]}}}}' - - - name: Check for default registry - command: > - {{ oc_cmd }} get -n default dc/docker-registry - register: _default_registry - failed_when: false - changed_when: false - - name: Update registry image to current version - when: _default_registry.rc == 0 - command: > - {{ oc_cmd }} patch dc/docker-registry -p - '{"spec":{"template":{"spec":{"containers":[{"name":"registry","image":"{{ registry_image }}"}]}}}}' - -- name: Update image streams and templates - hosts: masters[0] - vars: - openshift_examples_import_command: "update" - openshift_deployment_type: "{{ deployment_type }}" - roles: - - openshift_examples - -- name: Ensure master services enabled - hosts: masters - vars: - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - tasks: - - name: Enable master services - service: name="{{ openshift.common.service_type}}-master" state=started enabled=yes - when: not openshift_master_ha | bool - -- name: Ensure node services enabled - hosts: nodes - tasks: - - name: Restart node services - service: name="{{ openshift.common.service_type }}-node" state=started enabled=yes - diff --git a/playbooks/byo/openshift-cluster/upgrades/README.md b/playbooks/byo/openshift-cluster/upgrades/README.md new file mode 100644 index 000000000..ce7aebf8e --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/README.md @@ -0,0 +1,8 @@ +# Upgrade playbooks +The playbooks provided in this directory can be used for upgrading an existing +environment. Additional notes for the associated upgrade playbooks are +provided in their respective directories. + +# Upgrades available +- [OpenShift Enterprise 3.0 to latest minor release](v3_0_minor/README.md) +- [OpenShift Enterprise 3.0 to 3.1](v3_0_to_v3_1/README.md) diff --git a/playbooks/adhoc/upgrades/README.md b/playbooks/byo/openshift-cluster/upgrades/v3_0_minor/README.md index 6de8a970f..c91a6cb96 100644 --- a/playbooks/adhoc/upgrades/README.md +++ b/playbooks/byo/openshift-cluster/upgrades/v3_0_minor/README.md @@ -1,11 +1,11 @@ -# [NOTE] -This playbook will re-run installation steps overwriting any local +# v3.0 minor upgrade playbook +**Note:** This playbook will re-run installation steps overwriting any local modifications. You should ensure that your inventory has been updated with any modifications you've made after your initial installation. If you find any items that cannot be configured via ansible please open an issue at https://github.com/openshift/openshift-ansible -# Overview +## Overview This playbook is available as a technical preview. It currently performs the following steps. @@ -17,5 +17,5 @@ following steps. * Updates the default registry if one exists * Updates image streams and quickstarts -# Usage -ansible-playbook -i ~/ansible-inventory openshift-ansible/playbooks/adhoc/upgrades/upgrade.yml +## Usage +ansible-playbook -i ~/ansible-inventory openshift-ansible/playbooks/byo/openshift-cluster/upgrades/v3_0_minor/upgrade.yml diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_0_minor/upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/v3_0_minor/upgrade.yml new file mode 100644 index 000000000..76fa9ba22 --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/v3_0_minor/upgrade.yml @@ -0,0 +1,9 @@ +--- +- include: ../../../../common/openshift-cluster/upgrades/v3_0_minor/upgrade.yml + vars: + g_etcd_group: "{{ 'etcd' }}" + g_masters_group: "{{ 'masters' }}" + g_nodes_group: "{{ 'nodes' }}" + g_lb_group: "{{ 'lb' }}" + openshift_cluster_id: "{{ cluster_id | default('default') }}" + openshift_deployment_type: "{{ deployment_type }}" diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/README.md b/playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/README.md new file mode 100644 index 000000000..c434be5b7 --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/README.md @@ -0,0 +1,17 @@ +# v3.0 to v3.1 upgrade playbook + +## Overview +This playbook currently performs the +following steps. + +**TODO: update for current steps** + * Upgrade and restart master services + * Upgrade and restart node services + * Modifies the subset of the configuration necessary + * Applies the latest cluster policies + * Updates the default router if one exists + * Updates the default registry if one exists + * Updates image streams and quickstarts + +## Usage +ansible-playbook -i ~/ansible-inventory openshift-ansible/playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml new file mode 100644 index 000000000..b06442366 --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml @@ -0,0 +1,9 @@ +--- +- include: ../../../../common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml + vars: + g_etcd_group: "{{ 'etcd' }}" + g_masters_group: "{{ 'masters' }}" + g_nodes_group: "{{ 'nodes' }}" + g_lb_group: "{{ 'lb' }}" + openshift_cluster_id: "{{ cluster_id | default('default') }}" + openshift_deployment_type: "{{ deployment_type }}" diff --git a/playbooks/adhoc/upgrades/files/pre-upgrade-check b/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check index ed4ab6d1b..ed4ab6d1b 100644 --- a/playbooks/adhoc/upgrades/files/pre-upgrade-check +++ b/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check diff --git a/playbooks/adhoc/upgrades/files/versions.sh b/playbooks/common/openshift-cluster/upgrades/files/versions.sh index f90719cab..f90719cab 100644 --- a/playbooks/adhoc/upgrades/files/versions.sh +++ b/playbooks/common/openshift-cluster/upgrades/files/versions.sh diff --git a/playbooks/common/openshift-cluster/upgrades/filter_plugins b/playbooks/common/openshift-cluster/upgrades/filter_plugins new file mode 120000 index 000000000..b1213dedb --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/filter_plugins @@ -0,0 +1 @@ +../../../../filter_plugins
\ No newline at end of file diff --git a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py b/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py index a6721bb92..a6721bb92 100755 --- a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py +++ b/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py diff --git a/playbooks/common/openshift-cluster/upgrades/lookup_plugins b/playbooks/common/openshift-cluster/upgrades/lookup_plugins new file mode 120000 index 000000000..aff753026 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/lookup_plugins @@ -0,0 +1 @@ +../../../../lookup_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/roles b/playbooks/common/openshift-cluster/upgrades/roles new file mode 120000 index 000000000..4bdbcbad3 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/roles @@ -0,0 +1 @@ +../../../../roles
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_minor/filter_plugins b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/filter_plugins new file mode 120000 index 000000000..27ddaa18b --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/filter_plugins @@ -0,0 +1 @@ +../../../../../filter_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_minor/library b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/library new file mode 120000 index 000000000..53bed9684 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/library @@ -0,0 +1 @@ +../library
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_minor/lookup_plugins b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/lookup_plugins new file mode 120000 index 000000000..cf407f69b --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/lookup_plugins @@ -0,0 +1 @@ +../../../../../lookup_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_minor/roles b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/roles new file mode 120000 index 000000000..6bc1a7aef --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/roles @@ -0,0 +1 @@ +../../../../../roles
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_minor/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/upgrade.yml new file mode 100644 index 000000000..9f7e49b93 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/upgrade.yml @@ -0,0 +1,112 @@ +--- +- name: Evaluate groups + include: ../../evaluate_groups.yml + +- name: Re-Run cluster configuration to apply latest configuration changes + include: ../../config.yml + +- name: Upgrade masters + hosts: oo_masters_to_config + vars: + openshift_version: "{{ openshift_pkg_version | default('') }}" + tasks: + - name: Upgrade master packages + yum: pkg={{ openshift.common.service_type }}-master{{ openshift_version }} state=latest + - name: Restart master services + service: name="{{ openshift.common.service_type}}-master" state=restarted + +- name: Upgrade nodes + hosts: oo_nodes_to_config + vars: + openshift_version: "{{ openshift_pkg_version | default('') }}" + tasks: + - name: Upgrade node packages + yum: pkg={{ openshift.common.service_type }}-node{{ openshift_version }} state=latest + - name: Restart node services + service: name="{{ openshift.common.service_type }}-node" state=restarted + +- name: Determine new master version + hosts: oo_first_master + tasks: + - name: Determine new version + command: > + rpm -q --queryformat '%{version}' {{ openshift.common.service_type }}-master + register: _new_version + +- name: Ensure AOS 3.0.2 or Origin 1.0.6 + hosts: oo_first_master + tasks: + fail: This playbook requires Origin 1.0.6 or Atomic OpenShift 3.0.2 or later + when: _new_version.stdout | version_compare('1.0.6','<') or ( _new_version.stdout | version_compare('3.0','>=' and _new_version.stdout | version_compare('3.0.2','<') ) + +- name: Update cluster policy + hosts: oo_first_master + tasks: + - name: oadm policy reconcile-cluster-roles --confirm + command: > + {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig + policy reconcile-cluster-roles --confirm + +- name: Upgrade default router + hosts: oo_first_master + vars: + - router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + _new_version.stdout ) }}" + - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig" + tasks: + - name: Check for default router + command: > + {{ oc_cmd }} get -n default dc/router + register: _default_router + failed_when: false + changed_when: false + - name: Check for allowHostNetwork and allowHostPorts + when: _default_router.rc == 0 + shell: > + {{ oc_cmd }} get -o yaml scc/privileged | /usr/bin/grep -e allowHostPorts -e allowHostNetwork + register: _scc + - name: Grant allowHostNetwork and allowHostPorts + when: + - _default_router.rc == 0 + - "'false' in _scc.stdout" + command: > + {{ oc_cmd }} patch scc/privileged -p '{"allowHostPorts":true,"allowHostNetwork":true}' --loglevel=9 + - name: Update deployment config to 1.0.4/3.0.1 spec + when: _default_router.rc == 0 + command: > + {{ oc_cmd }} patch dc/router -p + '{"spec":{"strategy":{"rollingParams":{"updatePercent":-10},"spec":{"serviceAccount":"router","serviceAccountName":"router"}}}}' + - name: Switch to hostNetwork=true + when: _default_router.rc == 0 + command: > + {{ oc_cmd }} patch dc/router -p '{"spec":{"template":{"spec":{"hostNetwork":true}}}}' + - name: Update router image to current version + when: _default_router.rc == 0 + command: > + {{ oc_cmd }} patch dc/router -p + '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}"}]}}}}' + +- name: Upgrade default + hosts: oo_first_master + vars: + - registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', 'v' + _new_version.stdout ) }}" + - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig" + tasks: + - name: Check for default registry + command: > + {{ oc_cmd }} get -n default dc/docker-registry + register: _default_registry + failed_when: false + changed_when: false + - name: Update registry image to current version + when: _default_registry.rc == 0 + command: > + {{ oc_cmd }} patch dc/docker-registry -p + '{"spec":{"template":{"spec":{"containers":[{"name":"registry","image":"{{ registry_image }}"}]}}}}' + +- name: Update image streams and templates + hosts: oo_first_master + vars: + openshift_examples_import_command: "update" + openshift_deployment_type: "{{ deployment_type }}" + roles: + - openshift_examples diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/filter_plugins b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/filter_plugins new file mode 120000 index 000000000..27ddaa18b --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/filter_plugins @@ -0,0 +1 @@ +../../../../../filter_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/library b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/library new file mode 120000 index 000000000..53bed9684 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/library @@ -0,0 +1 @@ +../library
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/lookup_plugins b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/lookup_plugins new file mode 120000 index 000000000..cf407f69b --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/lookup_plugins @@ -0,0 +1 @@ +../../../../../lookup_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/roles b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/roles new file mode 120000 index 000000000..6bc1a7aef --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/roles @@ -0,0 +1 @@ +../../../../../roles
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml new file mode 100644 index 000000000..78797f8b8 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml @@ -0,0 +1,429 @@ +--- +############################################################################### +# Evaluate host groups and gather facts +############################################################################### +- name: Evaluate host groups + include: ../../evaluate_groups.yml + +- name: Load openshift_facts + hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config:oo_lb_to_config + roles: + - openshift_facts + +- name: Evaluate etcd_hosts_to_backup + hosts: localhost + tasks: + - name: Evaluate etcd_hosts_to_backup + add_host: + name: "{{ item }}" + groups: etcd_hosts_to_backup + with_items: groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master + + +############################################################################### +# Pre-upgrade checks +############################################################################### +- name: Verify upgrade can proceed + hosts: oo_first_master + vars: + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + gather_facts: no + tasks: + # Pacemaker is currently the only supported upgrade path for multiple masters + - fail: + msg: "openshift_master_cluster_method must be set to 'pacemaker'" + when: openshift_master_ha | bool and ((openshift_master_cluster_method is not defined) or (openshift_master_cluster_method is defined and openshift_master_cluster_method != "pacemaker")) + + - fail: + msg: > + This upgrade is only supported for origin and openshift-enterprise + deployment types + when: deployment_type not in ['origin','openshift-enterprise'] + + - fail: + msg: > + openshift_pkg_version is {{ openshift_pkg_version }} which is not a + valid version for a 3.1 upgrade + when: openshift_pkg_version is defined and openshift_pkg_version.split('-',1).1 | version_compare('3.0.2.900','<') + + # If this script errors out ansible will show the default stdout/stderr + # which contains details for the user: + - script: ../files/pre-upgrade-check + + +- name: Verify upgrade can proceed + hosts: masters:nodes + tasks: + - name: Clean yum cache + command: yum clean all + + - set_fact: + g_new_service_name: "{{ 'origin' if deployment_type =='origin' else 'atomic-openshift' }}" + + - name: Determine available versions + script: ../files/versions.sh {{ g_new_service_name }} openshift + register: g_versions_result + + - set_fact: + g_aos_versions: "{{ g_versions_result.stdout | from_yaml }}" + + - set_fact: + g_new_version: "{{ g_aos_versions.curr_version.split('-', 1).0 if g_aos_versions.avail_version is none else g_aos_versions.avail_version.split('-', 1).0 }}" + + - fail: + msg: This playbook requires Origin 1.0.6 or later + when: deployment_type == 'origin' and g_aos_versions.curr_version | version_compare('1.0.6','<') + + - fail: + msg: Atomic OpenShift 3.1 packages not found + when: g_aos_versions.curr_version | version_compare('3.0.2.900','<') and (g_aos_versions.avail_version is none or g_aos_versions.avail_version | version_compare('3.0.2.900','<')) + + +############################################################################### +# Backup etcd +############################################################################### +- name: Backup etcd + hosts: etcd_hosts_to_backup + vars: + embedded_etcd: "{{ openshift.master.embedded_etcd }}" + timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" + roles: + - openshift_facts + tasks: + - openshift_facts: + role: etcd + local_facts: {} + when: "'etcd' not in openshift" + + - stat: path=/var/lib/openshift + register: var_lib_openshift + + - stat: path=/var/lib/origin + register: var_lib_origin + + - name: Create origin symlink if necessary + file: src=/var/lib/openshift/ dest=/var/lib/origin state=link + when: var_lib_openshift.stat.exists == True and var_lib_origin.stat.exists == False + + # TODO: replace shell module with command and update later checks + # We assume to be using the data dir for all backups. + - name: Check available disk space for etcd backup + shell: df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1 + register: avail_disk + + # TODO: replace shell module with command and update later checks + - name: Check current embedded etcd disk usage + shell: du -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1 | cut -f1 + register: etcd_disk_usage + when: embedded_etcd | bool + + - name: Abort if insufficient disk space for etcd backup + fail: + msg: > + {{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup, + {{ avail_disk.stdout }} Kb available. + when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int) + + - name: Install etcd (for etcdctl) + yum: + pkg: etcd + state: latest + + - name: Generate etcd backup + command: > + etcdctl backup --data-dir={{ openshift.etcd.etcd_data_dir }} + --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }} + + - name: Display location of etcd backup + debug: + msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}" + + +############################################################################### +# Upgrade Masters +############################################################################### +- name: Create temp directory for syncing certs + hosts: localhost + gather_facts: no + tasks: + - name: Create local temp directory for syncing certs + local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX + register: g_master_mktemp + changed_when: False + +- name: Update deployment type + hosts: OSEv3 + roles: + - openshift_facts + post_tasks: + - openshift_facts: + role: common + local_facts: + deployment_type: "{{ deployment_type }}" + +- name: Upgrade master packages and configuration + hosts: oo_masters_to_config + vars: + openshift_version: "{{ openshift_pkg_version | default('') }}" + tasks: + - name: Upgrade to latest available kernel + yum: + pkg: kernel + state: latest + + - name: Upgrade master packages + command: yum update -y {{ openshift.common.service_type }}-master{{ openshift_version }} + + - name: Ensure python-yaml present for config upgrade + yum: + pkg: PyYAML + state: installed + + - name: Upgrade master configuration + openshift_upgrade_config: + from_version: '3.0' + to_version: '3.1' + role: master + config_base: "{{ hostvars[inventory_hostname].openshift.common.config_base }}" + + - set_fact: + master_certs_missing: True + master_cert_subdir: master-{{ openshift.common.hostname }} + master_cert_config_dir: "{{ openshift.common.config_base }}/master" + + +- name: Generate missing master certificates + hosts: oo_first_master + vars: + master_hostnames: "{{ hostvars + | oo_select_keys(groups.oo_masters_to_config) + | oo_collect('openshift.common.all_hostnames') + | oo_flatten | unique }}" + master_generated_certs_dir: "{{ openshift.common.config_base }}/generated-configs" + masters_needing_certs: "{{ hostvars + | oo_select_keys(groups.oo_masters_to_config) + | difference([groups.oo_first_master.0]) }}" + sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" + openshift_deployment_type: "{{ deployment_type }}" + roles: + - openshift_master_certificates + post_tasks: + - name: Remove generated etcd client certs when using external etcd + file: + path: "{{ master_generated_certs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}" + state: absent + when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config + with_nested: + - masters_needing_certs + - - master.etcd-client.crt + - master.etcd-client.key + + - name: Create a tarball of the master certs + command: > + tar -czvf {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz + -C {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }} . + with_items: masters_needing_certs + + - name: Retrieve the master cert tarball from the master + fetch: + src: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz" + dest: "{{ sync_tmpdir }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes + with_items: masters_needing_certs + + +- name: Sync generated certs, update service config and restart master services + hosts: oo_masters_to_config + vars: + sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + openshift_deployment_type: "{{ deployment_type }}" + tasks: + - name: Unarchive the tarball on the master + unarchive: + src: "{{ sync_tmpdir }}/{{ master_cert_subdir }}.tgz" + dest: "{{ master_cert_config_dir }}" + when: inventory_hostname != groups.oo_first_master.0 + + - name: Restart master service + service: name="{{ openshift.common.service_type}}-master" state=restarted + when: not openshift_master_ha | bool + + - name: Ensure the master service is enabled + service: name="{{ openshift.common.service_type}}-master" state=started enabled=yes + when: not openshift_master_ha | bool + + - name: Check for configured cluster + stat: + path: /etc/corosync/corosync.conf + register: corosync_conf + when: openshift_master_ha | bool + + - name: Destroy cluster + command: pcs cluster destroy --all + when: openshift_master_ha | bool and corosync_conf.stat.exists == true + run_once: true + + - name: Start pcsd + service: name=pcsd enabled=yes state=started + when: openshift_master_ha | bool + + +- name: Re-create cluster + hosts: oo_first_master + vars: + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + openshift_deployment_type: "{{ deployment_type }}" + omc_cluster_hosts: "{{ groups.oo_masters_to_config | join(' ') }}" + roles: + - role: openshift_master_cluster + when: openshift_master_ha | bool + + +- name: Delete temporary directory on localhost + hosts: localhost + gather_facts: no + tasks: + - file: name={{ g_master_mktemp.stdout }} state=absent + changed_when: False + + +############################################################################### +# Upgrade Nodes +############################################################################### +- name: Upgrade nodes + hosts: oo_nodes_to_config + vars: + openshift_version: "{{ openshift_pkg_version | default('') }}" + roles: + - openshift_facts + tasks: + - name: Upgrade node packages + command: yum update -y {{ openshift.common.service_type }}-node{{ openshift_version }} + + - name: Restart node service + service: name="{{ openshift.common.service_type }}-node" state=restarted + + - name: Ensure node service enabled + service: name="{{ openshift.common.service_type }}-node" state=started enabled=yes + + +############################################################################### +# Post upgrade - Reconcile Cluster Roles and Cluster Role Bindings +############################################################################### +- name: Reconcile Cluster Roles and Cluster Role Bindings + hosts: oo_masters_to_config + vars: + origin_reconcile_bindings: "{{ deployment_type == 'origin' and g_new_version | version_compare('1.0.6', '>') }}" + ent_reconcile_bindings: true + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + tasks: + - name: Reconcile Cluster Roles + command: > + {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig + policy reconcile-cluster-roles --confirm + run_once: true + + - name: Reconcile Cluster Role Bindings + command: > + {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig + policy reconcile-cluster-role-bindings + --exclude-groups=system:authenticated + --exclude-groups=system:unauthenticated + --exclude-users=system:anonymous + --additive-only=true --confirm + when: origin_reconcile_bindings | bool or ent_reconcile_bindings | bool + run_once: true + + - name: Restart master services + service: name="{{ openshift.common.service_type}}-master" state=restarted + when: not openshift_master_ha | bool + + - name: Restart master cluster + command: pcs resource restart master + when: openshift_master_ha | bool + run_once: true + + - name: Wait for the clustered master service to be available + wait_for: + host: "{{ openshift_master_cluster_vip }}" + port: 8443 + state: started + timeout: 180 + delay: 90 + when: openshift_master_ha | bool + run_once: true + + +############################################################################### +# Post upgrade - Upgrade default router, default registry and examples +############################################################################### +- name: Upgrade default router and default registry + hosts: oo_first_master + vars: + openshift_deployment_type: "{{ deployment_type }}" + registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', 'v' + g_new_version ) }}" + router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + g_new_version ) }}" + oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig" + roles: + # Create the new templates shipped in 3.1, existing templates are left + # unmodified. This prevents the subsequent role definition for + # openshift_examples from failing when trying to replace templates that do + # not already exist. We could have potentially done a replace --force to + # create and update in one step. + - openshift_examples + # Update the existing templates + - role: openshift_examples + openshift_examples_import_command: replace + pre_tasks: + - name: Check for default router + command: > + {{ oc_cmd }} get -n default dc/router + register: _default_router + failed_when: false + changed_when: false + + - name: Check for allowHostNetwork and allowHostPorts + when: _default_router.rc == 0 + shell: > + {{ oc_cmd }} get -o yaml scc/privileged | /usr/bin/grep -e allowHostPorts -e allowHostNetwork + register: _scc + + - name: Grant allowHostNetwork and allowHostPorts + when: + - _default_router.rc == 0 + - "'false' in _scc.stdout" + command: > + {{ oc_cmd }} patch scc/privileged -p '{"allowHostPorts":true,"allowHostNetwork":true}' --loglevel=9 + + - name: Update deployment config to 1.0.4/3.0.1 spec + when: _default_router.rc == 0 + command: > + {{ oc_cmd }} patch dc/router -p + '{"spec":{"strategy":{"rollingParams":{"updatePercent":-10},"spec":{"serviceAccount":"router","serviceAccountName":"router"}}}}' + + - name: Switch to hostNetwork=true + when: _default_router.rc == 0 + command: > + {{ oc_cmd }} patch dc/router -p '{"spec":{"template":{"spec":{"hostNetwork":true}}}}' + + - name: Update router image to current version + when: _default_router.rc == 0 + command: > + {{ oc_cmd }} patch dc/router -p + '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}"}]}}}}' + + - name: Check for default registry + command: > + {{ oc_cmd }} get -n default dc/docker-registry + register: _default_registry + failed_when: false + changed_when: false + + - name: Update registry image to current version + when: _default_registry.rc == 0 + command: > + {{ oc_cmd }} patch dc/docker-registry -p + '{"spec":{"template":{"spec":{"containers":[{"name":"registry","image":"{{ registry_image }}"}]}}}}' diff --git a/playbooks/common/openshift-etcd/config.yml b/playbooks/common/openshift-etcd/config.yml index 952960652..ed23ada88 100644 --- a/playbooks/common/openshift-etcd/config.yml +++ b/playbooks/common/openshift-etcd/config.yml @@ -13,6 +13,8 @@ hostname: "{{ openshift_hostname | default(None) }}" public_hostname: "{{ openshift_public_hostname | default(None) }}" deployment_type: "{{ openshift_deployment_type }}" + - role: etcd + local_facts: {} - name: Check status of etcd certificates stat: path: "{{ item }}" diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml index b1da85d5d..1b3fba3aa 100644 --- a/playbooks/common/openshift-master/config.yml +++ b/playbooks/common/openshift-master/config.yml @@ -51,6 +51,9 @@ console_url: "{{ openshift_master_console_url | default(None) }}" console_use_ssl: "{{ openshift_master_console_use_ssl | default(None) }}" public_console_url: "{{ openshift_master_public_console_url | default(None) }}" + - role: etcd + local_facts: {} + when: openshift.master.embedded_etcd | bool - name: Check status of external etcd certificatees stat: path: "{{ openshift.common.config_base }}/master/{{ item }}" |