diff options
Diffstat (limited to 'roles/openshift_node_upgrade')
14 files changed, 128 insertions, 85 deletions
diff --git a/roles/openshift_node_upgrade/README.md b/roles/openshift_node_upgrade/README.md index 8b388cc6a..4e6229bfb 100644 --- a/roles/openshift_node_upgrade/README.md +++ b/roles/openshift_node_upgrade/README.md @@ -84,6 +84,11 @@ Including an example of how to use your role (for instance, with variables passe command: > {{ hostvars[groups.oo_first_master.0].openshift.common.admin_binary }} drain {{ openshift.node.nodename | lower }} --force --delete-local-data --ignore-daemonsets delegate_to: "{{ groups.oo_first_master.0 }}" + register: l_docker_upgrade_drain_result + until: not l_docker_upgrade_drain_result | failed + retries: 60 + delay: 60 + roles: - openshift_facts diff --git a/roles/openshift_node_upgrade/handlers/main.yml b/roles/openshift_node_upgrade/handlers/main.yml index cb51416d4..d31b899cf 100644 --- a/roles/openshift_node_upgrade/handlers/main.yml +++ b/roles/openshift_node_upgrade/handlers/main.yml @@ -1,7 +1,16 @@ --- - name: restart openvswitch - systemd: name=openvswitch state=restarted - when: (not skip_node_svc_handlers | default(False) | bool) and not (ovs_service_status_changed | default(false) | bool) and openshift.common.use_openshift_sdn | bool + systemd: + name: openvswitch + state: restarted + when: + - not skip_node_svc_handlers | default(False) | bool + - not (ovs_service_status_changed | default(false) | bool) + - openshift.common.use_openshift_sdn | default(true) | bool + register: l_openshift_node_upgrade_stop_openvswitch_result + until: not l_openshift_node_upgrade_stop_openvswitch_result | failed + retries: 3 + delay: 30 notify: - restart openvswitch pause @@ -10,5 +19,18 @@ when: (not skip_node_svc_handlers | default(False) | bool) and openshift.common.is_containerized | bool - name: restart node - systemd: name={{ openshift.common.service_type }}-node state=restarted - when: (not skip_node_svc_handlers | default(False) | bool) and not (node_service_status_changed | default(false) | bool) + systemd: + name: "{{ openshift.common.service_type }}-node" + state: restarted + register: l_openshift_node_upgrade_restart_node_result + until: not l_openshift_node_upgrade_restart_node_result | failed + retries: 3 + delay: 30 + when: + - (not skip_node_svc_handlers | default(False) | bool) + - not (node_service_status_changed | default(false) | bool) + +# TODO(jchaloup): once it is verified the systemd module works as expected +# switch to it: http://docs.ansible.com/ansible/latest/systemd_module.html +- name: reload systemd units + command: systemctl daemon-reload diff --git a/roles/openshift_node_upgrade/tasks/config/configure-node-settings.yml b/roles/openshift_node_upgrade/tasks/config/configure-node-settings.yml new file mode 100644 index 000000000..1186062eb --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/config/configure-node-settings.yml @@ -0,0 +1,16 @@ +--- +- name: Configure Node settings + lineinfile: + dest: /etc/sysconfig/{{ openshift.common.service_type }}-node + regexp: "{{ item.regex }}" + line: "{{ item.line }}" + create: true + with_items: + - regex: '^OPTIONS=' + line: "OPTIONS=--loglevel={{ openshift.node.debug_level | default(2) }}" + - regex: '^CONFIG_FILE=' + line: "CONFIG_FILE={{ openshift.common.config_base }}/node/node-config.yaml" + - regex: '^IMAGE_VERSION=' + line: "IMAGE_VERSION={{ openshift_image_tag }}" + notify: + - restart node diff --git a/roles/openshift_node_upgrade/tasks/config/configure-proxy-settings.yml b/roles/openshift_node_upgrade/tasks/config/configure-proxy-settings.yml new file mode 100644 index 000000000..d60794305 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/config/configure-proxy-settings.yml @@ -0,0 +1,17 @@ +--- +- name: Configure Proxy Settings + lineinfile: + dest: /etc/sysconfig/{{ openshift.common.service_type }}-node + regexp: "{{ item.regex }}" + line: "{{ item.line }}" + create: true + with_items: + - regex: '^HTTP_PROXY=' + line: "HTTP_PROXY={{ openshift.common.http_proxy | default('') }}" + - regex: '^HTTPS_PROXY=' + line: "HTTPS_PROXY={{ openshift.common.https_proxy | default('') }}" + - regex: '^NO_PROXY=' + line: "NO_PROXY={{ openshift.common.no_proxy | default([]) }},{{ openshift.common.portal_net }},{{ hostvars[groups.oo_first_master.0].openshift.master.sdn_cluster_network_cidr }}" + when: ('http_proxy' in openshift.common and openshift.common.http_proxy != '') + notify: + - restart node diff --git a/roles/openshift_node_upgrade/tasks/config/install-node-deps-docker-service-file.yml b/roles/openshift_node_upgrade/tasks/config/install-node-deps-docker-service-file.yml new file mode 100644 index 000000000..ee91a88ab --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/config/install-node-deps-docker-service-file.yml @@ -0,0 +1,8 @@ +--- +- name: Install Node dependencies docker service file + template: + dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node-dep.service" + src: openshift.docker.node.dep.service + notify: + - reload systemd units + - restart node diff --git a/roles/openshift_node_upgrade/tasks/config/install-node-docker-service-file.yml b/roles/openshift_node_upgrade/tasks/config/install-node-docker-service-file.yml new file mode 100644 index 000000000..f92ff79b5 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/config/install-node-docker-service-file.yml @@ -0,0 +1,8 @@ +--- +- name: Install Node docker service file + template: + dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node.service" + src: openshift.docker.node.service + notify: + - reload systemd units + - restart node diff --git a/roles/openshift_node_upgrade/tasks/config/install-ovs-docker-service-file.yml b/roles/openshift_node_upgrade/tasks/config/install-ovs-docker-service-file.yml new file mode 100644 index 000000000..c2c5ea1d4 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/config/install-ovs-docker-service-file.yml @@ -0,0 +1,8 @@ +--- +- name: Install OpenvSwitch docker service file + template: + dest: "/etc/systemd/system/openvswitch.service" + src: openvswitch.docker.service + notify: + - reload systemd units + - restart openvswitch diff --git a/roles/openshift_node_upgrade/tasks/config/install-ovs-service-env-file.yml b/roles/openshift_node_upgrade/tasks/config/install-ovs-service-env-file.yml new file mode 100644 index 000000000..1d75a3355 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/config/install-ovs-service-env-file.yml @@ -0,0 +1,8 @@ +--- +- name: Create the openvswitch service env file + template: + src: openvswitch.sysconfig.j2 + dest: /etc/sysconfig/openvswitch + notify: + - reload systemd units + - restart openvswitch diff --git a/roles/openshift_node_upgrade/tasks/config/workaround-bz1331590-ovs-oom-fix.yml b/roles/openshift_node_upgrade/tasks/config/workaround-bz1331590-ovs-oom-fix.yml new file mode 100644 index 000000000..5df1abc79 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/config/workaround-bz1331590-ovs-oom-fix.yml @@ -0,0 +1,13 @@ +--- +# May be a temporary workaround. +# https://bugzilla.redhat.com/show_bug.cgi?id=1331590 +- name: Create OpenvSwitch service.d directory + file: path=/etc/systemd/system/openvswitch.service.d/ state=directory + +- name: Install OpenvSwitch service OOM fix + template: + dest: "/etc/systemd/system/openvswitch.service.d/01-avoid-oom.conf" + src: openvswitch-avoid-oom.conf + notify: + - reload systemd units + - restart openvswitch diff --git a/roles/openshift_node_upgrade/tasks/docker/upgrade.yml b/roles/openshift_node_upgrade/tasks/docker/upgrade.yml index 416cf605a..ebe87d6fd 100644 --- a/roles/openshift_node_upgrade/tasks/docker/upgrade.yml +++ b/roles/openshift_node_upgrade/tasks/docker/upgrade.yml @@ -26,7 +26,13 @@ - debug: var=docker_image_count.stdout when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool -- service: name=docker state=stopped +- service: + name: docker + state: stopped + register: l_openshift_node_upgrade_docker_stop_result + until: not l_openshift_node_upgrade_docker_stop_result | failed + retries: 3 + delay: 30 - name: Upgrade Docker package: name=docker{{ '-' + docker_version }} state=present diff --git a/roles/openshift_node_upgrade/tasks/main.yml b/roles/openshift_node_upgrade/tasks/main.yml index ac9ea32cb..bc092c26c 100644 --- a/roles/openshift_node_upgrade/tasks/main.yml +++ b/roles/openshift_node_upgrade/tasks/main.yml @@ -24,7 +24,6 @@ name: "{{ item }}" state: stopped with_items: - - "{{ openshift.common.service_type }}-master" - "{{ openshift.common.service_type }}-master-controllers" - "{{ openshift.common.service_type }}-master-api" - etcd_container @@ -43,7 +42,9 @@ docker pull {{ openshift.node.ovs_image }}:{{ openshift_image_tag }} register: pull_result changed_when: "'Downloaded newer image' in pull_result.stdout" - when: openshift.common.is_containerized | bool and openshift.common.use_openshift_sdn | bool + when: + - openshift.common.is_containerized | bool + - openshift.common.use_openshift_sdn | default(true) | bool - include: docker/upgrade.yml vars: @@ -79,7 +80,6 @@ with_items: - etcd_container - openvswitch - - "{{ openshift.common.service_type }}-master" - "{{ openshift.common.service_type }}-master-api" - "{{ openshift.common.service_type }}-master-controllers" - "{{ openshift.common.service_type }}-node" diff --git a/roles/openshift_node_upgrade/tasks/restart.yml b/roles/openshift_node_upgrade/tasks/restart.yml index 6947223af..a4fa51172 100644 --- a/roles/openshift_node_upgrade/tasks/restart.yml +++ b/roles/openshift_node_upgrade/tasks/restart.yml @@ -19,7 +19,7 @@ state: started register: docker_start_result until: not docker_start_result | failed - retries: 1 + retries: 3 delay: 30 - name: Update docker facts @@ -31,7 +31,6 @@ with_items: - etcd_container - openvswitch - - "{{ openshift.common.service_type }}-master" - "{{ openshift.common.service_type }}-master-api" - "{{ openshift.common.service_type }}-master-controllers" - "{{ openshift.common.service_type }}-node" diff --git a/roles/openshift_node_upgrade/tasks/systemd_units.yml b/roles/openshift_node_upgrade/tasks/systemd_units.yml index e8f017445..4e9550150 100644 --- a/roles/openshift_node_upgrade/tasks/systemd_units.yml +++ b/roles/openshift_node_upgrade/tasks/systemd_units.yml @@ -18,87 +18,20 @@ # This file is included both in the openshift_master role and in the upgrade # playbooks. -- name: Install Node dependencies docker service file - template: - dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node-dep.service" - src: openshift.docker.node.dep.service - register: install_node_dep_result +- include: config/install-node-deps-docker-service-file.yml when: openshift.common.is_containerized | bool -- name: Install Node docker service file - template: - dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node.service" - src: openshift.docker.node.service - register: install_node_result +- include: config/install-node-docker-service-file.yml when: openshift.common.is_containerized | bool -- name: Create the openvswitch service env file - template: - src: openvswitch.sysconfig.j2 - dest: /etc/sysconfig/openvswitch +- include: config/install-ovs-service-env-file.yml when: openshift.common.is_containerized | bool - register: install_ovs_sysconfig - notify: - - restart openvswitch -# May be a temporary workaround. -# https://bugzilla.redhat.com/show_bug.cgi?id=1331590 -- name: Create OpenvSwitch service.d directory - file: path=/etc/systemd/system/openvswitch.service.d/ state=directory +- include: config/workaround-bz1331590-ovs-oom-fix.yml when: openshift.common.use_openshift_sdn | default(true) | bool -- name: Install OpenvSwitch service OOM fix - template: - dest: "/etc/systemd/system/openvswitch.service.d/01-avoid-oom.conf" - src: openvswitch-avoid-oom.conf - when: openshift.common.use_openshift_sdn | default(true) | bool - register: install_oom_fix_result - notify: - - restart openvswitch - -- name: Install OpenvSwitch docker service file - template: - dest: "/etc/systemd/system/openvswitch.service" - src: openvswitch.docker.service +- include: config/install-ovs-docker-service-file.yml when: openshift.common.is_containerized | bool and openshift.common.use_openshift_sdn | default(true) | bool - notify: - - restart openvswitch - -- name: Configure Node settings - lineinfile: - dest: /etc/sysconfig/{{ openshift.common.service_type }}-node - regexp: "{{ item.regex }}" - line: "{{ item.line }}" - create: true - with_items: - - regex: '^OPTIONS=' - line: "OPTIONS=--loglevel={{ openshift.node.debug_level | default(2) }}" - - regex: '^CONFIG_FILE=' - line: "CONFIG_FILE={{ openshift.common.config_base }}/node/node-config.yaml" - - regex: '^IMAGE_VERSION=' - line: "IMAGE_VERSION={{ openshift_image_tag }}" - notify: - - restart node - -- name: Configure Proxy Settings - lineinfile: - dest: /etc/sysconfig/{{ openshift.common.service_type }}-node - regexp: "{{ item.regex }}" - line: "{{ item.line }}" - create: true - with_items: - - regex: '^HTTP_PROXY=' - line: "HTTP_PROXY={{ openshift.common.http_proxy | default('') }}" - - regex: '^HTTPS_PROXY=' - line: "HTTPS_PROXY={{ openshift.common.https_proxy | default('') }}" - - regex: '^NO_PROXY=' - line: "NO_PROXY={{ openshift.common.no_proxy | default([]) }},{{ openshift.common.portal_net }},{{ hostvars[groups.oo_first_master.0].openshift.master.sdn_cluster_network_cidr }}" - when: ('http_proxy' in openshift.common and openshift.common.http_proxy != '') - notify: - - restart node -- name: Reload systemd units - command: systemctl daemon-reload - when: (openshift.common.is_containerized | bool and (install_node_result | changed or install_ovs_sysconfig | changed or install_node_dep_result | changed)) or install_oom_fix_result | changed - notify: - - restart node +- include: config/configure-node-settings.yml +- include: config/configure-proxy-settings.yml diff --git a/roles/openshift_node_upgrade/templates/node.service.j2 b/roles/openshift_node_upgrade/templates/node.service.j2 index 1dbe58439..e12a52c15 100644 --- a/roles/openshift_node_upgrade/templates/node.service.j2 +++ b/roles/openshift_node_upgrade/templates/node.service.j2 @@ -24,8 +24,8 @@ WorkingDirectory=/var/lib/origin/ SyslogIdentifier={{ openshift.common.service_type }}-node Restart=always RestartSec=5s +TimeoutStartSec=300 OOMScoreAdjust=-999 -KillMode=process [Install] WantedBy=multi-user.target |