1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
|
---
###############################################################################
# Upgrade Masters
###############################################################################
- name: Evaluate additional groups for upgrade
hosts: localhost
connection: local
become: no
tasks:
- name: Evaluate etcd_hosts_to_backup
add_host:
name: "{{ item }}"
groups: etcd_hosts_to_backup
with_items: "{{ groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master }}"
changed_when: False
# If facts cache were for some reason deleted, this fact may not be set, and if not set
# it will always default to true. This causes problems for the etcd data dir fact detection
# so we must first make sure this is set correctly before attempting the backup.
- name: Set master embedded_etcd fact
hosts: oo_masters_to_config
roles:
- openshift_facts
tasks:
- openshift_facts:
role: master
local_facts:
embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
debug_level: "{{ openshift_master_debug_level | default(openshift.common.debug_level | default(2)) }}"
- name: Upgrade and backup etcd
include: ./etcd/main.yml
# Create service signer cert when missing. Service signer certificate
# is added to master config in the master_config_upgrade hook.
- name: Determine if service signer cert must be created
hosts: oo_first_master
tasks:
- name: Determine if service signer certificate must be created
stat:
path: "{{ openshift.common.config_base }}/master/service-signer.crt"
register: service_signer_cert_stat
changed_when: false
- include: create_service_signer_cert.yml
# Set openshift_master_facts separately. In order to reconcile
# admission_config's, we currently must run openshift_master_facts and
# then run openshift_facts.
- name: Set OpenShift master facts
hosts: oo_masters_to_config
roles:
- openshift_master_facts
# The main master upgrade play. Should handle all changes to the system in one pass, with
# support for optional hooks to be defined.
- name: Upgrade master
hosts: oo_masters_to_config
vars:
openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
serial: 1
handlers:
- include: ../../../../roles/openshift_master/handlers/main.yml
static: yes
roles:
- openshift_facts
post_tasks:
# Run the pre-upgrade hook if defined:
- debug: msg="Running master pre-upgrade hook {{ openshift_master_upgrade_pre_hook }}"
when: openshift_master_upgrade_pre_hook is defined
- include: "{{ openshift_master_upgrade_pre_hook }}"
when: openshift_master_upgrade_pre_hook is defined
- include: rpm_upgrade.yml component=master
when: not openshift.common.is_containerized | bool
- include_vars: ../../../../roles/openshift_master_facts/vars/main.yml
- include: upgrade_scheduler.yml
- include: "{{ master_config_hook }}"
when: master_config_hook is defined
- include_vars: ../../../../roles/openshift_master/vars/main.yml
- name: Update systemd units
include: ../../../../roles/openshift_master/tasks/systemd_units.yml
- name: Check for ca-bundle.crt
stat:
path: "{{ openshift.common.config_base }}/master/ca-bundle.crt"
register: ca_bundle_stat
failed_when: false
- name: Check for ca.crt
stat:
path: "{{ openshift.common.config_base }}/master/ca.crt"
register: ca_crt_stat
failed_when: false
- name: Migrate ca.crt to ca-bundle.crt
command: mv ca.crt ca-bundle.crt
args:
chdir: "{{ openshift.common.config_base }}/master"
when: ca_crt_stat.stat.isreg and not ca_bundle_stat.stat.exists
- name: Link ca.crt to ca-bundle.crt
file:
src: "{{ openshift.common.config_base }}/master/ca-bundle.crt"
path: "{{ openshift.common.config_base }}/master/ca.crt"
state: link
when: ca_crt_stat.stat.isreg and not ca_bundle_stat.stat.exists
# Run the upgrade hook prior to restarting services/system if defined:
- debug: msg="Running master upgrade hook {{ openshift_master_upgrade_hook }}"
when: openshift_master_upgrade_hook is defined
- include: "{{ openshift_master_upgrade_hook }}"
when: openshift_master_upgrade_hook is defined
- include: ../../openshift-master/restart_hosts.yml
when: openshift.common.rolling_restart_mode == 'system'
- include: ../../openshift-master/restart_services.yml
when: openshift.common.rolling_restart_mode == 'services'
# Run the post-upgrade hook if defined:
- debug: msg="Running master post-upgrade hook {{ openshift_master_upgrade_post_hook }}"
when: openshift_master_upgrade_post_hook is defined
- include: "{{ openshift_master_upgrade_post_hook }}"
when: openshift_master_upgrade_post_hook is defined
- set_fact:
master_update_complete: True
##############################################################################
# Gate on master update complete
##############################################################################
- name: Gate on master update
hosts: localhost
connection: local
become: no
tasks:
- set_fact:
master_update_completed: "{{ hostvars
| oo_select_keys(groups.oo_masters_to_config)
| oo_collect('inventory_hostname', {'master_update_complete': true}) }}"
- set_fact:
master_update_failed: "{{ groups.oo_masters_to_config | difference(master_update_completed) }}"
- fail:
msg: "Upgrade cannot continue. The following masters did not finish updating: {{ master_update_failed | join(',') }}"
when: master_update_failed | length > 0
###############################################################################
# Reconcile Cluster Roles, Cluster Role Bindings and Security Context Constraints
###############################################################################
- name: Reconcile Cluster Roles and Cluster Role Bindings and Security Context Constraints
hosts: oo_masters_to_config
roles:
- { role: openshift_cli }
vars:
origin_reconcile_bindings: "{{ deployment_type == 'origin' and openshift_version | version_compare('1.0.6', '>') }}"
ent_reconcile_bindings: true
openshift_docker_hosted_registry_network: "{{ hostvars[groups.oo_first_master.0].openshift.common.portal_net }}"
# Another spot where we assume docker is running and do not want to accidentally trigger an unsafe
# restart.
skip_docker_role: True
tasks:
- name: Reconcile Cluster Roles
command: >
{{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig
policy reconcile-cluster-roles --additive-only=true --confirm
run_once: true
- name: Reconcile Cluster Role Bindings
command: >
{{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig
policy reconcile-cluster-role-bindings
--exclude-groups=system:authenticated
--exclude-groups=system:authenticated:oauth
--exclude-groups=system:unauthenticated
--exclude-users=system:anonymous
--additive-only=true --confirm
when: origin_reconcile_bindings | bool or ent_reconcile_bindings | bool
run_once: true
- name: Reconcile Jenkins Pipeline Role Bindings
command: >
{{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig policy reconcile-cluster-role-bindings system:build-strategy-jenkinspipeline --confirm
run_once: true
when: openshift.common.version_gte_3_4_or_1_4 | bool
- name: Reconcile Security Context Constraints
command: >
{{ openshift.common.client_binary }} adm policy reconcile-sccs --confirm --additive-only=true
run_once: true
- set_fact:
reconcile_complete: True
##############################################################################
# Gate on reconcile
##############################################################################
- name: Gate on reconcile
hosts: localhost
connection: local
become: no
tasks:
- set_fact:
reconcile_completed: "{{ hostvars
| oo_select_keys(groups.oo_masters_to_config)
| oo_collect('inventory_hostname', {'reconcile_complete': true}) }}"
- set_fact:
reconcile_failed: "{{ groups.oo_masters_to_config | difference(reconcile_completed) }}"
- fail:
msg: "Upgrade cannot continue. The following masters did not finish reconciling: {{ reconcile_failed | join(',') }}"
when: reconcile_failed | length > 0
- name: Upgrade Docker on dedicated containerized etcd hosts
hosts: oo_etcd_to_config:!oo_nodes_to_upgrade
serial: 1
any_errors_fatal: true
roles:
- openshift_facts
tasks:
- include: docker/upgrade.yml
when: l_docker_upgrade is defined and l_docker_upgrade | bool and not openshift.common.is_atomic | bool
- name: Drain and upgrade master nodes
hosts: oo_masters_to_config:&oo_nodes_to_upgrade
# This var must be set with -e on invocation, as it is not a per-host inventory var
# and is evaluated early. Values such as "20%" can also be used.
serial: "{{ openshift_upgrade_nodes_serial | default(1) }}"
any_errors_fatal: true
pre_tasks:
- name: Load lib_openshift modules
include_role:
name: lib_openshift
# TODO: To better handle re-trying failed upgrades, it would be nice to check if the node
# or docker actually needs an upgrade before proceeding. Perhaps best to save this until
# we merge upgrade functionality into the base roles and a normal config.yml playbook run.
- name: Mark node unschedulable
oadm_manage_node:
node: "{{ openshift.node.nodename | lower }}"
schedulable: False
delegate_to: "{{ groups.oo_first_master.0 }}"
retries: 10
delay: 5
register: node_unschedulable
until: node_unschedulable|succeeded
- name: Drain Node for Kubelet upgrade
command: >
{{ hostvars[groups.oo_first_master.0].openshift.common.admin_binary }} drain {{ openshift.node.nodename | lower }} --force --delete-local-data
delegate_to: "{{ groups.oo_first_master.0 }}"
roles:
- lib_openshift
- openshift_facts
- docker
- openshift_node_upgrade
post_tasks:
- name: Set node schedulability
oadm_manage_node:
node: "{{ openshift.node.nodename | lower }}"
schedulable: True
delegate_to: "{{ groups.oo_first_master.0 }}"
retries: 10
delay: 5
register: node_schedulable
until: node_schedulable|succeeded
when: node_unschedulable|changed
|