blob: c37a5f9abdda28253988f77d2f1eb79da8af6fb3 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
|
---
###############################################################################
# Upgrade Masters
###############################################################################
# oc adm migrate storage should be run prior to etcd v3 upgrade
# See: https://github.com/openshift/origin/pull/14625#issuecomment-308467060
- name: Pre master upgrade - Upgrade all storage
hosts: oo_first_master
tasks:
- name: Upgrade all storage
command: >
{{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig
migrate storage --include=* --confirm
register: l_pb_upgrade_control_plane_pre_upgrade_storage
when: openshift_upgrade_pre_storage_migration_enabled | default(true) | bool
failed_when:
- openshift_upgrade_pre_storage_migration_enabled | default(true) | bool
- l_pb_upgrade_control_plane_pre_upgrade_storage.rc != 0
- openshift_upgrade_pre_storage_migration_fatal | default(true) | bool
# If facts cache were for some reason deleted, this fact may not be set, and if not set
# it will always default to true. This causes problems for the etcd data dir fact detection
# so we must first make sure this is set correctly before attempting the backup.
- name: Set master embedded_etcd fact
hosts: oo_masters_to_config
roles:
- openshift_facts
tasks:
- openshift_facts:
role: master
local_facts:
embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
- name: Upgrade and backup etcd
include: ./etcd/main.yml
# Create service signer cert when missing. Service signer certificate
# is added to master config in the master_config_upgrade hook.
- name: Determine if service signer cert must be created
hosts: oo_first_master
tasks:
- name: Determine if service signer certificate must be created
stat:
path: "{{ openshift.common.config_base }}/master/service-signer.crt"
register: service_signer_cert_stat
changed_when: false
- include: create_service_signer_cert.yml
# Set openshift_master_facts separately. In order to reconcile
# admission_config's, we currently must run openshift_master_facts and
# then run openshift_facts.
- name: Set OpenShift master facts
hosts: oo_masters_to_config
roles:
- openshift_master_facts
# The main master upgrade play. Should handle all changes to the system in one pass, with
# support for optional hooks to be defined.
- name: Upgrade master
hosts: oo_masters_to_config
vars:
openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
serial: 1
handlers:
- include: ../../../../roles/openshift_master/handlers/main.yml
static: yes
roles:
- openshift_facts
- lib_utils
post_tasks:
# Run the pre-upgrade hook if defined:
- debug: msg="Running master pre-upgrade hook {{ openshift_master_upgrade_pre_hook }}"
when: openshift_master_upgrade_pre_hook is defined
- include: "{{ openshift_master_upgrade_pre_hook }}"
when: openshift_master_upgrade_pre_hook is defined
- include: rpm_upgrade.yml component=master
when: not openshift.common.is_containerized | bool
- include_vars: ../../../../roles/openshift_master_facts/vars/main.yml
- include: upgrade_scheduler.yml
- include: "{{ master_config_hook }}"
when: master_config_hook is defined
- include_vars: ../../../../roles/openshift_master/vars/main.yml
- name: Remove any legacy systemd units and update systemd units
include: ../../../../roles/openshift_master/tasks/systemd_units.yml
- name: Check for ca-bundle.crt
stat:
path: "{{ openshift.common.config_base }}/master/ca-bundle.crt"
register: ca_bundle_stat
failed_when: false
- name: Check for ca.crt
stat:
path: "{{ openshift.common.config_base }}/master/ca.crt"
register: ca_crt_stat
failed_when: false
- name: Migrate ca.crt to ca-bundle.crt
command: mv ca.crt ca-bundle.crt
args:
chdir: "{{ openshift.common.config_base }}/master"
when: ca_crt_stat.stat.isreg and not ca_bundle_stat.stat.exists
- name: Link ca.crt to ca-bundle.crt
file:
src: "{{ openshift.common.config_base }}/master/ca-bundle.crt"
path: "{{ openshift.common.config_base }}/master/ca.crt"
state: link
when: ca_crt_stat.stat.isreg and not ca_bundle_stat.stat.exists
- name: Update oreg value
yedit:
src: "{{ openshift.common.config_base }}/master/master-config.yaml"
key: 'imageConfig.format'
value: "{{ oreg_url | default(oreg_url_master) }}"
when: oreg_url is defined or oreg_url_master is defined
# Run the upgrade hook prior to restarting services/system if defined:
- debug: msg="Running master upgrade hook {{ openshift_master_upgrade_hook }}"
when: openshift_master_upgrade_hook is defined
- include: "{{ openshift_master_upgrade_hook }}"
when: openshift_master_upgrade_hook is defined
- include: ../../openshift-master/restart_hosts.yml
when: openshift.common.rolling_restart_mode == 'system'
- include: ../../openshift-master/restart_services.yml
when: openshift.common.rolling_restart_mode == 'services'
# Run the post-upgrade hook if defined:
- debug: msg="Running master post-upgrade hook {{ openshift_master_upgrade_post_hook }}"
when: openshift_master_upgrade_post_hook is defined
- include: "{{ openshift_master_upgrade_post_hook }}"
when: openshift_master_upgrade_post_hook is defined
- name: Post master upgrade - Upgrade clusterpolicies storage
command: >
{{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig
migrate storage --include=clusterpolicies --confirm
register: l_pb_upgrade_control_plane_post_upgrade_storage
when: openshift_upgrade_post_storage_migration_enabled | default(true) | bool
failed_when:
- openshift_upgrade_post_storage_migration_enabled | default(true) | bool
- l_pb_upgrade_control_plane_post_upgrade_storage.rc != 0
- openshift_upgrade_post_storage_migration_fatal | default(false) | bool
run_once: true
delegate_to: "{{ groups.oo_first_master.0 }}"
- set_fact:
master_update_complete: True
##############################################################################
# Gate on master update complete
##############################################################################
- name: Gate on master update
hosts: localhost
connection: local
become: no
tasks:
- set_fact:
master_update_completed: "{{ hostvars
| oo_select_keys(groups.oo_masters_to_config)
| oo_collect('inventory_hostname', {'master_update_complete': true}) }}"
- set_fact:
master_update_failed: "{{ groups.oo_masters_to_config | difference(master_update_completed) | list }}"
- fail:
msg: "Upgrade cannot continue. The following masters did not finish updating: {{ master_update_failed | join(',') }}"
when: master_update_failed | length > 0
###############################################################################
# Reconcile Cluster Roles, Cluster Role Bindings and Security Context Constraints
###############################################################################
- name: Reconcile Cluster Roles and Cluster Role Bindings and Security Context Constraints
hosts: oo_masters_to_config
roles:
- { role: openshift_cli }
vars:
openshift_docker_hosted_registry_network: "{{ hostvars[groups.oo_first_master.0].openshift.common.portal_net }}"
# Another spot where we assume docker is running and do not want to accidentally trigger an unsafe
# restart.
skip_docker_role: True
__master_shared_resource_viewer_file: "shared_resource_viewer_role.yaml"
tasks:
- name: Reconcile Cluster Roles
command: >
{{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig
policy reconcile-cluster-roles --additive-only=true --confirm -o name
register: reconcile_cluster_role_result
when: not openshift.common.version_gte_3_7 | bool
changed_when:
- reconcile_cluster_role_result.stdout != ''
- reconcile_cluster_role_result.rc == 0
run_once: true
- name: Reconcile Cluster Role Bindings
command: >
{{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig
policy reconcile-cluster-role-bindings
--exclude-groups=system:authenticated
--exclude-groups=system:authenticated:oauth
--exclude-groups=system:unauthenticated
--exclude-users=system:anonymous
--additive-only=true --confirm -o name
when: not openshift.common.version_gte_3_7 | bool
register: reconcile_bindings_result
changed_when:
- reconcile_bindings_result.stdout != ''
- reconcile_bindings_result.rc == 0
run_once: true
- name: Reconcile Jenkins Pipeline Role Bindings
command: >
{{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig policy reconcile-cluster-role-bindings system:build-strategy-jenkinspipeline --confirm -o name
run_once: true
register: reconcile_jenkins_role_binding_result
changed_when:
- reconcile_jenkins_role_binding_result.stdout != ''
- reconcile_jenkins_role_binding_result.rc == 0
when: (not openshift.common.version_gte_3_7 | bool) and (openshift.common.version_gte_3_4_or_1_4 | bool)
- when: (openshift.common.version_gte_3_6 | bool) and (not openshift.common.version_gte_3_7 | bool)
block:
- name: Retrieve shared-resource-viewer
oc_obj:
state: list
kind: role
name: "shared-resource-viewer"
namespace: "openshift"
register: objout
- name: Determine if shared-resource-viewer is protected
set_fact:
__shared_resource_viewer_protected: true
when:
- "'results' in objout"
- "'results' in objout['results']"
- "'annotations' in objout['results']['results'][0]['metadata']"
- "'openshift.io/reconcile-protect' in objout['results']['results'][0]['metadata']['annotations']"
- "objout['results']['results'][0]['metadata']['annotations']['openshift.io/reconcile-protect'] == 'true'"
- copy:
src: "{{ item }}"
dest: "/tmp/{{ item }}"
with_items:
- "{{ __master_shared_resource_viewer_file }}"
when: __shared_resource_viewer_protected is not defined
- name: Fixup shared-resource-viewer role
oc_obj:
state: present
kind: role
name: "shared-resource-viewer"
namespace: "openshift"
files:
- "/tmp/{{ __master_shared_resource_viewer_file }}"
delete_after: true
when: __shared_resource_viewer_protected is not defined
- name: Reconcile Security Context Constraints
command: >
{{ openshift.common.client_binary }} adm policy --config={{ openshift.common.config_base }}/master/admin.kubeconfig reconcile-sccs --confirm --additive-only=true -o name
register: reconcile_scc_result
changed_when:
- reconcile_scc_result.stdout != ''
- reconcile_scc_result.rc == 0
run_once: true
- name: Migrate storage post policy reconciliation
command: >
{{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig
migrate storage --include=* --confirm
run_once: true
register: l_pb_upgrade_control_plane_post_upgrade_storage
when: openshift_upgrade_post_storage_migration_enabled | default(true) | bool
failed_when:
- openshift_upgrade_post_storage_migration_enabled | default(true) | bool
- l_pb_upgrade_control_plane_post_upgrade_storage.rc != 0
- openshift_upgrade_post_storage_migration_fatal | default(false) | bool
- set_fact:
reconcile_complete: True
##############################################################################
# Gate on reconcile
##############################################################################
- name: Gate on reconcile
hosts: localhost
connection: local
become: no
tasks:
- set_fact:
reconcile_completed: "{{ hostvars
| oo_select_keys(groups.oo_masters_to_config)
| oo_collect('inventory_hostname', {'reconcile_complete': true}) }}"
- set_fact:
reconcile_failed: "{{ groups.oo_masters_to_config | difference(reconcile_completed) | list }}"
- fail:
msg: "Upgrade cannot continue. The following masters did not finish reconciling: {{ reconcile_failed | join(',') }}"
when: reconcile_failed | length > 0
- name: Upgrade Docker on dedicated containerized etcd hosts
hosts: oo_etcd_to_config:!oo_nodes_to_upgrade
serial: 1
any_errors_fatal: true
roles:
- openshift_facts
tasks:
- include: docker/tasks/upgrade.yml
when: l_docker_upgrade is defined and l_docker_upgrade | bool and not openshift.common.is_atomic | bool
- name: Drain and upgrade master nodes
hosts: oo_masters_to_config:&oo_nodes_to_upgrade
# This var must be set with -e on invocation, as it is not a per-host inventory var
# and is evaluated early. Values such as "20%" can also be used.
serial: "{{ openshift_upgrade_control_plane_nodes_serial | default(1) }}"
max_fail_percentage: "{{ openshift_upgrade_control_plane_nodes_max_fail_percentage | default(0) }}"
pre_tasks:
- name: Load lib_openshift modules
include_role:
name: lib_openshift
# TODO: To better handle re-trying failed upgrades, it would be nice to check if the node
# or docker actually needs an upgrade before proceeding. Perhaps best to save this until
# we merge upgrade functionality into the base roles and a normal config.yml playbook run.
- name: Mark node unschedulable
oc_adm_manage_node:
node: "{{ openshift.node.nodename | lower }}"
schedulable: False
delegate_to: "{{ groups.oo_first_master.0 }}"
retries: 10
delay: 5
register: node_unschedulable
until: node_unschedulable|succeeded
- name: Drain Node for Kubelet upgrade
command: >
{{ hostvars[groups.oo_first_master.0].openshift.common.admin_binary }} drain {{ openshift.node.nodename | lower }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig --force --delete-local-data --ignore-daemonsets
delegate_to: "{{ groups.oo_first_master.0 }}"
register: l_upgrade_control_plane_drain_result
until: not l_upgrade_control_plane_drain_result | failed
retries: 60
delay: 60
roles:
- lib_openshift
- openshift_facts
- docker
- openshift_node_dnsmasq
- openshift_node_upgrade
post_tasks:
- name: Set node schedulability
oc_adm_manage_node:
node: "{{ openshift.node.nodename | lower }}"
schedulable: True
delegate_to: "{{ groups.oo_first_master.0 }}"
retries: 10
delay: 5
register: node_schedulable
until: node_schedulable|succeeded
when: node_unschedulable|changed
|