diff options
15 files changed, 99 insertions, 22 deletions
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml index 5de03951c..b30450def 100644 --- a/playbooks/common/openshift-master/config.yml +++ b/playbooks/common/openshift-master/config.yml @@ -5,6 +5,19 @@ t_oo_option_master_debug_level: "{{ lookup('oo_option', 'openshift_master_debug_level') }}" pre_tasks: + # Per https://bugzilla.redhat.com/show_bug.cgi?id=1469336 + # + # When scaling up a cluster upgraded from OCP <= 3.5, ensure that + # OPENSHIFT_DEFAULT_REGISTRY is present as defined on the existing + # masters, or absent if such is the case. + - name: Detect if this host is a new master in a scale up + set_fact: + g_openshift_master_is_scaleup: "{{ openshift.common.hostname in ( groups['new_masters'] | default([]) ) }}" + + - name: Scaleup Detection + debug: + var: g_openshift_master_is_scaleup + - name: Check for RPM generated config marker file .config_managed stat: path: /etc/origin/.config_managed @@ -69,7 +82,7 @@ ha: "{{ openshift_master_ha | default(groups.oo_masters | length > 1) }}" master_count: "{{ openshift_master_count | default(groups.oo_masters | length) }}" -- name: Inspect state of first master session secrets and config +- name: Inspect state of first master config settings hosts: oo_first_master roles: - role: openshift_facts @@ -98,6 +111,42 @@ set_fact: l_etcd3_enabled: "{{ etcd3_grep.rc == 0 | bool }}" + - name: Check if atomic-openshift-master sysconfig exists yet + stat: + path: /etc/sysconfig/atomic-openshift-master + register: l_aom_exists + + - name: Preserve OPENSHIFT_DEFAULT_REGISTRY master parameter if present + command: awk '/^OPENSHIFT_DEFAULT_REGISTRY/' /etc/sysconfig/atomic-openshift-master + register: l_default_registry_defined + when: l_aom_exists.stat.exists | bool + + - name: Check if atomic-openshift-master-api sysconfig exists yet + stat: + path: /etc/sysconfig/atomic-openshift-master-api + register: l_aom_api_exists + + - name: Preserve OPENSHIFT_DEFAULT_REGISTRY master-api parameter if present + command: awk '/^OPENSHIFT_DEFAULT_REGISTRY/' /etc/sysconfig/atomic-openshift-master-api + register: l_default_registry_defined_api + when: l_aom_api_exists.stat.exists | bool + + - name: Check if atomic-openshift-master-controllers sysconfig exists yet + stat: + path: /etc/sysconfig/atomic-openshift-master-controllers + register: l_aom_controllers_exists + + - name: Preserve OPENSHIFT_DEFAULT_REGISTRY master-controllers parameter if present + command: awk '/^OPENSHIFT_DEFAULT_REGISTRY/' /etc/sysconfig/atomic-openshift-master-controllers + register: l_default_registry_defined_controllers + when: l_aom_controllers_exists.stat.exists | bool + + - name: Update facts with OPENSHIFT_DEFAULT_REGISTRY value + set_fact: + l_default_registry_value: "{{ l_default_registry_defined.stdout | default('') }}" + l_default_registry_value_api: "{{ l_default_registry_defined_api.stdout | default('') }}" + l_default_registry_value_controllers: "{{ l_default_registry_defined_controllers.stdout | default('') }}" + - name: Generate master session secrets hosts: oo_first_master vars: @@ -145,6 +194,10 @@ etcd_cert_prefix: "master.etcd-" r_openshift_master_clean_install: "{{ hostvars[groups.oo_first_master.0].l_clean_install }}" r_openshift_master_etcd3_storage: "{{ hostvars[groups.oo_first_master.0].l_etcd3_enabled }}" + openshift_master_is_scaleup_host: "{{ g_openshift_master_is_scaleup | default(false) }}" + openshift_master_default_registry_value: "{{ hostvars[groups.oo_first_master.0].l_default_registry_value }}" + openshift_master_default_registry_value_api: "{{ hostvars[groups.oo_first_master.0].l_default_registry_value_api }}" + openshift_master_default_registry_value_controllers: "{{ hostvars[groups.oo_first_master.0].l_default_registry_value_controllers }}" - role: nuage_master when: openshift.common.use_nuage | bool - role: calico_master diff --git a/roles/openshift_logging/README.md b/roles/openshift_logging/README.md index 97650e2ce..84ead3548 100644 --- a/roles/openshift_logging/README.md +++ b/roles/openshift_logging/README.md @@ -135,16 +135,23 @@ Elasticsearch OPS too, if using an OPS cluster: secure_forward forwarder for the node agent Fluentd daemonsets running in the cluster. This can be used to reduce the number of connections to the OpenShift API server, by using `mux` and configuring each node Fluentd to - send raw logs to mux and turn off the k8s metadata plugin. + send raw logs to mux and turn off the k8s metadata plugin. This requires the + use of `openshift_logging_mux_client_mode` (see below). - `openshift_logging_mux_allow_external`: Default `False`. If this is `True`, the `mux` service will be deployed, and it will be configured to allow Fluentd clients running outside of the cluster to send logs using secure_forward. This allows OpenShift logging to be used as a central logging service for clients other than OpenShift, or other OpenShift clusters. -- `openshift_logging_use_mux_client`: Default `False`. If this is `True`, the - node agent Fluentd services will be configured to send logs to the mux - service rather than directly to Elasticsearch. +- `openshift_logging_mux_client_mode`: Values - `minimal`, `maximal`. + Default is unset. Setting this value will cause the Fluentd node agent to + send logs to mux rather than directly to Elasticsearch. The value + `maximal` means that Fluentd will do as much processing as possible at the + node before sending the records to mux. This is the current recommended + way to use mux due to current scaling issues. + The value `minimal` means that Fluentd will do *no* processing at all, and + send the raw logs to mux for processing. We do not currently recommend using + this mode, and ansible will warn you about this. - `openshift_logging_mux_hostname`: Default is "mux." + `openshift_master_default_subdomain`. This is the hostname *external*_ clients will use to connect to mux, and will be used in the TLS server cert diff --git a/roles/openshift_logging/defaults/main.yml b/roles/openshift_logging/defaults/main.yml index 1c243f934..8b0f4cb62 100644 --- a/roles/openshift_logging/defaults/main.yml +++ b/roles/openshift_logging/defaults/main.yml @@ -157,8 +157,6 @@ openshift_logging_storage_access_modes: "{{ openshift_hosted_logging_storage_acc # mux - secure_forward listener service openshift_logging_mux_allow_external: False openshift_logging_use_mux: "{{ openshift_logging_mux_allow_external | default(False) }}" -# this tells the fluentd node agent to use mux instead of sending directly to Elasticsearch -openshift_logging_use_mux_client: False openshift_logging_mux_hostname: "{{ 'mux.' ~ (openshift_master_default_subdomain | default('router.default.svc.cluster.local', true)) }}" openshift_logging_mux_port: 24284 openshift_logging_mux_cpu_limit: 500m diff --git a/roles/openshift_logging_fluentd/defaults/main.yml b/roles/openshift_logging_fluentd/defaults/main.yml index be9943b0d..a53bbd2df 100644 --- a/roles/openshift_logging_fluentd/defaults/main.yml +++ b/roles/openshift_logging_fluentd/defaults/main.yml @@ -48,7 +48,6 @@ openshift_logging_fluentd_aggregating_strict: "no" openshift_logging_fluentd_aggregating_cert_path: none openshift_logging_fluentd_aggregating_key_path: none openshift_logging_fluentd_aggregating_passphrase: none -openshift_logging_use_mux_client: False ### Deprecating in 3.6 openshift_logging_fluentd_es_copy: false diff --git a/roles/openshift_logging_fluentd/tasks/main.yaml b/roles/openshift_logging_fluentd/tasks/main.yaml index 55de2ae8d..9dfc6fc86 100644 --- a/roles/openshift_logging_fluentd/tasks/main.yaml +++ b/roles/openshift_logging_fluentd/tasks/main.yaml @@ -23,6 +23,14 @@ msg: openshift_hosted_logging_use_journal is deprecated. Fluentd will automatically detect which logging driver is being used. when: openshift_hosted_logging_use_journal is defined +- fail: + msg: Invalid openshift_logging_mux_client_mode [{{ openshift_logging_mux_client_mode }}], one of {{ __allowed_mux_client_modes }} allowed + when: openshift_logging_mux_client_mode is defined and not openshift_logging_mux_client_mode in __allowed_mux_client_modes + +- debug: + msg: WARNING Use of openshift_logging_mux_client_mode=minimal is not recommended due to current scaling issues + when: openshift_logging_mux_client_mode is defined and openshift_logging_mux_client_mode == 'minimal' + - include: determine_version.yaml # allow passing in a tempdir diff --git a/roles/openshift_logging_fluentd/templates/fluentd.j2 b/roles/openshift_logging_fluentd/templates/fluentd.j2 index a4cf9a149..39dffba19 100644 --- a/roles/openshift_logging_fluentd/templates/fluentd.j2 +++ b/roles/openshift_logging_fluentd/templates/fluentd.j2 @@ -64,7 +64,7 @@ spec: readOnly: true - name: filebufferstorage mountPath: /var/lib/fluentd -{% if openshift_logging_use_mux_client | bool %} +{% if openshift_logging_mux_client_mode is defined %} - name: muxcerts mountPath: /etc/fluent/muxkeys readOnly: true @@ -112,10 +112,12 @@ spec: resourceFieldRef: containerName: "{{ daemonset_container_name }}" resource: limits.memory - - name: "USE_MUX_CLIENT" - value: "{{ openshift_logging_use_mux_client | default('false') | lower }}" - name: "FILE_BUFFER_LIMIT" value: "{{ openshift_logging_fluentd_file_buffer_limit | default('1Gi') }}" +{% if openshift_logging_mux_client_mode is defined %} + - name: "MUX_CLIENT_MODE" + value: "{{ openshift_logging_mux_client_mode }}" +{% endif %} volumes: - name: runlogjournal hostPath: @@ -144,7 +146,7 @@ spec: - name: dockerdaemoncfg hostPath: path: /etc/docker -{% if openshift_logging_use_mux_client | bool %} +{% if openshift_logging_mux_client_mode is defined %} - name: muxcerts secret: secretName: logging-mux diff --git a/roles/openshift_logging_fluentd/vars/main.yml b/roles/openshift_logging_fluentd/vars/main.yml index ad3fb0bdd..ec8e565c3 100644 --- a/roles/openshift_logging_fluentd/vars/main.yml +++ b/roles/openshift_logging_fluentd/vars/main.yml @@ -2,3 +2,4 @@ __latest_fluentd_version: "3_5" __allowed_fluentd_versions: ["3_5", "3_6"] __allowed_fluentd_types: ["hosted", "secure-aggregator", "secure-host"] +__allowed_mux_client_modes: ["minimal", "maximal"] diff --git a/roles/openshift_logging_mux/defaults/main.yml b/roles/openshift_logging_mux/defaults/main.yml index 35fc7146f..7a3da9b4c 100644 --- a/roles/openshift_logging_mux/defaults/main.yml +++ b/roles/openshift_logging_mux/defaults/main.yml @@ -28,6 +28,7 @@ openshift_logging_mux_journal_source: "{{ openshift_hosted_logging_journal_sourc openshift_logging_mux_journal_read_from_head: "{{ openshift_hosted_logging_journal_read_from_head | default('') }}" openshift_logging_mux_allow_external: False +openshift_logging_use_mux: "{{ openshift_logging_mux_allow_external | default(False) }}" openshift_logging_mux_hostname: "{{ 'mux.' ~ (openshift_master_default_subdomain | default('router.default.svc.cluster.local', true)) }}" openshift_logging_mux_port: 24284 # the namespace to use for undefined projects should come first, followed by any diff --git a/roles/openshift_logging_mux/templates/mux.j2 b/roles/openshift_logging_mux/templates/mux.j2 index e43d9d397..70afe5cee 100644 --- a/roles/openshift_logging_mux/templates/mux.j2 +++ b/roles/openshift_logging_mux/templates/mux.j2 @@ -101,8 +101,6 @@ spec: value: "{{ openshift_logging_mux_port }}" - name: USE_MUX value: "true" - - name: MUX_ALLOW_EXTERNAL - value: "{{ openshift_logging_mux_allow_external | default('false') | lower }}" - name: "BUFFER_QUEUE_LIMIT" value: "{{ openshift_logging_mux_buffer_queue_limit }}" - name: "BUFFER_SIZE_LIMIT" diff --git a/roles/openshift_master/tasks/systemd_units.yml b/roles/openshift_master/tasks/systemd_units.yml index dfc255b3d..d71ad3459 100644 --- a/roles/openshift_master/tasks/systemd_units.yml +++ b/roles/openshift_master/tasks/systemd_units.yml @@ -23,7 +23,7 @@ when: openshift.common.is_containerized | bool and not openshift.common.is_master_system_container | bool # workaround for missing systemd unit files -- name: Create the systemd unit files +- name: "Create the {{ openshift.common.service_type }} systemd unit file" template: src: "master_docker/master.docker.service.j2" dest: "{{ containerized_svc_dir }}/{{ openshift.common.service_type }}-master.service" @@ -32,7 +32,7 @@ - not openshift.common.is_master_system_container | bool register: create_master_unit_file -- name: Install Master service file +- name: "Install {{ openshift.common.service_type }} systemd unit file" copy: dest: "/etc/systemd/system/{{ openshift.common.service_type }}-master.service" src: "{{ openshift.common.service_type }}-master.service" @@ -44,7 +44,7 @@ - command: systemctl daemon-reload when: create_master_unit_file | changed -- name: Create the ha systemd unit files +- name: Create the ha systemd unit files for api and controller services template: src: "{{ ha_svc_template_path }}/atomic-openshift-master-{{ item }}.service.j2" dest: "{{ containerized_svc_dir }}/{{ openshift.common.service_type }}-master-{{ item }}.service" diff --git a/roles/openshift_master/templates/atomic-openshift-master.j2 b/roles/openshift_master/templates/atomic-openshift-master.j2 index 850fae0e4..b931f1414 100644 --- a/roles/openshift_master/templates/atomic-openshift-master.j2 +++ b/roles/openshift_master/templates/atomic-openshift-master.j2 @@ -1,6 +1,9 @@ OPTIONS=--loglevel={{ openshift.master.debug_level | default(2) }} CONFIG_FILE={{ openshift_master_config_file }} -{% if openshift_push_via_dns | default(false) %} +{# Preserve existing OPENSHIFT_DEFAULT_REGISTRY settings in scale up runs #} +{% if openshift_master_is_scaleup_host %} +{{ openshift_master_default_registry_value }} +{% elif openshift_push_via_dns | default(false) %} OPENSHIFT_DEFAULT_REGISTRY=docker-registry.default.svc:5000 {% endif %} {% if openshift.common.is_containerized | bool %} diff --git a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2 index c05a27559..63eb3ea1b 100644 --- a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2 +++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2 @@ -1,6 +1,9 @@ OPTIONS=--loglevel={{ openshift.master.debug_level }} --listen={{ 'https' if openshift.master.api_use_ssl else 'http' }}://{{ openshift.master.bind_addr }}:{{ openshift.master.api_port }} --master={{ openshift.master.loopback_api_url }} CONFIG_FILE={{ openshift_master_config_file }} -{% if openshift_push_via_dns | default(false) %} +{# Preserve existing OPENSHIFT_DEFAULT_REGISTRY settings in scale up runs #} +{% if openshift_master_is_scaleup_host %} +{{ openshift_master_default_registry_value_api }} +{% elif openshift_push_via_dns | default(false) %} OPENSHIFT_DEFAULT_REGISTRY=docker-registry.default.svc:5000 {% endif %} {% if openshift.common.is_containerized | bool %} diff --git a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2 index a153fb33d..0adfd05b6 100644 --- a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2 +++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2 @@ -1,13 +1,16 @@ OPTIONS=--loglevel={{ openshift.master.debug_level }} --listen={{ 'https' if openshift.master.api_use_ssl else 'http' }}://{{ openshift.master.bind_addr }}:{{ openshift.master.controllers_port }} CONFIG_FILE={{ openshift_master_config_file }} -{% if openshift_push_via_dns | default(false) %} +{# Preserve existing OPENSHIFT_DEFAULT_REGISTRY settings in scale up runs #} +{% if openshift_master_is_scaleup_host %} +{{ openshift_master_default_registry_value_controllers }} +{% elif openshift_push_via_dns | default(false) %} OPENSHIFT_DEFAULT_REGISTRY=docker-registry.default.svc:5000 {% endif %} {% if openshift.common.is_containerized | bool %} IMAGE_VERSION={{ openshift_image_tag }} {% endif %} -{% if openshift_cloudprovider_kind | default('') == 'aws' and openshift_cloudprovider_aws_access_key is defined and openshift_cloudprovider_aws_access_key is defined %} +{% if openshift_cloudprovider_kind | default('') == 'aws' and openshift_cloudprovider_aws_access_key is defined and openshift_cloudprovider_aws_secret_key is defined %} AWS_ACCESS_KEY_ID={{ openshift_cloudprovider_aws_access_key }} AWS_SECRET_ACCESS_KEY={{ openshift_cloudprovider_aws_secret_key }} {% endif %} diff --git a/roles/openshift_master/vars/main.yml b/roles/openshift_master/vars/main.yml index c5ba20409..7745d014f 100644 --- a/roles/openshift_master/vars/main.yml +++ b/roles/openshift_master/vars/main.yml @@ -20,3 +20,4 @@ openshift_master_valid_grant_methods: - deny l_is_ha: "{{ openshift.master.ha is defined and openshift.master.ha | bool }}" +openshift_master_is_scaleup_host: False diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml index 600d8f676..19eb3cdf7 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml @@ -15,7 +15,7 @@ oc_project: state: present name: "{{ glusterfs_namespace }}" - when: glusterfs_is_native or glusterfs_heketi_is_native + when: glusterfs_is_native or glusterfs_heketi_is_native or glusterfs_storageclass - name: Delete pre-existing heketi resources oc_obj: |