From 17e38ad8c62ac8f1c8f983e63181f69879a45d85 Mon Sep 17 00:00:00 2001
From: Jiri Mencak <jmencak@redhat.com>
Date: Tue, 3 Oct 2017 19:57:01 +0200
Subject: Separate tuned daemon setup into a role.

Currently, profiles for the tuned daemon are set only for
OpenShift node(s).  This excludes the OpenShift loadbalancer.
As a result, ARP cache limits on loadbalancers are not raised.
This causes problems with HA setups where loadbalancers serve
1k+ OpenShift nodes.

This commit ensures the openshift-control-plane role is applied
to loadbalancers, masters and OpenShift infra nodes.  Regular
OpenShift worker nodes get the openshift-node profile.

Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1498213
---
 playbooks/common/openshift-loadbalancer/config.yml |  1 +
 playbooks/common/openshift-master/config.yml       |  1 +
 .../common/openshift-node/configure_nodes.yml      |  1 +
 roles/openshift_node/tasks/config.yml              |  4 ---
 roles/openshift_node/tasks/tuned.yml               | 41 ----------------------
 .../tuned/openshift-control-plane/tuned.conf       | 25 -------------
 .../templates/tuned/openshift-node/tuned.conf      | 10 ------
 .../templates/tuned/openshift/tuned.conf           | 24 -------------
 .../openshift_node/templates/tuned/recommend.conf  |  8 -----
 roles/tuned/defaults/main.yml                      |  3 ++
 roles/tuned/meta/main.yml                          | 13 +++++++
 roles/tuned/tasks/main.yml                         | 39 ++++++++++++++++++++
 .../templates/openshift-control-plane/tuned.conf   | 25 +++++++++++++
 roles/tuned/templates/openshift-node/tuned.conf    | 10 ++++++
 roles/tuned/templates/openshift/tuned.conf         | 24 +++++++++++++
 roles/tuned/templates/recommend.conf               | 11 ++++++
 16 files changed, 128 insertions(+), 112 deletions(-)
 delete mode 100644 roles/openshift_node/tasks/tuned.yml
 delete mode 100644 roles/openshift_node/templates/tuned/openshift-control-plane/tuned.conf
 delete mode 100644 roles/openshift_node/templates/tuned/openshift-node/tuned.conf
 delete mode 100644 roles/openshift_node/templates/tuned/openshift/tuned.conf
 delete mode 100644 roles/openshift_node/templates/tuned/recommend.conf
 create mode 100644 roles/tuned/defaults/main.yml
 create mode 100644 roles/tuned/meta/main.yml
 create mode 100644 roles/tuned/tasks/main.yml
 create mode 100644 roles/tuned/templates/openshift-control-plane/tuned.conf
 create mode 100644 roles/tuned/templates/openshift-node/tuned.conf
 create mode 100644 roles/tuned/templates/openshift/tuned.conf
 create mode 100644 roles/tuned/templates/recommend.conf

diff --git a/playbooks/common/openshift-loadbalancer/config.yml b/playbooks/common/openshift-loadbalancer/config.yml
index ecbb092bc..9e148e636 100644
--- a/playbooks/common/openshift-loadbalancer/config.yml
+++ b/playbooks/common/openshift-loadbalancer/config.yml
@@ -27,6 +27,7 @@
   roles:
   - role: os_firewall
   - role: openshift_loadbalancer
+  - role: tuned
 
 - name: Load Balancer Install Checkpoint End
   hosts: localhost
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index bc1fee982..04e2bdbbf 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -198,6 +198,7 @@
     openshift_master_default_registry_value: "{{ hostvars[groups.oo_first_master.0].l_default_registry_value }}"
     openshift_master_default_registry_value_api: "{{ hostvars[groups.oo_first_master.0].l_default_registry_value_api }}"
     openshift_master_default_registry_value_controllers: "{{ hostvars[groups.oo_first_master.0].l_default_registry_value_controllers }}"
+  - role: tuned
   - role: nuage_ca
     when: openshift_use_nuage | default(false) | bool
   - role: nuage_common
diff --git a/playbooks/common/openshift-node/configure_nodes.yml b/playbooks/common/openshift-node/configure_nodes.yml
index c96e4921c..17259422d 100644
--- a/playbooks/common/openshift-node/configure_nodes.yml
+++ b/playbooks/common/openshift-node/configure_nodes.yml
@@ -13,4 +13,5 @@
   roles:
   - role: os_firewall
   - role: openshift_node
+  - role: tuned
   - role: nickhammond.logrotate
diff --git a/roles/openshift_node/tasks/config.yml b/roles/openshift_node/tasks/config.yml
index e3898b520..e5fcaf9af 100644
--- a/roles/openshift_node/tasks/config.yml
+++ b/roles/openshift_node/tasks/config.yml
@@ -111,9 +111,5 @@
     msg: Node failed to start please inspect the logs and try again
   when: node_start_result | failed
 
-- name: Setup tuned
-  include: tuned.yml
-  static: yes
-
 - set_fact:
     node_service_status_changed: "{{ node_start_result | changed }}"
diff --git a/roles/openshift_node/tasks/tuned.yml b/roles/openshift_node/tasks/tuned.yml
deleted file mode 100644
index 425bf6a26..000000000
--- a/roles/openshift_node/tasks/tuned.yml
+++ /dev/null
@@ -1,41 +0,0 @@
----
-- name: Check for tuned package
-  command: rpm -q tuned
-  args:
-    warn: no
-  register: tuned_installed
-  changed_when: false
-  failed_when: false
-
-- name: Tuned service setup
-  block:
-  - name: Set tuned OpenShift variables
-    set_fact:
-      openshift_tuned_guest_profile: "{{ 'atomic-guest' if openshift.common.is_atomic else 'virtual-guest' }}"
-      tuned_etc_directory: '/etc/tuned'
-      tuned_templates_source: '../templates/tuned'
-
-  - name: Ensure directory structure exists
-    file:
-      state: directory
-      dest: '{{ tuned_etc_directory }}/{{ item.path }}'
-    with_filetree: '{{ tuned_templates_source }}'
-    when: item.state == 'directory'
-
-  - name: Ensure files are populated from templates
-    template:
-      src: '{{ item.src }}'
-      dest: '{{ tuned_etc_directory }}/{{ item.path }}'
-    with_filetree: '{{ tuned_templates_source }}'
-    when: item.state == 'file'
-
-  - name: Make tuned use the recommended tuned profile on restart
-    file: path=/etc/tuned/active_profile state=absent
-
-  - name: Restart tuned service
-    systemd:
-      state: restarted
-      daemon_reload: yes
-      name: tuned
-
-  when: tuned_installed.rc == 0 | bool
diff --git a/roles/openshift_node/templates/tuned/openshift-control-plane/tuned.conf b/roles/openshift_node/templates/tuned/openshift-control-plane/tuned.conf
deleted file mode 100644
index f22f21065..000000000
--- a/roles/openshift_node/templates/tuned/openshift-control-plane/tuned.conf
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# tuned configuration
-#
-
-[main]
-summary=Optimize systems running OpenShift control plane
-include=openshift
-
-[sysctl]
-# ktune sysctl settings, maximizing i/o throughput
-#
-# Minimal preemption granularity for CPU-bound tasks:
-# (default: 1 msec#  (1 + ilog(ncpus)), units: nanoseconds)
-kernel.sched_min_granularity_ns=10000000
-
-# The total time the scheduler will consider a migrated process
-# "cache hot" and thus less likely to be re-migrated
-# (system default is 500000, i.e. 0.5 ms)
-kernel.sched_migration_cost_ns=5000000
-
-# SCHED_OTHER wake-up granularity.
-#
-# Preemption granularity when tasks wake up.  Lower the value to improve 
-# wake-up latency and throughput for latency critical tasks.
-kernel.sched_wakeup_granularity_ns = 4000000
diff --git a/roles/openshift_node/templates/tuned/openshift-node/tuned.conf b/roles/openshift_node/templates/tuned/openshift-node/tuned.conf
deleted file mode 100644
index 78c7d19c9..000000000
--- a/roles/openshift_node/templates/tuned/openshift-node/tuned.conf
+++ /dev/null
@@ -1,10 +0,0 @@
-#
-# tuned configuration
-#
-
-[main]
-summary=Optimize systems running OpenShift nodes
-include=openshift
-
-[sysctl]
-net.ipv4.tcp_fastopen=3
diff --git a/roles/openshift_node/templates/tuned/openshift/tuned.conf b/roles/openshift_node/templates/tuned/openshift/tuned.conf
deleted file mode 100644
index 68ac5dadb..000000000
--- a/roles/openshift_node/templates/tuned/openshift/tuned.conf
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# tuned configuration
-#
-
-[main]
-summary=Optimize systems running OpenShift (parent profile)
-include=${f:virt_check:{{ openshift_tuned_guest_profile }}:throughput-performance}
-
-[selinux]
-avc_cache_threshold=65536
-
-[net]
-nf_conntrack_hashsize=131072
-
-[sysctl]
-kernel.pid_max=131072
-net.netfilter.nf_conntrack_max=1048576
-fs.inotify.max_user_watches=65536
-net.ipv4.neigh.default.gc_thresh1=8192
-net.ipv4.neigh.default.gc_thresh2=32768
-net.ipv4.neigh.default.gc_thresh3=65536
-net.ipv6.neigh.default.gc_thresh1=8192
-net.ipv6.neigh.default.gc_thresh2=32768
-net.ipv6.neigh.default.gc_thresh3=65536
diff --git a/roles/openshift_node/templates/tuned/recommend.conf b/roles/openshift_node/templates/tuned/recommend.conf
deleted file mode 100644
index 5fa765798..000000000
--- a/roles/openshift_node/templates/tuned/recommend.conf
+++ /dev/null
@@ -1,8 +0,0 @@
-[openshift-node]
-/etc/origin/node/node-config.yaml=.*region=primary
-
-[openshift-control-plane,master]
-/etc/origin/master/master-config.yaml=.*
-
-[openshift-control-plane,node]
-/etc/origin/node/node-config.yaml=.*region=infra
diff --git a/roles/tuned/defaults/main.yml b/roles/tuned/defaults/main.yml
new file mode 100644
index 000000000..418a4b521
--- /dev/null
+++ b/roles/tuned/defaults/main.yml
@@ -0,0 +1,3 @@
+---
+tuned_etc_directory: '/etc/tuned'
+tuned_templates_source: '../templates'
diff --git a/roles/tuned/meta/main.yml b/roles/tuned/meta/main.yml
new file mode 100644
index 000000000..833d94c13
--- /dev/null
+++ b/roles/tuned/meta/main.yml
@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Jiri Mencak
+  description: Restart the tuned daemon if present and make it use the recommended profile
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 2.3
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
diff --git a/roles/tuned/tasks/main.yml b/roles/tuned/tasks/main.yml
new file mode 100644
index 000000000..e95d274d5
--- /dev/null
+++ b/roles/tuned/tasks/main.yml
@@ -0,0 +1,39 @@
+---
+- name: Check for tuned package
+  command: rpm -q tuned
+  args:
+    warn: no
+  register: tuned_installed
+  changed_when: false
+  failed_when: false
+
+- name: Tuned service setup
+  block:
+  - name: Set tuned OpenShift variables
+    set_fact:
+      openshift_tuned_guest_profile: "{{ 'atomic-guest' if openshift.common.is_atomic else 'virtual-guest' }}"
+
+  - name: Ensure directory structure exists
+    file:
+      state: directory
+      dest: '{{ tuned_etc_directory }}/{{ item.path }}'
+    with_filetree: '{{ tuned_templates_source }}'
+    when: item.state == 'directory'
+
+  - name: Ensure files are populated from templates
+    template:
+      src: '{{ item.src }}'
+      dest: '{{ tuned_etc_directory }}/{{ item.path }}'
+    with_filetree: '{{ tuned_templates_source }}'
+    when: item.state == 'file'
+
+  - name: Make tuned use the recommended tuned profile on restart
+    file: path=/etc/tuned/active_profile state=absent
+
+  - name: Restart tuned service
+    systemd:
+      state: restarted
+      daemon_reload: yes
+      name: tuned
+
+  when: tuned_installed.rc == 0 | bool
diff --git a/roles/tuned/templates/openshift-control-plane/tuned.conf b/roles/tuned/templates/openshift-control-plane/tuned.conf
new file mode 100644
index 000000000..f22f21065
--- /dev/null
+++ b/roles/tuned/templates/openshift-control-plane/tuned.conf
@@ -0,0 +1,25 @@
+#
+# tuned configuration
+#
+
+[main]
+summary=Optimize systems running OpenShift control plane
+include=openshift
+
+[sysctl]
+# ktune sysctl settings, maximizing i/o throughput
+#
+# Minimal preemption granularity for CPU-bound tasks:
+# (default: 1 msec#  (1 + ilog(ncpus)), units: nanoseconds)
+kernel.sched_min_granularity_ns=10000000
+
+# The total time the scheduler will consider a migrated process
+# "cache hot" and thus less likely to be re-migrated
+# (system default is 500000, i.e. 0.5 ms)
+kernel.sched_migration_cost_ns=5000000
+
+# SCHED_OTHER wake-up granularity.
+#
+# Preemption granularity when tasks wake up.  Lower the value to improve 
+# wake-up latency and throughput for latency critical tasks.
+kernel.sched_wakeup_granularity_ns = 4000000
diff --git a/roles/tuned/templates/openshift-node/tuned.conf b/roles/tuned/templates/openshift-node/tuned.conf
new file mode 100644
index 000000000..78c7d19c9
--- /dev/null
+++ b/roles/tuned/templates/openshift-node/tuned.conf
@@ -0,0 +1,10 @@
+#
+# tuned configuration
+#
+
+[main]
+summary=Optimize systems running OpenShift nodes
+include=openshift
+
+[sysctl]
+net.ipv4.tcp_fastopen=3
diff --git a/roles/tuned/templates/openshift/tuned.conf b/roles/tuned/templates/openshift/tuned.conf
new file mode 100644
index 000000000..68ac5dadb
--- /dev/null
+++ b/roles/tuned/templates/openshift/tuned.conf
@@ -0,0 +1,24 @@
+#
+# tuned configuration
+#
+
+[main]
+summary=Optimize systems running OpenShift (parent profile)
+include=${f:virt_check:{{ openshift_tuned_guest_profile }}:throughput-performance}
+
+[selinux]
+avc_cache_threshold=65536
+
+[net]
+nf_conntrack_hashsize=131072
+
+[sysctl]
+kernel.pid_max=131072
+net.netfilter.nf_conntrack_max=1048576
+fs.inotify.max_user_watches=65536
+net.ipv4.neigh.default.gc_thresh1=8192
+net.ipv4.neigh.default.gc_thresh2=32768
+net.ipv4.neigh.default.gc_thresh3=65536
+net.ipv6.neigh.default.gc_thresh1=8192
+net.ipv6.neigh.default.gc_thresh2=32768
+net.ipv6.neigh.default.gc_thresh3=65536
diff --git a/roles/tuned/templates/recommend.conf b/roles/tuned/templates/recommend.conf
new file mode 100644
index 000000000..086e5673d
--- /dev/null
+++ b/roles/tuned/templates/recommend.conf
@@ -0,0 +1,11 @@
+[openshift-control-plane,master]
+/etc/origin/master/master-config.yaml=.*
+
+[openshift-control-plane,node]
+/etc/origin/node/node-config.yaml=.*region=infra
+
+[openshift-control-plane,lb]
+/etc/haproxy/haproxy.cfg=.*
+
+[openshift-node]
+/etc/origin/node/node-config.yaml=.*
-- 
cgit v1.2.3