Production

12 files changed, 251 insertions, 72 deletions

diff --git a/roles/ands_facts/defaults/main.yml b/roles/ands_facts/defaults/main.yml
index c74984e..5feb354 100644
--- a/roles/ands_facts/defaults/main.yml
+++ b/roles/ands_facts/defaults/main.yml
@@ -25,7 +25,9 @@ ands_cluster_domain: "{{ ansible_domain }}"
 ands_inner_domain: "{{ ands_cluster_domain }}"
 
 ands_default_ip: "{{ ansible_default_ipv4.address }}"
+ands_default_cidr: "{{ ansible_default_ipv4.address ~ '/' ~ ( ansible_default_ipv4.netmask | ipaddr('prefix') ) }}"
 ands_openshift_default_ip: "{{ ands_resolve_public_ip | default(false) | ternary(ands_default_ip, ands_none) }}"
+ands_openshift_default_cidr: "{{ ands_resolve_public_ip | default(false) | ternary(ands_default_cidr, ands_none) }}"
 ands_openshift_default_hostname: "{{ (ands_hostname_template is defined) | ternary(ands_hostname_template ~ ands_host_id, ansible_hostname) }}"
 
 ands_inner_lb: true
@@ -33,5 +35,5 @@ ands_use_inner_lb: false
 ands_inner_lb_id: 254
 ands_inner_lb_hostname: 'ands-lb'
 
-#ands_openshift_inner_interface:
-#ands_openshift_public_interface: 
+#ands_inner_interface:
+#ands_public_interface: 
diff --git a/roles/ands_facts/tasks/find_interface_by_net.yml b/roles/ands_facts/tasks/find_interface_by_net.yml
new file mode 100644
index 0000000..ad44578
--- /dev/null
+++ b/roles/ands_facts/tasks/find_interface_by_net.yml
@@ -0,0 +1,17 @@
+- name: "Looking for interface holding {{ net }}"
+  set_fact:
+    "{{ var }}": "{{ eth['device'] }}"
+  vars:
+    eth:     "{{ hostvars[inventory_hostname]['ansible_' + item] | default({}) }}"
+    ipv4:    "{{ eth['ipv4'] | default({}) }}"
+    q:       "{{ eth | json_query('ipv4_secondaries[*].network') }}"
+    sec:     "{{ ((q == ands_none) or (q == '')) | ternary([], q) }}"
+    nets:     "{{ sec  | union([ipv4.network]) }}"
+  when: 
+    - eth['type'] is defined
+    - eth['ipv4'] is defined
+    - eth['device'] is defined
+    - eth['type'] == 'ether'
+    - net | ipaddr('network') in nets
+  with_items:
+    - "{{ hostvars[inventory_hostname]['ansible_interfaces'] }}"
diff --git a/roles/ands_facts/tasks/network.yml b/roles/ands_facts/tasks/network.yml
index 808d7b6..1acafc1 100644
--- a/roles/ands_facts/tasks/network.yml
+++ b/roles/ands_facts/tasks/network.yml
@@ -12,10 +12,13 @@
     ands_use_inner_lb: "{{ ands_use_inner_lb }}"
     ands_inner_lb_ip: "{{ ands_openshift_network | ipaddr(ands_inner_lb_id) | ipaddr('address') }}"
     ands_inner_lb_hostname: "{{ ands_inner_lb_hostname }}"
+    ands_openshift_cidr: "{{ ands_openshift_network | ipaddr(ands_host_id) }}"
     ands_openshift_ip: "{{ ands_openshift_network | ipaddr(ands_host_id) | ipaddr('address') }}"
     ands_openshift_hostname: "{{ ands_openshift_hostname | default(ands_openshift_default_hostname) }}"
+    ands_openshift_public_cidr: "{{ (ands_openshift_public_network is defined) | ternary( ands_openshift_public_network | ipaddr(ands_host_id), ands_openshift_default_cidr) }}"
     ands_openshift_public_ip: "{{ (ands_openshift_public_network is defined) | ternary( ands_openshift_public_network | ipaddr(ands_host_id) | ipaddr('address'), ands_openshift_default_ip) }}"
     ands_openshift_public_hostname: "{{ ands_openshift_public_hostname | default(ands_openshift_default_hostname) }}"
+    ands_storage_cidr: "{{ ands_storage_network | default(ands_openshift_network) | ipaddr(ands_host_id) }}"
     ands_storage_ip: "{{ ands_storage_network | default(ands_openshift_network) | ipaddr(ands_host_id) | ipaddr('address') }}"
     ands_hostname_storage: "ands_storage{{ ands_host_id }}"
     ands_hostname_openshift: "ands_openshift{{ ands_host_id }}"
@@ -33,27 +36,37 @@
 - name: "Detect inner network interface"
   include_tasks: "find_interface_by_ip.yml"
   vars:
-    var: "ands_openshift_inner_interface"
+    var: "ands_inner_interface"
     ip: "{{ ands_openshift_ip }}"
   when: 
-    - ands_openshift_inner_interface is not defined
+    - ands_inner_interface is not defined
 
-- name: "Detect public network interface"
+- name: "Detect public network interface (using public ip)"
   include_tasks: "find_interface_by_ip.yml"
   vars:
-    var: "ands_openshift_public_interface"
+    var: "ands_public_interface"
     ip: "{{ (ands_openshift_public_ip == ands_none) | ternary(ands_default_ip, ands_openshift_public_ip) }}"
   when: 
-    - ands_openshift_public_interface is not defined
+    - ands_public_interface is not defined
 
-- name: Set ipfailover interface
-  set_fact: 
-    ands_ipfailover_interface: "{{ ands_openshift_public_interface }}"
-  when: ands_ipfailover_interface is not defined
+- name: "Detect public network interface (using global network)"
+  include_tasks: "find_interface_by_net.yml"
+  vars:
+    var: "ands_public_interface"
+    net: "{{ ands_global_network }}"
+  when: 
+    - ands_public_interface is not defined
+    - ands_global_network is defined
 
 - name: Set ipfailover inner interface
   set_fact: 
-    ands_ipfailover_inner_interface: "{{ ands_openshift_inner_interface }}"
+    ands_ipfailover_inner_interface: "{{ ands_inner_interface }}"
   when: ands_ipfailover_inner_interface is not defined
 
+- name: Set ipfailover interface
+  set_fact: 
+    ands_ipfailover_interface: "{{ ands_public_interface }}"
+  when: ands_ipfailover_interface is not defined
+
+
 #- debug: msg="{{ hostvars }}"
diff --git a/roles/ands_network/defaults/main.yml b/roles/ands_network/defaults/main.yml
new file mode 100644
index 0000000..139e8b3
--- /dev/null
+++ b/roles/ands_network/defaults/main.yml
@@ -0,0 +1 @@
+configure_network: "{{ ands_configure_network | default(false) }}"
diff --git a/roles/ands_network/tasks/nm_configure.yml b/roles/ands_network/tasks/nm_configure.yml
new file mode 100644
index 0000000..4482705
--- /dev/null
+++ b/roles/ands_network/tasks/nm_configure.yml
@@ -0,0 +1,39 @@
+
+- name: install needed network manager libs
+  yum: name='{{ item }}' state=installed
+  with_items:
+    - NetworkManager-glib
+    - nm-connection-editor
+    - libsemanage-python
+    - policycoreutils-python
+
+# Works in 2.4.3.0 with couple of upstream patches
+# Infiniband is not supported yet
+#- name: configure storage network interface
+#  nmcli: type=inifinband conn_name=storage ifname="{{ ands_storage_interface }}" ip4="{{ ands_storage_cidr }}" state="present" autoconnect="yes"
+
+
+- name: configure storage nework
+  include_tasks: nm_configure_connection.yml
+  vars:
+    name: "storage"
+    iface: "{{ ands_storage_interface }}"
+    cidr: "{{ ands_storage_cidr }}"
+    force: true
+
+- name: configure openshift nework
+  include_tasks: nm_configure_connection.yml
+  vars:
+    name: "openshift"
+    iface: "{{ ands_inner_interface }}"
+    cidr: "{{ ands_openshift_cidr }}"
+    force: true
+
+- name: configure public nework
+  include_tasks: nm_configure_connection.yml
+  vars:
+    name: "storage"
+    iface: "{{ ands_public_interface }}"
+    cidr: "{{ ands_openshift_public_cidr }}"
+    alias: true
+
diff --git a/roles/ands_network/tasks/nm_configure_connection.yml b/roles/ands_network/tasks/nm_configure_connection.yml
new file mode 100644
index 0000000..18fc91e
--- /dev/null
+++ b/roles/ands_network/tasks/nm_configure_connection.yml
@@ -0,0 +1,51 @@
+- name: "detect nm connection corresponding to interface '{{ iface }}'"
+  shell: "nmcli d show {{ iface | quote }} | grep CONNECTION | cut -d ':' -f 2- | sed -E -e 's/^[[:space:]]+//' | grep '^[[:alpha:]]'"
+  register: conres
+  failed_when: false
+  changed_when: false
+
+
+- name: "check if the requested ip '{{ cidr }}' is present on the interface '{{ iface }}'"
+  set_fact:
+    ip_present: "{{ cidr | ipaddr('address') in ips }}"
+  vars:
+    eth:     "{{ hostvars[inventory_hostname]['ansible_' + iface] | default({}) }}"
+    ipv4:    "{{ eth['ipv4'] | default({}) }}"
+    q:       "{{ eth | json_query('ipv4_secondaries[*].address') }}"
+    sec:     "{{ ((q == ands_none) or (q == '')) | ternary([], q) }}"
+    ips:     "{{ sec  | union([ipv4.address]) }}"
+  when: 
+    - conres.rc == 0
+#    - eth.ipv4 is defined
+
+- name: "destroy connection '{{ conres.stdout }}' if ip does not match"
+  command: "nmcli connection delete {{ conres.stdout | quote }}"
+  register: delres
+  when: 
+    - conres.rc == 0
+    - force | default(false)
+    - not (alias | default(false))
+    - not ip_present
+    
+- name: "configure storage network interface '{{ iface }}' to '{{ cidr }}'"
+  command: "nmcli connection add type infiniband ifname {{ iface | quote }} con-name {{ name }} ip4 {{ cidr }}"
+  when: 
+    - (conres.rc != 0) or (not (delres | skipped))
+    - (conres.rc != 0) or (not (alias | default(false)))
+
+- name: "add ip alias '{{ cidr }}' to connection '{{ conres.stdout }}' using interface '{{ iface }}'"
+  command: "nmcli connection modify {{ conres.stdout | quote }} +ipv4.address {{ cidr }}"
+  register: alres
+  when: 
+    - alias | default(false)
+    - conres.rc == 0
+    - not ip_present
+    
+
+- name: "add ip alias '{{ cidr }}' to network interface '{{ iface }}'"
+  command: "nmcli connection up {{ conres.stdout | quote }}"
+  register: alres
+  when: 
+    - not(alres | skipped)
+    - alres | succeeded
+    - not ip_present
diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml
index 30b1ff8..def846d 100644
--- a/roles/docker/defaults/main.yml
+++ b/roles/docker/defaults/main.yml
@@ -1,5 +1,7 @@
 docker_exclude_vgs: "{{ ands_data_vg is defined | ternary( [ ands_data_vg ], [] ) }}"
 docker_lv: "docker-pool"
+docker_root_lv: "docker-root-lv"
+docker_setup_root: "{{ docker_root_volume_size is defined }}"
 
 docker_min_size: 100
 docker_max_log_size: "2m"
diff --git a/roles/docker/tasks/configure.yml b/roles/docker/tasks/configure.yml
new file mode 100644
index 0000000..5d29291
--- /dev/null
+++ b/roles/docker/tasks/configure.yml
@@ -0,0 +1,63 @@
+---
+#- name: Remove docker
+#  yum: name="{{ item }}" state="absent"
+#  with_items: [ docker, docker-client, docker-common ]
+
+- name: install docker
+  include_tasks: install.yml
+
+- name: start docker
+  service: name="docker" state="started"
+
+- name: Configure bridge-nf-call-iptables with sysctl
+  sysctl: name="net.bridge.bridge-nf-call-iptables"  value=1 state=present sysctl_set=yes
+
+- name: Configure bridge-nf-call-ip6tables with sysctl
+  sysctl: name="net.bridge.bridge-nf-call-ip6tables"  value=1 state=present sysctl_set=yes
+
+- name: Determine if loopback
+  shell: docker info | grep 'Data file:.*loop'
+  register: loop_device_check
+  failed_when: false
+  changed_when: loop_device_check.rc == 0
+
+- set_fact:  docker_reinit="{{ (loop_device_check.rc == 0) or (vg == '') or (docker_setup_root and ((root_vg == '') or (vg != root_vg))) or (docker_storage_vg is defined and (docker_storage_vg != vg)) }}"
+  vars:
+    lv: "{{ ansible_lvm['lvs'][docker_lv] | default({}) }}"
+    vg: "{{ lv['vg'] | default('') }}"
+    root_lv: "{{ ansible_lvm['lvs'][docker_root_lv] | default({}) }}"
+    root_vg: "{{ root_lv['vg'] | default('') }}"
+
+- debug: msg="Re-initializing - {{ docker_reinit }}, Loopback check - {{ loop_device_check.stderr }}"
+  when: loop_device_check.stderr
+
+- import_tasks: storage.yml
+  when: docker_reinit
+
+- name: extend vg
+  lvol: vg="{{ ansible_lvm['lvs'][docker_lv]['vg'] }}" lv="{{ docker_lv }}" size="{{ docker_volume_size }}"
+  when: 
+    - not docker_reinit
+    - docker_volume_size is defined
+
+- name: extend root vg
+  lvol: vg="{{ ansible_lvm['lvs'][docker_root_lv]['vg'] }}" lv="{{ docker_root_lv }}" size="{{ docker_root_volume_size }}"
+  when: 
+    - not docker_reinit
+    - docker_setup_root
+    - docker_root_volume_size is defined
+
+# By default there is systemd driver installed. It is removed during OpenShift installation, but is still there during prepare stage
+# The parameters to docker can be set trough OpenShift and currently are moved there.
+#- name: Limit size of container log files
+#  ghetto_json:
+#    path: "/etc/docker/daemon.json"
+#    log-driver: "json-file"
+#    log-opts.max-size: "{{ docker_max_log_size }}" 
+#    log-opts.max-file: "{{ docker_max_log_files }}"
+#  notify:
+#    - restart docker
+
+- name: start docker
+  service: name="docker" enabled=yes state=started
+  
\ No newline at end of file
diff --git a/roles/docker/tasks/install.yml b/roles/docker/tasks/install.yml
new file mode 100644
index 0000000..41ae5a3
--- /dev/null
+++ b/roles/docker/tasks/install.yml
@@ -0,0 +1,13 @@
+- name: Remove versionlock from yum
+  command: yum versionlock delete docker docker-common docker-client
+  register: result
+  when: update | default(false)
+  failed_when: false
+  changed_when: result | failed
+
+- name: Ensure docker is installed
+  yum: name="docker{{ docker_version | default('') }}" state="{{ ((docker_version is defined) and (update | default(false))) | ternary('latest', 'present') }}"
+
+- name: Add versionlock to yum
+  command: yum versionlock add docker docker-common docker-client
+  when: docker_version is defined
diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml
index c03d897..c9d0251 100644
--- a/roles/docker/tasks/main.yml
+++ b/roles/docker/tasks/main.yml
@@ -1,57 +1,3 @@
 ---
-#- name: Remove docker
-#  yum: name="{{ item }}" state="absent"
-#  with_items: [ docker, docker-client, docker-common ]
-
-- name: Remove versionlock from yum
-  command: yum versionlock delete docker docker-common docker-client
-  register: result
-  failed_when: false
-  changed_when: result | failed
-
-- name: Ensure docker is installed
-  yum: name="docker{{ docker_version | default('') }}" state="{{ docker_version is defined | ternary('latest', 'present') }}"
-
-- name: Add versionlock to yum
-  command: yum versionlock add docker docker-common docker-client
-  when: docker_version is defined
-
-- name: start docker
-  service: name="docker" state="started"
-
-- name: Configure bridge-nf-call-iptables with sysctl
-  sysctl: name="net.bridge.bridge-nf-call-iptables"  value=1 state=present sysctl_set=yes
-
-- name: Configure bridge-nf-call-ip6tables with sysctl
-  sysctl: name="net.bridge.bridge-nf-call-ip6tables"  value=1 state=present sysctl_set=yes
-
-- name: Determine if loopback
-  shell: docker info | grep 'Data file:.*loop'
-  register: loop_device_check
-  failed_when: false
-  changed_when: loop_device_check.rc == 0
-
-- debug: msg="{{ loop_device_check.stderr }}"
-  when: loop_device_check.stderr
-
-- import_tasks: storage.yml
-  when: loop_device_check.rc == 0
-
-- name: extend the vg
-  lvol: vg="{{ ansible_lvm['lvs'][docker_lv]['vg'] }}" lv="docker_lv" size="{{ docker_volume_size }}"
-  when: docker_volume_size is defined
-
-# By default there is systemd driver installed. It is removed during OpenShift installation, but is still there during prepare stage
-# The parameters to docker can be set trough OpenShift and currently are moved there.
-#- name: Limit size of container log files
-#  ghetto_json:
-#    path: "/etc/docker/daemon.json"
-#    log-driver: "json-file"
-#    log-opts.max-size: "{{ docker_max_log_size }}" 
-#    log-opts.max-file: "{{ docker_max_log_files }}"
-#  notify:
-#    - restart docker
-
-- name: start docker
-  service: name="docker" enabled=yes state=started
-  
\ No newline at end of file
+- name: "Configuring network"
+  include_tasks: "{{ action | default('configure') }}.yml"
diff --git a/roles/docker/tasks/storage.yml b/roles/docker/tasks/storage.yml
index 5a5c858..e431030 100644
--- a/roles/docker/tasks/storage.yml
+++ b/roles/docker/tasks/storage.yml
@@ -1,11 +1,13 @@
 ---
+# We need to adjust docker_min_size if docker_volume_size set (min_size is just given in GB, but volume size
+# support suffixes and we need to handle this).
 - name: list volume groups
   set_fact: available_vg="{{ item.key }}"
   with_dict: "{{ ansible_lvm.vgs }}"
   when:
     - not available_vg is defined
     - not item.key in ( docker_exclude_vgs | default([]) )
-    - (item.value.free_g | int) > (docker_volume_size | default(docker_min_size))
+    - (item.value.free_g | int) > docker_min_size
 
 - set_fact: docker_storage_vg="{{ available_vg }}"
   when: (not docker_storage_vg is defined) and (available_vg is defined)
@@ -13,11 +15,13 @@
 - fail: msg="Can't detect Docker VG"
   when: not docker_storage_vg is defined
 
-- name: check to see if {{ docker_storage_device }} exists
+- name: "check to see if {{ docker_storage_device }} exists"
   command: "test -e {{ docker_storage_device }}"
   when: docker_storage_device is defined
 
-- set_fact: docker_storage_config="VG={{ docker_storage_vg }} AUTO_EXTEND_POOL=true"
+- set_fact: docker_storage_config="VG={{ docker_storage_vg }} DATA_SIZE={{ docker_volume_size | default(docker_min_size ~ 'G') }} AUTO_EXTEND_POOL=true"
+- set_fact: docker_storage_config="{{ docker_storage_config }} DOCKER_ROOT_VOLUME=yes DOCKER_ROOT_VOLUME_SIZE={{ docker_root_volume_size | default(docker_min_size ~ 'G') }}"
+  when: docker_setup_root
 
 - set_fact: docker_storage_config="{{ docker_storage_config }} DEVS={{ docker_storage_device }}"
   when: ( docker_storage_device is defined ) and ( ansible_lvm.vgs[docker_storage_vg] is not defined )
@@ -28,6 +32,29 @@
 - name: delete /var/lib/docker
   file: path="/var/lib/docker" state=absent
 
+- name: "delete {{ docker_lv }} lv"
+  lvol: vg="{{ ansible_lvm['lvs'][docker_lv]['vg'] }}" lv="{{ docker_lv }}" state="absent" force="yes"
+  when: ansible_lvm['lvs'][docker_lv] is defined
+
+- name: delete docker-pool lv
+  lvol: vg="{{ ansible_lvm['lvs']['docker-pool']['vg'] }}" lv="docker-pool" state="absent" force="yes"
+  when: ansible_lvm['lvs']['docker-pool'] is defined
+
+- name: unmount docker partition
+  mount: path="/var/lib/docker" state="unmounted" 
+
+- name: "delete {{ docker_root_lv }} lv"
+  lvol: vg="{{ ansible_lvm['lvs'][docker_root_lv]['vg'] }}" lv="{{ docker_root_lv }}" state="absent" force="yes"
+  when: ansible_lvm['lvs'][docker_root_lv] is defined
+
+- name: delete docker-pool lv
+  lvol: vg="{{ ansible_lvm['lvs']['docker-root-lv']['vg'] }}" lv="docker-root-lv" state="absent" force="yes"
+  when: ansible_lvm['lvs']['docker-root-lv'] is defined
+
+
+- name: remove old configuration
+  file: path="/etc/sysconfig/docker-storage" state="absent"
+
 - name: generate docker-storage-setup config file
   copy:
       content: "{{ docker_storage_config }}"
@@ -38,4 +65,3 @@
 
 - name: docker storage setup
   command: docker-storage-setup
-
diff --git a/roles/ofed/tasks/main.yml b/roles/ofed/tasks/main.yml
new file mode 100644
index 0000000..bd85d43
--- /dev/null
+++ b/roles/ofed/tasks/main.yml
@@ -0,0 +1,6 @@
+- name: install the 'Infiniband support' package group
+  yum: name="@Infiniband Support" state=present
+  
+- name: start rdma service
+  service: name="rdma" enabled=yes state=started
+  
\ No newline at end of file