Production

25 files changed, 409 insertions, 81 deletions

diff --git a/group_vars/baremetal.yml b/group_vars/baremetal.yml
index 032e53f..640a00a 100644
--- a/group_vars/baremetal.yml
+++ b/group_vars/baremetal.yml
@@ -1,3 +1,20 @@
 glusterfs_transport: rdma
 
+# Size in GB to detect main Ands device
+ands_data_device_threshold: 8192
+
+# Primary LVM devices
+ands_data_volume_size: "40T"
+ands_heketi_volume_size: "4T"
+docker_volume_size: "1T"
+docker_root_volume_size: "1T"
+docker_storage_vg: "ands"
+
+#Heketi Volumes
+ands_registry_volume_size: "2T"
+
+ands_storage_interface: "ib0"
+ands_inner_interface: "ib1"
+#ands_public_interface: "eth0"
+
 ands_host_id: "{{ ansible_hostname | regex_replace('^ipekatrin(\\d+)(\\.|$)',  '\\1') }}"
diff --git a/group_vars/production.yml b/group_vars/production.yml
index 24b189f..c731873 100644
--- a/group_vars/production.yml
+++ b/group_vars/production.yml
@@ -1,7 +1,27 @@
+ands_configure_network: true
 ands_storage_network: 192.168.12.0/24
 
+ands_cluster_domain: ipe.kit.edu
+ands_hostname_template: ipekatrin
 ands_openshift_lb: kaas.kit.edu
 ands_openshift_subdomain: kaas.kit.edu
-ands_openshift_network: 192.168.26.0/24
+ands_openshift_network: 192.168.13.0/24
 ands_openshift_public_network: 192.168.26.0/24
-ands_openshift_ingress_network: 192.168.16.0/22
+ands_openshift_ingress_network: 192.168.16.0/24
+ands_global_network: 141.52.64.0/23
+
+#ands_inner_domain: ""
+ands_openshift_set_hostname: false
+# if we provision inner_lb (default), we can turn it on and just re-run ands_network role (or maintain play)
+ands_use_inner_lb: false
+
+
+#ands_ipfailover_interface: eth1
+ands_ipfailover_vips: [141.52.64.15/23, 141.52.64.17/23]
+
+katrin_openvpn_subnet_bits: 24
+katrin_openvpn_subnet_offset: 111 
+katrin_openvpn_network: "192.168.0.0/16"
+
+
+ands_prefer_docker: true
diff --git a/inventories/production.erb b/inventories/production.erb
new file mode 100644
index 0000000..575a86f
--- /dev/null
+++ b/inventories/production.erb
@@ -0,0 +1,74 @@
+[masters]
+ipekatrin[1:2].ipe.kit.edu 
+
+[etcd]
+ipekatrin[1:3].ipe.kit.edu 
+
+[simple_storage_nodes]
+ipekatrin[3:3].ipe.kit.edu 
+
+[simple_nodes]
+
+[external_storage_servers]
+
+[production:children]
+nodes
+new_nodes
+etcd
+new_etcd
+external_storage_servers
+
+[baremetal:children]
+nodes
+new_nodes
+etcd
+new_etcd
+external_storage_servers
+
+[virtual]
+
+[OSEv3:children]
+masters
+new_masters
+nodes
+new_nodes
+etcd
+new_etcd
+
+[nodes:children]
+masters
+simple_storage_nodes
+simple_nodes
+
+[storage_nodes:children]
+masters
+new_masters
+simple_storage_nodes
+new_simple_storage_nodes
+
+[glusterfs:children]
+storage_nodes
+
+[ands_servers:children]
+nodes
+new_nodes
+etcd
+new_etcd
+external_storage_servers
+
+[ands_storage_servers:children]
+storage_nodes
+external_storage_servers
+
+[ands:children]
+ands_servers
+
+[new_masters]
+[new_etcd]
+[new_simple_storage_nodes]
+[new_simple_nodes]
+
+[new_nodes:children]
+new_masters
+new_simple_storage_nodes
+new_simple_nodes
diff --git a/inventories/staging.erb b/inventories/staging.erb
index aa9e935..3ce8a39 100644
--- a/inventories/staging.erb
+++ b/inventories/staging.erb
@@ -19,6 +19,8 @@ new_etcd
 external_storage_servers
 vagrant
 
+[baremetal]
+
 [virtual:children]
 nodes
 new_nodes
diff --git a/inventories/testing.erb b/inventories/testing.erb
index f9d27ae..9ee38ec 100644
--- a/inventories/testing.erb
+++ b/inventories/testing.erb
@@ -20,6 +20,8 @@ new_etcd
 external_storage_servers
 vagrant
 
+[baremetal]
+
 [virtual:children]
 nodes
 new_nodes
diff --git a/opts.sh b/opts.sh
index 9cfaf86..b17754e 100644
--- a/opts.sh
+++ b/opts.sh
@@ -17,6 +17,7 @@ Actions:
  Initial system installation
     all                 - full install, all the following steps in sequence:
     local               - configure local ssh client
+    network             - configure network interfaces
     prepare             - perform all required pre-configuration before setting openshift
     openshift           - setup OpenShift cluster
     gluster             - configure gluster software and volumes
diff --git a/playbooks/ands-network.yml b/playbooks/ands-network.yml
new file mode 100644
index 0000000..1788781
--- /dev/null
+++ b/playbooks/ands-network.yml
@@ -0,0 +1,19 @@
+- name: Determin Ands facts
+  hosts: ands
+  roles:
+    - role: ands_facts
+
+- name: Prepare virtual hosts
+  hosts: virtual
+  roles:
+    - role: ands_vagrant_vmconf
+
+- name: Configure ininiband infrastrcture
+  hosts: baremetal
+  roles:
+    - role: ofed
+
+- name: Network configuration
+  hosts: ands
+  roles:
+    - { role: ands_network, action: nm_configure }
diff --git a/playbooks/ands-prepare.yml b/playbooks/ands-prepare.yml
index 239d292..4b204d2 100644
--- a/playbooks/ands-prepare.yml
+++ b/playbooks/ands-prepare.yml
@@ -3,11 +3,6 @@
   roles:
     - role: ands_facts
 
-- name: Prepare virtual hosts
-  hosts: virtual
-  roles:
-    - role: ands_vagrant_vmconf
-
 - name: Common setup procedures
   hosts: ands
   roles:
diff --git a/playbooks/openshift-add-masters.yml b/playbooks/openshift-add-masters.yml
index 6878137..8c1672c 100644
--- a/playbooks/openshift-add-masters.yml
+++ b/playbooks/openshift-add-masters.yml
@@ -2,9 +2,10 @@
   hosts: nodes:new_nodes
   roles:
     - { role: ands_facts }
-    - { role: ands_common, os_update: true }
     - { role: ands_network, action: install_pre }
 
+- import_playbook: x-os-update.yml
+
 # etcd will provisioned as well if node is listed in new_etcd
 - import_playbook: ../anslib/openshift-ansible/playbooks/prerequisites.yml
 - import_playbook: ../anslib/openshift-ansible/playbooks/openshift-master/scaleup.yml
diff --git a/playbooks/openshift-add-nodes.yml b/playbooks/openshift-add-nodes.yml
index 3d3efc4..6d3ace6 100644
--- a/playbooks/openshift-add-nodes.yml
+++ b/playbooks/openshift-add-nodes.yml
@@ -2,9 +2,10 @@
   hosts: nodes:new_nodes
   roles:
     - { role: ands_facts }
-    - { role: ands_common, os_update: true }
     - { role: ands_network, action: install_pre }
 
+- import_playbook: x-os-update.yml
+
 # I am not sure if etcd will be automatic here. If not, we may need to run etcd scaleup afterwards
 # if node is also in new_etcd list
 - import_playbook: ../anslib/openshift-ansible/playbooks/prerequisites.yml
diff --git a/playbooks/x-os-update.yml b/playbooks/x-os-update.yml
new file mode 100644
index 0000000..b17ff20
--- /dev/null
+++ b/playbooks/x-os-update.yml
@@ -0,0 +1,9 @@
+- name: Configure cluster hosts names 
+  hosts: nodes:new_nodes
+  roles:
+    - { role: ands_common, os_update: true }
+
+- name: Configure cluster hosts names 
+  hosts: nodes:new_nodes
+  roles:
+    - { role: docker, action: install, update: true }
diff --git a/roles/ands_facts/defaults/main.yml b/roles/ands_facts/defaults/main.yml
index c74984e..5feb354 100644
--- a/roles/ands_facts/defaults/main.yml
+++ b/roles/ands_facts/defaults/main.yml
@@ -25,7 +25,9 @@ ands_cluster_domain: "{{ ansible_domain }}"
 ands_inner_domain: "{{ ands_cluster_domain }}"
 
 ands_default_ip: "{{ ansible_default_ipv4.address }}"
+ands_default_cidr: "{{ ansible_default_ipv4.address ~ '/' ~ ( ansible_default_ipv4.netmask | ipaddr('prefix') ) }}"
 ands_openshift_default_ip: "{{ ands_resolve_public_ip | default(false) | ternary(ands_default_ip, ands_none) }}"
+ands_openshift_default_cidr: "{{ ands_resolve_public_ip | default(false) | ternary(ands_default_cidr, ands_none) }}"
 ands_openshift_default_hostname: "{{ (ands_hostname_template is defined) | ternary(ands_hostname_template ~ ands_host_id, ansible_hostname) }}"
 
 ands_inner_lb: true
@@ -33,5 +35,5 @@ ands_use_inner_lb: false
 ands_inner_lb_id: 254
 ands_inner_lb_hostname: 'ands-lb'
 
-#ands_openshift_inner_interface:
-#ands_openshift_public_interface: 
+#ands_inner_interface:
+#ands_public_interface: 
diff --git a/roles/ands_facts/tasks/find_interface_by_net.yml b/roles/ands_facts/tasks/find_interface_by_net.yml
new file mode 100644
index 0000000..ad44578
--- /dev/null
+++ b/roles/ands_facts/tasks/find_interface_by_net.yml
@@ -0,0 +1,17 @@
+- name: "Looking for interface holding {{ net }}"
+  set_fact:
+    "{{ var }}": "{{ eth['device'] }}"
+  vars:
+    eth:     "{{ hostvars[inventory_hostname]['ansible_' + item] | default({}) }}"
+    ipv4:    "{{ eth['ipv4'] | default({}) }}"
+    q:       "{{ eth | json_query('ipv4_secondaries[*].network') }}"
+    sec:     "{{ ((q == ands_none) or (q == '')) | ternary([], q) }}"
+    nets:     "{{ sec  | union([ipv4.network]) }}"
+  when: 
+    - eth['type'] is defined
+    - eth['ipv4'] is defined
+    - eth['device'] is defined
+    - eth['type'] == 'ether'
+    - net | ipaddr('network') in nets
+  with_items:
+    - "{{ hostvars[inventory_hostname]['ansible_interfaces'] }}"
diff --git a/roles/ands_facts/tasks/network.yml b/roles/ands_facts/tasks/network.yml
index 808d7b6..1acafc1 100644
--- a/roles/ands_facts/tasks/network.yml
+++ b/roles/ands_facts/tasks/network.yml
@@ -12,10 +12,13 @@
     ands_use_inner_lb: "{{ ands_use_inner_lb }}"
     ands_inner_lb_ip: "{{ ands_openshift_network | ipaddr(ands_inner_lb_id) | ipaddr('address') }}"
     ands_inner_lb_hostname: "{{ ands_inner_lb_hostname }}"
+    ands_openshift_cidr: "{{ ands_openshift_network | ipaddr(ands_host_id) }}"
     ands_openshift_ip: "{{ ands_openshift_network | ipaddr(ands_host_id) | ipaddr('address') }}"
     ands_openshift_hostname: "{{ ands_openshift_hostname | default(ands_openshift_default_hostname) }}"
+    ands_openshift_public_cidr: "{{ (ands_openshift_public_network is defined) | ternary( ands_openshift_public_network | ipaddr(ands_host_id), ands_openshift_default_cidr) }}"
     ands_openshift_public_ip: "{{ (ands_openshift_public_network is defined) | ternary( ands_openshift_public_network | ipaddr(ands_host_id) | ipaddr('address'), ands_openshift_default_ip) }}"
     ands_openshift_public_hostname: "{{ ands_openshift_public_hostname | default(ands_openshift_default_hostname) }}"
+    ands_storage_cidr: "{{ ands_storage_network | default(ands_openshift_network) | ipaddr(ands_host_id) }}"
     ands_storage_ip: "{{ ands_storage_network | default(ands_openshift_network) | ipaddr(ands_host_id) | ipaddr('address') }}"
     ands_hostname_storage: "ands_storage{{ ands_host_id }}"
     ands_hostname_openshift: "ands_openshift{{ ands_host_id }}"
@@ -33,27 +36,37 @@
 - name: "Detect inner network interface"
   include_tasks: "find_interface_by_ip.yml"
   vars:
-    var: "ands_openshift_inner_interface"
+    var: "ands_inner_interface"
     ip: "{{ ands_openshift_ip }}"
   when: 
-    - ands_openshift_inner_interface is not defined
+    - ands_inner_interface is not defined
 
-- name: "Detect public network interface"
+- name: "Detect public network interface (using public ip)"
   include_tasks: "find_interface_by_ip.yml"
   vars:
-    var: "ands_openshift_public_interface"
+    var: "ands_public_interface"
     ip: "{{ (ands_openshift_public_ip == ands_none) | ternary(ands_default_ip, ands_openshift_public_ip) }}"
   when: 
-    - ands_openshift_public_interface is not defined
+    - ands_public_interface is not defined
 
-- name: Set ipfailover interface
-  set_fact: 
-    ands_ipfailover_interface: "{{ ands_openshift_public_interface }}"
-  when: ands_ipfailover_interface is not defined
+- name: "Detect public network interface (using global network)"
+  include_tasks: "find_interface_by_net.yml"
+  vars:
+    var: "ands_public_interface"
+    net: "{{ ands_global_network }}"
+  when: 
+    - ands_public_interface is not defined
+    - ands_global_network is defined
 
 - name: Set ipfailover inner interface
   set_fact: 
-    ands_ipfailover_inner_interface: "{{ ands_openshift_inner_interface }}"
+    ands_ipfailover_inner_interface: "{{ ands_inner_interface }}"
   when: ands_ipfailover_inner_interface is not defined
 
+- name: Set ipfailover interface
+  set_fact: 
+    ands_ipfailover_interface: "{{ ands_public_interface }}"
+  when: ands_ipfailover_interface is not defined
+
+
 #- debug: msg="{{ hostvars }}"
diff --git a/roles/ands_network/defaults/main.yml b/roles/ands_network/defaults/main.yml
new file mode 100644
index 0000000..139e8b3
--- /dev/null
+++ b/roles/ands_network/defaults/main.yml
@@ -0,0 +1 @@
+configure_network: "{{ ands_configure_network | default(false) }}"
diff --git a/roles/ands_network/tasks/nm_configure.yml b/roles/ands_network/tasks/nm_configure.yml
new file mode 100644
index 0000000..4482705
--- /dev/null
+++ b/roles/ands_network/tasks/nm_configure.yml
@@ -0,0 +1,39 @@
+
+- name: install needed network manager libs
+  yum: name='{{ item }}' state=installed
+  with_items:
+    - NetworkManager-glib
+    - nm-connection-editor
+    - libsemanage-python
+    - policycoreutils-python
+
+# Works in 2.4.3.0 with couple of upstream patches
+# Infiniband is not supported yet
+#- name: configure storage network interface
+#  nmcli: type=inifinband conn_name=storage ifname="{{ ands_storage_interface }}" ip4="{{ ands_storage_cidr }}" state="present" autoconnect="yes"
+
+
+- name: configure storage nework
+  include_tasks: nm_configure_connection.yml
+  vars:
+    name: "storage"
+    iface: "{{ ands_storage_interface }}"
+    cidr: "{{ ands_storage_cidr }}"
+    force: true
+
+- name: configure openshift nework
+  include_tasks: nm_configure_connection.yml
+  vars:
+    name: "openshift"
+    iface: "{{ ands_inner_interface }}"
+    cidr: "{{ ands_openshift_cidr }}"
+    force: true
+
+- name: configure public nework
+  include_tasks: nm_configure_connection.yml
+  vars:
+    name: "storage"
+    iface: "{{ ands_public_interface }}"
+    cidr: "{{ ands_openshift_public_cidr }}"
+    alias: true
+
diff --git a/roles/ands_network/tasks/nm_configure_connection.yml b/roles/ands_network/tasks/nm_configure_connection.yml
new file mode 100644
index 0000000..18fc91e
--- /dev/null
+++ b/roles/ands_network/tasks/nm_configure_connection.yml
@@ -0,0 +1,51 @@
+- name: "detect nm connection corresponding to interface '{{ iface }}'"
+  shell: "nmcli d show {{ iface | quote }} | grep CONNECTION | cut -d ':' -f 2- | sed -E -e 's/^[[:space:]]+//' | grep '^[[:alpha:]]'"
+  register: conres
+  failed_when: false
+  changed_when: false
+
+
+- name: "check if the requested ip '{{ cidr }}' is present on the interface '{{ iface }}'"
+  set_fact:
+    ip_present: "{{ cidr | ipaddr('address') in ips }}"
+  vars:
+    eth:     "{{ hostvars[inventory_hostname]['ansible_' + iface] | default({}) }}"
+    ipv4:    "{{ eth['ipv4'] | default({}) }}"
+    q:       "{{ eth | json_query('ipv4_secondaries[*].address') }}"
+    sec:     "{{ ((q == ands_none) or (q == '')) | ternary([], q) }}"
+    ips:     "{{ sec  | union([ipv4.address]) }}"
+  when: 
+    - conres.rc == 0
+#    - eth.ipv4 is defined
+
+- name: "destroy connection '{{ conres.stdout }}' if ip does not match"
+  command: "nmcli connection delete {{ conres.stdout | quote }}"
+  register: delres
+  when: 
+    - conres.rc == 0
+    - force | default(false)
+    - not (alias | default(false))
+    - not ip_present
+    
+- name: "configure storage network interface '{{ iface }}' to '{{ cidr }}'"
+  command: "nmcli connection add type infiniband ifname {{ iface | quote }} con-name {{ name }} ip4 {{ cidr }}"
+  when: 
+    - (conres.rc != 0) or (not (delres | skipped))
+    - (conres.rc != 0) or (not (alias | default(false)))
+
+- name: "add ip alias '{{ cidr }}' to connection '{{ conres.stdout }}' using interface '{{ iface }}'"
+  command: "nmcli connection modify {{ conres.stdout | quote }} +ipv4.address {{ cidr }}"
+  register: alres
+  when: 
+    - alias | default(false)
+    - conres.rc == 0
+    - not ip_present
+    
+
+- name: "add ip alias '{{ cidr }}' to network interface '{{ iface }}'"
+  command: "nmcli connection up {{ conres.stdout | quote }}"
+  register: alres
+  when: 
+    - not(alres | skipped)
+    - alres | succeeded
+    - not ip_present
diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml
index 30b1ff8..def846d 100644
--- a/roles/docker/defaults/main.yml
+++ b/roles/docker/defaults/main.yml
@@ -1,5 +1,7 @@
 docker_exclude_vgs: "{{ ands_data_vg is defined | ternary( [ ands_data_vg ], [] ) }}"
 docker_lv: "docker-pool"
+docker_root_lv: "docker-root-lv"
+docker_setup_root: "{{ docker_root_volume_size is defined }}"
 
 docker_min_size: 100
 docker_max_log_size: "2m"
diff --git a/roles/docker/tasks/configure.yml b/roles/docker/tasks/configure.yml
new file mode 100644
index 0000000..5d29291
--- /dev/null
+++ b/roles/docker/tasks/configure.yml
@@ -0,0 +1,63 @@
+---
+#- name: Remove docker
+#  yum: name="{{ item }}" state="absent"
+#  with_items: [ docker, docker-client, docker-common ]
+
+- name: install docker
+  include_tasks: install.yml
+
+- name: start docker
+  service: name="docker" state="started"
+
+- name: Configure bridge-nf-call-iptables with sysctl
+  sysctl: name="net.bridge.bridge-nf-call-iptables"  value=1 state=present sysctl_set=yes
+
+- name: Configure bridge-nf-call-ip6tables with sysctl
+  sysctl: name="net.bridge.bridge-nf-call-ip6tables"  value=1 state=present sysctl_set=yes
+
+- name: Determine if loopback
+  shell: docker info | grep 'Data file:.*loop'
+  register: loop_device_check
+  failed_when: false
+  changed_when: loop_device_check.rc == 0
+
+- set_fact:  docker_reinit="{{ (loop_device_check.rc == 0) or (vg == '') or (docker_setup_root and ((root_vg == '') or (vg != root_vg))) or (docker_storage_vg is defined and (docker_storage_vg != vg)) }}"
+  vars:
+    lv: "{{ ansible_lvm['lvs'][docker_lv] | default({}) }}"
+    vg: "{{ lv['vg'] | default('') }}"
+    root_lv: "{{ ansible_lvm['lvs'][docker_root_lv] | default({}) }}"
+    root_vg: "{{ root_lv['vg'] | default('') }}"
+
+- debug: msg="Re-initializing - {{ docker_reinit }}, Loopback check - {{ loop_device_check.stderr }}"
+  when: loop_device_check.stderr
+
+- import_tasks: storage.yml
+  when: docker_reinit
+
+- name: extend vg
+  lvol: vg="{{ ansible_lvm['lvs'][docker_lv]['vg'] }}" lv="{{ docker_lv }}" size="{{ docker_volume_size }}"
+  when: 
+    - not docker_reinit
+    - docker_volume_size is defined
+
+- name: extend root vg
+  lvol: vg="{{ ansible_lvm['lvs'][docker_root_lv]['vg'] }}" lv="{{ docker_root_lv }}" size="{{ docker_root_volume_size }}"
+  when: 
+    - not docker_reinit
+    - docker_setup_root
+    - docker_root_volume_size is defined
+
+# By default there is systemd driver installed. It is removed during OpenShift installation, but is still there during prepare stage
+# The parameters to docker can be set trough OpenShift and currently are moved there.
+#- name: Limit size of container log files
+#  ghetto_json:
+#    path: "/etc/docker/daemon.json"
+#    log-driver: "json-file"
+#    log-opts.max-size: "{{ docker_max_log_size }}" 
+#    log-opts.max-file: "{{ docker_max_log_files }}"
+#  notify:
+#    - restart docker
+
+- name: start docker
+  service: name="docker" enabled=yes state=started
+  
\ No newline at end of file
diff --git a/roles/docker/tasks/install.yml b/roles/docker/tasks/install.yml
new file mode 100644
index 0000000..41ae5a3
--- /dev/null
+++ b/roles/docker/tasks/install.yml
@@ -0,0 +1,13 @@
+- name: Remove versionlock from yum
+  command: yum versionlock delete docker docker-common docker-client
+  register: result
+  when: update | default(false)
+  failed_when: false
+  changed_when: result | failed
+
+- name: Ensure docker is installed
+  yum: name="docker{{ docker_version | default('') }}" state="{{ ((docker_version is defined) and (update | default(false))) | ternary('latest', 'present') }}"
+
+- name: Add versionlock to yum
+  command: yum versionlock add docker docker-common docker-client
+  when: docker_version is defined
diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml
index c03d897..c9d0251 100644
--- a/roles/docker/tasks/main.yml
+++ b/roles/docker/tasks/main.yml
@@ -1,57 +1,3 @@
 ---
-#- name: Remove docker
-#  yum: name="{{ item }}" state="absent"
-#  with_items: [ docker, docker-client, docker-common ]
-
-- name: Remove versionlock from yum
-  command: yum versionlock delete docker docker-common docker-client
-  register: result
-  failed_when: false
-  changed_when: result | failed
-
-- name: Ensure docker is installed
-  yum: name="docker{{ docker_version | default('') }}" state="{{ docker_version is defined | ternary('latest', 'present') }}"
-
-- name: Add versionlock to yum
-  command: yum versionlock add docker docker-common docker-client
-  when: docker_version is defined
-
-- name: start docker
-  service: name="docker" state="started"
-
-- name: Configure bridge-nf-call-iptables with sysctl
-  sysctl: name="net.bridge.bridge-nf-call-iptables"  value=1 state=present sysctl_set=yes
-
-- name: Configure bridge-nf-call-ip6tables with sysctl
-  sysctl: name="net.bridge.bridge-nf-call-ip6tables"  value=1 state=present sysctl_set=yes
-
-- name: Determine if loopback
-  shell: docker info | grep 'Data file:.*loop'
-  register: loop_device_check
-  failed_when: false
-  changed_when: loop_device_check.rc == 0
-
-- debug: msg="{{ loop_device_check.stderr }}"
-  when: loop_device_check.stderr
-
-- import_tasks: storage.yml
-  when: loop_device_check.rc == 0
-
-- name: extend the vg
-  lvol: vg="{{ ansible_lvm['lvs'][docker_lv]['vg'] }}" lv="docker_lv" size="{{ docker_volume_size }}"
-  when: docker_volume_size is defined
-
-# By default there is systemd driver installed. It is removed during OpenShift installation, but is still there during prepare stage
-# The parameters to docker can be set trough OpenShift and currently are moved there.
-#- name: Limit size of container log files
-#  ghetto_json:
-#    path: "/etc/docker/daemon.json"
-#    log-driver: "json-file"
-#    log-opts.max-size: "{{ docker_max_log_size }}" 
-#    log-opts.max-file: "{{ docker_max_log_files }}"
-#  notify:
-#    - restart docker
-
-- name: start docker
-  service: name="docker" enabled=yes state=started
-  
\ No newline at end of file
+- name: "Configuring network"
+  include_tasks: "{{ action | default('configure') }}.yml"
diff --git a/roles/docker/tasks/storage.yml b/roles/docker/tasks/storage.yml
index 5a5c858..e431030 100644
--- a/roles/docker/tasks/storage.yml
+++ b/roles/docker/tasks/storage.yml
@@ -1,11 +1,13 @@
 ---
+# We need to adjust docker_min_size if docker_volume_size set (min_size is just given in GB, but volume size
+# support suffixes and we need to handle this).
 - name: list volume groups
   set_fact: available_vg="{{ item.key }}"
   with_dict: "{{ ansible_lvm.vgs }}"
   when:
     - not available_vg is defined
     - not item.key in ( docker_exclude_vgs | default([]) )
-    - (item.value.free_g | int) > (docker_volume_size | default(docker_min_size))
+    - (item.value.free_g | int) > docker_min_size
 
 - set_fact: docker_storage_vg="{{ available_vg }}"
   when: (not docker_storage_vg is defined) and (available_vg is defined)
@@ -13,11 +15,13 @@
 - fail: msg="Can't detect Docker VG"
   when: not docker_storage_vg is defined
 
-- name: check to see if {{ docker_storage_device }} exists
+- name: "check to see if {{ docker_storage_device }} exists"
   command: "test -e {{ docker_storage_device }}"
   when: docker_storage_device is defined
 
-- set_fact: docker_storage_config="VG={{ docker_storage_vg }} AUTO_EXTEND_POOL=true"
+- set_fact: docker_storage_config="VG={{ docker_storage_vg }} DATA_SIZE={{ docker_volume_size | default(docker_min_size ~ 'G') }} AUTO_EXTEND_POOL=true"
+- set_fact: docker_storage_config="{{ docker_storage_config }} DOCKER_ROOT_VOLUME=yes DOCKER_ROOT_VOLUME_SIZE={{ docker_root_volume_size | default(docker_min_size ~ 'G') }}"
+  when: docker_setup_root
 
 - set_fact: docker_storage_config="{{ docker_storage_config }} DEVS={{ docker_storage_device }}"
   when: ( docker_storage_device is defined ) and ( ansible_lvm.vgs[docker_storage_vg] is not defined )
@@ -28,6 +32,29 @@
 - name: delete /var/lib/docker
   file: path="/var/lib/docker" state=absent
 
+- name: "delete {{ docker_lv }} lv"
+  lvol: vg="{{ ansible_lvm['lvs'][docker_lv]['vg'] }}" lv="{{ docker_lv }}" state="absent" force="yes"
+  when: ansible_lvm['lvs'][docker_lv] is defined
+
+- name: delete docker-pool lv
+  lvol: vg="{{ ansible_lvm['lvs']['docker-pool']['vg'] }}" lv="docker-pool" state="absent" force="yes"
+  when: ansible_lvm['lvs']['docker-pool'] is defined
+
+- name: unmount docker partition
+  mount: path="/var/lib/docker" state="unmounted" 
+
+- name: "delete {{ docker_root_lv }} lv"
+  lvol: vg="{{ ansible_lvm['lvs'][docker_root_lv]['vg'] }}" lv="{{ docker_root_lv }}" state="absent" force="yes"
+  when: ansible_lvm['lvs'][docker_root_lv] is defined
+
+- name: delete docker-pool lv
+  lvol: vg="{{ ansible_lvm['lvs']['docker-root-lv']['vg'] }}" lv="docker-root-lv" state="absent" force="yes"
+  when: ansible_lvm['lvs']['docker-root-lv'] is defined
+
+
+- name: remove old configuration
+  file: path="/etc/sysconfig/docker-storage" state="absent"
+
 - name: generate docker-storage-setup config file
   copy:
       content: "{{ docker_storage_config }}"
@@ -38,4 +65,3 @@
 
 - name: docker storage setup
   command: docker-storage-setup
-
diff --git a/roles/ofed/tasks/main.yml b/roles/ofed/tasks/main.yml
new file mode 100644
index 0000000..bd85d43
--- /dev/null
+++ b/roles/ofed/tasks/main.yml
@@ -0,0 +1,6 @@
+- name: install the 'Infiniband support' package group
+  yum: name="@Infiniband Support" state=present
+  
+- name: start rdma service
+  service: name="rdma" enabled=yes state=started
+  
\ No newline at end of file
diff --git a/setup.sh b/setup.sh
index 4ccf94d..69293aa 100755
--- a/setup.sh
+++ b/setup.sh
@@ -7,6 +7,7 @@ shift
 
 case "$action" in
     all)
+        ./setup.sh -i $inventory network "$@" || exit 1
         ./setup.sh -i $inventory prepare "$@" || exit 1
         ./setup.sh -i $inventory openshift "$@" || exit 1
         ./setup.sh -i $inventory gluster "$@" || exit 1
@@ -19,6 +20,9 @@ case "$action" in
     vm)
         apply playbooks/ands-vm-setup.yml "$@" || exit 1
         ;;
+    network)
+        apply playbooks/ands-network.yml "$@" || exit 1
+        ;;
     prepare)
         apply playbooks/ands-prepare.yml "$@" || exit 1
         ;;
@@ -64,12 +68,14 @@ case "$action" in
         apply playbooks/openshift-add-gluster.yml "$@" || exit 1
         ;;
     masters)
+        ./setup.sh -i $inventory network "$@" || exit 1
         ./setup.sh -i $inventory prepare "$@" || exit 1
         ./setup.sh -i $inventory openshift-masters "$@" || exit 1
         ./setup.sh -i $inventory gluster "$@" || exit 1
         ./setup.sh -i $inventory configure "$@" || exit 1
         ;;
     nodes)
+        ./setup.sh -i $inventory network "$@" || exit 1
         ./setup.sh -i $inventory prepare "$@" || exit 1
         ./setup.sh -i $inventory openshift-nodes "$@" || exit 1
         ./setup.sh -i $inventory gluster "$@" || exit 1
diff --git a/setup/users/htpasswd b/setup/users/htpasswd
index cf0d67e..cf0f1e6 100644
--- a/setup/users/htpasswd
+++ b/setup/users/htpasswd
@@ -1,3 +1,5 @@
 pdv:$apr1$ACvj6uUa$Nm1Vq8hZq3RzTtaYpAHv01
 csa:$apr1$IqEwdnzy$UAdd8ZSFnXommBbj29w3c0
 katrin:$apr1$/hxgbxC4$/MxeHtIYvAJcIQFR5Jz0E0
+ntj:$apr1$un8GkxMv$VZ36KeB90qwMMVpvHxOj8.
+kopmann:CZFk3ASLX0Vq6