Fix etcd cert generation when etcd_interface is defined

- Refactor certificate generation to properly accept overrides of etcd_interface
  per host and set the certificate SANS and peer URLs properly.

- Add sanity checking to user-set values of etcd_interface to provide a better
  error message

19 files changed, 157 insertions, 69 deletions

diff --git a/roles/etcd/README.md b/roles/etcd/README.md
index 49207c428..88e4ff874 100644
--- a/roles/etcd/README.md
+++ b/roles/etcd/README.md
@@ -17,7 +17,7 @@ TODO
 Dependencies
 ------------
 
-None
+etcd-common
 
 Example Playbook
 ----------------
diff --git a/roles/etcd/defaults/main.yaml b/roles/etcd/defaults/main.yaml
index 0f216b84e..0fd3de585 100644
--- a/roles/etcd/defaults/main.yaml
+++ b/roles/etcd/defaults/main.yaml
@@ -2,16 +2,8 @@
 etcd_interface: "{{ ansible_default_ipv4.interface }}"
 etcd_client_port: 2379
 etcd_peer_port: 2380
-etcd_peers_group: etcd
 etcd_url_scheme: http
 etcd_peer_url_scheme: http
-etcd_conf_dir: /etc/etcd
-etcd_ca_file: "{{ etcd_conf_dir }}/ca.crt"
-etcd_cert_file: "{{ etcd_conf_dir }}/server.crt"
-etcd_key_file: "{{ etcd_conf_dir }}/server.key"
-etcd_peer_ca_file: "{{ etcd_conf_dir }}/ca.crt"
-etcd_peer_cert_file: "{{ etcd_conf_dir }}/peer.crt"
-etcd_peer_key_file: "{{ etcd_conf_dir }}/peer.key"
 
 etcd_initial_cluster_state: new
 etcd_initial_cluster_token: etcd-cluster-1
diff --git a/roles/etcd/handlers/main.yml b/roles/etcd/handlers/main.yml
index b897913f9..4c0efb97b 100644
--- a/roles/etcd/handlers/main.yml
+++ b/roles/etcd/handlers/main.yml
@@ -1,3 +1,4 @@
 ---
 - name: restart etcd
   service: name=etcd state=restarted
+  when: not etcd_service_status_changed | default(false)
diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml
index 92d44ef4d..a71b36237 100644
--- a/roles/etcd/meta/main.yml
+++ b/roles/etcd/meta/main.yml
@@ -17,4 +17,4 @@ galaxy_info:
   - system
 dependencies:
 - { role: os_firewall }
-- { role: openshift_repos }
+- { role: etcd_common }
diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml
index 656901409..fcbdecd37 100644
--- a/roles/etcd/tasks/main.yml
+++ b/roles/etcd/tasks/main.yml
@@ -1,4 +1,12 @@
 ---
+- fail:
+    msg: Interface {{ etcd_interface }} not found
+  when: "'ansible_' ~ etcd_interface not in hostvars[inventory_hostname]"
+
+- fail:
+    msg: IPv4 address not found for {{ etcd_interface }}
+  when: "'ipv4' not in hostvars[inventory_hostname]['ansible_' ~ etcd_interface] or 'address' not in hostvars[inventory_hostname]['ansible_' ~ etcd_interface].ipv4"
+
 - name: Install etcd
   yum: pkg=etcd-2.* state=present
 
@@ -49,5 +57,5 @@
     enabled: yes
   register: start_result
 
-- pause: seconds=30
-  when: start_result | changed
+- set_fact:
+    etcd_service_status_changed = start_result | changed
diff --git a/roles/etcd/templates/etcd.conf.j2 b/roles/etcd/templates/etcd.conf.j2
index 9ac23b1dd..32577c96c 100644
--- a/roles/etcd/templates/etcd.conf.j2
+++ b/roles/etcd/templates/etcd.conf.j2
@@ -1,9 +1,9 @@
 {% macro initial_cluster() -%}
 {% for host in groups[etcd_peers_group] -%}
 {% if loop.last -%}
-{{ host }}={{ etcd_peer_url_scheme }}://{{ hostvars[host]['ansible_' + etcd_interface]['ipv4']['address'] }}:{{ etcd_peer_port }}
+{{ host }}={{ etcd_peer_url_scheme }}://{{ etcd_host_int_map[host].interface.ipv4.address }}:{{ etcd_peer_port }}
 {%- else -%}
-{{ host }}={{ etcd_peer_url_scheme }}://{{ hostvars[host]['ansible_' + etcd_interface]['ipv4']['address'] }}:{{ etcd_peer_port }},
+{{ host }}={{ etcd_peer_url_scheme }}://{{ etcd_host_int_map[host].interface.ipv4.address }}:{{ etcd_peer_port }},
 {%- endif -%}
 {% endfor -%}
 {% endmacro -%}
diff --git a/roles/etcd_ca/meta/main.yml b/roles/etcd_ca/meta/main.yml
index fb9280c9e..d02456ca3 100644
--- a/roles/etcd_ca/meta/main.yml
+++ b/roles/etcd_ca/meta/main.yml
@@ -13,4 +13,4 @@ galaxy_info:
   - cloud
   - system
 dependencies:
-- { role: openshift_repos }
+- { role: etcd_common }
diff --git a/roles/etcd_ca/tasks/main.yml b/roles/etcd_ca/tasks/main.yml
index 625756867..d32f5e48c 100644
--- a/roles/etcd_ca/tasks/main.yml
+++ b/roles/etcd_ca/tasks/main.yml
@@ -1,14 +1,14 @@
 ---
 - file:
-    path: "{{ etcd_ca_dir }}/{{ item }}"
+    path: "{{ item }}"
     state: directory
     mode: 0700
     owner: root
     group: root
   with_items:
-  - certs
-  - crl
-  - fragments
+  - "{{ etcd_ca_new_certs_dir }}"
+  - "{{ etcd_ca_crl_dir }}"
+  - "{{ etcd_ca_dir }}/fragments"
 
 - command: cp /etc/pki/tls/openssl.cnf ./
   args:
@@ -22,25 +22,25 @@
 
 - assemble:
     src: "{{ etcd_ca_dir }}/fragments"
-    dest: "{{ etcd_ca_dir }}/openssl.cnf"
+    dest: "{{ etcd_openssl_conf }}"
 
-- command: touch index.txt
+- command: touch {{ etcd_ca_db }}
   args:
-    chdir: "{{ etcd_ca_dir }}"
-    creates: "{{ etcd_ca_dir }}/index.txt"
+    creates: "{{ etcd_ca_db }}"
 
 - copy:
-    dest: "{{ etcd_ca_dir }}/serial"
+    dest: "{{ etcd_ca_serial }}"
     content: "01"
     force: no
 
 - command: >
-    openssl req -config openssl.cnf -newkey rsa:4096
-    -keyout ca.key -new -out ca.crt -x509 -extensions etcd_v3_ca_self
-    -batch -nodes -subj /CN=etcd-signer@{{ ansible_date_time.epoch }}
-    -days 365
+    openssl req -config {{ etcd_openssl_conf }} -newkey rsa:4096
+    -keyout {{ etcd_ca_key }} -new -out {{ etcd_ca_cert }}
+    -x509 -extensions {{ etcd_ca_exts_self }} -batch -nodes
+    -days {{ etcd_ca_default_days }}
+    -subj /CN=etcd-signer@{{ ansible_date_time.epoch }}
   args:
     chdir: "{{ etcd_ca_dir }}"
-    creates: "{{ etcd_ca_dir }}/ca.crt"
+    creates: "{{ etcd_ca_cert }}"
   environment:
-    SAN: ''
+    SAN: 'etcd-signer'
diff --git a/roles/etcd_ca/templates/openssl_append.j2 b/roles/etcd_ca/templates/openssl_append.j2
index de2adaead..f28316fc2 100644
--- a/roles/etcd_ca/templates/openssl_append.j2
+++ b/roles/etcd_ca/templates/openssl_append.j2
@@ -1,20 +1,20 @@
 
-[ etcd_v3_req ]
+[ {{ etcd_req_ext }} ]
 basicConstraints = critical,CA:FALSE
 keyUsage         = digitalSignature,keyEncipherment
 subjectAltName   = ${ENV::SAN}
 
-[ etcd_ca ]
+[ {{ etcd_ca_name }} ]
 dir             = {{ etcd_ca_dir }}
-crl_dir         = $dir/crl
-database        = $dir/index.txt
-new_certs_dir   = $dir/certs
-certificate     = $dir/ca.crt
-serial          = $dir/serial
-private_key     = $dir/ca.key
-crl_number      = $dir/crlnumber
-x509_extensions = etcd_v3_ca_client
-default_days    = 365
+crl_dir         = {{ etcd_ca_crl_dir }}
+database        = {{ etcd_ca_db }}
+new_certs_dir   = {{ etcd_ca_new_certs_dir }}
+certificate     = {{ etcd_ca_cert }}
+serial          = {{ etcd_ca_serial }}
+private_key     = {{ etcd_ca_key }}
+crl_number      = {{ etcd_ca_crl_number }}
+x509_extensions = {{ etcd_ca_exts_client }}
+default_days    = {{ etcd_ca_default_days }}
 default_md      = sha256
 preserve        = no
 name_opt        = ca_default
@@ -23,27 +23,27 @@ policy          = policy_anything
 unique_subject  = no
 copy_extensions = copy
 
-[ etcd_v3_ca_self ]
+[ {{ etcd_ca_exts_self }} ]
 authorityKeyIdentifier = keyid,issuer
 basicConstraints       = critical,CA:TRUE,pathlen:0
 keyUsage               = critical,digitalSignature,keyEncipherment,keyCertSign
 subjectKeyIdentifier   = hash
 
-[ etcd_v3_ca_peer ]
+[ {{ etcd_ca_exts_peer }} ]
 authorityKeyIdentifier = keyid,issuer:always
 basicConstraints       = critical,CA:FALSE
 extendedKeyUsage       = clientAuth,serverAuth
 keyUsage               = digitalSignature,keyEncipherment
 subjectKeyIdentifier   = hash
 
-[ etcd_v3_ca_server ]
+[ {{ etcd_ca_exts_server }} ]
 authorityKeyIdentifier = keyid,issuer:always
 basicConstraints       = critical,CA:FALSE
 extendedKeyUsage       = serverAuth
 keyUsage               = digitalSignature,keyEncipherment
 subjectKeyIdentifier   = hash
 
-[ etcd_v3_ca_client ]
+[ {{ etcd_ca_exts_client }} ]
 authorityKeyIdentifier = keyid,issuer:always
 basicConstraints       = critical,CA:FALSE
 extendedKeyUsage       = clientAuth
diff --git a/roles/etcd_ca/vars/main.yml b/roles/etcd_ca/vars/main.yml
deleted file mode 100644
index 901e95027..000000000
--- a/roles/etcd_ca/vars/main.yml
+++ /dev/null
@@ -1,3 +0,0 @@
----
-etcd_conf_dir: /etc/etcd
-etcd_ca_dir: /etc/etcd/ca
diff --git a/roles/etcd_certificates/tasks/client.yml b/roles/etcd_certificates/tasks/client.yml
index 28f33f442..6aa4883e0 100644
--- a/roles/etcd_certificates/tasks/client.yml
+++ b/roles/etcd_certificates/tasks/client.yml
@@ -32,7 +32,7 @@
     creates: "{{ etcd_generated_certs_dir ~ '/' ~  item.etcd_cert_subdir ~ '/'
                  ~ item.etcd_cert_prefix ~ 'client.crt' }}"
   environment:
-    SAN: ''
+    SAN: "IP:{{ item.openshift.common.ip }}"
   with_items: etcd_needing_client_certs
 
 - file:
diff --git a/roles/etcd_certificates/tasks/main.yml b/roles/etcd_certificates/tasks/main.yml
index da875e8ea..3bb715943 100644
--- a/roles/etcd_certificates/tasks/main.yml
+++ b/roles/etcd_certificates/tasks/main.yml
@@ -4,6 +4,3 @@
 
 - include: server.yml
   when: etcd_needing_server_certs is defined and etcd_needing_server_certs
-
-
-
diff --git a/roles/etcd_certificates/tasks/server.yml b/roles/etcd_certificates/tasks/server.yml
index 727b7fa2c..3499dcbef 100644
--- a/roles/etcd_certificates/tasks/server.yml
+++ b/roles/etcd_certificates/tasks/server.yml
@@ -18,7 +18,7 @@
     creates: "{{ etcd_generated_certs_dir ~ '/' ~  item.etcd_cert_subdir ~ '/'
                  ~ item.etcd_cert_prefix ~ 'server.csr' }}"
   environment:
-    SAN: "IP:{{ item.openshift.common.ip }}"
+    SAN: "IP:{{ etcd_host_int_map[item.inventory_hostname].interface.ipv4.address }}"
   with_items: etcd_needing_server_certs
 
 - name: Sign and create the server crt
@@ -32,7 +32,7 @@
     creates: "{{ etcd_generated_certs_dir ~ '/' ~  item.etcd_cert_subdir ~ '/'
                  ~ item.etcd_cert_prefix ~ 'server.crt' }}"
   environment:
-    SAN: ''
+    SAN: "IP:{{ etcd_host_int_map[item.inventory_hostname].interface.ipv4.address }}"
   with_items: etcd_needing_server_certs
 
 - name: Create the peer csr
@@ -47,7 +47,7 @@
     creates: "{{ etcd_generated_certs_dir ~ '/' ~  item.etcd_cert_subdir ~ '/'
                  ~ item.etcd_cert_prefix ~ 'peer.csr' }}"
   environment:
-    SAN: "IP:{{ item.openshift.common.ip }}"
+    SAN: "IP:{{ etcd_host_int_map[item.inventory_hostname].interface.ipv4.address }}"
   with_items: etcd_needing_server_certs
 
 - name: Sign and create the peer crt
@@ -61,7 +61,7 @@
     creates: "{{ etcd_generated_certs_dir ~ '/' ~  item.etcd_cert_subdir ~ '/'
                  ~ item.etcd_cert_prefix ~ 'peer.crt' }}"
   environment:
-    SAN: ''
+    SAN: "IP:{{ etcd_host_int_map[item.inventory_hostname].interface.ipv4.address }}"
   with_items: etcd_needing_server_certs
 
 - file:
@@ -69,5 +69,3 @@
     dest: "{{ etcd_generated_certs_dir}}/{{ item.etcd_cert_subdir }}/{{ item.etcd_cert_prefix }}ca.crt"
     state: hard
   with_items: etcd_needing_server_certs
-
-
diff --git a/roles/etcd_certificates/vars/main.yml b/roles/etcd_certificates/vars/main.yml
deleted file mode 100644
index 0eaeeb82b..000000000
--- a/roles/etcd_certificates/vars/main.yml
+++ /dev/null
@@ -1,11 +0,0 @@
----
-etcd_conf_dir: /etc/etcd
-etcd_ca_dir: /etc/etcd/ca
-etcd_generated_certs_dir: /etc/etcd/generated_certs
-etcd_ca_cert: "{{ etcd_ca_dir }}/ca.crt"
-etcd_ca_key: "{{ etcd_ca_dir }}/ca.key"
-etcd_openssl_conf: "{{ etcd_ca_dir }}/openssl.cnf"
-etcd_ca_name: etcd_ca
-etcd_req_ext: etcd_v3_req
-etcd_ca_exts_peer: etcd_v3_ca_peer
-etcd_ca_exts_server: etcd_v3_ca_server
diff --git a/roles/etcd_common/README.md b/roles/etcd_common/README.md
new file mode 100644
index 000000000..131a01490
--- /dev/null
+++ b/roles/etcd_common/README.md
@@ -0,0 +1,34 @@
+etcd_common
+========================
+
+TODO
+
+Requirements
+------------
+
+TODO
+
+Role Variables
+--------------
+
+TODO
+
+Dependencies
+------------
+
+openshift-repos
+
+Example Playbook
+----------------
+
+TODO
+
+License
+-------
+
+Apache License Version 2.0
+
+Author Information
+------------------
+
+Jason DeTiberus (jdetiber@redhat.com)
diff --git a/roles/etcd_common/defaults/main.yml b/roles/etcd_common/defaults/main.yml
new file mode 100644
index 000000000..96f4b63af
--- /dev/null
+++ b/roles/etcd_common/defaults/main.yml
@@ -0,0 +1,30 @@
+---
+etcd_peers_group: etcd
+
+# etcd server vars
+etcd_conf_dir: /etc/etcd
+etcd_ca_file: "{{ etcd_conf_dir }}/ca.crt"
+etcd_cert_file: "{{ etcd_conf_dir }}/server.crt"
+etcd_key_file: "{{ etcd_conf_dir }}/server.key"
+etcd_peer_ca_file: "{{ etcd_conf_dir }}/ca.crt"
+etcd_peer_cert_file: "{{ etcd_conf_dir }}/peer.crt"
+etcd_peer_key_file: "{{ etcd_conf_dir }}/peer.key"
+
+# etcd ca vars
+etcd_ca_dir: "{{ etcd_conf_dir}}/ca"
+etcd_generated_certs_dir: "{{ etcd_conf_dir }}/generated_certs"
+etcd_ca_cert: "{{ etcd_ca_dir }}/ca.crt"
+etcd_ca_key: "{{ etcd_ca_dir }}/ca.key"
+etcd_openssl_conf: "{{ etcd_ca_dir }}/openssl.cnf"
+etcd_ca_name: etcd_ca
+etcd_req_ext: etcd_v3_req
+etcd_ca_exts_peer: etcd_v3_ca_peer
+etcd_ca_exts_server: etcd_v3_ca_server
+etcd_ca_exts_self: etcd_v3_ca_self
+etcd_ca_exts_client: etcd_v3_ca_client
+etcd_ca_crl_dir: "{{ etcd_ca_dir }}/crl"
+etcd_ca_new_certs_dir: "{{ etcd_ca_dir }}/certs"
+etcd_ca_db: "{{ etcd_ca_dir }}/index.txt"
+etcd_ca_serial: "{{ etcd_ca_dir }}/serial"
+etcd_ca_crl_number: "{{ etcd_ca_dir }}/crlnumber"
+etcd_ca_default_days: 365
diff --git a/roles/etcd_common/meta/main.yml b/roles/etcd_common/meta/main.yml
new file mode 100644
index 000000000..fb9280c9e
--- /dev/null
+++ b/roles/etcd_common/meta/main.yml
@@ -0,0 +1,16 @@
+---
+galaxy_info:
+  author: Jason DeTiberus
+  description:
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.9
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+  - system
+dependencies:
+- { role: openshift_repos }
diff --git a/roles/etcd_common/tasks/main.yml b/roles/etcd_common/tasks/main.yml
new file mode 100644
index 000000000..cd108495d
--- /dev/null
+++ b/roles/etcd_common/tasks/main.yml
@@ -0,0 +1,13 @@
+---
+- set_fact:
+    etcd_host_int_map: "{{ lookup('template', '../templates/host_int_map.j2') | from_yaml }}"
+
+- fail:
+    msg: "Interface {{ item.value.etcd_interface }} not found on host {{ item.key }}"
+  when: "'etcd_interface' in item.value and 'interface' not in item.value"
+  with_dict: etcd_host_int_map
+
+- fail:
+    msg: IPv4 address not found for {{ item.value.interface.device }} on host {{ item.key }}
+  when: "'ipv4' not in item.value.interface or 'address' not in item.value.interface.ipv4"
+  with_dict: etcd_host_int_map
diff --git a/roles/etcd_common/templates/host_int_map.j2 b/roles/etcd_common/templates/host_int_map.j2
new file mode 100644
index 000000000..9c9c76413
--- /dev/null
+++ b/roles/etcd_common/templates/host_int_map.j2
@@ -0,0 +1,13 @@
+---
+{% for host in groups[etcd_peers_group] %}
+{% set entry=hostvars[host] %}
+{{ entry.inventory_hostname }}:
+{% if 'etcd_interface' in entry %}
+  etcd_interface: {{ entry.etcd_interface }}
+{% if entry.etcd_interface in entry.ansible_interfaces %}
+  interface: {{ entry['ansible_' ~ entry.etcd_interface] | to_json }}
+{% endif %}
+{% else %}
+  interface: {{ entry['ansible_' ~ entry.ansible_default_ipv4.interface] | to_json }}
+{% endif %}
+{% endfor %}