diff options
Diffstat (limited to 'roles/ands_openshift')
19 files changed, 590 insertions, 0 deletions
diff --git a/roles/ands_openshift/defaults/main.yml b/roles/ands_openshift/defaults/main.yml new file mode 100644 index 0000000..857c389 --- /dev/null +++ b/roles/ands_openshift/defaults/main.yml @@ -0,0 +1,11 @@ +openshift_all_subroles: "{{ [ 'hostnames', 'users', 'ssh', 'storage', 'heketi' ] }}" +openshift_subroles: "{{ ( subrole is defined ) | ternary( [ subrole ], openshift_all_subroles ) }}" + +openshift_namespace: "default" +ands_disable_dynamic_provisioning: false + +ssh_template_path: "{{ ands_paths.provision }}/ssh/" +storage_template_path: "{{ ands_paths.provision }}/gfs/" +heketi_template_path: "{{ ands_paths.provision }}/heketi/" + +openshift_storage_nodes: "{{ groups.storage_nodes | map('extract', hostvars, 'ands_storage_hostname') | list }}" diff --git a/roles/ands_openshift/files/gfs-svc.yml b/roles/ands_openshift/files/gfs-svc.yml new file mode 100644 index 0000000..359f3b1 --- /dev/null +++ b/roles/ands_openshift/files/gfs-svc.yml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: Template +metadata: + name: gfs + annotations: + descriptions: "GlusterFS endpoints & service" + tags: glusterfs +objects: + - apiVersion: v1 + kind: Service + metadata: + name: gfs + spec: + ports: + - port: 1 diff --git a/roles/ands_openshift/files/heketi/heketi.json b/roles/ands_openshift/files/heketi/heketi.json new file mode 100644 index 0000000..9efe610 --- /dev/null +++ b/roles/ands_openshift/files/heketi/heketi.json @@ -0,0 +1,23 @@ +{ + "_port_comment": "Heketi Server Port Number", + "port" : "8080", + + "use_auth" : false, + "jwt" : { + "admin" : { + "key" : "My Secret" + }, + "user" : { + "key" : "My Secret" + } + }, + + "glusterfs" : { + "executor" : "ssh", + "sshexec": { + "keyfile": "/etc/heketi_keys/id_rsa", + "user": "root" + }, + "db" : "/var/lib/heketi/heketi.db" + } +} diff --git a/roles/ands_openshift/handlers/main.yml b/roles/ands_openshift/handlers/main.yml new file mode 100644 index 0000000..e46b2a9 --- /dev/null +++ b/roles/ands_openshift/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: heketi_topology + debug: msg="heketi-cli -s http://heketi.{{ openshift_master_default_subdomain }} --user=admin --secret={{ ands_secrets.heketi.admin | quote }} topology load --json={{ heketi_template_path }}/topology.json" +# command: heketi-cli -s "http://heketi.{{ openshift_master_default_subdomain }}" --user="admin" --secret={{ ands_secrets.heketi.admin | quote }} topology load --json="{{ heketi_template_path }}/topology.json" diff --git a/roles/ands_openshift/tasks/heketi.yml b/roles/ands_openshift/tasks/heketi.yml new file mode 100644 index 0000000..149f85d --- /dev/null +++ b/roles/ands_openshift/tasks/heketi.yml @@ -0,0 +1,13 @@ +--- +- block: + - name: Ensure all required packages are installed + yum: name={{item}} state=present + with_items: + - heketi-client + + - include: heketi_resources.yml + run_once: true + delegate_to: "{{ groups.masters[0] }}" + when: ansible_lvm.lvs.{{ ands_heketi_lv }} is defined + + when: ansible_lvm.lvs.{{ ands_heketi_lv }} is defined diff --git a/roles/ands_openshift/tasks/heketi_perms.yml b/roles/ands_openshift/tasks/heketi_perms.yml new file mode 100644 index 0000000..4df6260 --- /dev/null +++ b/roles/ands_openshift/tasks/heketi_perms.yml @@ -0,0 +1,9 @@ +--- +- name: Mount heketidb volume + mount: name="{{ heketi_template_path }}/heketidbstorage" src="localhost:heketidbstorage" fstype="glusterfs" opts="defaults,_netdev" state="mounted" + +- name: Allow writting to heketidb + file: path="{{ heketi_template_path }}/heketidbstorage" owner="root" group="root" mode=0777 + +- name: Mount heketidb volume + mount: name="{{ heketi_template_path }}/heketidbstorage" state="absent" diff --git a/roles/ands_openshift/tasks/heketi_resources.yml b/roles/ands_openshift/tasks/heketi_resources.yml new file mode 100644 index 0000000..06ae6b3 --- /dev/null +++ b/roles/ands_openshift/tasks/heketi_resources.yml @@ -0,0 +1,74 @@ +--- +- name: Ensure heketi configuration directory exists + file: path="{{ heketi_template_path }}" state="directory" mode=0600 owner=root group=root + +- name: Check if secret exists + command: oc -n "{{ openshift_namespace }}" get secret/heketi + register: result + failed_when: false + changed_when: (result | failed) + +- name: Create secret for dynamic volume provisioning + command: "kubectl create secret generic heketi --type=kubernetes.io/glusterfs --from-literal=key={{ ands_secrets.heketi.admin | quote }} --from-literal=user={{ ands_secrets.heketi.user | quote }} --namespace={{ openshift_namespace }}" + when: (result | changed) + +- name: Copy Heketi configuration + copy: src="heketi/heketi.json" dest="{{ heketi_template_path }}/heketi.json" owner=root group=root mode="0644" + register: result1 + +- name: Check if configMap exists + command: oc -n "{{ openshift_namespace }}" get cm/heketi + register: result2 + failed_when: false + changed_when: (result2 | failed) + +- name: Desotry existing Heketi configuration + command: oc -n "{{ openshift_namespace }}" delete cm/heketi + when: ( result1 | changed ) and (not (result2 | changed)) + +- name: Create heketiConfigmap + command: oc -n "{{ openshift_namespace }}" create cm heketi --from-file="{{ heketi_template_path }}/heketi.json" + when: (result1 | changed) or (result2 | changed) + +- name: Check if Heketi POD is running + command: oc -n "{{ openshift_namespace }}" get dc/heketi --template "{{ '{{.status.availableReplicas}}' }}" + register: result + failed_when: false + changed_when: (result | failed) or ((result.stdout | int) < 1) + +- name: Fix GlusterFS volume permissions + include: heketi_perms.yml + run_once: true + delegate_to: "{{ groups.masters[0] }}" + when: (result | changed) + +- name: Copy Heketi Template + template: src="heketi/heketi_template.json.j2" dest="{{ heketi_template_path }}/heketi_template.json" owner=root group=root mode="0644" + register: result + +- name: Create Heketi Pod + include_role: name="openshift_resource" + vars: + template: heketi_template.json + template_path: "{{ heketi_template_path }}" + project: "{{ openshift_namespace }}" + recreate: "{{ result | changed | ternary (true, false) }}" + +- name: Wait until heketi service is running + wait_for: host="heketi.{{ openshift_master_default_subdomain }}" port=80 state=present + +- name: Copy Heketi topology + template: src="heketi/topology.json.j2" dest="{{ heketi_template_path }}/topology.json" owner=root group=root mode="0644" + notify: heketi_topology + +- name: Copy Heketi storage class + template: src="heketi/heketi-sc.yml.j2" dest="{{ heketi_template_path }}/heketi-sc.yml" owner=root group=root mode="0644" + register: result + +- name: Setup Heketi-based dynamic volume provisioning + include_role: name="openshift_resource" + vars: + template: heketi-sc.yml + template_path: "{{ heketi_template_path }}" + project: "{{ openshift_namespace }}" + recreate: "{{ result | changed | ternary (true, false) }}" diff --git a/roles/ands_openshift/tasks/hostnames.yml b/roles/ands_openshift/tasks/hostnames.yml new file mode 100644 index 0000000..e489a8c --- /dev/null +++ b/roles/ands_openshift/tasks/hostnames.yml @@ -0,0 +1,15 @@ +--- +#- name: Remove obsolte hostnames from /etc/hosts +# lineinfile: dest="/etc/hosts" regexp="{{ hostvars[item]['openshift_hostname'] }}" state="absent" +# with_inventory_hostnames: +# - nodes + + +- name: Configure all cluster hostnames in /etc/hosts + lineinfile: dest="/etc/hosts" line="{{ hostvars[item]['openshift_ip'] }} {{ hostvars[item]['openshift_public_hostname'] }} {{ hostvars[item]['openshift_hostname'] }}" regexp="{{ hostvars[item]['openshift_hostname'] }}" state="present" + with_inventory_hostnames: + - nodes + +- name: Provision /etc/hosts to ensure that all masters servers are accessing Master API on loopback device + lineinfile: dest="/etc/hosts" line="127.0.0.1 {{ openshift_master_cluster_hostname }}" regexp=".*{{ openshift_master_cluster_hostname }}$" state="present" + when: "'masters' in group_names" diff --git a/roles/ands_openshift/tasks/main.yml b/roles/ands_openshift/tasks/main.yml new file mode 100644 index 0000000..f72123f --- /dev/null +++ b/roles/ands_openshift/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: "Configuring OpenShift" + include: "{{ current_subrole }}.yml" + with_items: "{{ openshift_subroles }}" + loop_control: + loop_var: current_subrole diff --git a/roles/ands_openshift/tasks/ssh.yml b/roles/ands_openshift/tasks/ssh.yml new file mode 100644 index 0000000..7d8d99d --- /dev/null +++ b/roles/ands_openshift/tasks/ssh.yml @@ -0,0 +1,21 @@ +--- +- name: Check if ssh secret exists + run_once: true + delegate_to: "{{ groups.masters[0] }}" + command: oc -n "{{ openshift_namespace }}" get secret/ands-ssh + register: result + changed_when: (result | failed) + failed_when: false + +- include: ssh_keygen.yml + run_once: true + delegate_to: "{{ groups.masters[0] }}" + when: (result | changed) + +- name: Read SSH public key + shell: cat "{{ ssh_template_path }}/id_rsa.pub" + changed_when: false + register: result + +- name: Distribute public keys + authorized_key: user="root" key="{{result.stdout}}" state=present manage_dir=yes exclusive=no diff --git a/roles/ands_openshift/tasks/ssh_keygen.yml b/roles/ands_openshift/tasks/ssh_keygen.yml new file mode 100644 index 0000000..21a7b0a --- /dev/null +++ b/roles/ands_openshift/tasks/ssh_keygen.yml @@ -0,0 +1,12 @@ +--- +- name: Ensure ssh directory exists + file: path="{{ ssh_template_path }}" state="directory" mode=0600 owner=root group=root + +- name: Generate ssh-key + command: ssh-keygen -t rsa -C "ands-ssh@ipe.kit.edu" -N "" -f "{{ ssh_template_path }}"/id_rsa creates="{{ ssh_template_path }}/id_rsa" + +- name: Create ssh secret + command: oc -n "{{ openshift_namespace }}" secrets new ands-ssh id_rsa="{{ ssh_template_path }}"/id_rsa id_rsa_pub="{{ ssh_template_path }}/id_rsa.pub" + +- name: Ensure ssh secret key is removed + file: path="{{ ssh_template_path }}/id_rsa" state=absent diff --git a/roles/ands_openshift/tasks/storage.yml b/roles/ands_openshift/tasks/storage.yml new file mode 100644 index 0000000..be2583a --- /dev/null +++ b/roles/ands_openshift/tasks/storage.yml @@ -0,0 +1,4 @@ +--- +- include: storage_resources.yml + run_once: true + delegate_to: "{{ groups.masters[0] }}" diff --git a/roles/ands_openshift/tasks/storage_resources.yml b/roles/ands_openshift/tasks/storage_resources.yml new file mode 100644 index 0000000..5adf69e --- /dev/null +++ b/roles/ands_openshift/tasks/storage_resources.yml @@ -0,0 +1,33 @@ +--- +- name: Ensure OpenShift template directory exists + file: path="{{ storage_template_path }}" state="directory" mode=0644 owner=root group=root + +- name: Copy GlusterFS service template + copy: src="gfs-svc.yml" dest="{{ storage_template_path }}/gfs-svc.yml" owner=root group=root mode="0644" + register: result + +- name: Configure GFS service & endpoints + include_role: name="openshift_resource" + vars: + template: gfs-svc.yml + template_path: "{{ storage_template_path }}" + project: "{{ prj_item }}" + recreate: "{{ result | changed | ternary (true, false) }}" + with_items: "{{ ands_openshift_projects.keys() | union(['default']) }}" + loop_control: + loop_var: prj_item + +- name: Configure GlusterFS end-points + template: src="gfs-ep.yml.j2" dest="{{ storage_template_path }}/gfs-ep.yml" owner=root group=root mode="0644" + register: result + +- name: Configure GFS service & endpoints + include_role: name="openshift_resource" + vars: + template: gfs-ep.yml + template_path: "{{ storage_template_path }}" + project: "{{ prj_item }}" + recreate: "{{ result | changed | ternary (true, false) }}" + with_items: "{{ ands_openshift_projects.keys() | union(['default']) }}" + loop_control: + loop_var: prj_item diff --git a/roles/ands_openshift/tasks/users.yml b/roles/ands_openshift/tasks/users.yml new file mode 100644 index 0000000..c816203 --- /dev/null +++ b/roles/ands_openshift/tasks/users.yml @@ -0,0 +1,8 @@ +--- +- name: Copy htpasswd to /etc/origin/master + copy: src="users/htpasswd" dest="/etc/origin/master/htpasswd" mode=0644 owner=root group=root force=yes backup=no + when: "'masters' in group_names" + +- include: users_resources.yml + run_once: true + delegate_to: "{{ groups.masters[0] }}" diff --git a/roles/ands_openshift/tasks/users_resources.yml b/roles/ands_openshift/tasks/users_resources.yml new file mode 100644 index 0000000..35323cb --- /dev/null +++ b/roles/ands_openshift/tasks/users_resources.yml @@ -0,0 +1,40 @@ +--- +- name: Configure cluster roles + command: "oc adm policy add-cluster-role-to-user {{ item.key.split('/')[0] }} {{ item.value.replace(' ','').split(',') | join(' ') }}" + with_dict: "{{ ands_openshift_roles }}" + when: "{{ item.key.split('/') | length == 1 }}" + +- name: Get project list + command: "oc get projects -o json" + changed_when: false + register: results + +- name: Find missing projects + set_fact: new_projects="{{ ands_openshift_projects.keys() | difference (results.stdout | from_json | json_query('items[*].metadata.name')) }}" + when: (results | succeeded) + +- name: Create missing projects + command: "oc adm new-project --description '{{ ands_openshift_projects[item] }}' {{ item }}" + with_items: "{{ new_projects | default([]) }}" + +- name: Configure per project roles + command: "oc adm policy add-role-to-user -n {{ item.key.split('/')[0] }} {{ item.key.split('/')[1] }} {{ item.value.replace(' ','').split(',') | join(' ') }}" + with_dict: "{{ ands_openshift_roles }}" + when: "{{ item.key.split('/') | length == 2 }}" + +- name: Get user list + command: "oc get users -o json" + changed_when: false + register: results + +- name: Find removed users + set_fact: removed_users="{{ results.stdout | from_json | json_query('items[*].metadata.name') | difference(ands_openshift_users.keys()) }}" + when: (results | succeeded) + +- name: Create missing projects + command: "oc delete identity htpasswd_auth:{{ item }}" + with_items: "{{ removed_users | default([]) }}" + +- name: Create missing projects + command: "oc delete user {{ item }}" + with_items: "{{ removed_users | default([]) }}" diff --git a/roles/ands_openshift/templates/gfs-ep.yml.j2 b/roles/ands_openshift/templates/gfs-ep.yml.j2 new file mode 100644 index 0000000..de3acac --- /dev/null +++ b/roles/ands_openshift/templates/gfs-ep.yml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: Template +metadata: + name: gfs + annotations: + descriptions: "GlusterFS endpoints & service" + tags: glusterfs +objects: + - apiVersion: v1 + kind: Endpoints + metadata: + name: gfs + subsets: +{% for node in openshift_storage_nodes %} + - addresses: + - ip: {{ node }} + ports: + - port: 1 +{% endfor %} diff --git a/roles/ands_openshift/templates/heketi/heketi-sc.yml.j2 b/roles/ands_openshift/templates/heketi/heketi-sc.yml.j2 new file mode 100644 index 0000000..23ce6ce --- /dev/null +++ b/roles/ands_openshift/templates/heketi/heketi-sc.yml.j2 @@ -0,0 +1,21 @@ +--- +apiVersion: v1 +kind: Template +metadata: + name: heketi-sc + annotations: + descriptions: "Heketi Dynamic Volume Provisioning" + tags: heketi +objects: + - apiVersion: storage.k8s.io/v1beta1 + kind: StorageClass + metadata: + name: heketi + annotations: + storageclass.beta.kubernetes.io/is-default-class: "true" + provisioner: kubernetes.io/glusterfs + parameters: + resturl: "http://heketi.{{ openshift_master_default_subdomain }}" + restuser: "admin" + secretName: "heketi" + secretNamespace: "default" diff --git a/roles/ands_openshift/templates/heketi/heketi_template.json.j2 b/roles/ands_openshift/templates/heketi/heketi_template.json.j2 new file mode 100644 index 0000000..221662b --- /dev/null +++ b/roles/ands_openshift/templates/heketi/heketi_template.json.j2 @@ -0,0 +1,232 @@ +{ + "kind": "Template", + "apiVersion": "v1", + "metadata": { + "name": "heketi", + "labels": { + "glusterfs": "heketi-template" + }, + "annotations": { + "description": "Heketi service deployment template", + "tags": "glusterfs,heketi" + } + }, + "labels": { + "template": "heketi" + }, + "objects": [ + { + "kind": "PersistentVolume", + "apiVersion": "v1", + "metadata": { + "name": "heketidb" + }, + "spec": { + "persistentVolumeReclaimPolicy": "Retain", + "glusterfs": { + "endpoints": "gfs", + "path": "heketidbstorage" + }, + "accessModes": [ "ReadWriteMany" ], + "capacity": { + "storage": "1Gi" + }, + "claimRef": { + "name": "heketidb", + "namespace": "default" + } + } + }, + { + "kind": "PersistentVolumeClaim", + "apiVersion": "v1", + "metadata": { + "name": "heketidb" + }, + "spec": { + "volumeName": "heketidb", + "accessModes": [ "ReadWriteMany" ], + "resources": { + "requests": { + "storage": "1Gi" + } + } + } + }, + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "heketi", + "labels": { + "glusterfs": "heketi" + }, + "annotations": { + "description": "Exposes Heketi service" + } + }, + "spec": { + "ports": [ + { + "name": "heketi", + "port": 8080, + "targetPort": 8080 + } + ], + "selector": { + "name": "heketi" + } + } + }, + { + "kind": "Route", + "apiVersion": "v1", + "metadata": { + "name": "heketi", + "labels": { + "glusterfs": "heketi" + } + }, + "spec": { + "host": "heketi.{{ openshift_master_default_subdomain }}", + "to": { + "kind": "Service", + "name": "heketi" + } + } + }, + { + "kind": "DeploymentConfig", + "apiVersion": "v1", + "metadata": { + "name": "heketi", + "labels": { + "glusterfs": "heketi" + }, + "annotations": { + "description": "Defines how to deploy Heketi" + } + }, + "spec": { + "replicas": 1, + "selector": { + "name": "heketi" + }, + "template": { + "metadata": { + "name": "heketi", + "labels": { + "name": "heketi", + "glusterfs": "heketi" + } + }, + "triggers": [ + { + "type": "ConfigChange" + } + ], + "strategy": { + "type": "Recreate" + }, + "spec": { + "nodeSelector": { + "master": "1" + }, + "containers": [ + { + "name": "heketi", + "image": "heketi/heketi:dev", + "imagePullPolicy": "Always", + "env": [ + { + "name": "HEKETI_USER_KEY", + "valueFrom": { + "secretKeyRef": { + "name": "heketi", + "key": "user" + } + } + }, + { + "name": "HEKETI_ADMIN_KEY", + "valueFrom": { + "secretKeyRef": { + "name": "heketi", + "key": "key" + } + } + }, + { + "name": "HEKETI_FSTAB", + "value": "/var/lib/heketi/fstab" + }, + { + "name": "HEKETI_SNAPSHOT_LIMIT", + "value": "14" + } + ], + "ports": [ + { + "containerPort": 8080 + } + ], + "volumeMounts": [ + { + "name": "config", + "mountPath": "/etc/heketi", + "readOnly": true + }, + { + "name": "ssh", + "mountPath": "/etc/heketi_keys", + "readOnly": true + }, + { + "name": "db", + "mountPath": "/var/lib/heketi" + } + ], + "readinessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 3, + "httpGet": { + "path": "/hello", + "port": 8080 + } + }, + "livenessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 30, + "httpGet": { + "path": "/hello", + "port": 8080 + } + } + } + ], + "volumes": [ + { + "name": "ssh", + "secret": { + "secretName": "ands-ssh" + } + }, + { + "name": "config", + "configMap": { + "name" : "heketi" + } + }, + { + "name": "db", + "persistentVolumeClaim": { + "claimName" : "heketidb" + } + } + ] + } + } + } + } + ] +}
\ No newline at end of file diff --git a/roles/ands_openshift/templates/heketi/topology.json.j2 b/roles/ands_openshift/templates/heketi/topology.json.j2 new file mode 100644 index 0000000..53d683e --- /dev/null +++ b/roles/ands_openshift/templates/heketi/topology.json.j2 @@ -0,0 +1,28 @@ + +{ + "clusters": [ + { + "nodes": [ +{% set comma = joiner(",") %} +{% for node in openshift_storage_nodes %} + {{ comma() }} { + "node": { + "hostnames": { + "manage": [ + "{{ node }}" + ], + "storage": [ + "{{ node }}" + ] + }, + "zone": 1 + }, + "devices": [ + "/dev/{{ansible_lvm.lvs[ands_heketi_lv].vg}}/{{ ands_heketi_lv }}" + ] + } +{% endfor %} + ] + } + ] +} |