From afd6a03b071eced6bd0940bb96a2a39233739523 Mon Sep 17 00:00:00 2001 From: Tomas Sedovic Date: Tue, 12 Sep 2017 17:05:56 +0200 Subject: Support Cinder-backed Openshift registry (#707) * Attach and detach a volume, wait for it to be accessible This is mostly just handling the attach/detach code, making sure the necessary vars are accessible where they need to be as well as finding out the correct device name the volume is attached as. * Create temp directory for mounts, remove some debug info * add the fs actions * Remove debug * Prepare the volume automatically if possible * Add docs and sample inventory * Read OS_* creds from shell in sample inventory * Fix yamlint complaint * Update readme This mentions the potential pitfalls when using devstack. * Better check for the router deployment in CI * Set the openshift_hoster*_wait vars to True * Fix typo --- playbooks/provisioning/openstack/README.md | 78 ++++++++++++++++++++++ .../openstack/post-provision-openstack.yml | 3 + .../prepare-and-format-cinder-volume.yaml | 75 +++++++++++++++++++++ .../sample-inventory/group_vars/OSEv3.yml | 20 ++++++ .../openstack/sample-inventory/group_vars/all.yml | 7 ++ 5 files changed, 183 insertions(+) create mode 100644 playbooks/provisioning/openstack/prepare-and-format-cinder-volume.yaml diff --git a/playbooks/provisioning/openstack/README.md b/playbooks/provisioning/openstack/README.md index 8b9a37537..267176eec 100644 --- a/playbooks/provisioning/openstack/README.md +++ b/playbooks/provisioning/openstack/README.md @@ -295,6 +295,7 @@ variables for the `inventory/group_vars/OSEv3.yml`, `all.yml`: deployment_type: origin openshift_deployment_type: "{{ deployment_type }}" + #### Setting a custom entrypoint In order to set a custom entrypoint, update `openshift_master_cluster_public_hostname` @@ -304,6 +305,83 @@ In order to set a custom entrypoint, update `openshift_master_cluster_public_hos Note than an empty hostname does not work, so if your domain is `openshift.example.com`, you cannot set this value to simply `openshift.example.com`. +### Use an existing Cinder volume for the OpenShift registry + +You can optionally use an existing Cinder volume for the storage of +your OpenShift registry. + +To do that, you need to have a Cinder volume (you can create one by +running: + + openstack volume create --size + +The volume needs to have a file system created before you put it to +use. We can do prepare it for you if you put this in inventory/group_vars/all.yml: + + prepare_and_format_registry_volume: true + +**NOTE:** doing so **will destroy any data that's currently on the volume**! + +You can also run the registry setup playbook directly: + + ansible-playbook -i inventory playbooks/provisioning/openstack/prepare-and-format-cinder-volume.yaml + +(the provisioning phase must be completed, first) + + +To instruct OpenShift to actually use the volume, you must first configure it +with the OpenStack credentials by putting the following to `OSEv3.yml`: + + ## Openstack credentials + #openshift_cloudprovider_kind=openstack + #openshift_cloudprovider_openstack_auth_url=http://openstack.example.com:35357/v2.0/ + #openshift_cloudprovider_openstack_username=username + #openshift_cloudprovider_openstack_password=password + #openshift_cloudprovider_openstack_domain_id=domain_id + #openshift_cloudprovider_openstack_domain_name=domain_name + #openshift_cloudprovider_openstack_tenant_id=tenant_id + #openshift_cloudprovider_openstack_tenant_name=tenant_name + #openshift_cloudprovider_openstack_region=region + +Note that these credentials may be different from the ones you used for +provisioning (say for quota or access control reasons). To use the same +OpenStack credentials for both, take a look at the `sample-inventory`. It shows +how to read the values from your shell environment. + +Make sure to only set the values you need from (e.g. your keystonerc or +clouds.yaml). Some of the options ar keystone V2 or V3 specific. + +**NOTE**: If you're testing this on (DevStack)[devstack], you must +explicitly set your Keystone API version to v2 (e.g. +`OS_AUTH_URL=http://10.20.30.40/identity/v2.0`) instead of the default +value provided by `openrc`. You may also encounter the following issue +with Cinder: + +https://github.com/kubernetes/kubernetes/issues/50461 + + +[devstack]: https://docs.openstack.org/devstack/latest/ + + +You can read the (OpenShift documentation on configuring +OpenStack)[openstack] for more information. + +[openstack]: https://docs.openshift.org/latest/install_config/configuring_openstack.html + + +Next we need to instruct openshift-ansible to use the Cinder volume +for it's registry. Again in `OSEv3.yml`: + + ## Use Cinder volume for Openshift registry: + #openshift_hosted_registry_storage_kind: openstack + #openshift_hosted_registry_storage_access_modes: ['ReadWriteOnce'] + #openshift_hosted_registry_storage_openstack_filesystem: xfs + #openshift_hosted_registry_storage_openstack_volumeID: e0ba2d73-d2f9-4514-a3b2-a0ced507fa05 + #openshift_hosted_registry_storage_volume_size: 10Gi + +The **Cinder volume ID**, **filesystem** and **volume size** variables must +correspond to the values in your volume. + ### Configure static inventory and access via a bastion node Example inventory variables: diff --git a/playbooks/provisioning/openstack/post-provision-openstack.yml b/playbooks/provisioning/openstack/post-provision-openstack.yml index 28f3e5fcf..116eb1244 100644 --- a/playbooks/provisioning/openstack/post-provision-openstack.yml +++ b/playbooks/provisioning/openstack/post-provision-openstack.yml @@ -84,3 +84,6 @@ line: "IP4_NAMESERVERS={{ hostvars['localhost'].private_dns_server }}" roles: - node-network-manager + +- include: prepare-and-format-cinder-volume.yaml + when: prepare_and_format_registry_volume|default(False) diff --git a/playbooks/provisioning/openstack/prepare-and-format-cinder-volume.yaml b/playbooks/provisioning/openstack/prepare-and-format-cinder-volume.yaml new file mode 100644 index 000000000..2d630f79d --- /dev/null +++ b/playbooks/provisioning/openstack/prepare-and-format-cinder-volume.yaml @@ -0,0 +1,75 @@ +--- +- hosts: localhost + gather_facts: False + become: False + tasks: + - set_fact: + cinder_volume: "{{ hostvars[groups.masters[0]].openshift_hosted_registry_storage_openstack_volumeID }}" + cinder_fs: "{{ hostvars[groups.masters[0]].openshift_hosted_registry_storage_openstack_filesystem }}" + + - name: Attach the volume to the VM + os_server_volume: + state: present + server: "{{ groups['masters'][0] }}" + volume: "{{ cinder_volume }}" + register: volume_attachment + + - set_fact: + attached_device: >- + {{ volume_attachment['attachments']|json_query("[?volume_id=='" + cinder_volume + "'].device | [0]") }} + + +- hosts: masters[0] + gather_facts: False + become: True + tasks: + - name: Wait for the device to appear + wait_for: path={{ hostvars['localhost'].attached_device }} + + - name: Create a temp directory for mounting the volume + tempfile: + prefix: cinder-volume + state: directory + register: cinder_mount_dir + + - name: Format the device + filesystem: + fstype: "{{ openshift_hosted_registry_storage_openstack_filesystem }}" + dev: "{{ hostvars['localhost'].attached_device }}" + + - name: Mount the device + mount: + name: "{{ cinder_mount_dir.path }}" + src: "{{ hostvars['localhost'].attached_device }}" + state: mounted + fstype: "{{ openshift_hosted_registry_storage_openstack_filesystem }}" + + - name: Change mode on the filesystem + file: + path: "{{ cinder_mount_dir.path }}" + state: directory + recurse: true + mode: 0777 + + - name: Unmount the device + mount: + name: "{{ cinder_mount_dir.path }}" + src: "{{ hostvars['localhost'].attached_device }}" + state: absent + fstype: "{{ openshift_hosted_registry_storage_openstack_filesystem }}" + + - name: Delete the temp directory + file: + name: "{{ cinder_mount_dir.path }}" + state: absent + + +- hosts: localhost + gather_facts: False + become: False + tasks: + - name: Detach the volume from the VM + os_server_volume: + state: absent + server: "{{ groups['masters'][0] }}" + volume: "{{ cinder_volume }}" diff --git a/playbooks/provisioning/openstack/sample-inventory/group_vars/OSEv3.yml b/playbooks/provisioning/openstack/sample-inventory/group_vars/OSEv3.yml index 4d27ae873..874ea7126 100644 --- a/playbooks/provisioning/openstack/sample-inventory/group_vars/OSEv3.yml +++ b/playbooks/provisioning/openstack/sample-inventory/group_vars/OSEv3.yml @@ -10,6 +10,26 @@ openshift_master_cluster_public_hostname: "{{ groups.lb.0|default(groups.masters osm_default_node_selector: 'region=primary' +openshift_hosted_router_wait: True +openshift_hosted_registry_wait: True + +## Openstack credentials +#openshift_cloudprovider_kind=openstack +#openshift_cloudprovider_openstack_auth_url: "{{ lookup('env','OS_AUTH_URL') }}" +#openshift_cloudprovider_openstack_username: "{{ lookup('env','OS_USERNAME') }}" +#openshift_cloudprovider_openstack_password: "{{ lookup('env','OS_PASSWORD') }}" +#openshift_cloudprovider_openstack_tenant_name: "{{ lookup('env','OS_TENANT_NAME') }}" +#openshift_cloudprovider_openstack_region="{{ lookup('env', 'OS_REGION_NAME') }}" + + +## Use Cinder volume for Openshift registry: +#openshift_hosted_registry_storage_kind: openstack +#openshift_hosted_registry_storage_access_modes: ['ReadWriteOnce'] +#openshift_hosted_registry_storage_openstack_filesystem: xfs +#openshift_hosted_registry_storage_openstack_volumeID: e0ba2d73-d2f9-4514-a3b2-a0ced507fa05 +#openshift_hosted_registry_storage_volume_size: 10Gi + + # NOTE(shadower): the hostname check seems to always fail because the # host's floating IP address doesn't match the address received from # inside the host. diff --git a/playbooks/provisioning/openstack/sample-inventory/group_vars/all.yml b/playbooks/provisioning/openstack/sample-inventory/group_vars/all.yml index 0e198342c..2e73d2e26 100644 --- a/playbooks/provisioning/openstack/sample-inventory/group_vars/all.yml +++ b/playbooks/provisioning/openstack/sample-inventory/group_vars/all.yml @@ -62,6 +62,13 @@ openstack_default_flavor: "m1.medium" #docker_lb_volume_size: "5" docker_volume_size: "15" +## Set up a filesystem on the cinder volume specified in `OSEv3.yaml`. +## You need to specify the file system and volume ID in OSEv3 via +## `openshift_hosted_registry_storage_openstack_filesystem` and +## `openshift_hosted_registry_storage_openstack_volumeID`. +## WARNING: This will delete any data on the volume! +#prepare_and_format_registry_volume: False + openstack_subnet_prefix: "192.168.99" # # Red Hat subscription -- cgit v1.2.3