diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 5cb69cc947..4c6a321070 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -1259,6 +1259,8 @@ horizon_enable_tls_backend: "{{ kolla_enable_tls_backend }}" # External Ceph - cephx auth enabled (this is the standard nowadays, defaults to yes) external_ceph_cephx_enabled: "yes" +ceph_cluster: "ceph" + # External Ceph pool names ceph_cinder_pool_name: "volumes" ceph_cinder_backup_pool_name: "backups" @@ -1273,14 +1275,6 @@ ceph_gnocchi_user: "gnocchi" ceph_manila_user: "manila" ceph_nova_user: "{{ ceph_cinder_user }}" -# External Ceph keyrings -ceph_cinder_keyring: "client.{{ ceph_cinder_user }}.keyring" -ceph_cinder_backup_keyring: "client.{{ ceph_cinder_backup_user }}.keyring" -ceph_glance_keyring: "client.{{ ceph_glance_user }}.keyring" -ceph_gnocchi_keyring: "client.{{ ceph_gnocchi_user }}.keyring" -ceph_manila_keyring: "client.{{ ceph_manila_user }}.keyring" -ceph_nova_keyring: "{{ ceph_cinder_keyring }}" - ##################### # VMware support ###################### diff --git a/ansible/roles/cinder/defaults/main.yml b/ansible/roles/cinder/defaults/main.yml index 289c95e507..797dd7d66a 100644 --- a/ansible/roles/cinder/defaults/main.yml +++ b/ansible/roles/cinder/defaults/main.yml @@ -264,11 +264,13 @@ cinder_backend_pure_nvme_tcp_name: "Pure-FlashArray-nvme-tcp" cinder_ceph_backends: - name: "{{ cinder_backend_ceph_name }}" - cluster: "ceph" + cluster: "{{ ceph_cluster }}" + user: "{{ ceph_cinder_user }}" + pool: "{{ ceph_cinder_pool_name }}" enabled: "{{ cinder_backend_ceph | bool }}" cinder_backup_backend_ceph_name: "rbd-1" -cinder_backup_ceph_backend: "{{ cinder_ceph_backends | selectattr('name', 'equalto', cinder_backup_backend_ceph_name) | list | first }}" +cinder_backup_ceph_backend: "{{ cinder_ceph_backends | selectattr('name', 'equalto', cinder_backup_backend_ceph_name) | list | first | combine({'pool': ceph_cinder_backup_pool_name, 'user': ceph_cinder_backup_user }) }}" skip_cinder_backend_check: False diff --git a/ansible/roles/cinder/tasks/external_ceph.yml b/ansible/roles/cinder/tasks/external_ceph.yml index 96bc8cc8cd..4b5b8dcd33 100644 --- a/ansible/roles/cinder/tasks/external_ceph.yml +++ b/ansible/roles/cinder/tasks/external_ceph.yml @@ -34,13 +34,13 @@ - service_name in services_need_config with_nested: - "{{ cinder_services | dict2items }}" - - "{{ cinder_ceph_backends }}" + - "{{ cinder_ceph_backends + [cinder_backup_ceph_backend] }}" notify: - "Restart {{ service_name }} container" - name: Copy over Ceph keyring files for cinder-volume vars: - keyring: "{{ item.cluster }}.{{ ceph_cinder_keyring }}" + keyring: "{{ item.cluster }}.client.{{ item.user }}.keyring" service: "{{ cinder_services['cinder-volume'] }}" template: src: "{{ node_custom_config }}/cinder/cinder-volume/{{ keyring }}" @@ -57,15 +57,15 @@ - name: Copy over Ceph keyring files for cinder-backup vars: service: "{{ cinder_services['cinder-backup'] }}" + keyring: "{{ item.cluster }}.client.{{ item.user }}.keyring" template: - src: "{{ node_custom_config }}/cinder/cinder-backup/{{ item }}" - dest: "{{ node_config_directory }}/cinder-backup/ceph/{{ item }}" + src: "{{ node_custom_config }}/cinder/cinder-backup/{{ keyring }}" + dest: "{{ node_config_directory }}/cinder-backup/ceph/{{ keyring }}" mode: "0660" become: true - register: cinder_backup_ceph_keyring with_items: - - "{{ cinder_backup_ceph_backend.cluster }}.{{ ceph_cinder_keyring }}" - - "{{ cinder_backup_ceph_backend.cluster }}.{{ ceph_cinder_backup_keyring }}" + - "{{ cinder_ceph_backends }}" + - "{{ cinder_backup_ceph_backend }}" when: - external_ceph_cephx_enabled | bool - service | service_enabled_and_mapped_to_host diff --git a/ansible/roles/cinder/templates/cinder.conf.j2 b/ansible/roles/cinder/templates/cinder.conf.j2 index c9f582f5f4..d7efe8cdc7 100644 --- a/ansible/roles/cinder/templates/cinder.conf.j2 +++ b/ansible/roles/cinder/templates/cinder.conf.j2 @@ -25,16 +25,18 @@ cluster = {{ cinder_cluster_name }} {% endif %} {% if cinder_enabled_backends %} +{% if service_name == 'cinder-volume' %} enabled_backends = {{ cinder_enabled_backends|map(attribute='name')|join(',') }} {% endif %} +{% endif %} {% if service_name == "cinder-backup" and enable_cinder_backup | bool %} {% if cinder_backup_driver == "ceph" %} backup_driver = cinder.backup.drivers.ceph.CephBackupDriver -backup_ceph_conf = /etc/ceph/{{ cinder_backup_ceph_backend.cluster }}.conf -backup_ceph_user = {{ ceph_cinder_backup_user }} +backup_ceph_conf = /etc/ceph/{{ cinder_backup_ceph_backend['cluster'] }}.conf +backup_ceph_user = {{ cinder_backup_ceph_backend['user'] }} backup_ceph_chunk_size = 134217728 -backup_ceph_pool = {{ ceph_cinder_backup_pool_name }} +backup_ceph_pool = {{ cinder_backup_ceph_backend['pool'] }} backup_ceph_stripe_unit = 0 backup_ceph_stripe_count = 0 restore_discard_excess_bytes = true @@ -150,16 +152,17 @@ target_protocol = iscsi {% endif %} {% if cinder_backend_ceph | bool %} +{% if service_name == 'cinder-volume' %} {% for backend in cinder_ceph_backends %} [{{ backend.name }}] volume_driver = cinder.volume.drivers.rbd.RBDDriver volume_backend_name = {{ backend.name }} -rbd_pool = {{ ceph_cinder_pool_name }} +rbd_pool = {{ backend.pool }} rbd_ceph_conf = /etc/ceph/{{ backend.cluster }}.conf rados_connect_timeout = 5 -rbd_user = {{ ceph_cinder_user }} +rbd_user = {{ backend.user }} rbd_cluster_name = {{ backend.cluster }} -rbd_keyring_conf = /etc/ceph/{{ backend.cluster }}.{{ ceph_cinder_keyring }} +rbd_keyring_conf = /etc/ceph/{{ backend.cluster }}.client.{{ backend.user }}.keyring rbd_secret_uuid = {{ cinder_rbd_secret_uuid }} report_discard_supported = True {% if backend.availability_zone is defined %} @@ -167,6 +170,7 @@ backend_availability_zone = {{ backend.availability_zone }} {% endif %} {% endfor %} {% endif %} +{% endif %} {% if enable_cinder_backend_nfs | bool %} [{{ cinder_backend_nfs_name }}] diff --git a/ansible/roles/glance/defaults/main.yml b/ansible/roles/glance/defaults/main.yml index 6bad90254a..21289fee7b 100644 --- a/ansible/roles/glance/defaults/main.yml +++ b/ansible/roles/glance/defaults/main.yml @@ -238,7 +238,9 @@ glance_backends: glance_ceph_backends: - name: "rbd" type: "rbd" - cluster: "ceph" + cluster: "{{ ceph_cluster }}" + pool: "{{ ceph_glance_pool_name }}" + user: "{{ ceph_glance_user }}" enabled: "{{ glance_backend_ceph | bool }}" glance_store_backends: "{{ glance_backends | selectattr('enabled', 'equalto', true) | list + glance_ceph_backends | selectattr('enabled', 'equalto', true) | list }}" diff --git a/ansible/roles/glance/tasks/external_ceph.yml b/ansible/roles/glance/tasks/external_ceph.yml index ad3587ec34..8137f4541f 100644 --- a/ansible/roles/glance/tasks/external_ceph.yml +++ b/ansible/roles/glance/tasks/external_ceph.yml @@ -25,9 +25,11 @@ - Restart glance-api container - name: Copy over ceph Glance keyrings + vars: + keyring: "{{ item.cluster }}.client.{{ item.user }}.keyring" template: - src: "{{ node_custom_config }}/glance/{{ item.cluster }}.{{ ceph_glance_keyring }}" - dest: "{{ node_config_directory }}/glance-api/ceph/{{ item.cluster }}.{{ ceph_glance_keyring }}" + src: "{{ node_custom_config }}/glance/{{ keyring }}" + dest: "{{ node_config_directory }}/glance-api/ceph/{{ keyring }}" mode: "0660" become: true with_items: "{{ glance_ceph_backends }}" diff --git a/ansible/roles/glance/templates/glance-api.conf.j2 b/ansible/roles/glance/templates/glance-api.conf.j2 index d760dfcc16..99631a4eb7 100644 --- a/ansible/roles/glance/templates/glance-api.conf.j2 +++ b/ansible/roles/glance/templates/glance-api.conf.j2 @@ -68,8 +68,8 @@ filesystem_store_datadir = /var/lib/glance/images/ {% if glance_backend_ceph | bool %} {% for backend in glance_ceph_backends %} [{{ backend.name }}] -rbd_store_user = {{ ceph_glance_user }} -rbd_store_pool = {{ ceph_glance_pool_name }} +rbd_store_user = {{ backend.user }} +rbd_store_pool = {{ backend.pool }} rbd_store_ceph_conf = /etc/ceph/{{ backend.cluster }}.conf {% endfor %} {% endif %} diff --git a/ansible/roles/gnocchi/defaults/main.yml b/ansible/roles/gnocchi/defaults/main.yml index ef18743ebf..c27147cf1d 100644 --- a/ansible/roles/gnocchi/defaults/main.yml +++ b/ansible/roles/gnocchi/defaults/main.yml @@ -189,5 +189,3 @@ gnocchi_ks_users: user: "{{ gnocchi_keystone_user }}" password: "{{ gnocchi_keystone_password }}" role: "admin" - -gnocchi_ceph_cluster: "ceph" diff --git a/ansible/roles/gnocchi/tasks/external_ceph.yml b/ansible/roles/gnocchi/tasks/external_ceph.yml index 6f01494e71..d05629f9f2 100644 --- a/ansible/roles/gnocchi/tasks/external_ceph.yml +++ b/ansible/roles/gnocchi/tasks/external_ceph.yml @@ -12,9 +12,9 @@ - name: Copy over ceph config for Gnocchi merge_configs: sources: - - "{{ node_custom_config }}/gnocchi/{{ gnocchi_ceph_cluster }}.conf" - - "{{ node_custom_config }}/gnocchi/{{ item.key }}/{{ gnocchi_ceph_cluster }}.conf" - dest: "{{ node_config_directory }}/{{ item.key }}/ceph/{{ gnocchi_ceph_cluster }}.conf" + - "{{ node_custom_config }}/gnocchi/{{ ceph_cluster }}.conf" + - "{{ node_custom_config }}/gnocchi/{{ item.key }}/{{ ceph_cluster }}.conf" + dest: "{{ node_config_directory }}/{{ item.key }}/ceph/{{ ceph_cluster }}.conf" mode: "0660" become: true with_dict: "{{ gnocchi_services | select_services_enabled_and_mapped_to_host }}" @@ -23,8 +23,8 @@ - name: Copy over ceph Gnocchi keyrings template: - src: "{{ node_custom_config }}/gnocchi/{{ gnocchi_ceph_cluster }}.{{ ceph_gnocchi_keyring }}" - dest: "{{ node_config_directory }}/{{ item.key }}/ceph/{{ gnocchi_ceph_cluster }}.{{ ceph_gnocchi_keyring }}" + src: "{{ node_custom_config }}/gnocchi/{{ ceph_cluster }}.client.{{ ceph_gnocchi_user }}.keyring" + dest: "{{ node_config_directory }}/{{ item.key }}/ceph/{{ ceph_cluster }}.client.{{ ceph_gnocchi_user }}.keyring" mode: "0660" become: true with_dict: "{{ gnocchi_services | select_services_enabled_and_mapped_to_host }}" diff --git a/ansible/roles/gnocchi/templates/gnocchi.conf.j2 b/ansible/roles/gnocchi/templates/gnocchi.conf.j2 index 2eb1fdaa71..a23a1170fb 100644 --- a/ansible/roles/gnocchi/templates/gnocchi.conf.j2 +++ b/ansible/roles/gnocchi/templates/gnocchi.conf.j2 @@ -82,8 +82,8 @@ file_basepath = /var/lib/gnocchi driver = ceph ceph_pool = {{ ceph_gnocchi_pool_name }} ceph_username = {{ ceph_gnocchi_user }} -ceph_keyring = /etc/ceph/{{ gnocchi_ceph_cluster }}.{{ ceph_gnocchi_keyring }} -ceph_conffile = /etc/ceph/{{ gnocchi_ceph_cluster }}.conf +ceph_keyring = /etc/ceph/{{ ceph_cluster }}.client.{{ ceph_gnocchi_user }}.keyring +ceph_conffile = /etc/ceph/{{ ceph_cluster }}.conf {% elif gnocchi_backend_storage == 'swift' %} driver = swift swift_authurl = {{ keystone_internal_url }} diff --git a/ansible/roles/manila/defaults/main.yml b/ansible/roles/manila/defaults/main.yml index a2459e3f10..3ad48f651a 100644 --- a/ansible/roles/manila/defaults/main.yml +++ b/ansible/roles/manila/defaults/main.yml @@ -235,14 +235,14 @@ manila_ceph_backends: - name: "cephfsnative1" share_name: "CEPHFS1" driver: "cephfsnative" - cluster: "ceph" + cluster: "{{ ceph_cluster }}" enabled: "{{ enable_manila_backend_cephfs_native | bool }}" protocols: - "CEPHFS" - name: "cephfsnfs1" share_name: "CEPHFSNFS1" driver: "cephfsnfs" - cluster: "ceph" + cluster: "{{ ceph_cluster }}" enabled: "{{ enable_manila_backend_cephfs_nfs | bool }}" protocols: - "NFS" diff --git a/ansible/roles/manila/tasks/external_ceph.yml b/ansible/roles/manila/tasks/external_ceph.yml index 729a14769d..8a66ed370f 100644 --- a/ansible/roles/manila/tasks/external_ceph.yml +++ b/ansible/roles/manila/tasks/external_ceph.yml @@ -28,8 +28,8 @@ - name: Copy over ceph Manila keyrings template: - src: "{{ node_custom_config }}/manila/{{ item.cluster }}.{{ ceph_manila_keyring }}" - dest: "{{ node_config_directory }}/manila-share/ceph/{{ item.cluster }}.{{ ceph_manila_keyring }}" + src: "{{ node_custom_config }}/manila/{{ item.cluster }}.client.{{ ceph_manila_user }}.keyring" + dest: "{{ node_config_directory }}/manila-share/ceph/{{ item.cluster }}.client.{{ ceph_manila_user }}.keyring" mode: "0660" become: true with_items: "{{ manila_ceph_backends }}" diff --git a/ansible/roles/nova-cell/defaults/main.yml b/ansible/roles/nova-cell/defaults/main.yml index 48115e62c3..a497a5fdcb 100644 --- a/ansible/roles/nova-cell/defaults/main.yml +++ b/ansible/roles/nova-cell/defaults/main.yml @@ -85,6 +85,15 @@ nova_cell_config_validation: # qemu (1, 6, 0) or later. Set to "" to disable. nova_hw_disk_discard: "unmap" +nova_cell_ceph_backend: + cluster: "{{ ceph_cluster }}" + vms: + user: "{{ ceph_nova_user }}" + pool: "{{ ceph_nova_pool_name }}" + volumes: + user: "{{ ceph_cinder_user }}" + pool: "{{ ceph_cinder_pool_name }}" + #################### # Cells Options #################### @@ -528,8 +537,6 @@ nova_notification_topics: nova_enabled_notification_topics: "{{ nova_notification_topics | selectattr('enabled', 'equalto', true) | list }}" -nova_ceph_cluster: "ceph" - #################### # VMware #################### diff --git a/ansible/roles/nova-cell/tasks/external_ceph.yml b/ansible/roles/nova-cell/tasks/external_ceph.yml index f0a9aa1a7f..fdf1aa6b80 100644 --- a/ansible/roles/nova-cell/tasks/external_ceph.yml +++ b/ansible/roles/nova-cell/tasks/external_ceph.yml @@ -1,7 +1,7 @@ --- - name: Check nova keyring file vars: - keyring: "{{ nova_ceph_cluster }}.{{ ceph_nova_keyring }}" + keyring: "{{ nova_cell_ceph_backend['cluster'] }}.client.{{ nova_cell_ceph_backend['vms']['user'] }}.keyring" paths: - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ keyring }}" - "{{ node_custom_config }}/nova/{{ keyring }}" @@ -16,7 +16,7 @@ - name: Check cinder keyring file vars: - keyring: "{{ nova_ceph_cluster }}.{{ ceph_cinder_keyring }}" + keyring: "{{ nova_cell_ceph_backend['cluster'] }}.client.{{ nova_cell_ceph_backend['volumes']['user'] }}.keyring" paths: - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ keyring }}" - "{{ node_custom_config }}/nova/{{ keyring }}" @@ -85,8 +85,8 @@ vars: service: "{{ nova_cell_services[item] }}" paths: - - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ nova_ceph_cluster }}.conf" - - "{{ node_custom_config }}/nova/{{ nova_ceph_cluster }}.conf" + - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ nova_cell_ceph_backend['cluster'] }}.conf" + - "{{ node_custom_config }}/nova/{{ nova_cell_ceph_backend['cluster'] }}.conf" template: src: "{{ lookup('first_found', paths) }}" dest: "{{ node_config_directory }}/{{ item }}/" @@ -107,8 +107,8 @@ - name: Ensure /etc/ceph directory exists (host libvirt) vars: paths: - - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ nova_ceph_cluster }}.conf" - - "{{ node_custom_config }}/nova/{{ nova_ceph_cluster }}.conf" + - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ nova_cell_ceph_backend['cluster'] }}.conf" + - "{{ node_custom_config }}/nova/{{ nova_cell_ceph_backend['cluster'] }}.conf" file: path: "/etc/ceph/" state: "directory" @@ -120,11 +120,11 @@ - name: Copy over ceph.conf (host libvirt) vars: paths: - - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ nova_ceph_cluster }}.conf" - - "{{ node_custom_config }}/nova/{{ nova_ceph_cluster }}.conf" + - "{{ node_custom_config }}/nova/{{ inventory_hostname }}/{{ nova_cell_ceph_backend['cluster'] }}.conf" + - "{{ node_custom_config }}/nova/{{ nova_cell_ceph_backend['cluster'] }}.conf" template: src: "{{ lookup('first_found', paths) }}" - dest: "/etc/ceph/{{ nova_ceph_cluster }}.conf" + dest: "/etc/ceph/{{ nova_cell_ceph_backend['cluster'] }}.conf" owner: "root" group: "root" mode: "0644" diff --git a/ansible/roles/nova-cell/templates/nova-compute.json.j2 b/ansible/roles/nova-cell/templates/nova-compute.json.j2 index 1069c541d0..cf437bf7aa 100644 --- a/ansible/roles/nova-cell/templates/nova-compute.json.j2 +++ b/ansible/roles/nova-cell/templates/nova-compute.json.j2 @@ -14,14 +14,14 @@ "perm": "0600" }{% endif %}{% if nova_backend == "rbd" %}, { - "source": "{{ container_config_directory }}/ceph.{{ ceph_nova_keyring }}", - "dest": "/etc/ceph/ceph.{{ ceph_nova_keyring }}", + "source": "{{ container_config_directory }}/{{ nova_cell_ceph_backend['cluster'] }}.client.{{ nova_cell_ceph_backend['vms']['user'] }}.keyring", + "dest": "/etc/ceph/{{ nova_cell_ceph_backend['cluster'] }}.client.{{ nova_cell_ceph_backend['vms']['user'] }}.keyring", "owner": "nova", "perm": "0600" }, { - "source": "{{ container_config_directory }}/{{ nova_ceph_cluster }}.conf", - "dest": "/etc/ceph/{{ nova_ceph_cluster }}.conf", + "source": "{{ container_config_directory }}/{{ nova_cell_ceph_backend['cluster'] }}.conf", + "dest": "/etc/ceph/{{ nova_cell_ceph_backend['cluster'] }}.conf", "owner": "nova", "perm": "0600" }{% endif %}{% if nova_compute_virt_type == "vmware" and not vmware_vcenter_insecure | bool %}, diff --git a/ansible/roles/nova-cell/templates/nova-libvirt.json.j2 b/ansible/roles/nova-cell/templates/nova-libvirt.json.j2 index d2ddc9e6f9..a7df6a275d 100644 --- a/ansible/roles/nova-cell/templates/nova-libvirt.json.j2 +++ b/ansible/roles/nova-cell/templates/nova-libvirt.json.j2 @@ -51,8 +51,8 @@ "merge": true }{% endif %}{% if nova_backend == "rbd" %}, { - "source": "{{ container_config_directory }}/ceph.conf", - "dest": "/etc/ceph/ceph.conf", + "source": "{{ container_config_directory }}/{{ nova_cell_ceph_backend['cluster'] }}.conf", + "dest": "/etc/ceph/{{ nova_cell_ceph_backend['cluster'] }}.conf", "owner": "nova", "perm": "0600" }{% endif %}{% if libvirt_enable_sasl | bool %}, diff --git a/ansible/roles/nova-cell/templates/nova.conf.d/libvirt.conf.j2 b/ansible/roles/nova-cell/templates/nova.conf.d/libvirt.conf.j2 index 2a81411319..7c23ff8306 100644 --- a/ansible/roles/nova-cell/templates/nova.conf.d/libvirt.conf.j2 +++ b/ansible/roles/nova-cell/templates/nova.conf.d/libvirt.conf.j2 @@ -9,9 +9,9 @@ live_migration_inbound_addr = "{{ migration_interface_address }}" {% endif %} {% if nova_backend == "rbd" %} images_type = rbd -images_rbd_pool = {{ ceph_nova_pool_name }} -images_rbd_ceph_conf = /etc/ceph/ceph.conf -rbd_user = {{ ceph_nova_user }} +images_rbd_pool = {{ nova_cell_ceph_backend['vms']['pool'] }} +images_rbd_ceph_conf = /etc/ceph/{{ nova_cell_ceph_backend['cluster'] }}.conf +rbd_user = {{ nova_cell_ceph_backend['vms']['user'] }} disk_cachemodes="network=writeback" {% if nova_hw_disk_discard != '' %} hw_disk_discard = {{ nova_hw_disk_discard }} diff --git a/ansible/roles/zun/defaults/main.yml b/ansible/roles/zun/defaults/main.yml index fe89a54329..009368cb1f 100644 --- a/ansible/roles/zun/defaults/main.yml +++ b/ansible/roles/zun/defaults/main.yml @@ -244,5 +244,3 @@ zun_ks_users: user: "{{ zun_keystone_user }}" password: "{{ zun_keystone_password }}" role: "admin" - -zun_ceph_cluster: "ceph" diff --git a/ansible/roles/zun/tasks/external_ceph.yml b/ansible/roles/zun/tasks/external_ceph.yml index 3f9bdadeab..324f28bb4a 100644 --- a/ansible/roles/zun/tasks/external_ceph.yml +++ b/ansible/roles/zun/tasks/external_ceph.yml @@ -1,7 +1,7 @@ --- - name: Copying over ceph.conf for Zun copy: - src: "{{ node_custom_config }}/zun/zun-compute/{{ zun_ceph_cluster }}.conf" + src: "{{ node_custom_config }}/zun/zun-compute/{{ ceph_cluster }}.conf" dest: "{{ node_config_directory }}/zun-compute/" mode: "0660" become: true @@ -10,7 +10,7 @@ - name: Copy over Ceph keyring files for zun-compute copy: - src: "{{ node_custom_config }}/zun/zun-compute/{{ zun_ceph_cluster }}.{{ ceph_cinder_keyring }}" + src: "{{ node_custom_config }}/zun/zun-compute/{{ ceph_cluster }}.client.{{ ceph_cinder_user }}.keyring" dest: "{{ node_config_directory }}/zun-compute/" mode: "0660" become: true diff --git a/ansible/roles/zun/templates/zun-compute.json.j2 b/ansible/roles/zun/templates/zun-compute.json.j2 index 266a5283f0..4854a86f92 100644 --- a/ansible/roles/zun/templates/zun-compute.json.j2 +++ b/ansible/roles/zun/templates/zun-compute.json.j2 @@ -8,15 +8,15 @@ "perm": "0600" }, { - "source": "{{ container_config_directory }}/{{ zun_ceph_cluster }}.{{ ceph_cinder_keyring }}", - "dest": "/etc/ceph/{{ zun_ceph_cluster }}.{{ ceph_cinder_keyring }}", + "source": "{{ container_config_directory }}/{{ ceph_cluster }}.client.{{ ceph_cinder_user }}.keyring", + "dest": "/etc/ceph/{{ ceph_cluster }}.client.{{ ceph_cinder_user }}.keyring", "owner": "zun", "perm": "0600", "optional": {{ (not zun_configure_for_cinder_ceph | bool) | string | lower }} }, { - "source": "{{ container_config_directory }}/{{ zun_ceph_cluster }}.conf", - "dest": "/etc/ceph/{{ zun_ceph_cluster }}.conf", + "source": "{{ container_config_directory }}/{{ ceph_cluster }}.conf", + "dest": "/etc/ceph/{{ ceph_cluster }}.conf", "owner": "zun", "perm": "0600", "optional": {{ (not zun_configure_for_cinder_ceph | bool) | string | lower }} diff --git a/doc/source/reference/storage/external-ceph-guide.rst b/doc/source/reference/storage/external-ceph-guide.rst index 585caf48ab..cf3bd0e5a8 100644 --- a/doc/source/reference/storage/external-ceph-guide.rst +++ b/doc/source/reference/storage/external-ceph-guide.rst @@ -37,6 +37,36 @@ Ceph integration is configured for different OpenStack services independently. Be sure to remove the leading tabs from your ``ceph.conf`` files when copying them in the following sections. +When openstack services access Ceph via a Ceph client, the Ceph client will +look for a local keyring. Ceph presets the keyring setting with four keyring +names by default. + +* The four default keyring names are as follows: + + * ``/etc/ceph/$cluster.$name.keyring`` + * ``/etc/ceph/$cluster.keyring`` + * ``/etc/ceph/keyring`` + * ``/etc/ceph/keyring.bin`` + +The ``$cluster`` metavariable found in the first two default keyring names +above is your Ceph cluster name as defined by the name of the Ceph +configuration file: for example, if the Ceph configuration file is named +``ceph.conf``, then your Ceph cluster name is ceph and the second name above +would be ``ceph.keyring``. The ``$name`` metavariable is the user type and +user ID: for example, given the user ``client.admin``, the first name above +would be ``ceph.client.admin.keyring``. This principle is applied in the +services documentation below. + +.. note:: + + More information about user configuration and related keyrings can be found in the + official Ceph documentation at https://docs.ceph.com/en/latest/rados/operations/user-management/#keyring-management + +.. note:: + + Below examples uses default ``$cluster`` and ``$user`` which can be configured + via kolla-ansible by setting ``ceph_cluster``,``$user`` per project or on the + host level (nova) in inventory file. Glance ------ @@ -52,7 +82,6 @@ for Ceph includes the following steps: * Configure Ceph authentication details in ``/etc/kolla/globals.yml``: - * ``ceph_glance_keyring`` (default: ``client.glance.keyring``) * ``ceph_glance_user`` (default: ``glance``) * ``ceph_glance_pool_name`` (default: ``images``) @@ -70,7 +99,7 @@ for Ceph includes the following steps: auth_service_required = cephx auth_client_required = cephx -* Copy Ceph keyring to ``/etc/kolla/config/glance/ceph.`` +* Copy Ceph keyring to ``/etc/kolla/config/glance/ceph.client.glance.keyring`` To configure multiple Ceph backends with Glance, which is useful for multistore: @@ -78,28 +107,28 @@ for multistore: * Copy the Ceph configuration files into ``/etc/kolla/config/glance/`` using different names for each - ``/etc/kolla/config/glance/ceph.conf`` + ``/etc/kolla/config/glance/ceph1.conf`` - .. path /etc/kolla/config/glance/ceph.conf + .. path /etc/kolla/config/glance/ceph1.conf .. code-block:: ini [global] fsid = 1d89fec3-325a-4963-a950-c4afedd37fe3 - keyring = /etc/ceph/ceph.client.glance.keyring + keyring = /etc/ceph/ceph1.client.glance.keyring mon_initial_members = ceph-0 mon_host = 192.168.0.56 auth_cluster_required = cephx auth_service_required = cephx auth_client_required = cephx - ``/etc/kolla/config/glance/rbd1.conf`` + ``/etc/kolla/config/glance/ceph2.conf`` - .. path /etc/kolla/config/glance/rbd1.conf + .. path /etc/kolla/config/glance/ceph2.conf .. code-block:: ini [global] fsid = dbfea068-89ca-4d04-bba0-1b8a56c3abc8 - keyring = /etc/ceph/rbd1.client.glance.keyring + keyring = /etc/ceph/ceph2.client.glance.keyring mon_initial_members = ceph-0 mon_host = 192.10.0.100 auth_cluster_required = cephx @@ -111,17 +140,21 @@ for multistore: .. code-block:: yaml glance_ceph_backends: - - name: "rbd" + - name: "ceph1-rbd" type: "rbd" - cluster: "ceph" + cluster: "ceph1" + user: "glance" + pool: "images" enabled: "{{ glance_backend_ceph | bool }}" - - name: "another-rbd" + - name: "ceph2-rbd" type: "rbd" - cluster: "rbd1" + cluster: "ceph2" + user: "glance" + pool: "images" enabled: "{{ glance_backend_ceph | bool }}" -* Copy Ceph keyring to ``/etc/kolla/config/glance/ceph.`` - and analogously to ``/etc/kolla/config/glance/rbd1.`` +* Copy Ceph keyring to ``/etc/kolla/config/glance/ceph1.client.glance.keyring`` + and analogously to ``/etc/kolla/config/glance/ceph2.client.glance.keyring`` * For copy-on-write set following in ``/etc/kolla/config/glance.conf``: @@ -161,11 +194,8 @@ Cinder for Ceph includes following steps: * Configure Ceph authentication details in ``/etc/kolla/globals.yml``: - * ``ceph_cinder_keyring`` (default: ``client.cinder.keyring``) * ``ceph_cinder_user`` (default: ``cinder``) * ``ceph_cinder_pool_name`` (default: ``volumes``) - * ``ceph_cinder_backup_keyring`` - (default: ``client.cinder-backup.keyring``) * ``ceph_cinder_backup_user`` (default: ``cinder-backup``) * ``ceph_cinder_backup_pool_name`` (default: ``backups``) @@ -179,15 +209,15 @@ Cinder for Ceph includes following steps: * Copy Ceph keyring files to: - * ``/etc/kolla/config/cinder/cinder-volume/ceph.`` - * ``/etc/kolla/config/cinder/cinder-backup/ceph.`` - * ``/etc/kolla/config/cinder/cinder-backup/ceph. - `` + * ``/etc/kolla/config/cinder/cinder-volume/ceph.client.cinder.keyring`` + * ``/etc/kolla/config/cinder/cinder-backup/ceph.client.cinder.keyring`` + * ``/etc/kolla/config/cinder/cinder-backup/ + ceph.client.cinder-backup.keyring`` .. note:: - ``cinder-backup`` requires two keyrings for accessing volumes - and backup pool. + ``cinder-backup`` requires keyrings for accessing volumes + and backups pools. To configure ``multiple Ceph backends`` with Cinder, which is useful for the use with availability zones: @@ -195,9 +225,9 @@ the use with availability zones: * Copy their Ceph configuration files into ``/etc/kolla/config/cinder/`` using different names for each - ``/etc/kolla/config/cinder/ceph.conf`` + ``/etc/kolla/config/cinder/ceph1.conf`` - .. path /etc/kolla/config/cinder/ceph.conf + .. path /etc/kolla/config/cinder/ceph1.conf .. code-block:: ini [global] @@ -208,9 +238,9 @@ the use with availability zones: auth_service_required = cephx auth_client_required = cephx - ``/etc/kolla/config/cinder/rbd2.conf`` + ``/etc/kolla/config/cinder/ceph2.conf`` - .. path /etc/kolla/config/cinder/rbd2.conf + .. path /etc/kolla/config/cinder/ceph2.conf .. code-block:: ini [global] @@ -226,46 +256,63 @@ the use with availability zones: .. code-block:: yaml cinder_ceph_backends: - - name: "rbd-1" - cluster: "ceph" + - name: "ceph1-rbd" + cluster: "ceph1" + user: "cinder" + pool: "volumes" enabled: "{{ cinder_backend_ceph | bool }}" - - name: "rbd-2" - cluster: "rbd2" + - name: "ceph2-rbd" + cluster: "ceph2" + user: "cinder" + pool: "volumes" availability_zone: "az2" enabled: "{{ cinder_backend_ceph | bool }}" + cinder_backup_ceph_backend: + name: "ceph2-backup-rbd" + cluster: "ceph2" + user: "cinder-backup" + pool: "backups" + type: rbd + enabled: "{{ enable_cinder_backup | bool }}" + * Copy Ceph keyring files for all Ceph backends: - * ``/etc/kolla/config/cinder/cinder-volume/ceph.`` - * ``/etc/kolla/config/cinder/cinder-backup/ceph.`` - * ``/etc/kolla/config/cinder/cinder-backup/ceph. - `` - * ``/etc/kolla/config/cinder/cinder-volume/rbd2.`` - * ``/etc/kolla/config/cinder/cinder-backup/rbd2.`` - * ``/etc/kolla/config/cinder/cinder-backup/rbd2. - `` + * ``/etc/kolla/config/cinder/cinder-volume/ceph1.client.cinder.keyring`` + * ``/etc/kolla/config/cinder/cinder-backup/ceph1.client.cinder.keyring`` + * ``/etc/kolla/config/cinder/cinder-backup/ceph2.client.cinder.keyring`` + * ``/etc/kolla/config/cinder/cinder-backup/ + ceph2.client.cinder-backup.keyring`` .. note:: - ``cinder-backup`` requires two keyrings for accessing volumes - and backup pool. + ``cinder-backup`` requires keyrings for accessing volumes + and backups pool. Nova must also be configured to allow access to Cinder volumes: -* Configure Ceph authentication details in ``/etc/kolla/globals.yml``: +* Copy Ceph config and keyring file(s) to: - * ``ceph_cinder_keyring`` (default: ``client.cinder.keyring``) - -* Copy Ceph keyring file(s) to: - - * ``/etc/kolla/config/nova/ceph.`` + * ``/etc/kolla/config/nova/ceph.conf`` + * ``/etc/kolla/config/nova/ceph.client.cinder.keyring`` To configure ``different Ceph backend`` for nova-compute host, which is useful for the use with availability zones: -* Copy Ceph keyring file to: +* Edit inventory file in the way described below: - * ``/etc/kolla/config/nova//ceph.`` + .. code-block:: ini + + [compute] + hostname1 ceph_cluster=ceph1 + hostname2 ceph_cluster=ceph2 + +* Copy Ceph config and keyring file(s): + + * ``/etc/kolla/config/nova//ceph1.conf`` + * ``/etc/kolla/config/nova//ceph1.client.cinder.keyring`` + * ``/etc/kolla/config/nova//ceph2.conf`` + * ``/etc/kolla/config/nova//ceph2.client.cinder.keyring`` If ``zun`` is enabled, and you wish to use cinder volumes with zun, it must also be configured to allow access to Cinder volumes: @@ -282,7 +329,7 @@ it must also be configured to allow access to Cinder volumes: * Copy Ceph keyring file(s) to: - * ``/etc/kolla/config/zun/zun-compute/ceph.`` + * ``/etc/kolla/config/zun/zun-compute/ceph.client.cinder.keyring`` Nova @@ -303,30 +350,37 @@ Configuring Nova for Ceph includes following steps: * Configure Ceph authentication details in ``/etc/kolla/globals.yml``: - * ``ceph_nova_keyring`` (by default it's the same as - ``ceph_cinder_keyring``) * ``ceph_nova_user`` (by default it's the same as ``ceph_cinder_user``) * ``ceph_nova_pool_name`` (default: ``vms``) * Copy Ceph configuration file to ``/etc/kolla/config/nova/ceph.conf`` * Copy Ceph keyring file(s) to: - * ``/etc/kolla/config/nova/ceph.`` + * ``/etc/kolla/config/nova/ceph.client.nova.keyring`` .. note:: If you are using a Ceph deployment tool that generates separate Ceph keys for Cinder and Nova, you will need to override - ``ceph_nova_keyring`` and ``ceph_nova_user`` to match. + ``ceph_nova_user`` to match. To configure ``different Ceph backend`` for nova-compute host, which is useful for the use with availability zones: -* Copy Ceph configuration file to ``/etc/kolla/config/nova/ - /ceph.conf`` -* Copy Ceph keyring file(s) to: +Edit inventory file in the way described below: - * ``/etc/kolla/config/nova//ceph.`` + .. code-block:: ini + + [compute] + hostname1 ceph_cluster=ceph1 + hostname2 ceph_cluster=ceph2 + +* Copy Ceph config and keyring file(s): + + * ``/etc/kolla/config/nova//ceph1.conf`` + * ``/etc/kolla/config/nova//ceph1.client.nova.keyring`` + * ``/etc/kolla/config/nova//ceph2.conf`` + * ``/etc/kolla/config/nova//ceph2.client.nova.keyring`` Gnocchi ------- @@ -342,17 +396,13 @@ Configuring Gnocchi for Ceph includes following steps: * Configure Ceph authentication details in ``/etc/kolla/globals.yml``: - * ``ceph_gnocchi_keyring`` - (default: ``client.gnocchi.keyring``) * ``ceph_gnocchi_user`` (default: ``gnocchi``) * ``ceph_gnocchi_pool_name`` (default: ``gnocchi``) - * ``ceph_gnocchi_conf`` - (default: ``ceph.conf``) * Copy Ceph configuration file to - ``/etc/kolla/config/gnocchi/`` + ``/etc/kolla/config/gnocchi/ceph.conf`` * Copy Ceph keyring to - ``/etc/kolla/config/gnocchi/ceph.`` + ``/etc/kolla/config/gnocchi/ceph.client.gnocchi.keyring`` Manila ------ @@ -368,7 +418,6 @@ for Ceph includes following steps: * Configure Ceph authentication details in ``/etc/kolla/globals.yml``: - * ``ceph_manila_keyring`` (default: ``client.manila.keyring``) * ``ceph_manila_user`` (default: ``manila``) .. note:: @@ -377,7 +426,7 @@ for Ceph includes following steps: :manila-doc:`CephFS Native driver `. * Copy Ceph configuration file to ``/etc/kolla/config/manila/ceph.conf`` -* Copy Ceph keyring to ``/etc/kolla/config/manila/ceph.`` +* Copy Ceph keyring to ``/etc/kolla/config/manila/ceph.client.manila.keyring`` To configure ``multiple Ceph backends`` with Manila, which is useful for the use with availability zones: @@ -385,9 +434,9 @@ the use with availability zones: * Copy their Ceph configuration files into ``/etc/kolla/config/manila/`` using different names for each - ``/etc/kolla/config/manila/ceph.conf`` + ``/etc/kolla/config/manila/ceph1.conf`` - .. path /etc/kolla/config/manila/ceph.conf + .. path /etc/kolla/config/manila/ceph1.conf .. code-block:: ini [global] @@ -398,9 +447,9 @@ the use with availability zones: auth_service_required = cephx auth_client_required = cephx - ``/etc/kolla/config/manila/rbd2.conf`` + ``/etc/kolla/config/manila/ceph2.conf`` - .. path /etc/kolla/config/manila/rbd2.conf + .. path /etc/kolla/config/manila/ceph2.conf .. code-block:: ini [global] @@ -419,14 +468,14 @@ the use with availability zones: - name: "cephfsnative1" share_name: "CEPHFS1" driver: "cephfsnative" - cluster: "ceph" + cluster: "ceph1" enabled: "{{ enable_manila_backend_cephfs_native | bool }}" protocols: - "CEPHFS" - name: "cephfsnative2" share_name: "CEPHFS2" driver: "cephfsnative" - cluster: "rbd2" + cluster: "ceph2" enabled: "{{ enable_manila_backend_cephfs_native | bool }}" protocols: - "CEPHFS" @@ -441,7 +490,7 @@ the use with availability zones: - name: "cephfsnfs2" share_name: "CEPHFSNFS2" driver: "cephfsnfs" - cluster: "rbd2" + cluster: "ceph2" enabled: "{{ enable_manila_backend_cephfs_nfs | bool }}" protocols: - "NFS" @@ -449,8 +498,8 @@ the use with availability zones: * Copy Ceph keyring files for all Ceph backends: - * ``/etc/kolla/config/manila/manila-share/ceph.`` - * ``/etc/kolla/config/manila/manila-share/rbd2.`` + * ``/etc/kolla/config/manila/manila-share/ceph1.client.manila.keyring`` + * ``/etc/kolla/config/manila/manila-share/ceph2.client.manila.keyring`` * If using multiple filesystems (Ceph Pacific+), set ``manila_cephfs_filesystem_name`` in ``/etc/kolla/globals.yml`` to the diff --git a/doc/source/user/operating-kolla.rst b/doc/source/user/operating-kolla.rst index 98e766458b..a54d21eefb 100644 --- a/doc/source/user/operating-kolla.rst +++ b/doc/source/user/operating-kolla.rst @@ -75,8 +75,7 @@ Limitations and Recommendations .. note:: If you have separate keys for nova and cinder, please be sure to set - ``ceph_nova_keyring: ceph.client.nova.keyring`` and ``ceph_nova_user: nova`` - in ``/etc/kolla/globals.yml`` + ``ceph_nova_user: nova`` in ``/etc/kolla/globals.yml`` Preparation (the foreword) -------------------------- diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 1f4a0b9594..1c0ee0371b 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -465,26 +465,20 @@ workaround_ansible_issue_8743: yes # Glance #ceph_glance_user: "glance" -#ceph_glance_keyring: "client.{{ ceph_glance_user }}.keyring" #ceph_glance_pool_name: "images" # Cinder #ceph_cinder_user: "cinder" -#ceph_cinder_keyring: "client.{{ ceph_cinder_user }}.keyring" #ceph_cinder_pool_name: "volumes" #ceph_cinder_backup_user: "cinder-backup" -#ceph_cinder_backup_keyring: "client.{{ ceph_cinder_backup_user }}.keyring" #ceph_cinder_backup_pool_name: "backups" # Nova -#ceph_nova_keyring: "{{ ceph_cinder_keyring }}" #ceph_nova_user: "{{ ceph_cinder_user }}" #ceph_nova_pool_name: "vms" # Gnocchi #ceph_gnocchi_user: "gnocchi" -#ceph_gnocchi_keyring: "client.{{ ceph_gnocchi_user }}.keyring" #ceph_gnocchi_pool_name: "gnocchi" # Manila #ceph_manila_user: "manila" -#ceph_manila_keyring: "client.{{ ceph_manila_user }}.keyring" ############################# # Keystone - Identity Options diff --git a/releasenotes/notes/enhance-ceph-multiple-clusters-integration-d29815a12c223152.yaml b/releasenotes/notes/enhance-ceph-multiple-clusters-integration-d29815a12c223152.yaml new file mode 100644 index 0000000000..bd13b78ce5 --- /dev/null +++ b/releasenotes/notes/enhance-ceph-multiple-clusters-integration-d29815a12c223152.yaml @@ -0,0 +1,17 @@ +--- +features: + - | + Improved the handling of multiple Ceph clusters in Kolla-Ansible + by allowing explicit configuration of users, pools, and cluster + names, following the official Ceph keyring format + ``$cluster.client.$user.keyring``. +upgrade: + - | + The variables ``ceph_cinder_keyring``, ``ceph_cinder_backup_keyring``, + ``ceph_glance_keyring``, ``ceph_gnocchi_keyring``, ``ceph_manila_keyring``, + and ``ceph_nova_keyring`` have been removed, and their values are now + automatically derived from the configurable Ceph users. Users who have + relied on completely different keyrings or custom user configurations + should ensure their setups are correctly aligned with the new convention + as per `documentation + `__.