From 8c5012e940e569057f69851c9f3280e8b6a14bf8 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 19 Feb 2021 13:39:06 +0000 Subject: [PATCH] Add support for Ceph RadosGW integration * Register Swift-compatible endpoints in Keystone * Load balance across RadosGW API servers using HAProxy The support is exercised in the cephadm CI jobs, but since RGW is not currently enabled via cephadm, it is not yet tested. https://docs.ceph.com/en/latest/radosgw/keystone/ Implements: blueprint ceph-rgw Change-Id: I891c3ed4ed93512607afe65a42dd99596fd4dbf9 --- ansible/group_vars/all.yml | 6 ++ ansible/roles/ceph-rgw/defaults/main.yml | 92 +++++++++++++++++++ ansible/roles/ceph-rgw/tasks/check.yml | 1 + ansible/roles/ceph-rgw/tasks/config.yml | 1 + .../ceph-rgw/tasks/deploy-containers.yml | 1 + ansible/roles/ceph-rgw/tasks/deploy.yml | 2 + ansible/roles/ceph-rgw/tasks/loadbalancer.yml | 7 ++ ansible/roles/ceph-rgw/tasks/main.yml | 2 + ansible/roles/ceph-rgw/tasks/precheck.yml | 10 ++ ansible/roles/ceph-rgw/tasks/pull.yml | 1 + ansible/roles/ceph-rgw/tasks/reconfigure.yml | 2 + ansible/roles/ceph-rgw/tasks/register.yml | 9 ++ ansible/roles/ceph-rgw/tasks/stop.yml | 1 + ansible/roles/ceph-rgw/tasks/upgrade.yml | 1 + ansible/roles/loadbalancer/tasks/precheck.yml | 14 +++ ansible/site.yml | 19 ++++ .../reference/storage/external-ceph-guide.rst | 71 ++++++++++++++ etc/kolla/globals.yml | 2 + etc/kolla/passwords.yml | 5 + .../notes/ceph-rgw-062e0544a004f7b1.yaml | 4 + tests/templates/globals-default.j2 | 8 ++ 21 files changed, 259 insertions(+) create mode 100644 ansible/roles/ceph-rgw/defaults/main.yml create mode 100644 ansible/roles/ceph-rgw/tasks/check.yml create mode 100644 ansible/roles/ceph-rgw/tasks/config.yml create mode 100644 ansible/roles/ceph-rgw/tasks/deploy-containers.yml create mode 100644 ansible/roles/ceph-rgw/tasks/deploy.yml create mode 100644 ansible/roles/ceph-rgw/tasks/loadbalancer.yml create mode 100644 ansible/roles/ceph-rgw/tasks/main.yml create mode 100644 ansible/roles/ceph-rgw/tasks/precheck.yml create mode 100644 ansible/roles/ceph-rgw/tasks/pull.yml create mode 100644 ansible/roles/ceph-rgw/tasks/reconfigure.yml create mode 100644 ansible/roles/ceph-rgw/tasks/register.yml create mode 100644 ansible/roles/ceph-rgw/tasks/stop.yml create mode 100644 ansible/roles/ceph-rgw/tasks/upgrade.yml create mode 100644 releasenotes/notes/ceph-rgw-062e0544a004f7b1.yaml diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index c685db2f66..813b3b68cc 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -285,6 +285,10 @@ barbican_api_listen_port: "{{ barbican_api_port }}" blazar_api_port: "1234" +ceph_rgw_internal_fqdn: "{{ kolla_internal_fqdn }}" +ceph_rgw_external_fqdn: "{{ kolla_external_fqdn }}" +ceph_rgw_port: "6780" + cinder_internal_fqdn: "{{ kolla_internal_fqdn }}" cinder_external_fqdn: "{{ kolla_external_fqdn }}" cinder_api_port: "8776" @@ -601,6 +605,8 @@ enable_ceilometer: "no" enable_ceilometer_ipmi: "no" enable_cells: "no" enable_central_logging: "no" +enable_ceph_rgw: "no" +enable_ceph_rgw_loadbalancer: "{{ enable_ceph_rgw | bool }}" enable_chrony: "no" enable_cinder: "no" enable_cinder_backup: "yes" diff --git a/ansible/roles/ceph-rgw/defaults/main.yml b/ansible/roles/ceph-rgw/defaults/main.yml new file mode 100644 index 0000000000..07345434d8 --- /dev/null +++ b/ansible/roles/ceph-rgw/defaults/main.yml @@ -0,0 +1,92 @@ +--- +project_name: "ceph-rgw" + +ceph_rgw_services: + # NOTE(mgoddard): There is no container deployment, this is used for load + # balancer configuration. + ceph-rgw: + group: "all" + enabled: "{{ enable_ceph_rgw | bool }}" + haproxy: + radosgw: + enabled: "{{ enable_ceph_rgw_loadbalancer | bool }}" + mode: "http" + external: false + port: "{{ ceph_rgw_port }}" + custom_member_list: "{{ ceph_rgw_haproxy_members }}" + radosgw_external: + enabled: "{{ enable_ceph_rgw_loadbalancer | bool }}" + mode: "http" + external: true + port: "{{ ceph_rgw_port }}" + custom_member_list: "{{ ceph_rgw_haproxy_members }}" + +#################### +# Load balancer +#################### + +# List of Ceph hosts to use as HAProxy backends. Each item should contain +# 'host' and 'port'` keys. The 'ip' and 'port' keys are optional. If 'ip' is +# not specified, the 'host' values should be resolvable from the host running +# HAProxy. If the ``port`` is not specified, the default HTTP (80) or HTTPS +# (443) port will be used. +ceph_rgw_hosts: [] +ceph_rgw_haproxy_members: >- + {%- set members = [] -%} + {%- for host in ceph_rgw_hosts -%} + {%- set port = (":" ~ host.port) if host.port is defined else "" -%} + {%- set member = "server " ~ host.host ~ " " ~ host.ip | default(host.host) ~ port ~ " " ~ ceph_rgw_haproxy_healthcheck -%} + {%- set _ = members.append(member) -%} + {%- endfor -%} + {{ members }} +ceph_rgw_haproxy_healthcheck: "check inter 2000 rise 2 fall 5" + + +#################### +# OpenStack +#################### + +# Whether to register Ceph RadosGW swift-compatible endpoints in Keystone. +enable_ceph_rgw_keystone: "{{ enable_ceph_rgw | bool }}" + +# Enable/disable ceph-rgw compatibility with OpenStack Swift. +# This should match the configuration used by Ceph RadosGW. +ceph_rgw_swift_compatibility: false + +# Enable/disable including the account (project) in the endpoint URL. This +# allows for cross-project and public object access. +# This should match the 'rgw_swift_account_in_url' config option used by Ceph +# RadosGW. +ceph_rgw_swift_account_in_url: false + +ceph_rgw_endpoint_path: "{{ '/' if ceph_rgw_swift_compatibility | bool else '/swift/' }}v1{% if ceph_rgw_swift_account_in_url | bool %}/AUTH_%(project_id)s{% endif %}" + +ceph_rgw_admin_endpoint: "{{ admin_protocol }}://{{ ceph_rgw_internal_fqdn | put_address_in_context('url') }}:{{ ceph_rgw_port }}{{ ceph_rgw_endpoint_path }}" +ceph_rgw_internal_endpoint: "{{ internal_protocol }}://{{ ceph_rgw_internal_fqdn | put_address_in_context('url') }}:{{ ceph_rgw_port }}{{ ceph_rgw_endpoint_path }}" +ceph_rgw_public_endpoint: "{{ public_protocol }}://{{ ceph_rgw_external_fqdn | put_address_in_context('url') }}:{{ ceph_rgw_port }}{{ ceph_rgw_endpoint_path }}" + +ceph_rgw_keystone_user: "ceph_rgw" + +openstack_ceph_rgw_auth: "{{ openstack_auth }}" + + +#################### +# Keystone +#################### +ceph_rgw_ks_services: + - name: "swift" + type: "object-store" + description: "Openstack Object Storage" + endpoints: + - {'interface': 'admin', 'url': '{{ ceph_rgw_admin_endpoint }}'} + - {'interface': 'internal', 'url': '{{ ceph_rgw_internal_endpoint }}'} + - {'interface': 'public', 'url': '{{ ceph_rgw_public_endpoint }}'} + +ceph_rgw_ks_users: + - project: "service" + user: "{{ ceph_rgw_keystone_user }}" + password: "{{ ceph_rgw_keystone_password }}" + role: "admin" + +ceph_rgw_ks_roles: + - "ResellerAdmin" diff --git a/ansible/roles/ceph-rgw/tasks/check.yml b/ansible/roles/ceph-rgw/tasks/check.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/check.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/ceph-rgw/tasks/config.yml b/ansible/roles/ceph-rgw/tasks/config.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/config.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/ceph-rgw/tasks/deploy-containers.yml b/ansible/roles/ceph-rgw/tasks/deploy-containers.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/deploy-containers.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/ceph-rgw/tasks/deploy.yml b/ansible/roles/ceph-rgw/tasks/deploy.yml new file mode 100644 index 0000000000..40daddd63b --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/deploy.yml @@ -0,0 +1,2 @@ +--- +- import_tasks: register.yml diff --git a/ansible/roles/ceph-rgw/tasks/loadbalancer.yml b/ansible/roles/ceph-rgw/tasks/loadbalancer.yml new file mode 100644 index 0000000000..d29f3e56d1 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/loadbalancer.yml @@ -0,0 +1,7 @@ +--- +- name: "Configure haproxy for {{ project_name }}" + import_role: + role: haproxy-config + vars: + project_services: "{{ ceph_rgw_services }}" + tags: always diff --git a/ansible/roles/ceph-rgw/tasks/main.yml b/ansible/roles/ceph-rgw/tasks/main.yml new file mode 100644 index 0000000000..bc5d1e6257 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/main.yml @@ -0,0 +1,2 @@ +--- +- include_tasks: "{{ kolla_action }}.yml" diff --git a/ansible/roles/ceph-rgw/tasks/precheck.yml b/ansible/roles/ceph-rgw/tasks/precheck.yml new file mode 100644 index 0000000000..5430f4837a --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/precheck.yml @@ -0,0 +1,10 @@ +--- +- name: Fail if load balancer members not set + fail: + msg: >- + Ceph RadosGW load balancer configuration is enabled + (enable_ceph_rgw_loadbalancer) but no HAProxy members are configured. + Have you set ceph_rgw_hosts? + when: + - enable_ceph_rgw_loadbalancer | bool + - ceph_rgw_haproxy_members | length == 0 diff --git a/ansible/roles/ceph-rgw/tasks/pull.yml b/ansible/roles/ceph-rgw/tasks/pull.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/pull.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/ceph-rgw/tasks/reconfigure.yml b/ansible/roles/ceph-rgw/tasks/reconfigure.yml new file mode 100644 index 0000000000..5b10a7e111 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/reconfigure.yml @@ -0,0 +1,2 @@ +--- +- import_tasks: deploy.yml diff --git a/ansible/roles/ceph-rgw/tasks/register.yml b/ansible/roles/ceph-rgw/tasks/register.yml new file mode 100644 index 0000000000..c33683163c --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/register.yml @@ -0,0 +1,9 @@ +--- +- import_role: + name: service-ks-register + vars: + service_ks_register_auth: "{{ openstack_ceph_rgw_auth }}" + service_ks_register_services: "{{ ceph_rgw_ks_services }}" + service_ks_register_users: "{{ ceph_rgw_ks_users }}" + service_ks_register_roles: "{{ ceph_rgw_ks_roles }}" + when: enable_ceph_rgw_keystone | bool diff --git a/ansible/roles/ceph-rgw/tasks/stop.yml b/ansible/roles/ceph-rgw/tasks/stop.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/stop.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/ceph-rgw/tasks/upgrade.yml b/ansible/roles/ceph-rgw/tasks/upgrade.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/ceph-rgw/tasks/upgrade.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/loadbalancer/tasks/precheck.yml b/ansible/roles/loadbalancer/tasks/precheck.yml index 10d13d45e1..6f5b2742f1 100644 --- a/ansible/roles/loadbalancer/tasks/precheck.yml +++ b/ansible/roles/loadbalancer/tasks/precheck.yml @@ -208,6 +208,20 @@ - haproxy_stat.find('blazar_api') == -1 - haproxy_vip_prechecks +- name: Checking free port for Ceph RadosGW HAProxy + wait_for: + host: "{{ kolla_internal_vip_address }}" + port: "{{ ceph_rgw_port }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - enable_ceph_rgw | bool + - enable_ceph_rgw_loadbalancer | bool + - inventory_hostname in groups['loadbalancer'] + - haproxy_stat.find('radosgw') == -1 + - haproxy_vip_prechecks + - name: Checking free port for Cinder API HAProxy wait_for: host: "{{ kolla_internal_vip_address }}" diff --git a/ansible/site.yml b/ansible/site.yml index 832f3912a7..ca177a6d77 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -22,6 +22,7 @@ - enable_barbican_{{ enable_barbican | bool }} - enable_blazar_{{ enable_blazar | bool }} - enable_ceilometer_{{ enable_ceilometer | bool }} + - enable_ceph_rgw_{{ enable_ceph_rgw | bool }} - enable_chrony_{{ enable_chrony | bool }} - enable_cinder_{{ enable_cinder | bool }} - enable_cloudkitty_{{ enable_cloudkitty | bool }} @@ -143,6 +144,11 @@ tasks_from: loadbalancer tags: blazar when: enable_blazar | bool + - include_role: + name: ceph-rgw + tasks_from: loadbalancer + tags: ceph-rgw + when: enable_ceph_rgw | bool - include_role: name: cinder tasks_from: loadbalancer @@ -603,6 +609,19 @@ tags: swift, when: enable_swift | bool } +- name: Apply role ceph-rgw + gather_facts: false + hosts: + # NOTE(mgoddard): This is only used to register Keystone services, and + # can run on any host running kolla-toolbox. + - kolla-toolbox + - '&enable_ceph_rgw_True' + serial: '{{ kolla_serial|default("0") }}' + roles: + - { role: ceph-rgw, + tags: ceph-rgw, + when: enable_ceph_rgw | bool } + - name: Apply role glance gather_facts: false hosts: diff --git a/doc/source/reference/storage/external-ceph-guide.rst b/doc/source/reference/storage/external-ceph-guide.rst index 87e085856e..304a2ed060 100644 --- a/doc/source/reference/storage/external-ceph-guide.rst +++ b/doc/source/reference/storage/external-ceph-guide.rst @@ -211,3 +211,74 @@ type ``default_share_type``, please see :doc:`Manila in Kolla `. For more details on the CephFS Native driver, please see :manila-doc:`CephFS Native driver `. + +RadosGW +------- + +As of the Xena 13.0.0 release, Kolla Ansible supports integration with Ceph +RadosGW. This includes: + +* Registration of Swift-compatible endpoints in Keystone +* Load balancing across RadosGW API servers using HAProxy + +See the `Ceph documentation +`__ for further information, +including changes that must be applied to the Ceph cluster configuration. + +Enable Ceph RadosGW integration: + +.. code-block:: yaml + + enable_ceph_rgw: true + +Keystone integration +==================== + +A Keystone user and endpoints are registered by default, however this may be +avoided by setting ``enable_ceph_rgw_keystone`` to ``false``. If registration +is enabled, the username is defined via ``ceph_rgw_keystone_user``, and this +defaults to ``ceph_rgw``. The hostnames used by the endpoints default to +``ceph_rgw_external_fqdn`` and ``ceph_rgw_internal_fqdn`` for the public and +internal endpoints respectively. These default to ``kolla_external_fqdn`` and +``kolla_internal_fqdn`` respectively. The port used by the endpoints is defined +via ``ceph_rgw_port``, and defaults to 6780. + +By default RadosGW supports both Swift and S3 API, and it is not completely +compatible with Swift API. The option ``ceph_rgw_swift_compatibility`` can +enable/disable complete RadosGW compatibility with Swift API. This should +match the configuration used by Ceph RadosGW. After changing the value, run +the ``kolla-ansible deploy`` command to enable. + +By default, the RadosGW endpoint URL does not include the project (account) ID. +This prevents cross-project and public object access. This can be resolved by +setting ``ceph_rgw_swift_account_in_url`` to ``true``. This should match the +``rgw_swift_account_in_url`` configuration option in Ceph RadosGW. + +Load balancing +============== + +.. warning:: + + Users of Ceph RadosGW can generate very high volumes of traffic. It is + advisable to use a separate load balancer for RadosGW for anything other + than small or lightly utilised RadosGW deployments, however this is + currently out of scope for Kolla Ansible. + +Load balancing is enabled by default, however this may be avoided by setting +``enable_ceph_rgw_loadbalancer`` to ``false``. If using load balancing, the +RadosGW hosts and ports must be configured. Each item should contain +``host`` and ``port`` keys. The ``ip`` and ``port`` keys are optional. If +``ip`` is not specified, the ``host`` values should be resolvable from the host +running HAProxy. If the ``port`` is not specified, the default HTTP (80) or +HTTPS (443) port will be used. For example: + +.. code-block:: yaml + + ceph_rgw_hosts: + - host: rgw-host-1 + - host: rgw-host-2 + ip: 10.0.0.42 + port: 8080 + +The HAProxy frontend port is defined via ``ceph_rgw_port``, and defaults to +6780. diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index b71d573569..ab7250d996 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -296,6 +296,8 @@ #enable_ceilometer_ipmi: "no" #enable_cells: "no" #enable_central_logging: "no" +#enable_ceph_rgw: "no" +#enable_ceph_rgw_loadbalancer: "{{ enable_ceph_rgw | bool }}" #enable_chrony: "no" #enable_cinder: "no" #enable_cinder_backup: "yes" diff --git a/etc/kolla/passwords.yml b/etc/kolla/passwords.yml index 684d313531..9b0dd99257 100644 --- a/etc/kolla/passwords.yml +++ b/etc/kolla/passwords.yml @@ -246,3 +246,8 @@ prometheus_alertmanager_password: # OpenStack identity federation ############################### keystone_federation_openid_crypto_password: + +#################### +# Ceph RadosGW options +#################### +ceph_rgw_keystone_password: diff --git a/releasenotes/notes/ceph-rgw-062e0544a004f7b1.yaml b/releasenotes/notes/ceph-rgw-062e0544a004f7b1.yaml new file mode 100644 index 0000000000..b17de1205a --- /dev/null +++ b/releasenotes/notes/ceph-rgw-062e0544a004f7b1.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds support for integration with Ceph RadosGW. diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2 index 87fa60051b..3aa8f2793e 100644 --- a/tests/templates/globals-default.j2 +++ b/tests/templates/globals-default.j2 @@ -132,6 +132,14 @@ nova_backend_ceph: "yes" # TODO(yoctozepto): Remove this in the Xena cycle. # cephadm doesn't support chrony in a container (checks for chrony.service) enable_chrony: "no" + +enable_ceph_rgw: {{ not is_upgrade or previous_release != 'wallaby' }} +ceph_rgw_hosts: +{% for host in hostvars %} + - host: {{ host }} + ip: {{ hostvars[host]['ansible_host'] }} + port: 6780 +{% endfor %} {% endif %} {% if tls_enabled %}