Make OpenStack use Ceph

This implements all the openstack pieces needed to make ceph work.

DocImpact
Change-Id: I1d24476a966602cf955e5ef872b0efb01319894a
Partially-Implements: blueprint ceph-container
Implements: blueprint kolla-live-migration

ansible/group_vars/all.yml

@@ -163,4 +163,4 @@ haproxy_user: "openstack"
 #################################
 # Cinder - Block Storage options
 #################################
-cinder_volume_driver: "lvm"
+cinder_volume_driver: "{{ 'ceph' if enable_ceph | bool else 'lvm' }}"

ansible/roles/cinder/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: common, project_yaml: 'cinder.yml'  }

ansible/roles/cinder/tasks/ceph.yml

@@ -0,0 +1,60 @@
+---
+- name: Ensuring config directory exists
+  file:
+    path: "{{ node_config_directory }}/{{ item }}"
+    state: "directory"
+  with_items:
+    - "cinder-volume"
+    - "cinder-backup"
+  when: inventory_hostname in groups['cinder-volume']
+
+- name: Copying over config(s)
+  template:
+    src: roles/ceph/templates/ceph.conf.j2
+    dest: "{{ node_config_directory }}/{{ item }}/ceph.conf"
+  with_items:
+    - "cinder-volume"
+    - "cinder-backup"
+  when: inventory_hostname in groups['cinder-volume']
+
+# TODO(SamYaple): Improve failed_when and changed_when tests
+- name: Creating ceph pool for cinder
+  command: docker exec -it ceph_mon ceph osd pool create volumes 128
+  delegate_to: "{{ groups['ceph-mon'][0] }}"
+  changed_when: False
+  failed_when: False
+  run_once: True
+
+# TODO(SamYaple): Improve failed_when and changed_when tests
+- name: Creating ceph pool for cinder-backup
+  command: docker exec -it ceph_mon ceph osd pool create backups 128
+  delegate_to: "{{ groups['ceph-mon'][0] }}"
+  changed_when: False
+  failed_when: False
+  run_once: True
+
+# TODO(SamYaple): Improve failed_when and changed_when tests
+- name: Pulling cephx keyring for cinder
+  command: docker exec -it ceph_mon ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=vms, allow rx pool=images'
+  register: cephx_key_cinder
+  delegate_to: "{{ groups['ceph-mon'][0] }}"
+  changed_when: False
+  run_once: True
+
+# TODO(SamYaple): Improve failed_when and changed_when tests
+- name: Pulling cephx keyring for cinder-backup
+  command: docker exec -it ceph_mon ceph auth get-or-create client.cinder-backup mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=backups'
+  register: cephx_key_cinder_backup
+  delegate_to: "{{ groups['ceph-mon'][0] }}"
+  changed_when: False
+  run_once: True
+
+- name: Pushing cephx keyring
+  copy:
+    content: "{{ item.content }}\n\r"
+    dest: "{{ node_config_directory }}/{{ item.service_name }}/ceph.client.{{ item.key_name }}.keyring"
+    mode: "0600"
+  with_items:
+    - { service_name: "cinder-volume", key_name: "cinder", content: "{{ cephx_key_cinder.stdout }}" }
+    - { service_name: "cinder-backup", key_name: "cinder-backup", content: "{{ cephx_key_cinder_backup.stdout }}" }
+  when: inventory_hostname in groups['cinder-volume']

ansible/roles/cinder/tasks/main.yml

@@ -1,8 +1,27 @@
 ---
+- include: ceph.yml
+  when: enable_ceph | bool
+
 - include: register.yml
+  when: inventory_hostname in groups['cinder-api'] or
+        inventory_hostname in groups['cinder-volume'] or
+        inventory_hostname in groups['cinder-scheduler'] or
+        inventory_hostname in groups['cinder-backup']
 
 - include: config.yml
+  when: inventory_hostname in groups['cinder-api'] or
+        inventory_hostname in groups['cinder-volume'] or
+        inventory_hostname in groups['cinder-scheduler'] or
+        inventory_hostname in groups['cinder-backup']
 
 - include: bootstrap.yml
+  when: inventory_hostname in groups['cinder-api'] or
+        inventory_hostname in groups['cinder-volume'] or
+        inventory_hostname in groups['cinder-scheduler'] or
+        inventory_hostname in groups['cinder-backup']
 
 - include: start.yml
+  when: inventory_hostname in groups['cinder-api'] or
+        inventory_hostname in groups['cinder-volume'] or
+        inventory_hostname in groups['cinder-scheduler'] or
+        inventory_hostname in groups['cinder-backup']

ansible/roles/cinder/templates/cinder-volume.json.j2

@@ -6,6 +6,18 @@
             "dest": "/etc/cinder/cinder.conf",
             "owner": "cinder",
             "perm": "0600"
-        }
+        }{% if enable_ceph | bool %},
+        {
+            "source": "/opt/kolla/config_files/ceph.client.cinder.keyring",
+            "dest": "/etc/ceph/ceph.client.cinder.keyring",
+            "owner": "cinder",
+            "perm": "0600"
+        },
+        {
+            "source": "/opt/kolla/config_files/ceph.conf",
+            "dest": "/etc/ceph/ceph.conf",
+            "owner": "cinder",
+            "perm": "0600"
+        }{% endif %}
     ]
 }

ansible/roles/cinder/templates/cinder.conf.j2

@@ -1,12 +1,18 @@
 [DEFAULT]
 verbose = true
 debug = true
+
 use_syslog = True
 syslog_log_facility = LOG_LOCAL0
+
 enable_v1_api=false
 volume_name_template = %s
+
 glance_api_servers = http://{{ kolla_internal_address }}:{{ glance_api_port }}
+glance_api_version = 2
+
 os_region_name = {{ openstack_region_name }}
+
 {% if cinder_volume_driver == "lvm" %}
 default_volume_type = lvmdriver-1
 enabled_backends = lvmdriver-1
@@ -14,10 +20,24 @@ enabled_backends = lvmdriver-1
 default_volume_type = rbd-1
 enabled_backends = rbd-1
 {% endif %}
+
+{% if service_name == "cinder-backup" and cinder_volume_driver == "ceph" %}
+backup_driver = cinder.backup.drivers.ceph
+backup_ceph_conf = /etc/ceph/ceph.conf
+backup_ceph_user = cinder-backup
+backup_ceph_chunk_size = 134217728
+backup_ceph_pool = backups
+backup_ceph_stripe_unit = 0
+backup_ceph_stripe_count = 0
+restore_discard_excess_bytes = true
+{% endif %}
+
 osapi_volume_listen = {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
 osapi_volume_listen_port = {{ cinder_api_port }}
+
 api_paste_config = /etc/cinder/api-paste.ini
 nova_catalog_info = compute:nova:internalURL
+
 auth_strategy = keystone
 
 [database]
@@ -52,7 +72,12 @@ volume_backend_name = lvmdriver-1
 [rbd-1]
 volume_driver = cinder.volume.drivers.rbd.RBDDriver
 rbd_pool = volumes
+rbd_ceph_conf = /etc/ceph/ceph.conf
+rbd_flatten_volume_from_snapshot = false
+rbd_max_clone_depth = 5
+rbd_store_chunk_size = 4
+rados_connect_timeout = -1
 rbd_user = cinder
-rbd_secret_uuid = {{ metadata_secret }}
+rbd_secret_uuid = {{ rbd_secret_uuid }}
 {% endif %}
 

ansible/roles/common/tasks/main.yml

@@ -2,4 +2,3 @@
 - include: config.yml
 
 - include: start.yml
-

ansible/roles/glance/meta/main.yml

@@ -1,3 +1,3 @@
 ---
 dependencies:
-  - { role: common }
+  - { role: common, project_yaml: 'glance.yml' }

ansible/roles/glance/tasks/ceph.yml

@@ -0,0 +1,35 @@
+---
+- name: Ensuring config directory exists
+  file:
+    path: "{{ node_config_directory }}/glance-api"
+    state: "directory"
+  when: inventory_hostname in groups['glance-api']
+
+- name: Copying over config(s)
+  template:
+    src: roles/ceph/templates/ceph.conf.j2
+    dest: "{{ node_config_directory }}/glance-api/ceph.conf"
+  when: inventory_hostname in groups['glance-api']
+
+# TODO(SamYaple): Improve failed_when and changed_when tests
+- name: Creating ceph pool for glance
+  command: docker exec -it ceph_mon ceph osd pool create images 128
+  delegate_to: "{{ groups['ceph-mon'][0] }}"
+  changed_when: False
+  failed_when: False
+  run_once: True
+
+# TODO(SamYaple): Improve failed_when and changed_when tests
+- name: Pulling cephx keyring
+  command: docker exec -it ceph_mon ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=images'
+  register: cephx_key
+  delegate_to: "{{ groups['ceph-mon'][0] }}"
+  changed_when: False
+  run_once: True
+
+- name: Pushing cephx keyring
+  copy:
+    content: "{{ cephx_key.stdout }}\n\r"
+    dest: "{{ node_config_directory }}/glance-api/ceph.client.glance.keyring"
+    mode: "0600"
+  when: inventory_hostname in groups['glance-api']

ansible/roles/glance/tasks/main.yml

@@ -1,8 +1,19 @@
 ---
+- include: ceph.yml
+  when: enable_ceph | bool
+
 - include: register.yml
+  when: inventory_hostname in groups['glance-api'] or
+        inventory_hostname in groups['glance-registry']
 
 - include: config.yml
+  when: inventory_hostname in groups['glance-api'] or
+        inventory_hostname in groups['glance-registry']
 
 - include: bootstrap.yml
+  when: inventory_hostname in groups['glance-api'] or
+        inventory_hostname in groups['glance-registry']
 
 - include: start.yml
+  when: inventory_hostname in groups['glance-api'] or
+        inventory_hostname in groups['glance-registry']

ansible/roles/glance/templates/glance-api.conf.j2

@@ -12,6 +12,10 @@ registry_host = {{ kolla_internal_address }}
 use_syslog = True
 syslog_log_facility = LOG_LOCAL0
 
+{% if enable_ceph | bool %}
+show_image_direct_url= True
+{% endif %}
+
 [database]
 connection = mysql://{{ glance_database_user }}:{{ glance_database_password }}@{{ glance_database_address }}/{{ glance_database_name }}
 
@@ -29,5 +33,13 @@ password = {{ glance_keystone_password }}
 flavor = keystone
 
 [glance_store]
+{% if enable_ceph | bool %}
+default_store = rbd
+stores = rbd
+rbd_store_user = glance
+rbd_store_pool = images
+rbd_store_chunk_size = 8
+{% else %}
 default_store = file
 filesystem_store_datadir = /var/lib/glance/images/
+{% endif %}

ansible/roles/glance/templates/glance-api.json.j2

@@ -6,6 +6,18 @@
             "dest": "/etc/glance/glance-api.conf",
             "owner": "glance",
             "perm": "0600"
-        }
+        }{% if enable_ceph | bool %},
+        {
+            "source": "/opt/kolla/config_files/ceph.client.glance.keyring",
+            "dest": "/etc/ceph/ceph.client.glance.keyring",
+            "owner": "glance",
+            "perm": "0600"
+        },
+        {
+            "source": "/opt/kolla/config_files/ceph.conf",
+            "dest": "/etc/ceph/ceph.conf",
+            "owner": "glance",
+            "perm": "0600"
+        }{% endif %}
     ]
 }

ansible/roles/nova/meta/main.yml

@@ -1,3 +1,3 @@
 ---
 dependencies:
-  - { role: common }
+  - { role: common, project_yaml: 'nova.yml'  }

ansible/roles/nova/tasks/ceph.yml

@@ -0,0 +1,63 @@
+---
+- name: Ensuring config directory exists
+  file:
+    path: "{{ node_config_directory }}/{{ item }}"
+    state: "directory"
+  with_items:
+    - "nova-compute"
+    - "nova-libvirt/secrets"
+  when: inventory_hostname in groups['compute']
+
+- name: Copying over config(s)
+  template:
+    src: roles/ceph/templates/ceph.conf.j2
+    dest: "{{ node_config_directory }}/{{ item }}/ceph.conf"
+  with_items:
+    - "nova-compute"
+    - "nova-libvirt"
+  when: inventory_hostname in groups['compute']
+
+# TODO(SamYaple): Improve failed_when and changed_when tests
+- name: Creating ceph pool for vms
+  command: docker exec -it ceph_mon ceph osd pool create vms 128
+  delegate_to: "{{ groups['ceph-mon'][0] }}"
+  changed_when: False
+  failed_when: False
+  run_once: True
+
+# TODO(SamYaple): Improve failed_when and changed_when tests
+- name: Pulling cephx keyring for nova
+  command: docker exec -it ceph_mon ceph auth get-or-create client.nova mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=vms, allow rx pool=images'
+  register: cephx_key
+  delegate_to: "{{ groups['ceph-mon'][0] }}"
+  changed_when: False
+  run_once: True
+
+# TODO(SamYaple): Improve failed_when and changed_when tests
+- name: Pulling cephx keyring for libvirt
+  command: docker exec -it ceph_mon ceph auth get-key client.nova
+  register: cephx_raw_key
+  delegate_to: "{{ groups['ceph-mon'][0] }}"
+  changed_when: False
+  run_once: True
+
+- name: Pushing cephx keyring for nova
+  copy:
+    content: "{{ cephx_key.stdout }}\n\r"
+    dest: "{{ node_config_directory }}/nova-compute/ceph.client.nova.keyring"
+    mode: "0600"
+  when: inventory_hostname in groups['compute']
+
+- name: Pushing secrets xml for libvirt
+  template:
+    src: roles/nova/templates/secret.xml.j2
+    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ rbd_secret_uuid }}.xml"
+    mode: "0600"
+  when: inventory_hostname in groups['compute']
+
+- name: Pushing secrets key for libvirt
+  copy:
+    content: "{{ cephx_raw_key.stdout }}"
+    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ rbd_secret_uuid }}.base64"
+    mode: "0600"
+  when: inventory_hostname in groups['compute']

ansible/roles/nova/tasks/config.yml

@@ -129,6 +129,12 @@
     dest: "{{ node_config_directory }}/nova-compute/config.json"
   when: inventory_hostname in groups['compute']
 
+- name: Copying over config(s)
+  template:
+    src: "libvirtd.conf.j2"
+    dest: "{{ node_config_directory }}/nova-libvirt/libvirtd.conf"
+  when: inventory_hostname in groups['compute']
+
 - include: ../../config.yml
   vars:
     service_name: "nova-novncproxy"

ansible/roles/nova/tasks/main.yml

@@ -1,8 +1,31 @@
 ---
+- include: ceph.yml
+  when: enable_ceph | bool
+
 - include: register.yml
+  when: inventory_hostname in groups['nova-api'] or
+        inventory_hostname in groups['nova-consoleauth'] or
+        inventory_hostname in groups['nova-novncproxy'] or
+        inventory_hostname in groups['nova-scheduler'] or
+        inventory_hostname in groups['nova-compute']
 
 - include: config.yml
+  when: inventory_hostname in groups['nova-api'] or
+        inventory_hostname in groups['nova-consoleauth'] or
+        inventory_hostname in groups['nova-novncproxy'] or
+        inventory_hostname in groups['nova-scheduler'] or
+        inventory_hostname in groups['nova-compute']
 
 - include: bootstrap.yml
+  when: inventory_hostname in groups['nova-api'] or
+        inventory_hostname in groups['nova-consoleauth'] or
+        inventory_hostname in groups['nova-novncproxy'] or
+        inventory_hostname in groups['nova-scheduler'] or
+        inventory_hostname in groups['nova-compute']
 
 - include: start.yml
+  when: inventory_hostname in groups['nova-api'] or
+        inventory_hostname in groups['nova-consoleauth'] or
+        inventory_hostname in groups['nova-novncproxy'] or
+        inventory_hostname in groups['nova-scheduler'] or
+        inventory_hostname in groups['nova-compute']

ansible/roles/nova/tasks/start.yml

@@ -19,6 +19,7 @@
       - "{{ node_config_directory }}/nova-libvirt/:/opt/kolla/config_files/:ro"
       - "/run:/run"
       - "/sys/fs/cgroup:/sys/fs/cgroup"
+      - "/lib/modules:/lib/modules:ro"
     volumes_from:
       - nova_data
     env:

ansible/roles/nova/templates/libvirtd.conf.j2

@@ -0,0 +1,11 @@
+listen_tcp = 1
+auth_tcp = "none"
+ca_file = ""
+log_level = 2
+log_outputs = "2:file:/var/log/libvirt/libvirtd.log"
+listen_addr = "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+unix_sock_group = "root"
+unix_sock_ro_perms = "0777"
+unix_sock_rw_perms = "0770"
+auth_unix_ro = "none"
+auth_unix_rw = "none"

ansible/roles/nova/templates/nova-compute.json.j2

@@ -6,6 +6,18 @@
             "dest": "/etc/nova/nova.conf",
             "owner": "nova",
             "perm": "0600"
-        }
+        }{% if enable_ceph | bool %},
+        {
+            "source": "/opt/kolla/config_files/ceph.client.nova.keyring",
+            "dest": "/etc/ceph/ceph.client.nova.keyring",
+            "owner": "nova",
+            "perm": "0600"
+        },
+        {
+            "source": "/opt/kolla/config_files/ceph.conf",
+            "dest": "/etc/ceph/ceph.conf",
+            "owner": "nova",
+            "perm": "0600"
+        }{% endif %}
     ]
 }

ansible/roles/nova/templates/nova-libvirt.json.j2

@@ -1,4 +1,23 @@
 {
-    "command": "/usr/sbin/libvirtd",
+    "command": "/usr/sbin/libvirtd -l",
-    "config_files": []
+    "config_files": [
+        {
+            "source": "/opt/kolla/config_files/libvirt.conf",
+            "dest": "/etc/libvirt/libvirtd.conf",
+            "owner": "root",
+            "perm": "0600"
+        }{% if enable_ceph | bool %},
+        {
+            "source": "/opt/kolla/config_files/secrets",
+            "dest": "/etc/libvirt/secrets",
+            "owner": "root",
+            "perm": "0600"
+        },
+        {
+            "source": "/opt/kolla/config_files/ceph.conf",
+            "dest": "/etc/ceph/ceph.conf",
+            "owner": "root",
+            "perm": "0600"
+        }{% endif %}
+    ]
 }

ansible/roles/nova/templates/nova.conf.j2

@@ -87,3 +87,15 @@ user_domain_id = default
 project_name = service
 username = nova
 password = {{ nova_keystone_password }}
+
+{% if enable_ceph | bool %}
+[libvirt]
+images_type = rbd
+images_rbd_pool = vms
+images_rbd_ceph_conf = /etc/ceph/ceph.conf
+rbd_user = nova
+rbd_secret_uuid = {{ rbd_secret_uuid }}
+disk_cachemodes="network=writeback"
+live_migration_flag="VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"
+hw_disk_discard = unmap
+{% endif %}

ansible/roles/nova/templates/secret.xml.j2

@@ -0,0 +1,6 @@
+<secret ephemeral='no' private='no'>
+  <uuid>{{ rbd_secret_uuid }}</uuid>
+  <usage type='ceph'>
+    <name>client.nova secret</name>
+  </usage>
+</secret>

ansible/site.yml

@@ -24,11 +24,11 @@
   roles:
     - { role: swift, tags: swift, when: enable_swift | bool }
 
-- hosts: [glance-api, glance-registry]
+- hosts: [glance-api, glance-registry, ceph-mon]
   roles:
     - { role: glance, tags: glance, when: enable_glance | bool }
 
-- hosts: [nova-api, nova-conductor, nova-consoleauth, nova-novncproxy, nova-scheduler, compute]
+- hosts: [nova-api, nova-conductor, nova-consoleauth, nova-novncproxy, nova-scheduler, compute, ceph-mon]
   roles:
     - { role: nova, tags: nova, when: enable_nova | bool }
 
@@ -36,7 +36,7 @@
   roles:
     - { role: neutron, tags: neutron, when: enable_neutron | bool }
 
-- hosts: [cinder-api, cinder-backup, cinder-scheduler, cinder-volume]
+- hosts: [cinder-api, cinder-backup, cinder-scheduler, cinder-volume, ceph-mon]
   roles:
     - { role: cinder, tags: cinder, when: enable_cinder | bool }
 

docker/cinder/cinder-base/Dockerfile.j2

@@ -28,8 +28,11 @@ RUN yum -y install lvm2 \
 
      {% elif base_distro in ['ubuntu', 'debian'] %}
 
-RUN apt-get install -y --no-install-recommends lvm2 \
+RUN apt-get install -y --no-install-recommends \
-   && apt-get clean
+        lvm2 \
+        ceph-common \
+    && apt-get clean \
+    && mkdir -p /etc/ceph
 
     {% endif %}
 

docker/glance/glance-base/Dockerfile.j2

@@ -19,6 +19,15 @@ RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \
 
     {% endif %}
 {% elif install_type == 'source' %}
+    {% if base_distro in ['ubuntu', 'debian'] %}
+
+RUN apt-get install -y --no-install-recommends \
+        python-rbd \
+        python-rados \
+    && apt-get clean \
+    && mkdir -p /etc/ceph/
+
+    {% endif %}
 
 ADD glance-base-archive /glance-base-source
 RUN ln -s glance-base-source/* glance \

docker/nova/nova-compute/Dockerfile.j2

@@ -22,8 +22,11 @@ RUN yum -y install \
 
     {% elif base_distro in ['ubuntu', 'debian'] %}
 
-RUN apt-get install -y --no-install-recommends qemu-utils \
+RUN apt-get install -y --no-install-recommends \
-    && apt-get clean
+        qemu-utils \
+        ceph-common \
+    && apt-get clean \
+    && mkdir -p /etc/ceph
 
     {% endif %}
 {% endif %}

docker/nova/nova-libvirt/Dockerfile.j2

@@ -15,12 +15,15 @@ RUN yum -y install \
 {% elif base_distro in ['ubuntu', 'debian'] %}
 
 RUN apt-get install -y --no-install-recommends \
+        ceph-common \
         libvirt-bin \
         dmidecode \
         pm-utils \
         qemu \
+        qemu-block-extra \
         ebtables \
-    && apt-get clean
+    && apt-get clean \
+    && mkdir -p /etc/ceph
 
 {% endif %}
 

docker/nova/nova-libvirt/start.sh

@@ -8,12 +8,5 @@ source /opt/kolla/kolla-common.sh
 python /opt/kolla/set_configs.py
 CMD=$(cat /run_command)
 
-# TODO(SamYaple): Tweak libvirt.conf rather than change permissions.
-# Fix permissions for libvirt
-if [[ -c /dev/kvm ]]; then
-    chmod 660 /dev/kvm
-    chown root:kvm /dev/kvm
-fi
-
 echo "Running command: ${CMD}"
 exec $CMD

etc/kolla/passwords.yml

@@ -7,6 +7,7 @@
 # Ceph options
 ####################
 ceph_cluster_fsid: "5fba2fbc-551d-11e5-a8ce-01ef4c5cf93c"
+rbd_secret_uuid: "bbc5b4d5-6fca-407d-807d-06a4f4a7bccb"
 
 
 ###################