From d4686cebe598cbb8b56c669f511c2313adce7e27 Mon Sep 17 00:00:00 2001 From: Thirunavukkarasu Palani Date: Thu, 17 Dec 2020 00:02:59 +0000 Subject: [PATCH] Libvirt daemonset for the VINO operator Change-Id: I47273f6f8cf7325764d1786b87e032e9bd1aa046 --- .../libvirt-sushy/libvirt-bin-configmap.yaml | 148 ++++++++++++++++++ config/libvirt-sushy/libvirt.yaml | 93 +++++++++++ .../libvirt-sushy/libvirt_conf_configmap.yaml | 10 ++ config/libvirt-sushy/qemu_conf_configmap.yaml | 10 ++ 4 files changed, 261 insertions(+) create mode 100644 config/libvirt-sushy/libvirt-bin-configmap.yaml create mode 100644 config/libvirt-sushy/libvirt.yaml create mode 100644 config/libvirt-sushy/libvirt_conf_configmap.yaml create mode 100644 config/libvirt-sushy/qemu_conf_configmap.yaml diff --git a/config/libvirt-sushy/libvirt-bin-configmap.yaml b/config/libvirt-sushy/libvirt-bin-configmap.yaml new file mode 100644 index 0000000..f370dfb --- /dev/null +++ b/config/libvirt-sushy/libvirt-bin-configmap.yaml @@ -0,0 +1,148 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: libvirt-bin +data: + libvirt.sh: | + #!/bin/bash + set -ex + + if [ -n "$(cat /proc/*/comm 2>/dev/null | grep -w libvirtd)" ]; then + set +x + for proc in $(ls /proc/*/comm 2>/dev/null); do + if [ "x$(cat $proc 2>/dev/null | grep -w libvirtd)" == "xlibvirtd" ]; then + set -x + libvirtpid=$(echo $proc | cut -f 3 -d '/') + echo "WARNING: libvirtd daemon already running on host" 1>&2 + echo "$(cat "/proc/${libvirtpid}/status" 2>/dev/null | grep State)" 1>&2 + kill -9 "$libvirtpid" || true + set +x + fi + done + set -x + fi + + rm -f /var/run/libvirtd.pid + + if [[ -c /dev/kvm ]]; then + chmod 660 /dev/kvm + chown root:kvm /dev/kvm + fi + + CGROUPS="" + for CGROUP in cpu rdma hugetlb; do + if [ -d /sys/fs/cgroup/${CGROUP} ]; then + CGROUPS+="${CGROUP}," + fi + done + cgcreate -g ${CGROUPS%,}:/osh-libvirt + + hp_count="$(cat /proc/meminfo | grep HugePages_Total | tr -cd '[:digit:]')" + if [ 0"$hp_count" -gt 0 ]; then + + echo "INFO: Detected hugepage count of '$hp_count'. Enabling hugepage settings for libvirt/qemu." + + if [ -n "$(grep KVM_HUGEPAGES=0 /etc/default/qemu-kvm)" ]; then + sed -i 's/.*KVM_HUGEPAGES=0.*/KVM_HUGEPAGES=1/g' /etc/default/qemu-kvm + else + echo KVM_HUGEPAGES=1 >> /etc/default/qemu-kvm + fi + + if [ ! -d /dev/hugepages ]; then + echo "ERROR: Hugepages configured in kernel, but libvirtd container cannot access /dev/hugepages" + exit 1 + fi + + if [ -d /sys/fs/cgroup/hugetlb ]; then + limits="$(ls /sys/fs/cgroup/hugetlb/{{ .Values.conf.kubernetes.cgroup }}/hugetlb.*.limit_in_bytes)" || \ + (echo "ERROR: Failed to locate any hugetable limits. Did you set the correct cgroup in your values used for this chart?" + exit 1) + for limit in $limits; do + target="/sys/fs/cgroup/hugetlb/$(dirname $(awk -F: '($2~/hugetlb/){print $3}' /proc/self/cgroup))/$(basename $limit)" + if [ ! -f "$target" ]; then + echo "ERROR: Could not find write target for hugepage limit: $target" + fi + + echo "$(cat $limit)" > "$target" + done + fi + + default_hp_kb="$(cat /proc/meminfo | grep Hugepagesize | tr -cd '[:digit:]')" + + num_free_pages="$(cat /sys/kernel/mm/hugepages/hugepages-${default_hp_kb}kB/free_hugepages | tr -cd '[:digit:]')" + echo "INFO: '$num_free_pages' free hugepages of size ${default_hp_kb}kB" + if [ 0"$num_free_pages" -gt 0 ]; then + (fallocate -o0 -l "$default_hp_kb" /dev/hugepages/foo && rm /dev/hugepages/foo) || \ + (echo "ERROR: fallocate failed test at /dev/hugepages with size ${default_hp_kb}kB" + rm /dev/hugepages/foo + exit 1) + fi + fi + + if [ -n "${LIBVIRT_CEPH_CINDER_SECRET_UUID}" ] ; then + cgexec -g ${CGROUPS%,}:/osh-libvirt systemd-run --scope --slice=system libvirtd --listen & + + tmpsecret=$(mktemp --suffix .xml) + if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then + tmpsecret2=$(mktemp --suffix .xml) + fi + function cleanup { + rm -f "${tmpsecret}" + if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then + rm -f "${tmpsecret2}" + fi + } + trap cleanup EXIT + + TIMEOUT=60 + while [[ ! -f /var/run/libvirtd.pid ]]; do + if [[ ${TIMEOUT} -gt 0 ]]; then + let TIMEOUT-=1 + sleep 1 + else + echo "ERROR: libvirt did not start in time (pid file missing)" + exit 1 + fi + done + + TIMEOUT=10 + while [[ ! -e /var/run/libvirt/libvirt-sock ]]; do + if [[ ${TIMEOUT} -gt 0 ]]; then + let TIMEOUT-=1 + sleep 1 + else + echo "ERROR: libvirt did not start in time (socket missing)" + exit 1 + fi + done + + function create_virsh_libvirt_secret { + sec_user=$1 + sec_uuid=$2 + sec_ceph_keyring=$3 + cat > ${tmpsecret} < + ${sec_uuid} + + client.${sec_user}. secret + + + EOF + virsh secret-define --file ${tmpsecret} + virsh secret-set-value --secret "${sec_uuid}" --base64 "${sec_ceph_keyring}" + } + + if [ -z "${CEPH_CINDER_KEYRING}" ] ; then + CEPH_CINDER_KEYRING=$(awk '/key/{print $3}' /etc/ceph/ceph.client.${CEPH_CINDER_USER}.keyring) + fi + create_virsh_libvirt_secret ${CEPH_CINDER_USER} ${LIBVIRT_CEPH_CINDER_SECRET_UUID} ${CEPH_CINDER_KEYRING} + + if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then + EXTERNAL_CEPH_CINDER_KEYRING=$(cat /tmp/external-ceph-client-keyring) + create_virsh_libvirt_secret ${EXTERNAL_CEPH_CINDER_USER} ${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID} ${EXTERNAL_CEPH_CINDER_KEYRING} + fi + + wait + else + exec cgexec -g ${CGROUPS%,}:/osh-libvirt systemd-run --scope --slice=system libvirtd + fi diff --git a/config/libvirt-sushy/libvirt.yaml b/config/libvirt-sushy/libvirt.yaml new file mode 100644 index 0000000..2f22336 --- /dev/null +++ b/config/libvirt-sushy/libvirt.yaml @@ -0,0 +1,93 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: libvirt +spec: + selector: + matchLabels: + kubernetes.io/os: linux + template: + metadata: + labels: + kubernetes.io/os: linux + spec: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + hostNetwork: true + hostPID: true + hostIPC: true + containers: + - name: libvirt + command: + - /tmp/libvirt.sh + image: docker.io/openstackhelm/libvirt:ubuntu_xenial-20190903 + securityContext: + privileged: true + runAsUser: 0 + readOnlyRootFilesystem: false + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: libvirt-bin + mountPath: /tmp/libvirt.sh + subPath: libvirt.sh + readOnly: true + - name: libvirt-etc + mountPath: /etc/libvirt/libvirtd.conf + subPath: libvirtd.conf + - name: libvirt-qemu + mountPath: /etc/libvirt/qemu.conf + subPath: qemu.conf + - mountPath: /lib/modules + name: libmodules + readOnly: true + - name: var-lib-libvirt + mountPath: /var/lib/libvirt + mountPropagation: Bidirectional + - name: var-lib-libvirt-images + mountPath: /var/lib/libvirt/images + - name: run + mountPath: /run + - name: dev + mountPath: /dev + - name: cgroup + mountPath: /sys/fs/cgroup + - name: logs + mountPath: /var/log/libvirt + volumes: + - name: pod-tmp + emptyDir: {} + - name: libvirt-bin + configMap: + name: libvirt-bin + defaultMode: 0555 + - name: libvirt-etc + configMap: + name: libvirt-etc + defaultMode: 0444 + - name: libvirt-qemu + configMap: + name: libvirt-qemu + defaultMode: 0444 + - name: libmodules + hostPath: + path: /lib/modules + - name: var-lib-libvirt + hostPath: + path: /var/lib/libvirt + - name: var-lib-libvirt-images + hostPath: + path: /var/lib/libvirt/images + - name: run + hostPath: + path: /run + - name: dev + hostPath: + path: /dev + - name: logs + hostPath: + path: /var/log/libvirt + - name: cgroup + hostPath: + path: /sys/fs/cgroup \ No newline at end of file diff --git a/config/libvirt-sushy/libvirt_conf_configmap.yaml b/config/libvirt-sushy/libvirt_conf_configmap.yaml new file mode 100644 index 0000000..7bd9478 --- /dev/null +++ b/config/libvirt-sushy/libvirt_conf_configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: libvirt-etc +data: + libvirtd.conf: | + unix_sock_group = "libvirtd" + unix_sock_ro_perms = "0777" + unix_sock_rw_perms = "0770" + log_outputs = "1:file:/var/log/libvirt/libvirtd.log" \ No newline at end of file diff --git a/config/libvirt-sushy/qemu_conf_configmap.yaml b/config/libvirt-sushy/qemu_conf_configmap.yaml new file mode 100644 index 0000000..aea439a --- /dev/null +++ b/config/libvirt-sushy/qemu_conf_configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: libvirt-qemu +data: + qemu.conf: | + stdio_handler = "file" + user = "libvirt-qemu" + group = "kvm" + security_driver = "none" \ No newline at end of file