commit fcd40818cdf0872393e65628b0a93d4e6a29a9f0 Author: babak sarashki Date: Wed Oct 2 13:20:53 2019 -0700 StarlingX base Initial checkin Enables: kubernetes docker ceph (ceph-mgr, reconfigured with python3) diff --git a/README.md b/README.md new file mode 100644 index 0000000..299c747 --- /dev/null +++ b/README.md @@ -0,0 +1,106 @@ +meta-stx +========= + +Introduction +------------------------ + +This layer is intended to enable starlingx on poky. + + +Dependencies +------------------------- + +This layer depends on: + +``` + URI: git://git.openembedded.org/meta-openembedded/ + revision: HEAD + + layes: meta-oe + meta-python + meta-networking +``` +You are solely responsible for determining the appropriateness of using or redistributing the above dependencies and assume any risks associated with your exercise of permissions under the license. + +Maintenance +------------------------- + +Maintainer: + Babak Sarashki + + +Building the meta-stx layer +--------------------------- + + +Use Case: Virtual Network Setup +------------------------------- + +This script is inteded to setup virtual network between ns's on the host. + +``` +mkdir -p layers +cd layers + +git clone --branch thud git://git.yoctoproject.org/poky.git +git clone --branch thud git://git.openembedded.org/meta-openembedded +git clone --branch thud git://git.yoctoproject.org/meta-virtualization +git clone --branch thud git://git.yoctoproject.org/meta-cloud-services +git clone --branch thud git://git.yoctoproject.org/meta-intel +git clone --branch thud git://git.yoctoproject.org/meta-intel-qat + +git clone --branch thud git://git.yoctoproject.org/meta-selinux +git clone --branch thud git://git.yoctoproject.org/meta-security +git clone --branch thud https://github.com/jiazhang0/meta-secure-core.git +git clone --branch thud https://github.com/intel-iot-devkit/meta-iot-cloud.git +git clone --branch thud https://github.com/rauc/meta-rauc.git +git clone https://github.com/zbsarashki/meta-stx.git + +cd poky +. ./oe-init-build-env /path/to/prj/dir + +``` +Add the following layers to conf/bblayers.conf +``` +PATH_TO_LOCAL_REPO/layers/poky/meta +PATH_TO_LOCAL_REPO/layers/poky/meta-poky +PATH_TO_LOCAL_REPO/layers/poky/meta-yocto-bsp +PATH_TO_LOCAL_REPO/layers/meta-openembedded/meta-oe +PATH_TO_LOCAL_REPO/layers/meta-openembedded/meta-networking +PATH_TO_LOCAL_REPO/layers/meta-openembedded/meta-filesystems +PATH_TO_LOCAL_REPO/layers/meta-openembedded/meta-perl +PATH_TO_LOCAL_REPO/layers/meta-openembedded/meta-python +PATH_TO_LOCAL_REPO/layers/meta-openembedded/meta-webserver +PATH_TO_LOCAL_REPO/layers/meta-openembedded/meta-initramfs +PATH_TO_LOCAL_REPO/layers/meta-openembedded/meta-gnome +PATH_TO_LOCAL_REPO/layers/meta-virtualization +PATH_TO_LOCAL_REPO/layers/meta-cloud-services +PATH_TO_LOCAL_REPO/layers/meta-cloud-services/meta-openstack +PATH_TO_LOCAL_REPO/layers/meta-cloud-services/meta-openstack-aio-deploy +PATH_TO_LOCAL_REPO/layers/meta-cloud-services/meta-openstack-compute-deploy +PATH_TO_LOCAL_REPO/layers/meta-cloud-services/meta-openstack-controller-deploy +PATH_TO_LOCAL_REPO/layers/meta-cloud-services/meta-openstack-qemu +PATH_TO_LOCAL_REPO/layers/meta-cloud-services/meta-openstack-swift-deploy +PATH_TO_LOCAL_REPO/layers/meta-measured +PATH_TO_LOCAL_REPO/layers/meta-secure-core/meta-signing-key +PATH_TO_LOCAL_REPO/layers/meta-secure-core/meta-efi-secure-boot +PATH_TO_LOCAL_REPO/layers/meta-secure-core/meta-encrypted-storage +PATH_TO_LOCAL_REPO/layers/meta-secure-core/meta-integrity +PATH_TO_LOCAL_REPO/layers/meta-secure-core/meta-tpm2 +PATH_TO_LOCAL_REPO/layers/meta-secure-core/meta +PATH_TO_LOCAL_REPO/layers/meta-security +PATH_TO_LOCAL_REPO/layers/meta-security/meta-security-compliance +PATH_TO_LOCAL_REPO/layers/meta-selinux +PATH_TO_LOCAL_REPO/layers/meta-intel +PATH_TO_LOCAL_REPO/layers/meta-intel-qat +PATH_TO_LOCAL_REPO/layers/meta-rauc +PATH_TO_LOCAL_REPO/layers/meta-stx +PATH_TO_LOCAL_REPO/layers/local +PATH_TO_LOCAL_REPO/layers/meta-iot-cloud +``` + +# Legal Notices + +All product names, logos, and brands are property of their respective owners. All company, product and service names used in this software are for identification purposes only. Wind River is a registered trademarks of Wind River Systems, Inc. Linux is a registered trademark of Linus Torvalds. + +Disclaimer of Warranty / No Support: Wind River does not provide support and maintenance services for this software, under Wind River’s standard Software Support and Maintenance Agreement or otherwise. Unless required by applicable law, Wind River provides the software (and each contributor provides its contribution) on an “AS IS” BASIS, WITHOUT WARRANTIES OF ANY KIND, either express or implied, including, without limitation, any warranties of TITLE, NONINFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the software and assume any risks associated with your exercise of permissions under the license. diff --git a/conf/layer.conf b/conf/layer.conf new file mode 100644 index 0000000..5502c98 --- /dev/null +++ b/conf/layer.conf @@ -0,0 +1,93 @@ +# We have a conf and classes directory, add to BBPATH +BBPATH .= ":${LAYERDIR}" + +# We have recipes-* directories, add to BBFILES +BBFILES += "${LAYERDIR}/recipes-*/*/*.bb \ + ${LAYERDIR}/recipes-*/*/*.bbappend" + +BBFILE_COLLECTIONS += "stx-layer" +BBFILE_PATTERN_stx-layer := "^${LAYERDIR}/" +BBFILE_PRIORITY_stx-layer = "5" + +DISTRO_FEATURES_append = " x11 opengl" +EXTRA_IMAGE_FEATURES_append = " x11-base" +VIRTUAL-RUNTIME_graphical_init_manager = "lxdm" + +# INITRAMFS_IMAGE = "secure-core-image-initramfs" +DISTRO_FEATURES_NATIVE_append += "systemd ima tpm tpm2 efi-secure-boot luks" +DISTRO_FEATURES_append += "systemd ima tpm tpm2 efi-secure-boot luks" +#DISTRO_FEATURES_append += "systemd ima tpm tpm2 efi-secure-boot luks modsign" +MACHINE_FEATURES_NATIVE_append += "efi" +MACHINE_FEATURES_append += "efi" +#PACKAGE_CLASSES = "package_rpm" +#INHERIT += "sign_rpm_ext" + +#SECURE_CORE_IMAGE_EXTRA_INSTALL ?= "\ +# packagegroup-efi-secure-boot \ +# packagegroup-tpm \ +# packagegroup-tpm2 \ +# packagegroup-ima \ +# packagegroup-luks \ +# " + +# For iso image +#KERNEL_FEATURES += "features/overlayfs/overlayfs.scc" + +#INITRAMFS_SCRIPTS = "initramfs-live-boot \ +# initramfs-live-install \ +# initramfs-live-install-efi \ +# " + +LAYERDEPENDS_stx-layer = "\ + core \ + networking-layer \ + openembedded-layer \ + networking-layer \ + filesystems-layer \ + perl-layer \ + meta-python \ + webserver \ + virtualization-layer \ + cloud-services-layer \ + openstack-layer \ + openstack-aio-deploy-layer \ + openstack-compute-deploy-layer \ + openstack-controller-deploy-layer \ + openstack-qemu-layer \ + openstack-swift-deploy-layer \ + signing-key \ + efi-secure-boot \ + encrypted-storage \ + integrity \ + tpm2 \ + secure-core \ + selinux \ +" +# tpm-layer + +# openstack-compute-test-config-layer +## openstack-controller-test-config-layer + +# This should only be incremented on significant changes that will +# cause compatibility issues with other layers +LAYERVERSION_stx-version = "1" +LAYERSERIES_COMPAT_stx-layer = "thud" + +VIRTUAL-RUNTIME_init_manager = "systemd" +DISTRO_FEATURES_append = " systemd" +DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" + +DISTRO_FEATURES_append = " bluez pam largefile opengl" +DISTRO_FEATURES_append = " virtualization" +DISTRO_FEATURES_append = " openstack" +DISTRO_FEATURES_append = " selinux" +DISTRO_FEATURES_append = " kvm" + +PREFERRED_PROVIDER_virtual/containerd = "containerd-opencontainers" +PREFERRED_PROVIDER_virtual/kernel = "linux-yocto" + +PREFERRED_VERSION_keyutils = "1.5.10" +PREFERRED_VERSION_python3-cherrypy = "git" +PREFERRED_VERSION_python-cheroot = "git" +PREFERRED_VERSION_python3-cheroot = "git" +PREFERRED_VERSION_python-cherrypy = "git" diff --git a/recipes-core/images/stx-image-aio.bb b/recipes-core/images/stx-image-aio.bb new file mode 100644 index 0000000..c1e426a --- /dev/null +++ b/recipes-core/images/stx-image-aio.bb @@ -0,0 +1,25 @@ +SUMMARY = " StarlingX Single Server" + +LICENSE = "MIT" + +IMAGE_INSTALL_append = " \ + ${CORE_IMAGE_BASE_INSTALL} \ + packagegroup-core-full-cmdline \ + packagegroup-core-security \ + packagegroup-core-selinux \ + packagegroup-stx-kube \ + packagegroup-stx-ceph \ + " +IMAGE_FEATURES += " \ + package-management \ + ssh-server-openssh \ + " + +inherit core-image +inherit openstack-base +inherit identity +inherit monitor + +# check for 5G of free space, so we use 5G as a starting point. +# IMAGE_ROOTFS_EXTRA_SPACE_append += "+ 5000000" +# POST_KEYSTONE_SETUP_COMMAND = "/etc/keystone/hybrid-backend-setup" diff --git a/recipes-core/packagegroups/packagegroup-stx.bb b/recipes-core/packagegroups/packagegroup-stx.bb new file mode 100644 index 0000000..293fd18 --- /dev/null +++ b/recipes-core/packagegroups/packagegroup-stx.bb @@ -0,0 +1,77 @@ +SUMMARY = "StarlingX stx packages" + +PR = "r0" + +# +# packages which content depend on MACHINE_FEATURES need to be MACHINE_ARCH +# + +PACKAGE_ARCH = "${MACHINE_ARCH}" + + +inherit packagegroup + +PROVIDES = "${PACKAGES}" +PACKAGES = " \ + packagegroup-stx-kube \ + packagegroup-stx-misc \ + packagegroup-stx-ceph \ + " + +RDEPENDS_packagegroup-stx-kube = "\ + kubernetes \ + kubernetes-misc \ + kubeadm \ + kubelet \ + kubectl \ + docker \ + util-linux-unshare \ + containerd-opencontainers \ + runc-docker \ + docker \ + packagegroup-stx-misc \ + " + +RDEPENDS_packagegroup-stx-misc = "\ + vim \ + vim-common \ + ntp \ + python3-pip \ + " + +RDEPENDS_packagegroup-stx-ceph = "\ + ceph \ + ceph-python \ + openldap \ + rocksdb \ + snappy \ + lz4 \ + oath \ + python-prettytable \ + python-eventlet \ + util-linux-uuidgen \ + rdma-core \ + python3-pyopenssl \ + python3-bcrypt \ + python3-werkzeug \ + python3-pyroute2 \ + python3-requests \ + python3-cherrypy \ + python3-six \ + python3-mako \ + python3-pecan \ + python3-prettytable \ + python3-pycparser \ + python3-cffi \ + python3-cryptography \ + python3-more-itertools \ + python3-pytz \ + python3-jaraco-functools \ + python3-tempora \ + python3-portend \ + python3-zc-lockfile \ + python-oslo.messaging \ + " +# tsconfig \ +# ceph-manager \ +# sysinv diff --git a/recipes-devtools/python/python-prettytable.inc b/recipes-devtools/python/python-prettytable.inc new file mode 100644 index 0000000..6004ecd --- /dev/null +++ b/recipes-devtools/python/python-prettytable.inc @@ -0,0 +1,25 @@ +SUMMARY = "Python library for displaying tabular data in a ASCII table format" +HOMEPAGE = "http://code.google.com/p/prettytable" +LICENSE = "BSD" +LIC_FILES_CHKSUM = "file://COPYING;md5=3e73500ffa52de5071cff65990055282" + +SRC_URI[md5sum] = "0c1361104caff8b09f220748f9d69899" +SRC_URI[sha256sum] = "a53da3b43d7a5c229b5e3ca2892ef982c46b7923b51e98f0db49956531211c4f" + +SRCNAME = "prettytable" + +SRC_URI = "https://pypi.python.org/packages/source/P/PrettyTable/${SRCNAME}-${PV}.zip" + +S = "${WORKDIR}/${SRCNAME}-${PV}" + +do_install_append() { + perm_files=`find "${D}${PYTHON_SITEPACKAGES_DIR}/" -name "*.txt" -o -name "PKG-INFO"` + for f in $perm_files; do + chmod 644 "${f}" + done +} + +UPSTREAM_CHECK_URI = "https://pypi.python.org/pypi/PrettyTable/" +UPSTREAM_CHECK_REGEX = "/PrettyTable/(?P(\d+[\.\-_]*)+)" + +BBCLASSEXTEND = "native nativesdk" diff --git a/recipes-devtools/python/python3-cheroot_git.bb b/recipes-devtools/python/python3-cheroot_git.bb new file mode 100644 index 0000000..3724b49 --- /dev/null +++ b/recipes-devtools/python/python3-cheroot_git.bb @@ -0,0 +1,36 @@ +SUMMARY = " Lang - Python Language Constraints" +DESCRIPTION = "\ + Lang is a Python module that allows enforcing programming language constraints. Lang was \ + built using a Java like mindset, so many of the constraints that are supported are mirrors \ + of constraints in the Java programming language. \ + " + + +LICENSE = "BSD" +LIC_FILES_CHKSUM = "file://LICENSE.md;md5=beeffd9dfcc746ed5a91921f1acc2746" + +SRCREV = "c7ca7ff0bcebb53e1bed783280a3bb5db35f900f" +PROTOCOL = "https" +BRANCH = "master" +S = "${WORKDIR}/git" + +SRC_URI = "git://github.com/cherrypy/cheroot.git;protocol=${PROTOCOL};rev=${SRCREV};branch=${BRANCH}" + + +DEPENDS += " \ + python3 \ + python3-setuptools-scm-native \ + python3-setuptools-scm-git-archive-native \ + " + +RDEPENDS_${PN} += " \ + python3-email \ + python3-fcntl \ + python3-io \ + python3-logging \ + python3-unixadmin \ + python3-pyopenssl \ + python3-six \ + " + +inherit setuptools3 distutils3 pkgconfig diff --git a/recipes-devtools/python/python3-cherrypy_git.bb b/recipes-devtools/python/python3-cherrypy_git.bb new file mode 100644 index 0000000..85c2e75 --- /dev/null +++ b/recipes-devtools/python/python3-cherrypy_git.bb @@ -0,0 +1,52 @@ +SUMMARY = " CherryPy is a pythonic, object-oriented HTTP framework" +DESCRIPTION = "\ + It allows building web applications in much the same way one would build any other object-oriented program. \ + This design results in less and more readable code being developed faster. It's all just properties and methods. \ + It is now more than ten years old and has proven fast and very stable. \ + It is being used in production by many sites, from the simplest to the most demanding. \ + " + + +LICENSE = "BSD" +LIC_FILES_CHKSUM = "file://LICENSE.md;md5=a8cbc5da4e6892b15a972a0b18622b2b" + +SRCREV = "994803e4923e53b7079c79f4e9b502cc1b8d0aa6" +PROTOCOL = "https" +BRANCH = "master" +S = "${WORKDIR}/git" + +SRC_URI = "git://github.com/cherrypy/cherrypy.git;protocol=${PROTOCOL};rev=${SRCREV};branch=${BRANCH}" + + +DEPENDS += " python3 python3-setuptools-scm-native" +inherit setuptools3 distutils3 pkgconfig + +RDEPENDS_${PN} += " \ + python3-compression \ + python3-crypt \ + python3-datetime \ + python3-email \ + python3-fcntl \ + python3-html \ + python3-io \ + python3-json \ + python3-logging \ + python3-netclient \ + python3-netserver \ + python3-profile \ + python3-pydoc \ + python3-xml \ + python3-unixadmin \ + " + +RDEPENDS_${PN} += " \ + python3-cheroot \ + python3-contextlib2 \ + python3-memcached \ + python3-portend \ + python3-pyopenssl \ + python3-routes \ + python3-simplejson \ + python3-six \ + python3-zc-lockfile \ + " diff --git a/recipes-devtools/python/python3-lang_git.bb b/recipes-devtools/python/python3-lang_git.bb new file mode 100644 index 0000000..8c5a198 --- /dev/null +++ b/recipes-devtools/python/python3-lang_git.bb @@ -0,0 +1,21 @@ +SUMMARY = " Lang - Python Language Constraints" +DESCRIPTION = "\ + Lang is a Python module that allows enforcing programming language constraints. Lang was \ + built using a Java like mindset, so many of the constraints that are supported are mirrors \ + of constraints in the Java programming language. \ + " + + +LICENSE = "Apache-2.0" +LIC_FILES_CHKSUM = "file://LICENSE;md5=fa818a259cbed7ce8bc2a22d35a464fc" + +SRCREV = "feb4c638ebc581d9913f440965e83558fd10018c" +PROTOCOL = "https" +BRANCH = "master" +S = "${WORKDIR}/git/src" + +SRC_URI = "git://github.com//amitassaraf/lang.git;protocol=${PROTOCOL};rev=${SRCREV};branch=${BRANCH}" + + +DEPENDS += " python3 python3-setuptools-scm-native" +inherit setuptools3 distutils3 pkgconfig diff --git a/recipes-devtools/python/python3-logutils_0.3.5.bb b/recipes-devtools/python/python3-logutils_0.3.5.bb new file mode 100644 index 0000000..c006855 --- /dev/null +++ b/recipes-devtools/python/python3-logutils_0.3.5.bb @@ -0,0 +1,16 @@ +DESCRIPTION = "Set of handlers for the Python standard library's logging package" +HOMEPAGE = "https://pypi.python.org/pypi/logutils" +SECTION = "devel/python" +LICENSE = "BSD" +LIC_FILES_CHKSUM = "file://LICENSE.txt;md5=44c35f0b8e2a27a2f33a4e4a5c65d014" + +SRC_URI[md5sum] = "fcd2f8e9217bfa0b580f253b35a9d954" +SRC_URI[sha256sum] = "bc058a25d5c209461f134e1f03cab637d66a7a5ccc12e593db56fbb279899a82" + +inherit setuptools3 pypi + +# avoid "error: option --single-version-externally-managed not recognized" +DISTUTILS_INSTALL_ARGS = "--root=${D} \ + --prefix=${prefix} \ + --install-lib=${PYTHON_SITEPACKAGES_DIR} \ + --install-data=${datadir}" diff --git a/recipes-devtools/python/python3-pecan_git.bb b/recipes-devtools/python/python3-pecan_git.bb new file mode 100644 index 0000000..fb75a7b --- /dev/null +++ b/recipes-devtools/python/python3-pecan_git.bb @@ -0,0 +1,22 @@ +CRIPTION = "WSGI object-dispatching web framework" + +HOMEPAGE = "https://pypi.python.org/pypi/pecan/" +SECTION = "devel/python" +LICENSE = "BSD" +LIC_FILES_CHKSUM = "file://LICENSE;md5=d846877d24bbb3d7a00a985c90378e8c" + +SRCREV = "da15e06d783e2cf569b39ba506e68e4e1e85568d" +PROTOCOL = "https" +BRANCH = "master" +S = "${WORKDIR}/git" + +SRC_URI = "git://github.com/pecan/pecan.git;protocol=${PROTOCOL};rev=${SRCREV};branch=${BRANCH}" + + +inherit setuptools3 distutils3 + +RDEPENDS_${PN} = "python3-mako \ + python3-six \ + python3-logutils \ + python3-webtest \ + " diff --git a/recipes-devtools/python/python3-prettytable_0.7.2.bb b/recipes-devtools/python/python3-prettytable_0.7.2.bb new file mode 100644 index 0000000..5445f10 --- /dev/null +++ b/recipes-devtools/python/python3-prettytable_0.7.2.bb @@ -0,0 +1,3 @@ +inherit setuptools3 +require python-prettytable.inc + diff --git a/recipes-devtools/python/python3-webtest_2.0.33.bb b/recipes-devtools/python/python3-webtest_2.0.33.bb new file mode 100644 index 0000000..5374617 --- /dev/null +++ b/recipes-devtools/python/python3-webtest_2.0.33.bb @@ -0,0 +1,13 @@ +DESCRIPTION = "Helper to test WSGI applications" +HOMEPAGE = "https://pypi.python.org/pypi/WebTest/" +SECTION = "devel/python" +LICENSE = "Apache-2.0" +LIC_FILES_CHKSUM = "file://README.rst;md5=e3e00192f000e33de99fd5a3cb32a8fc" + +PYPI_PACKAGE = "WebTest" + +RDEPENDS_${PN} += "${PYTHON_PN}-beautifulsoup4" + +SRC_URI[md5sum] = "dd0385c725b85ac1e8079f38d2acd7b1" +SRC_URI[sha256sum] = "41348efe4323a647a239c31cde84e5e440d726ca4f449859264e538d39037fd0" +inherit setuptools3 pypi diff --git a/recipes-extended/ceph/ceph_13.2.2.bbappend b/recipes-extended/ceph/ceph_13.2.2.bbappend new file mode 100644 index 0000000..1afbada --- /dev/null +++ b/recipes-extended/ceph/ceph_13.2.2.bbappend @@ -0,0 +1,141 @@ +FILESEXTRAPATHS_prepend := "${THISDIR}/patches:${THISDIR}/files:" + +inherit python3native python3-dir + +SRC_URI += "\ + file://ceph/0001-Add-hooks-for-orderly-shutdown-on-controller.patch \ + file://ceph/ceph.conf \ + file://ceph/ceph-init-wrapper.sh \ + file://ceph/ceph-preshutdown.sh \ + file://ceph/ceph.service \ + file://ceph/mgr-restful-plugin.py \ + file://ceph/starlingx-docker-override.conf \ + file://ceph/ceph.conf.pmon \ + file://ceph/ceph-manage-journal.py \ + file://ceph/ceph-radosgw.service \ + file://ceph/ceph.sh \ + file://ceph/mgr-restful-plugin.service \ + file://ceph/ceph-mon_config.sh \ + file://ceph/ceph-mgr_manual.sh \ + file://ceph/ceph-volume_manual.sh \ + " + +DEPENDS = "boost rdma-core bzip2 curl expat gperf-native \ + keyutils libaio lz4 \ + nspr nss oath openldap openssl \ + python python3 python3-cython-native rocksdb snappy udev \ + python-cython-native valgrind xfsprogs zlib \ + " +RDEPENDS_${PN} += " rdma-core python3-core python3" + + +EXTRA_OECMAKE = "-DWITH_MANPAGE=OFF \ + -DWITH_FUSE=OFF \ + -DWITH_SPDK=OFF \ + -DWITH_LEVELDB=OFF \ + -DWITH_LTTNG=OFF \ + -DWITH_BABELTRACE=OFF \ + -DWITH_TESTS=OFF \ + -DWITH_MGR=ON \ + -DWITH_MGR_DASHBOARD_FRONTEND=OFF \ + -DWITH_PYTHON2=OFF \ + -DWITH_PYTHON3=ON \ + -DMGR_PYTHON_VERSION=3 \ + -DWITH_SYSTEM_BOOST=ON \ + -DWITH_SYSTEM_ROCKSDB=ON \ + " + +# TODO: Should be fixed in either boost package or CMake files. +do_configure_prepend() { + ln -s ${WORKDIR}/recipe-sysroot/usr/lib/libboost_python35.so \ + ${WORKDIR}/recipe-sysroot/usr/lib/libboost_python.so +} +do_install_append () { + install -d ${D}${sysconfdir}/ceph + install -m 0644 ${WORKDIR}/ceph/ceph.conf ${D}${sysconfdir}/ceph/ + install -m 0644 ${WORKDIR}/ceph/ceph-radosgw.service ${D}${systemd_system_unitdir}/ceph-radosgw@.service + install -m 0644 ${WORKDIR}/ceph/ceph.service ${D}${systemd_system_unitdir} + install -m 0644 ${WORKDIR}/ceph/mgr-restful-plugin.service ${D}${systemd_system_unitdir} + + install -m 0700 ${WORKDIR}/ceph/ceph-manage-journal.py ${D}${sbindir}/ceph-manage-journal + install -Dm 0750 ${WORKDIR}/ceph/mgr-restful-plugin.py ${D}${sysconfdir}/rc.d/init.d/mgr-restful-plugin + install -m 0750 ${WORKDIR}/ceph/ceph.conf.pmon ${D}${sysconfdir}/ceph/ + + install -d -m 0750 ${D}${sysconfdir}/services.d/controller + install -d -m 0750 ${D}${sysconfdir}/services.d/storage + install -d -m 0750 ${D}${sysconfdir}/services.d/worker + + install -m 0750 ${WORKDIR}/ceph/ceph.sh ${D}${sysconfdir}/services.d/controller + install -m 0750 ${WORKDIR}/ceph/ceph.sh ${D}${sysconfdir}/services.d/storage + install -m 0750 ${WORKDIR}/ceph/ceph.sh ${D}${sysconfdir}/services.d/worker + + install -Dm 0750 ${WORKDIR}/ceph/ceph-init-wrapper.sh ${D}${sysconfdir}/rc.d/init.d/ceph-init-wrapper + install -m 0700 ${WORKDIR}/ceph/ceph-preshutdown.sh ${D}${sbindir}/ceph-preshutdown.sh + + install -Dm 0644 ${WORKDIR}/ceph/starlingx-docker-override.conf ${D}${systemd_system_unitdir}/docker.service.d/starlingx-docker-override.conf + + install -m 0644 -D ${S}/src/etc-rbdmap ${D}${sysconfdir}/ceph/rbdmap + install -m 0644 -D ${S}/etc/sysconfig/ceph ${D}${sysconfdir}/sysconfig/ceph + install -m 0644 -D ${S}/src/logrotate.conf ${D}${sysconfdir}/logrotate.d/ceph + + install -m 0644 -D ${S}/COPYING ${D}${docdir}/ceph/COPYING + install -m 0644 -D ${S}/etc/sysctl/90-ceph-osd.conf ${D}${libdir}/sysctl.d/90-ceph-osd.conf + install -m 0644 -D ${S}/udev/50-rbd.rules ${D}${libdir}/udev/rules.d/50-rbd.rules + install -m 0644 -D ${S}/udev/60-ceph-by-parttypeuuid.rules ${D}${libdir}/udev/rules.d/60-ceph-by-parttypeuuid.rules + + mkdir -p ${D}${localstatedir}/ceph + #TODO: + #mkdir -p /run/ceph must be done at init time. + #mkdir -p ${D}${localstatedir}/run/ceph + mkdir -p ${D}${localstatedir}/log/ceph + mkdir -p ${D}${localstatedir}/lib/ceph/tmp + mkdir -p ${D}${localstatedir}/lib/ceph/mon + mkdir -p ${D}${localstatedir}/lib/ceph/osd + mkdir -p ${D}${localstatedir}/lib/ceph/mds + mkdir -p ${D}${localstatedir}/lib/ceph/mgr + mkdir -p ${D}${localstatedir}/lib/ceph/radosgw + mkdir -p ${D}${localstatedir}/lib/ceph/bootstrap-osd + mkdir -p ${D}${localstatedir}/lib/ceph/bootstrap-mds + mkdir -p ${D}${localstatedir}/lib/ceph/bootstrap-rgw + mkdir -p ${D}${localstatedir}/lib/ceph/bootstrap-mgr + mkdir -p ${D}${localstatedir}/lib/ceph/bootstrap-rbd + + install -m 0750 -D ${S}/src/init-radosgw ${D}${sysconfdir}/rc.d/init.d/ceph-radosgw + sed -i '/### END INIT INFO/a SYSTEMCTL_SKIP_REDIRECT=1' ${D}${sysconfdir}/rc.d/init.d/ceph-radosgw + install -m 0750 -D ${S}/src/init-rbdmap ${D}${sysconfdir}/rc.d/init.d/rbdmap + install -m 0750 -D ${B}/bin/init-ceph ${D}${sysconfdir}/rc.d/init.d/ceph + install -m 0750 -D ${B}/bin/init-ceph ${D}${sysconfdir}/init.d/ceph + install -d -m 0750 ${D}${localstatedir}/log/radosgw + install -d -m 0750 ${D}/home/root/cluster + cp ${WORKDIR}/ceph/ceph-mon_config.sh ${D}/home/root/cluster + cp ${WORKDIR}/ceph/ceph-mgr_manual.sh ${D}/home/root/cluster + cp ${WORKDIR}/ceph/ceph-volume_manual.sh ${D}/home/root/cluster + chmod 755 ${D}/home/root/cluster/* + sed -i -e 's:${WORKDIR}.*python3:${bindir}/python3:' ${D}${bindir}/ceph + sed -i -e 's:${WORKDIR}.*python3:${bindir}/python3:' ${D}${bindir}/ceph-disk + sed -i -e 's:${WORKDIR}.*python3:${bindir}/python3:' ${D}${bindir}/ceph-detect-init + sed -i -e '1s:python$:python3:' ${D}${bindir}/ceph-volume + sed -i -e '1s:python$:python3:' ${D}${bindir}/ceph-volume-systemd + sed -i -e 's:/sbin/:/bin/:' ${D}${systemd_system_unitdir}/ceph-volume@.service +} + +TARGET_CC_ARCH += "${LDFLAGS}" +RDEPENDS_${PN} += "\ + bash \ +" + +FILES_${PN} += "\ + ${localstatedir} \ + ${docdir}/ceph/COPYING \ + ${libdir}/sysctl.d/90-ceph-osd.conf \ + ${libdir}/udev/rules.d/50-rbd.rules \ + ${libdir}/udev/rules.d/60-ceph-by-parttypeuuid.rules \ + ${systemd_system_unitdir}/mgr-restful-plugin.service \ + ${systemd_system_unitdir}/ceph-radosgw@.service \ + ${systemd_system_unitdir}/ceph.service \ + ${systemd_system_unitdir}/docker.service.d/starlingx-docker-override.conf \ + home/root/cluster/ceph-mon_config.sh \ + home/root/cluster/ceph-mgr_manual.sh \ + home/root/cluster/ceph-volume_manual.sh \ +" +# /run/ceph diff --git a/recipes-extended/ceph/files/ceph/0001-Add-hooks-for-orderly-shutdown-on-controller.patch b/recipes-extended/ceph/files/ceph/0001-Add-hooks-for-orderly-shutdown-on-controller.patch new file mode 100644 index 0000000..15bb7c3 --- /dev/null +++ b/recipes-extended/ceph/files/ceph/0001-Add-hooks-for-orderly-shutdown-on-controller.patch @@ -0,0 +1,59 @@ +From 03340eaf0004e3cc8e3f8991ea96a46757d92830 Mon Sep 17 00:00:00 2001 +From: Don Penney +Date: Sat, 26 Jan 2019 13:34:55 -0500 +Subject: [PATCH] Add hooks for orderly shutdown on controller + +Hook the ceph init script to add systemd overrides to define +an orderly shutdown for StarlingX controllers. + +Signed-off-by: Don Penney +--- + src/init-ceph.in | 32 ++++++++++++++++++++++++++++++++ + 1 file changed, 32 insertions(+) + +diff --git a/src/init-ceph.in b/src/init-ceph.in +index 1fdb4b3..515d818 100644 +--- a/src/init-ceph.in ++++ b/src/init-ceph.in +@@ -861,6 +861,38 @@ for name in $what; do + fi + fi + ++ . /etc/platform/platform.conf ++ if [ "${nodetype}" = "controller" ]; then ++ # StarlingX: Hook the transient services launched by systemd-run ++ # to allow for proper cleanup and orderly shutdown ++ ++ # Set nullglob so wildcards will return empty string if no match ++ shopt -s nullglob ++ ++ OSD_SERVICES=$(for svc in /run/systemd/system/ceph-osd*.service; do basename $svc; done | xargs echo) ++ for d in /run/systemd/system/ceph-osd*.d; do ++ cat < $d/starlingx-overrides.conf ++[Unit] ++Before=docker.service ++After=sm-shutdown.service ++ ++EOF ++ done ++ ++ for d in /run/systemd/system/ceph-mon*.d; do ++ cat < $d/starlingx-overrides.conf ++[Unit] ++Before=docker.service ++After=sm-shutdown.service ${OSD_SERVICES} ++ ++EOF ++ done ++ ++ shopt -u nullglob ++ ++ systemctl daemon-reload ++ fi ++ + [ -n "$post_start" ] && do_cmd "$post_start" + [ -n "$lockfile" ] && [ "$?" -eq 0 ] && touch $lockfile + ;; +-- +1.8.3.1 + diff --git a/recipes-extended/ceph/files/ceph/ceph-init-wrapper.sh b/recipes-extended/ceph/files/ceph/ceph-init-wrapper.sh new file mode 100755 index 0000000..0a5cd53 --- /dev/null +++ b/recipes-extended/ceph/files/ceph/ceph-init-wrapper.sh @@ -0,0 +1,282 @@ +#!/bin/bash +# +# Copyright (c) 2019 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# This script is a helper wrapper for pmon monitoring of ceph +# processes. The "/etc/init.d/ceph" script does not know if ceph is +# running on the node. For example when the node is locked, ceph +# processes are not running. In that case we do not want pmond to +# monitor these processes. +# +# The script "/etc/services.d//ceph.sh" will create the file +# "/var/run/.ceph_started" when ceph is running and remove it when +# is not. +# +# The script also extracts one or more ceph process names that are +# reported as 'not running' or 'dead' or 'failed' by '/etc/intit.d/ceph status' +# and writes the names to a text file: /tmp/ceph_status_failure.txt for +# pmond to access. The pmond adds the text to logs and alarms. Example of text +# samples written to file by this script are: +# 'osd.1' +# 'osd.1, osd.2' +# 'mon.storage-0' +# 'mon.storage-0, osd.2' +# +# Moreover, for processes that are reported as 'hung' by '/etc/intit.d/ceph status' +# the script will try increase their logging to 'debug' for a configurable interval. +# With logging increased it will outputs a few stack traces then, at the end of this +# interval, it dumps its stack core and kills it. +# +# Return values; +# zero - /etc/init.d/ceph returned success or ceph is not running on the node +# non-zero /etc/init.d/ceph returned a failure or invalid syntax +# + +source /usr/bin/tsconfig +source /etc/platform/platform.conf + +CEPH_SCRIPT="/etc/init.d/ceph" +CEPH_FILE="$VOLATILE_PATH/.ceph_started" +CEPH_RESTARTING_FILE="$VOLATILE_PATH/.ceph_restarting" +CEPH_GET_STATUS_FILE="$VOLATILE_PATH/.ceph_getting_status" +CEPH_STATUS_FAILURE_TEXT_FILE="/tmp/ceph_status_failure.txt" + +BINDIR=/usr/bin +SBINDIR=/usr/sbin +LIBDIR=/usr/lib64/ceph +ETCDIR=/etc/ceph +source $LIBDIR/ceph_common.sh + +LOG_PATH=/var/log/ceph +LOG_FILE=$LOG_PATH/ceph-process-states.log +LOG_LEVEL=NORMAL # DEBUG +verbose=0 + +DATA_PATH=$VOLATILE_PATH/ceph_hang # folder where we keep state information +mkdir -p $DATA_PATH # make sure folder exists + +MONITORING_INTERVAL=15 +TRACE_LOOP_INTERVAL=5 +GET_STATUS_TIMEOUT=120 +CEPH_STATUS_TIMEOUT=20 + +WAIT_FOR_CMD=1 + +RC=0 + +args=("$@") + +if [ ! -z $ARGS ]; then + IFS=";" read -r -a new_args <<< "$ARGS" + args+=("${new_args[@]}") +fi + +wait_for_status () +{ + timeout=$GET_STATUS_TIMEOUT # wait for status no more than $timeout seconds + while [ -f ${CEPH_GET_STATUS_FILE} ] && [ $timeout -gt 0 ]; do + sleep 1 + let timeout-=1 + done + if [ $timeout -eq 0 ]; then + wlog "-" "WARN" "Getting status takes more than ${GET_STATUS_TIMEOUT}s, continuing" + rm -f $CEPH_GET_STATUS_FILE + fi +} + +start () +{ + if [ -f ${CEPH_FILE} ]; then + wait_for_status + ${CEPH_SCRIPT} start $1 + RC=$? + else + # Ceph is not running on this node, return success + exit 0 + fi +} + +stop () +{ + wait_for_status + ${CEPH_SCRIPT} stop $1 +} + +restart () +{ + if [ -f ${CEPH_FILE} ]; then + wait_for_status + touch $CEPH_RESTARTING_FILE + ${CEPH_SCRIPT} restart $1 + rm -f $CEPH_RESTARTING_FILE + else + # Ceph is not running on this node, return success + exit 0 + fi + +} + +log_and_restart_blocked_osds () +{ + # Log info about the blocked osd daemons and then restart it + local names=$1 + for name in $names; do + wlog $name "INFO" "Restarting OSD with blocked operations" + ${CEPH_SCRIPT} restart $name + done +} + +log_and_kill_hung_procs () +{ + # Log info about the hung processes and then kill them; later on pmon will restart them + local names=$1 + for name in $names; do + type=`echo $name | cut -c 1-3` # e.g. 'mon', if $item is 'mon1' + id=`echo $name | cut -c 4- | sed 's/^\\.//'` + get_conf run_dir "/var/run/ceph" "run dir" + get_conf pid_file "$run_dir/$type.$id.pid" "pid file" + pid=$(cat $pid_file) + wlog $name "INFO" "Dealing with hung process (pid:$pid)" + + # monitoring interval + wlog $name "INFO" "Increasing log level" + execute_ceph_cmd ret $name "ceph daemon $name config set debug_$type 20/20" + monitoring=$MONITORING_INTERVAL + while [ $monitoring -gt 0 ]; do + if [ $(($monitoring % $TRACE_LOOP_INTERVAL)) -eq 0 ]; then + date=$(date "+%Y-%m-%d_%H-%M-%S") + log_file="$LOG_PATH/hang_trace_${name}_${pid}_${date}.log" + wlog $name "INFO" "Dumping stack trace to: $log_file" + $(pstack $pid >$log_file) & + fi + let monitoring-=1 + sleep 1 + done + wlog $name "INFO" "Trigger core dump" + kill -ABRT $pid &>/dev/null + rm -f $pid_file # process is dead, core dump is archiving, preparing for restart + # Wait for pending systemd core dumps + sleep 2 # hope systemd_coredump has started meanwhile + deadline=$(( $(date '+%s') + 300 )) + while [[ $(date '+%s') -lt "${deadline}" ]]; do + systemd_coredump_pid=$(pgrep -f "systemd-coredump.*${pid}.*ceph-${type}") + [[ -z "${systemd_coredump_pid}" ]] && break + wlog $name "INFO" "systemd-coredump ceph-${type} in progress: pid ${systemd_coredump_pid}" + sleep 2 + done + kill -KILL $pid &>/dev/null + done +} + + +status () +{ + if [[ "$system_type" == "All-in-one" ]] && [[ "$system_mode" != "simplex" ]] && [[ "$1" == "osd" ]]; then + timeout $CEPH_STATUS_TIMEOUT ceph -s + if [ "$?" -ne 0 ]; then + # Ceph cluster is not accessible. Don't panic, controller swact + # may be in progress. + wlog "-" INFO "Ceph is down, ignoring OSD status." + exit 0 + fi + fi + + if [ -f ${CEPH_RESTARTING_FILE} ]; then + # Ceph is restarting, we don't report state changes on the first pass + rm -f ${CEPH_RESTARTING_FILE} + exit 0 + fi + if [ -f ${CEPH_FILE} ]; then + # Make sure the script does not 'exit' between here and the 'rm -f' below + # or the checkpoint file will be left behind + touch -f ${CEPH_GET_STATUS_FILE} + result=`${CEPH_SCRIPT} status $1` + RC=$? + if [ "$RC" -ne 0 ]; then + erred_procs=`echo "$result" | sort | uniq | awk ' /not running|dead|failed/ {printf "%s ", $1}' | sed 's/://g' | sed 's/, $//g'` + hung_procs=`echo "$result" | sort | uniq | awk ' /hung/ {printf "%s ", $1}' | sed 's/://g' | sed 's/, $//g'` + blocked_ops_procs=`echo "$result" | sort | uniq | awk ' /blocked ops/ {printf "%s ", $1}' | sed 's/://g' | sed 's/, $//g'` + invalid=0 + host=`hostname` + if [[ "$system_type" == "All-in-one" ]] && [[ "$system_mode" != "simplex" ]]; then + # On 2 node configuration we have a floating monitor + host="controller" + fi + for i in $(echo $erred_procs $hung_procs); do + if [[ "$i" =~ osd.?[0-9]?[0-9]|mon.$host ]]; then + continue + else + invalid=1 + fi + done + + log_and_restart_blocked_osds $blocked_ops_procs + log_and_kill_hung_procs $hung_procs + + hung_procs_text="" + for i in $(echo $hung_procs); do + hung_procs_text+="$i(process hung) " + done + + rm -f $CEPH_STATUS_FAILURE_TEXT_FILE + if [ $invalid -eq 0 ]; then + text="" + for i in $erred_procs; do + text+="$i, " + done + for i in $hung_procs; do + text+="$i (process hang), " + done + echo "$text" | tr -d '\n' > $CEPH_STATUS_FAILURE_TEXT_FILE + else + echo "$host: '${CEPH_SCRIPT} status $1' result contains invalid process names: $erred_procs" + echo "Undetermined osd or monitor id" > $CEPH_STATUS_FAILURE_TEXT_FILE + fi + fi + + rm -f ${CEPH_GET_STATUS_FILE} + + if [[ $RC == 0 ]] && [[ "$1" == "mon" ]] && [[ "$system_type" == "All-in-one" ]] && [[ "$system_mode" != "simplex" ]]; then + # SM needs exit code != 0 from 'status mon' argument of the init script on + # standby controller otherwise it thinks that the monitor is running and + # tries to stop it. + # '/etc/init.d/ceph status mon' checks the status of monitors configured in + # /etc/ceph/ceph.conf and if it should be running on current host. + # If it should not be running it just exits with code 0. This is what + # happens on the standby controller. + # When floating monitor is running on active controller /var/lib/ceph/mon of + # standby is not mounted (Ceph monitor partition is DRBD synced). + test -e "/var/lib/ceph/mon/ceph-controller" + if [ "$?" -ne 0 ]; then + exit 3 + fi + fi + else + # Ceph is not running on this node, return success + exit 0 + fi +} + + +case "${args[0]}" in + start) + start ${args[1]} + ;; + stop) + stop ${args[1]} + ;; + restart) + restart ${args[1]} + ;; + status) + status ${args[1]} + ;; + *) + echo "Usage: $0 {start|stop|restart|status} [{mon|osd|osd.|mon.}]" + exit 1 + ;; +esac + +exit $RC diff --git a/recipes-extended/ceph/files/ceph/ceph-manage-journal.py b/recipes-extended/ceph/files/ceph/ceph-manage-journal.py new file mode 100644 index 0000000..f91cbc1 --- /dev/null +++ b/recipes-extended/ceph/files/ceph/ceph-manage-journal.py @@ -0,0 +1,334 @@ +#!/usr/bin/python +# +# Copyright (c) 2019 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import ast +import os +import os.path +import re +import subprocess +import sys + +DEVICE_NAME_NVME = "nvme" + +######### +# Utils # +######### + + +def command(arguments, **kwargs): + """Execute e command and capture stdout, stderr & return code""" + process = subprocess.Popen( + arguments, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **kwargs) + out, err = process.communicate() + return out, err, process.returncode + + +def get_input(arg, valid_keys): + """Convert the input to a dict and perform basic validation""" + json_string = arg.replace("\\n", "\n") + try: + input_dict = ast.literal_eval(json_string) + if not all(k in input_dict for k in valid_keys): + return None + except Exception: + return None + + return input_dict + + +def get_partition_uuid(dev): + output, _, _ = command(['blkid', dev]) + try: + return re.search('PARTUUID=\"(.+?)\"', output).group(1) + except AttributeError: + return None + + +def device_path_to_device_node(device_path): + try: + output, _, _ = command(["udevadm", "settle", "-E", device_path]) + out, err, retcode = command(["readlink", "-f", device_path]) + out = out.rstrip() + except Exception as e: + return None + + return out + + +########################################### +# Manage Journal Disk Partitioning Scheme # +########################################### + +DISK_BY_PARTUUID = "/dev/disk/by-partuuid/" +JOURNAL_UUID = '45b0969e-9b03-4f30-b4c6-b4b80ceff106' # Type of a journal partition + + +def is_partitioning_correct(disk_path, partition_sizes): + """Validate the existence and size of journal partitions""" + + # Obtain the device node from the device path. + disk_node = device_path_to_device_node(disk_path) + + # Check that partition table format is GPT + output, _, _ = command(["udevadm", "settle", "-E", disk_node]) + output, _, _ = command(["parted", "-s", disk_node, "print"]) + if not re.search('Partition Table: gpt', output): + print("Format of disk node %s is not GPT, zapping disk" % disk_node) + return False + + # Check each partition size + partition_index = 1 + for size in partition_sizes: + # Check that each partition size matches the one in input + if DEVICE_NAME_NVME in disk_node: + partition_node = '{}p{}'.format(disk_node, str(partition_index)) + else: + partition_node = '{}{}'.format(disk_node, str(partition_index)) + + output, _, _ = command(["udevadm", "settle", "-E", partition_node]) + cmd = ["parted", "-s", partition_node, "unit", "MiB", "print"] + output, _, _ = command(cmd) + + regex = ("^Disk " + str(partition_node) + ":\\s*" + + str(size) + "[\\.0]*MiB") + if not re.search(regex, output, re.MULTILINE): + print("Journal partition %(node)s size is not %(size)s, " + "zapping disk" % {"node": partition_node, "size": size}) + return False + + partition_index += 1 + + output, _, _ = command(["udevadm", "settle", "-t", "10"]) + return True + + +def create_partitions(disk_path, partition_sizes): + """Recreate partitions""" + + # Obtain the device node from the device path. + disk_node = device_path_to_device_node(disk_path) + + # Issue: After creating a new partition table on a device, Udev does not + # always remove old symlinks (i.e. to previous partitions on that device). + # Also, even if links are erased before zapping the disk, some of them will + # be recreated even though there is no partition to back them! + # Therefore, we have to remove the links AFTER we erase the partition table + # Issue: DISK_BY_PARTUUID directory is not present at all if there are no + # GPT partitions on the storage node so nothing to remove in this case + links = [] + if os.path.isdir(DISK_BY_PARTUUID): + links = [os.path.join(DISK_BY_PARTUUID, l) for l in os.listdir(DISK_BY_PARTUUID) + if os.path.islink(os.path.join(DISK_BY_PARTUUID, l))] + + # Erase all partitions on current node by creating a new GPT table + _, err, ret = command(["parted", "-s", disk_node, "mktable", "gpt"]) + if ret: + print("Error erasing partition table of %(node)s\n" + "Return code: %(ret)s reason: %(reason)s" % + {"node": disk_node, "ret": ret, "reason": err}) + exit(1) + + # Erase old symlinks + for l in links: + if disk_node in os.path.realpath(l): + os.remove(l) + + # Create partitions in order + used_space_mib = 1 # leave 1 MB at the beginning of the disk + num = 1 + for size in partition_sizes: + cmd = ['parted', '-s', disk_node, 'unit', 'mib', + 'mkpart', 'primary', + str(used_space_mib), str(used_space_mib + size)] + _, err, ret = command(cmd) + parms = {"disk_node": disk_node, + "start": used_space_mib, + "end": used_space_mib + size, + "reason": err} + print("Created partition from start=%(start)s MiB to end=%(end)s MiB" + " on %(disk_node)s" % parms) + if ret: + print("Failed to create partition with " + "start=%(start)s, end=%(end)s " + "on %(disk_node)s reason: %(reason)s" % parms) + exit(1) + # Set partition type to ceph journal + # noncritical operation, it makes 'ceph-disk list' output correct info + cmd = ['sgdisk', + '--change-name={num}:ceph journal'.format(num=num), + '--typecode={num}:{uuid}'.format( + num=num, + uuid=JOURNAL_UUID, + ), + disk_node] + _, err, ret = command(cmd) + if ret: + print("WARNINIG: Failed to set partition name and typecode") + used_space_mib += size + num += 1 + + +########################### +# Manage Journal Location # +########################### + +OSD_PATH = "/var/lib/ceph/osd/" + + +def mount_data_partition(data_path, osdid): + """Mount an OSD data partition and return the mounted path""" + + # Obtain the device node from the device path. + data_node = device_path_to_device_node(data_path) + + mount_path = OSD_PATH + "ceph-" + str(osdid) + output, _, _ = command(['mount']) + regex = "^" + data_node + ".*" + mount_path + if not re.search(regex, output, re.MULTILINE): + cmd = ['mount', '-t', 'xfs', data_node, mount_path] + _, _, ret = command(cmd) + params = {"node": data_node, "path": mount_path} + if ret: + print("Failed to mount %(node)s to %(path), aborting" % params) + exit(1) + else: + print("Mounted %(node)s to %(path)s" % params) + return mount_path + + +def is_location_correct(path, journal_path, osdid): + """Check if location points to the correct device""" + + # Obtain the device node from the device path. + journal_node = device_path_to_device_node(journal_path) + + cur_node = os.path.realpath(path + "/journal") + if cur_node == journal_node: + return True + else: + return False + + +def fix_location(mount_point, journal_path, osdid): + """Move the journal to the new partition""" + + # Obtain the device node from the device path. + journal_node = device_path_to_device_node(journal_path) + + # Fix symlink + path = mount_point + "/journal" # 'journal' symlink path used by ceph-osd + journal_uuid = get_partition_uuid(journal_node) + new_target = DISK_BY_PARTUUID + journal_uuid + params = {"path": path, "target": new_target} + try: + if os.path.lexists(path): + os.unlink(path) # delete the old symlink + os.symlink(new_target, path) + print("Symlink created: %(path)s -> %(target)s" % params) + except: + print("Failed to create symlink: %(path)s -> %(target)s" % params) + exit(1) + # Fix journal_uuid + path = mount_point + "/journal_uuid" + try: + with open(path, 'w') as f: + f.write(journal_uuid) + except Exception as ex: + # The operation is noncritical, it only makes 'ceph-disk list' + # display complete output. We log and continue. + params = {"path": path, "uuid": journal_uuid} + print("WARNING: Failed to set uuid of %(path)s to %(uuid)s" % params) + + # Clean the journal partition + # even if erasing the partition table, if another journal was present here + # it's going to be reused. Journals are always bigger than 100MB. + command(['dd', 'if=/dev/zero', 'of=%s' % journal_node, + 'bs=1M', 'count=100']) + + # Format the journal + cmd = ['/usr/bin/ceph-osd', '-i', str(osdid), + '--pid-file', '/var/run/ceph/osd.%s.pid' % osdid, + '-c', '/etc/ceph/ceph.conf', + '--cluster', 'ceph', + '--mkjournal'] + out, err, ret = command(cmd) + params = {"journal_node": journal_node, + "osdid": osdid, + "ret": ret, + "reason": err} + if not ret: + print("Prepared new journal partition: %(journal_node)s " + "for osd id: %(osdid)s" % params) + else: + print("Error initializing journal node: " + "%(journal_node)s for osd id: %(osdid)s " + "ceph-osd return code: %(ret)s reason: %(reason)s" % params) + + +######## +# Main # +######## + +def main(argv): + # parse and validate arguments + err = False + partitions = None + location = None + if len(argv) != 2: + err = True + elif argv[0] == "partitions": + valid_keys = ['disk_path', 'journals'] + partitions = get_input(argv[1], valid_keys) + if not partitions: + err = True + elif not isinstance(partitions['journals'], list): + err = True + elif argv[0] == "location": + valid_keys = ['data_path', 'journal_path', 'osdid'] + location = get_input(argv[1], valid_keys) + if not location: + err = True + elif not isinstance(location['osdid'], int): + err = True + else: + err = True + if err: + print("Command intended for internal use only") + exit(-1) + + if partitions: + # Recreate partitions only if the existing ones don't match input + if not is_partitioning_correct(partitions['disk_path'], + partitions['journals']): + create_partitions(partitions['disk_path'], partitions['journals']) + else: + print("Partition table for %s is correct, " + "no need to repartition" % + device_path_to_device_node(partitions['disk_path'])) + elif location: + # we need to have the data partition mounted & we can let it mounted + mount_point = mount_data_partition(location['data_path'], + location['osdid']) + # Update journal location only if link point to another partition + if not is_location_correct(mount_point, + location['journal_path'], + location['osdid']): + print("Fixing journal location for " + "OSD id: %(id)s" % {"node": location['data_path'], + "id": location['osdid']}) + fix_location(mount_point, + location['journal_path'], + location['osdid']) + else: + print("Journal location for %s is correct," + "no need to change it" % location['data_path']) + + +main(sys.argv[1:]) diff --git a/recipes-extended/ceph/files/ceph/ceph-mds_manual.sh b/recipes-extended/ceph/files/ceph/ceph-mds_manual.sh new file mode 100644 index 0000000..e38cf66 --- /dev/null +++ b/recipes-extended/ceph/files/ceph/ceph-mds_manual.sh @@ -0,0 +1,2 @@ + +ceph-mds --cluster ceph -i starlingx-host-aio/keyring -m 10.30.0.110:6789 diff --git a/recipes-extended/ceph/files/ceph/ceph-mgr_manual.sh b/recipes-extended/ceph/files/ceph/ceph-mgr_manual.sh new file mode 100644 index 0000000..84fd323 --- /dev/null +++ b/recipes-extended/ceph/files/ceph/ceph-mgr_manual.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +mkdir -p /var/lib/ceph/mgr/ceph-starlingx-host-aio +chown ceph:ceph /var/lib/ceph/mgr +chown ceph:ceph /var/lib/ceph/mgr/ceph-starlingx-host-aio +ceph auth get-or-create mgr.starlingx-host-aio mon 'allow profile mgr' osd ' allow *' mds ' allow *' +#systemctl enable ceph-mgr@starlingx-host-aio +systemctl start ceph-mgr@starlingx-host-aio +sleep 4 +ceph mgr module enable status +ceph mgr module enable dashboard +ceph dashboard create-self-signed-cert +ceph dashboard set-login-credentials sysadmin sysadmin diff --git a/recipes-extended/ceph/files/ceph/ceph-mon_config.sh b/recipes-extended/ceph/files/ceph/ceph-mon_config.sh new file mode 100644 index 0000000..cccc4f9 --- /dev/null +++ b/recipes-extended/ceph/files/ceph/ceph-mon_config.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +rm -rf /tmp/ceph.mon.keyring /etc/ceph/ceph.client.admin.keyring /tmp/monmap +rm -rf /var/lib/ceph/mon/ceph-starlingx-host-aio +mkdir -p /var/lib/ceph/mon/ceph-starlingx-host-aio +mkdir -p /var/lib/ceph/mon/ceph-starlingx-host-aio +mkdir -p /var/run/ceph +chown ceph:ceph /var/run/ceph + +ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *' + +ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring \ + --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' \ + --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' + +ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring \ + --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' + +ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring +ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring +monmaptool --create --add starlingx-host-aio 10.30.0.109 --fsid b4fb230f-a5e7-4d72-ba62-ecc8c7dacac5 /tmp/monmap +chown -R ceph:ceph /tmp/monmap +chown -R ceph:ceph /var/lib/ceph/mon/ceph-starlingx-host-aio +chown -R ceph:ceph /tmp/ceph.mon.keyring +sudo -u ceph ceph-mon --mkfs -i starlingx-host-aio --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring +sudo -u ceph touch /var/lib/ceph/mon/ceph-starlingx-host-aio/done +systemctl enable ceph-mon@starlingx-host-aio +systemctl start ceph-mon@starlingx-host-aio diff --git a/recipes-extended/ceph/files/ceph/ceph-preshutdown.sh b/recipes-extended/ceph/files/ceph/ceph-preshutdown.sh new file mode 100644 index 0000000..5f59bd1 --- /dev/null +++ b/recipes-extended/ceph/files/ceph/ceph-preshutdown.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Copyright (c) 2019 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +script=$(basename $0) + +# Set nullglob so wildcards will return empty string if no match +shopt -s nullglob + +for dev in /dev/rbd[0-9]*; do + for mnt in $(mount | awk -v dev=$dev '($1 == dev) {print $3}'); do + logger -t ${script} "Unmounting $mnt" + /usr/bin/umount $mnt + done + logger -t ${script} "Unmounted $dev" +done + +for dev in /dev/rbd[0-9]*; do + /usr/bin/rbd unmap -o force $dev + logger -t ${script} "Unmapped $dev" +done + +lsmod | grep -q '^rbd\>' && /usr/sbin/modprobe -r rbd +lsmod | grep -q '^libceph\>' && /usr/sbin/modprobe -r libceph + +exit 0 + diff --git a/recipes-extended/ceph/files/ceph/ceph-radosgw.service b/recipes-extended/ceph/files/ceph/ceph-radosgw.service new file mode 100644 index 0000000..391ecf6 --- /dev/null +++ b/recipes-extended/ceph/files/ceph/ceph-radosgw.service @@ -0,0 +1,18 @@ +[Unit] +Description=radosgw RESTful rados gateway +After=network.target +#After=remote-fs.target nss-lookup.target network-online.target time-sync.target +#Wants=network-online.target + +[Service] +Type=forking +Restart=no +KillMode=process +RemainAfterExit=yes +ExecStart=/etc/rc.d/init.d/ceph-radosgw start +ExecStop=/etc/rc.d/init.d/ceph-radosgw stop +ExecReload=/etc/rc.d/init.d/ceph-radosgw reload + +[Install] +WantedBy=multi-user.target + diff --git a/recipes-extended/ceph/files/ceph/ceph-volume_manual.sh b/recipes-extended/ceph/files/ceph/ceph-volume_manual.sh new file mode 100644 index 0000000..2115879 --- /dev/null +++ b/recipes-extended/ceph/files/ceph/ceph-volume_manual.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +ceph-volume lvm create --data /dev/sda12 +OSDFSID=$(ceph-volume lvm list | sed -n -e "s/^.*osd fsid.* \([a-z0-9].*$\)/\1/pg") +ceph-volume lvm create --filestore --data /dev/sda12 --journal /dev/sda13 +systemctl enable ceph-volume@lvm-0-$OSDFSID +#ceph-volume lvm activate 0 $OSDFSID +#ceph-volume lvm activate 0 $OSDFSID diff --git a/recipes-extended/ceph/files/ceph/ceph.conf b/recipes-extended/ceph/files/ceph/ceph.conf new file mode 100644 index 0000000..690b8e6 --- /dev/null +++ b/recipes-extended/ceph/files/ceph/ceph.conf @@ -0,0 +1,65 @@ +[global] + # Unique ID for the cluster. + fsid = b4fb230f-a5e7-4d72-ba62-ecc8c7dacac5 + # Public network where the monitor is connected to, i.e, 128.224.0.0/16 + public network = 10.30.0.0/24 + #public network = 127.0.0.1/24 + # For version 0.55 and beyond, you must explicitly enable + # or disable authentication with "auth" entries in [global]. + auth_cluster_required = cephx + auth_service_required = cephx + auth_client_required = cephx + osd_journal_size = 1024 + + # Uncomment the following line if you are mounting with ext4 + # filestore xattr use omap = true + + # Number of replicas of objects. Write an object 2 times. + # Cluster cannot reach an active + clean state until there's enough OSDs + # to handle the number of copies of an object. In this case, it requires + # at least 2 OSDs + osd_pool_default_size = 1 + + # Allow writing one copy in a degraded state. + osd_pool_default_min_size = 1 + + # Ensure you have a realistic number of placement groups. We recommend + # approximately 100 per OSD. E.g., total number of OSDs multiplied by 100 + # divided by the number of replicas (i.e., osd pool default size). So for + # 2 OSDs and osd pool default size = 2, we'd recommend approximately + # (100 * 2) / 2 = 100. + osd_pool_default_pg_num = 64 + osd_pool_default_pgp_num = 64 + osd_crush_chooseleaf_type = 1 + setuser match path = /var/lib/ceph/$type/$cluster-$id + + # Override Jewel default of 2 reporters. StarlingX has replication factor 2 + mon_osd_min_down_reporters = 1 + + # Use Hammer's report interval default value + osd_mon_report_interval_max = 120 + + # Configure max PGs per OSD to cover worst-case scenario of all possible + # StarlingX deployments i.e. AIO-SX with one OSD. Otherwise using + # the default value provided by Ceph Mimic leads to "too many PGs per OSD" + # health warning as the pools needed by stx-openstack are being created. + mon_max_pg_per_osd = 2048 + osd_max_pg_per_osd_hard_ratio = 1.2 +# auth_supported = true +# ms_bind_ipv6 = false + mon initial members = starlingx-host-aio + mon host = 10.30.0.102:6789 + +[osd] + osd_mkfs_type = xfs + osd_mkfs_options_xfs = "-f" + osd_mount_options_xfs = "rw,noatime,inode64,logbufs=8,logbsize=256k" + +[mon] + mon warn on legacy crush tunables = false + # Quiet new warnings on move to Hammer + mon pg warn max per osd = 2048 + mon pg warn max object skew = 0 + mgr initial modules = restful + mon clock drift allowed = .1 +# mon host = starlingx-host-aio diff --git a/recipes-extended/ceph/files/ceph/ceph.conf.pmon b/recipes-extended/ceph/files/ceph/ceph.conf.pmon new file mode 100644 index 0000000..00418b2 --- /dev/null +++ b/recipes-extended/ceph/files/ceph/ceph.conf.pmon @@ -0,0 +1,26 @@ +[process] +process = ceph +script = /etc/init.d/ceph-init-wrapper + +style = lsb +severity = major ; minor, major, critical +restarts = 3 ; restart retries before error assertion +interval = 30 ; number of seconds to wait between restarts + +mode = status ; Monitoring mode: passive (default) or active + ; passive: process death monitoring (default: always) + ; active : heartbeat monitoring, i.e. request / response messaging + ; status : determine process health with executing "status" command + ; "start" is used to start the process(es) again + ; ignore : do not monitor or stop monitoring + +; Status and Active Monitoring Options + +period = 30 ; monitor period in seconds +timeout = 120 ; for active mode, messaging timeout period in seconds, must be shorter than period + ; for status mode, max amount of time for a command to execute + +; Status Monitoring Options +start_arg = start ; start argument for the script +status_arg = status ; status argument for the script +status_failure_text = /tmp/ceph_status_failure.txt ; text to be added to alarms or logs, this is optional diff --git a/recipes-extended/ceph/files/ceph/ceph.service b/recipes-extended/ceph/files/ceph/ceph.service new file mode 100644 index 0000000..d3c2acc --- /dev/null +++ b/recipes-extended/ceph/files/ceph/ceph.service @@ -0,0 +1,16 @@ +[Unit] +Description=StarlingX Ceph Startup +After=network.target + +[Service] +Type=forking +Restart=no +KillMode=process +RemainAfterExit=yes +ExecStart=/etc/rc.d/init.d/ceph start +ExecStop=/etc/rc.d/init.d/ceph stop +PIDFile=/var/run/ceph/ceph.pid + +[Install] +WantedBy=multi-user.target + diff --git a/recipes-extended/ceph/files/ceph/ceph.sh b/recipes-extended/ceph/files/ceph/ceph.sh new file mode 100644 index 0000000..e7e6ecd --- /dev/null +++ b/recipes-extended/ceph/files/ceph/ceph.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +INITDIR=/etc/init.d +LOGFILE=/var/log/ceph/ceph-init.log +CEPH_FILE=/var/run/.ceph_started + +# Get our nodetype +. /etc/platform/platform.conf + +# Exit immediately if ceph not configured (i.e. no mon in the config file) +if ! grep -q "mon\." /etc/ceph/ceph.conf +then + exit 0 +fi + +logecho () +{ + echo $1 + date >> ${LOGFILE} + echo $1 >> ${LOGFILE} +} + +start () +{ + logecho "Starting ceph services..." + ${INITDIR}/ceph start >> ${LOGFILE} 2>&1 + RC=$? + + if [ ! -f ${CEPH_FILE} ]; then + touch ${CEPH_FILE} + fi +} + +stop () +{ + if [[ "$system_type" == "All-in-one" ]] && [[ "$system_mode" == "simplex" ]]; then + logecho "Ceph services will continue to run on node" + exit 0 + fi + + logecho "Stopping ceph services..." + + if [ -f ${CEPH_FILE} ]; then + rm -f ${CEPH_FILE} + fi + + ${INITDIR}/ceph stop >> ${LOGFILE} 2>&1 + RC=$? +} + +RC=0 + +case "$1" in + start) + start + ;; + stop) + stop + ;; + *) + echo "Usage: $0 {start|stop}" + exit 1 + ;; +esac + +logecho "RC was: $RC" +exit $RC diff --git a/recipes-extended/ceph/files/ceph/mgr-restful-plugin.py b/recipes-extended/ceph/files/ceph/mgr-restful-plugin.py new file mode 100644 index 0000000..c1cae60 --- /dev/null +++ b/recipes-extended/ceph/files/ceph/mgr-restful-plugin.py @@ -0,0 +1,1056 @@ +#!/usr/bin/python +# +# Copyright (c) 2019 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +### BEGIN INIT INFO +# Provides: ceph/mgr RESTful API plugin +# Required-Start: $ceph +# Required-Stop: $ceph +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Ceph MGR RESTful API plugin +# Description: Ceph MGR RESTful API plugin +### END INIT INFO + +import argparse +import contextlib +import errno +import fcntl +import inspect +import json +import logging +import multiprocessing +import os +import signal +import socket +import subprocess +import sys +import time + +import daemon +import psutil +import requests + +# 'timeout' command returns exit status 124 +# if command times out (see man page) +GNU_TIMEOUT_EXPIRED_RETCODE = 124 + + +def psutil_terminate_kill(target, timeout): + + """Extend psutil functionality to stop a process. + + SIGINT is sent to each target then after a grace period SIGKILL + is sent to the ones that are still running. + """ + + if not isinstance(target, list): + target = [target] + _, target = psutil.wait_procs(target, timeout=0) + for action in [lambda p: p.terminate(), lambda p: p.kill()]: + for proc in target: + action(proc) + _, target = psutil.wait_procs( + target, timeout=timeout) + + +class Config(object): + + """ceph-mgr service wrapper configuration options. + + In the future we may want to load them from a configuration file + (for example /etc/ceph/mgr-restful-plugin.conf ) + """ + + def __init__(self): + self.log_level = logging.INFO + self.log_dir = '/var/log' + + self.ceph_mgr_service = '/usr/bin/ceph-mgr' + self.ceph_mgr_cluster = 'ceph' + self.ceph_mgr_rundir = '/var/run/ceph/mgr' + self.ceph_mgr_identity = socket.gethostname() + + self.service_name = 'mgr-restful-plugin' + self.service_socket = os.path.join( + self.ceph_mgr_rundir, '{}.socket'.format(self.service_name)) + self.service_lock = os.path.join( + self.ceph_mgr_rundir, '{}.lock'.format(self.service_name)) + self.service_pid_file = os.path.join( + '/var/run/ceph', '{}.pid'.format(self.service_name)) + + self.restful_plugin_port = 5001 + + # maximum size of a message received/sent via + # service monitor control socket + self.service_socket_bufsize = 1024 + + # maximum time to wait for ceph cli to exit + self.ceph_cli_timeout_sec = 30 + + # how much time to wait after ceph cli commands fail with timeout + # before running any other commands + self.cluster_grace_period_sec = 30 + + # after ceph-mgr is started it goes through an internal initialization + # phase before; how much time to wait before querying ceph-mgr + self.ceph_mgr_grace_period_sec = 15 + + # after sending SIGTERM to ceph-mgr how much time to wait before + # sending SIGKILL (maximum time allowed for ceph-mgr cleanup) + self.ceph_mgr_kill_delay_sec = 5 + + # if service monitor is running a recovery procedure it reports + # status OK even if ceph-mgr is currently down. This sets the + # maximum number of consecutive ceph-mgr failures before reporting + # status error + self.ceph_mgr_fail_count_report_error = 3 + + # maximum number of consecutive ceph-mgr failures before + # stopping mgr-restful-plugin service + self.ceph_mgr_fail_count_exit = 5 + + # maximum time allowed for ceph-mgr to respond to a REST API request + self.rest_api_timeout_sec = 15 + + # interval between consecutive REST API requests (ping's). A smaller + # value here triggers more requests to ceph-mgr restful plugin. A + # higher value makes recovery slower when services become unavailable + self.restful_plugin_ping_delay_sec = 3 + + # where to save the self-signed certificate generated by ceph-mgr + self.restful_plugin_cert_path = os.path.join( + self.ceph_mgr_rundir, 'restful.crt') + + # time to wait after enabling restful plugin + self.restful_plugin_grace_period_sec = 3 + + # after how many REST API ping failures to restart ceph-mgr + self.ping_fail_count_restart_mgr = 3 + + # after how many REST API ping failures to report status error. + # Until then service monitor reports status OK just in case + # restful plugin recovers + self.ping_fail_count_report_error = 5 + + @staticmethod + def load(): + return Config() + + +def setup_logging(name=None, cleanup_handlers=False): + if not name: + name = CONFIG.service_name + log = logging.getLogger(name) + log.setLevel(CONFIG.log_level) + if cleanup_handlers: + try: + for handler in log.handlers: + if isinstance(handler, logging.StreamHandler): + handler.flush() + if isinstance(handler, logging.FileHandler): + handler.close() + log.handlers = [] + except Exception: + pass + elif log.handlers: + return log + handler = logging.FileHandler( + os.path.join(CONFIG.log_dir, + '{}.log'.format(CONFIG.service_name))) + handler.setFormatter( + logging.Formatter('%(asctime)s %(process)s %(levelname)s %(name)s %(message)s')) + log.addHandler(handler) + return log + + +CONFIG = Config.load() +LOG = setup_logging(name='init-wrapper') + + +class ServiceException(Exception): + + """Generic mgr-restful-plugin service exception. + + Build exception string based on static (per exception class) + string plus args, keyword args passed to exception constructor. + """ + + message = "" + + def __init__(self, *args, **kwargs): + if "message" not in kwargs: + try: + message = self.message.format(*args, **kwargs) + except Exception: # noqa + message = '{}, args:{}, kwargs: {}'.format( + self.message, args, kwargs) + else: + message = kwargs["message"] + super(ServiceException, self).__init__(message) + + +class ServiceAlreadyStarted(ServiceException): + message = ('Service monitor already started') + + +class ServiceLockFailed(ServiceException): + message = ('Unable to lock service monitor: ' + 'reason={reason}') + + +class ServiceNoSocket(ServiceException): + message = ('Unable to create service monitor socket: ' + 'reason={reason}') + + +class ServiceSocketBindFailed(ServiceException): + message = ('Failed to bind service monitor socket: ' + 'path={path}, reason={reason}') + + +class ServiceNoPidFile(ServiceException): + message = ('Failed to update pid file: ' + 'path={path}, reason={reason}') + + +class CommandFailed(ServiceException): + message = ('Command failed: command={command}, ' + 'reason={reason}, out={out}') + + +class CommandTimeout(ServiceException): + message = ('Command timeout: command={command}, ' + 'timeout={timeout}') + + +class CephMgrStartFailed(ServiceException): + message = ('Failed to start ceph_mgr: ' + 'reason={reason}') + + +class CephRestfulPluginFailed(ServiceException): + message = ('Failed to start restful plugin: ' + 'reason={reason}') + + +class RestApiPingFailed(ServiceException): + message = ('REST API ping failed: ' + 'reason={reason}') + + +class ServiceMonitor(object): + + """Configure and monitor ceph-mgr and restful plugin (Ceph REST API) + + 1. process init script service requests: status, stop. Requests are + received via a control socket. Stop has priority over whatever + the monitor is doing currently. Any ceph command that may be running + is terminated/killed. Note that while ceph-mgr and restful plugin + configuration is in progress ServiceMonitor reports status OK to + avoid being restarted by SM. + + 2. configure ceph-mgr and mgr restful plugin: authentication, REST API + service port, self signed certificate. This runs as a separate + process so it can be stopped when init script requests it. + + 3. periodically check (ping) REST API responds to HTTPS requests. + Recovery actions are taken if REST API fails to respond: restart + ceph-mgr, wait for cluster to become available again. + """ + + def __init__(self): + # process running configuration & REST API ping loop + self.monitor = None + + # command socket used by init script + self.command = None + + # ceph-mgr process + self.ceph_mgr = None + + # consecutive ceph-mgr/restful-plugin start failures. Service monitor + # reports failure after CONFIG.ceph_mgr_max_failure_count + self.ceph_mgr_failure_count = 0 + + # consecutive REST API ping failures. ceph-mgr service is restarted + # after CONFIG.ping_fail_count_restart_mgr threshold is exceeded + self.ping_failure_count = 0 + + # REST API url reported by ceph-mgr after enabling restful plugin + self.restful_plugin_url = '' + + # REST API self signed certificate generated by restful plugin + self.certificate = '' + + def run(self): + self.disable_certificate_check() + with self.service_lock(), self.service_socket(), \ + self.service_pid_file(): + self.start_monitor() + self.server_loop() + + def disable_certificate_check(self): + # ceph-mgr restful plugin is configured with a self-signed + # certificate. Certificate host is hard-coded to "ceph-restful" + # which causes HTTPS requests to fail because they don't + # match current host name ("controller-..."). Disable HTTPS + # certificates check in urllib3 + LOG.warning('Disable urllib3 certifcates check') + requests.packages.urllib3.disable_warnings() + + def server_loop(self): + self.command.listen(2) + while True: + try: + client, _ = self.command.accept() + request = client.recv(CONFIG.service_socket_bufsize) + LOG.debug('Monitor command socket: request=%s', str(request)) + cmd = request.split(' ') + cmd, args = cmd[0], cmd[1:] + if cmd == 'status': + self.send_response(client, request, self.status()) + elif cmd == 'stop': + self.stop() + self.send_response(client, request, 'OK') + break + elif cmd == 'restful-url': + try: + self.restful_plugin_url = args[0] + self.send_response(client, request, 'OK') + except IndexError: + LOG.warning('Failed to update restful plugin url: ' + 'args=%s', str(args)) + self.send_response(client, request, 'ERR') + elif cmd == 'certificate': + try: + self.certificate = args[0] if args else '' + self.send_response(client, request, 'OK') + except IndexError: + LOG.warning('Failed to update certificate path: ' + 'args=%s', str(args)) + self.send_response(client, request, 'ERR') + elif cmd == 'ceph-mgr-failures': + try: + self.ceph_mgr_failure_count = int(args[0]) + self.send_response(client, request, 'OK') + if self.ceph_mgr_failure_count >= CONFIG.ceph_mgr_fail_count_exit: + self.stop() + break + except (IndexError, ValueError): + LOG.warning('Failed to update ceph-mgr failures: ' + 'args=%s', str(args)) + self.send_response(client, request, 'ERR') + elif cmd == 'ping-failures': + try: + self.ping_failure_count = int(args[0]) + self.send_response(client, request, 'OK') + except (IndexError, ValueError): + LOG.warning('Failed to update ping failures: ' + 'args=%s', str(args)) + self.send_response(client, request, 'ERR') + except Exception as err: + LOG.exception(err) + + @staticmethod + def send_response(client, request, response): + try: + client.send(response) + except socket.error as err: + LOG.warning('Failed to send response back. ' + 'request=%s, response=%s, reason=%s', + request, response, err) + + def status(self): + if not self.restful_plugin_url: + if self.ceph_mgr_failure_count < CONFIG.ceph_mgr_fail_count_report_error \ + and self.ping_failure_count < CONFIG.ping_fail_count_report_error: + LOG.debug('Monitor is starting services. Report status OK') + return 'OK' + LOG.debug('Too many failures: ' + 'ceph_mgr=%d < %d, ping=%d < %d. ' + 'Report status ERR', + self.ceph_mgr_failure_count, + CONFIG.ceph_mgr_fail_count_report_error, + self.ping_failure_count, + CONFIG.ping_fail_count_report_error) + return 'ERR.down' + try: + self.restful_plugin_ping() + LOG.debug('Restful plugin ping successful. Report status OK') + return 'OK' + except (CommandFailed, RestApiPingFailed): + if self.ceph_mgr_failure_count < CONFIG.ceph_mgr_fail_count_report_error \ + and self.ping_failure_count < CONFIG.ping_fail_count_report_error: + LOG.info('Restful plugin does not respond but failure ' + 'count is within acceptable limits: ' + ' ceph_mgr=%d < %d, ping=%d < %d. ' + 'Report status OK', + self.ceph_mgr_failure_count, + CONFIG.ceph_mgr_fail_count_report_error, + self.ping_failure_count, + CONFIG.ping_fail_count_report_error) + return 'OK' + LOG.debug('Restful does not respond (ping failure count %d). ' + 'Report status ERR', self.ping_failure_count) + return 'ERR.ping_failed' + + def stop(self): + if not self.monitor: + return + LOG.info('Stop monitor with SIGTERM to process group %d', + self.monitor.pid) + try: + os.killpg(self.monitor.pid, signal.SIGTERM) + except OSError as err: + LOG.info('Stop monitor failed: reason=%s', str(err)) + return + time.sleep(CONFIG.ceph_mgr_kill_delay_sec) + LOG.info('Stop monitor with SIGKILL to process group %d', + self.monitor.pid) + try: + os.killpg(self.monitor.pid, signal.SIGKILL) + os.waitpid(self.monitor.pid, 0) + except OSError as err: + LOG.info('Stop monitor failed: reason=%s', str(err)) + return + LOG.info('Monitor stopped: pid=%d', self.monitor.pid) + + @contextlib.contextmanager + def service_lock(self): + LOG.info('Take service lock: path=%s', CONFIG.service_lock) + try: + os.makedirs(os.path.dirname(CONFIG.service_lock)) + except OSError: + pass + lock_file = open(CONFIG.service_lock, 'w') + try: + fcntl.flock(lock_file.fileno(), + fcntl.LOCK_EX | fcntl.LOCK_NB) + except (IOError, OSError) as err: + if err.errno == errno.EAGAIN: + raise ServiceAlreadyStarted() + else: + raise ServiceLockFailed(reason=str(err)) + # even if we have the lock here there might be another service manager + # running whose CONFIG.ceph_mgr_rundir was removed before starting + # this instance. Make sure there is only one service manager running + self.stop_other_service_managers() + try: + yield + finally: + os.unlink(CONFIG.service_lock) + lock_file.close() + LOG.info('Release service lock: path=%s', CONFIG.service_lock) + + def stop_other_service_managers(self): + service = os.path.join('/etc/init.d', CONFIG.service_name) + for p in psutil.process_iter(): + if p.cmdline()[:2] not in [[service], ['/usr/bin/python', service]]: + continue + if p.pid == os.getpid(): + continue + p.kill() + + @contextlib.contextmanager + def service_socket(self): + LOG.info('Create service socket') + try: + self.command = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) + except socket.error as err: + raise ServiceNoSocket(reason=str(err)) + LOG.info('Remove existing socket files') + try: + os.unlink(CONFIG.service_socket) + except OSError: + pass + LOG.info('Bind service socket: path=%s', CONFIG.service_socket) + try: + self.command.bind(CONFIG.service_socket) + except socket.error as err: + raise ServiceSocketBindFailed( + path=CONFIG.service_socket, reason=str(err)) + try: + yield + finally: + LOG.info('Close service socket and remove file: path=%s', + CONFIG.service_socket) + self.command.close() + os.unlink(CONFIG.service_socket) + + @contextlib.contextmanager + def service_pid_file(self): + LOG.info('Update service pid file: path=%s', CONFIG.service_pid_file) + try: + pid_file = open(CONFIG.service_pid_file, 'w') + pid_file.write(str(os.getpid())) + except OSError as err: + raise ServiceNoPidFile( + path=CONFIG.service_pid_file, reason=str(err)) + try: + yield + finally: + LOG.info('Remove service pid file: path=%s', + CONFIG.service_pid_file) + try: + os.unlink(CONFIG.service_pid_file) + except OSError: + pass + + def start_monitor(self): + LOG.info('Start monitor loop') + self.monitor = multiprocessing.Process(target=self.monitor_loop) + self.monitor.start() + + def stop_unmanaged_ceph_mgr(self): + LOG.info('Stop unmanaged running ceph-mgr processes') + service_name = os.path.basename(CONFIG.ceph_mgr_service) + if self.ceph_mgr: + psutil_terminate_kill( + [proc for proc in psutil.process_iter() + if (proc.name() == service_name + and proc.pid != self.ceph_mgr.pid)], + CONFIG.ceph_mgr_kill_delay_sec) + else: + psutil_terminate_kill( + [proc for proc in psutil.process_iter() + if proc.name() == service_name], + CONFIG.ceph_mgr_kill_delay_sec) + + def monitor_loop(self): + + """Bring up and monitor ceph-mgr restful plugin. + + Steps: + - wait for Ceph cluster to become available + - configure and start ceph-mgr + - configure and enable restful plugin + - send periodic requests to REST API + - recover from failures + + Note: because this runs as a separate process it + must send status updates to service monitor + via control socket for: ping_failure_count, + restful_plugin_url and certificate. + """ + + # Promote to process group leader so parent (service monitor) + # can kill the monitor plus processes spawned by it. Otherwise + # children of monitor_loop() will keep running in background and + # will be reaped by init when they finish but by then they might + # interfere with any new service instance. + os.setpgrp() + + # Ignoring SIGTERM here ensures process group is not reused by + # the time parent (service monitor) issues the final SIGKILL. + signal.signal(signal.SIGTERM, signal.SIG_IGN) + + while True: + try: + # steps to configure/start ceph-mgr and restful plugin + self.ceph_fsid_get() + self.ceph_mgr_auth_create() + self.ceph_mgr_start() + self.restful_plugin_set_server_port() + self.restful_plugin_enable() + self.restful_plugin_create_certificate() + self.restful_plugin_create_admin_key() + self.restful_plugin_get_url() + self.restful_plugin_get_certificate() + + # REST API should be available now + # start making periodic requests (ping) + while True: + try: + self.restful_plugin_ping() + self.ping_failure_count = 0 + self.request_update_ping_failures( + self.ping_failure_count) + self.ceph_mgr_failure_count = 0 + self.request_update_ceph_mgr_failures( + self.ceph_mgr_failure_count) + time.sleep(CONFIG.restful_plugin_ping_delay_sec) + continue + except RestApiPingFailed as err: + LOG.warning(str(err)) + + LOG.info('REST API ping failure count=%d', + self.ping_failure_count) + self.ping_failure_count += 1 + self.request_update_ping_failures( + self.ping_failure_count) + + # maybe request failed because ceph-mgr is not running + if not self.ceph_mgr_is_running(): + self.ceph_mgr_failure_count += 1 + self.request_update_ceph_mgr_failures( + self.ceph_mgr_failure_count) + self.ceph_mgr_start() + time.sleep(CONFIG.ceph_mgr_grace_period_sec) + continue + + # maybe request failed because cluster health is not ok + if not self.ceph_fsid_get(): + LOG.info('Unable to get cluster fsid. ' + 'Sleep for a while') + time.sleep(CONFIG.cluster_grace_period_sec) + break + + # too many failures? Restart ceph-mgr and go again + # through configuration steps + if (self.ping_failure_count + % CONFIG.ping_fail_count_restart_mgr == 0): + LOG.info('Too many consecutive REST API failures. ' + 'Restart ceph-mgr. Update service ' + 'url and certificate') + self.ceph_mgr_stop() + self.restful_plugin_url = '' + self.request_update_plugin_url(self.restful_plugin_url) + self.certificate = '' + self.request_update_certificate(self.certificate) + break + + time.sleep(CONFIG.restful_plugin_ping_delay_sec) + + except CommandFailed as err: + LOG.warning(str(err)) + time.sleep(CONFIG.cluster_grace_period_sec) + except CommandTimeout as err: + LOG.warning(str(err)) + except (CephMgrStartFailed, CephRestfulPluginFailed) as err: + LOG.warning(str(err)) + self.ceph_mgr_failure_count += 1 + self.request_update_ceph_mgr_failures( + self.ceph_mgr_failure_count) + time.sleep(CONFIG.ceph_mgr_grace_period_sec) + except Exception as err: + LOG.exception(err) + time.sleep(CONFIG.cluster_grace_period_sec) + + @staticmethod + def run_with_timeout(command, timeout, stderr=subprocess.STDOUT): + try: + LOG.info('Run command: %s', ' '.join(command)) + return subprocess.check_output( + ['/usr/bin/timeout', str(timeout)] + command, + stderr=stderr, shell=False).strip() + except subprocess.CalledProcessError as err: + if err.returncode == GNU_TIMEOUT_EXPIRED_RETCODE: + raise CommandTimeout(command=err.cmd, timeout=timeout) + raise CommandFailed(command=err.cmd, reason=str(err), + out=err.output) + + def ceph_fsid_get(self): + return self.run_with_timeout(['/usr/bin/ceph', 'fsid'], + CONFIG.ceph_cli_timeout_sec) + + def ceph_mgr_has_auth(self): + path = '{}/ceph-{}'.format( + CONFIG.ceph_mgr_rundir, CONFIG.ceph_mgr_identity) + try: + os.makedirs(path) + except OSError as err: + pass + try: + self.run_with_timeout( + ['/usr/bin/ceph', 'auth', 'get', + 'mgr.{}'.format(CONFIG.ceph_mgr_identity), + '-o', '{}/keyring'.format(path)], + CONFIG.ceph_cli_timeout_sec) + return True + except CommandFailed as err: + if 'ENOENT' in str(err): + return False + raise + + def ceph_mgr_auth_create(self): + if self.ceph_mgr_has_auth(): + return + LOG.info('Create ceph-mgr authentication') + self.run_with_timeout( + ['/usr/bin/ceph', 'auth', 'get-or-create', + 'mgr.{}'.format(CONFIG.ceph_mgr_identity), + 'mon', 'allow *', 'osd', 'allow *'], + CONFIG.ceph_cli_timeout_sec) + + def ceph_mgr_is_running(self): + if not self.ceph_mgr: + return None + try: + self.ceph_mgr.wait(timeout=0) + except psutil.TimeoutExpired: + return True + return False + + def ceph_mgr_start(self): + if self.ceph_mgr_is_running(): + return + self.stop_unmanaged_ceph_mgr() + LOG.info('Start ceph-mgr daemon') + try: + with open(os.devnull, 'wb') as null: + self.ceph_mgr = psutil.Popen( + [CONFIG.ceph_mgr_service, + '--cluster', CONFIG.ceph_mgr_cluster, + '--id', CONFIG.ceph_mgr_identity, + '-f'], + close_fds=True, + stdout=null, + stderr=null, + shell=False) + except (OSError, ValueError) as err: + raise CephMgrStartFailed(reason=str(err)) + time.sleep(CONFIG.ceph_mgr_grace_period_sec) + + def ceph_mgr_stop(self): + if not self.ceph_mgr: + return + LOG.info('Stop ceph-mgr') + psutil_terminate_kill(self.ceph_mgr, CONFIG.ceph_mgr_kill_delay_sec) + + def restful_plugin_has_server_port(self): + try: + with open(os.devnull, 'wb') as null: + out = self.run_with_timeout( + ['/usr/bin/ceph', 'config-key', 'get', + 'config/mgr/mgr/restful/server_port'], + CONFIG.ceph_cli_timeout_sec, stderr=null) + if out == str(CONFIG.restful_plugin_port): + return True + LOG.warning('Restful plugin port mismatch: ' + 'current=%d, expected=%d', out, + CONFIG.restful_plugin_port) + except CommandFailed as err: + LOG.warning('Failed to get restful plugin port: ' + 'reason=%s', str(err)) + return False + + def restful_plugin_set_server_port(self): + if self.restful_plugin_has_server_port(): + return + LOG.info('Set restful plugin port=%d', CONFIG.restful_plugin_port) + self.run_with_timeout( + ['/usr/bin/ceph', 'config', 'set', 'mgr', + 'mgr/restful/server_port', str(CONFIG.restful_plugin_port)], + CONFIG.ceph_cli_timeout_sec) + + def restful_plugin_has_admin_key(self): + try: + self.run_with_timeout( + ['/usr/bin/ceph', 'config-key', 'get', + 'mgr/restful/keys/admin'], + CONFIG.ceph_cli_timeout_sec) + return True + except CommandFailed: + pass + return False + + def restful_plugin_create_admin_key(self): + if self.restful_plugin_has_admin_key(): + return + LOG.info('Create restful plugin admin key') + self.run_with_timeout( + ['/usr/bin/ceph', 'restful', + 'create-key', 'admin'], + CONFIG.ceph_cli_timeout_sec) + + def restful_plugin_has_certificate(self): + try: + self.run_with_timeout( + ['/usr/bin/ceph', 'config-key', 'get', + 'mgr/restful/{}/crt'.format(CONFIG.ceph_mgr_identity)], + CONFIG.ceph_cli_timeout_sec) + return True + except CommandFailed: + pass + return False + + def restful_plugin_create_certificate(self): + if self.restful_plugin_has_certificate(): + return + LOG.info('Create restful plugin self signed certificate') + self.run_with_timeout( + ['/usr/bin/ceph', 'restful', + 'create-self-signed-cert'], + CONFIG.ceph_cli_timeout_sec) + + def restful_plugin_is_enabled(self): + command = ['/usr/bin/ceph', 'mgr', 'module', 'ls', + '--format', 'json'] + with open(os.devnull, 'wb') as null: + out = self.run_with_timeout( + command, CONFIG.ceph_cli_timeout_sec, stderr=null) + try: + if 'restful' in json.loads(out)['enabled_modules']: + return True + except ValueError as err: + raise CommandFailed( + command=' '.join(command), + reason='unable to decode json: {}'.format(err), out=out) + except KeyError as err: + raise CommandFailed( + command=' '.join(command), + reason='missing expected key: {}'.format(err), out=out) + return False + + def restful_plugin_enable(self): + if not self.restful_plugin_is_enabled(): + LOG.info('Enable restful plugin') + self.run_with_timeout( + ['/usr/bin/ceph', 'mgr', + 'module', 'enable', 'restful'], + CONFIG.ceph_cli_timeout_sec) + time.sleep(CONFIG.restful_plugin_grace_period_sec) + + def restful_plugin_get_url(self): + command = ['/usr/bin/ceph', 'mgr', 'services', + '--format', 'json'] + with open(os.devnull, 'wb') as null: + out = self.run_with_timeout( + command, CONFIG.ceph_cli_timeout_sec, stderr=null) + try: + self.restful_plugin_url = json.loads(out)['restful'] + except ValueError as err: + raise CephRestfulPluginFailed( + reason='unable to decode json: {} output={}'.format(err, out)) + except KeyError as err: + raise CephRestfulPluginFailed( + reason='missing expected key: {} in ouput={}'.format(err, out)) + self.request_update_plugin_url(self.restful_plugin_url) + + def restful_plugin_get_certificate(self): + command = ['/usr/bin/ceph', 'config-key', 'get', + 'mgr/restful/controller-0/crt'] + with open(os.devnull, 'wb') as null: + certificate = self.run_with_timeout( + command, CONFIG.ceph_cli_timeout_sec, stderr=null) + with open(CONFIG.restful_plugin_cert_path, 'wb') as cert_file: + cert_file.write(certificate) + self.certificate = CONFIG.restful_plugin_cert_path + self.request_update_certificate( + self.certificate) + + def restful_plugin_ping(self): + if not self.restful_plugin_url: + raise RestApiPingFailed(reason='missing service url') + if not self.certificate: + raise RestApiPingFailed(reason='missing certificate') + LOG.debug('Ping restful plugin: url=%d', self.restful_plugin_url) + try: + response = requests.request( + 'GET', self.restful_plugin_url, verify=False, + timeout=CONFIG.rest_api_timeout_sec) + if not response.ok: + raise RestApiPingFailed( + reason='response not ok ({})'.format(response)) + LOG.debug('Ping restful plugin OK') + except (requests.ConnectionError, + requests.Timeout, + requests.HTTPError) as err: + raise RestApiPingFailed(reason=str(err)) + + @staticmethod + def _make_client_socket(): + sock = socket.socket( + socket.AF_UNIX, socket.SOCK_SEQPACKET) + sock.settimeout(2 * CONFIG.rest_api_timeout_sec) + sock.connect(CONFIG.service_socket) + return sock + + @staticmethod + def request_status(): + try: + with contextlib.closing( + ServiceMonitor._make_client_socket()) as sock: + sock.send('status') + status = sock.recv(CONFIG.service_socket_bufsize) + LOG.debug('Status %s', status) + return status.startswith('OK') + except socket.error as err: + LOG.error('Status error: reason=%s', err) + return False + + @staticmethod + def request_stop(): + try: + with contextlib.closing( + ServiceMonitor._make_client_socket()) as sock: + sock.send('stop') + response = sock.recv(CONFIG.service_socket_bufsize) + LOG.debug('Stop response: %s', response) + return True + except socket.error as err: + LOG.error('Stop error: reason=%s', err) + return False + + @staticmethod + def request_update_ceph_mgr_failures(count): + try: + with contextlib.closing( + ServiceMonitor._make_client_socket()) as sock: + sock.send('ceph-mgr-failures {}'.format(count)) + sock.recv(CONFIG.service_socket_bufsize) + return True + except socket.error as err: + LOG.error('Stop error: reason=%s', err) + return False + + @staticmethod + def request_update_ping_failures(count): + try: + with contextlib.closing( + ServiceMonitor._make_client_socket()) as sock: + sock.send('ping-failures {}'.format(count)) + sock.recv(CONFIG.service_socket_bufsize) + return True + except socket.error as err: + LOG.error('Stop error: reason=%s', err) + return False + + @staticmethod + def request_update_plugin_url(url): + try: + with contextlib.closing( + ServiceMonitor._make_client_socket()) as sock: + sock.send('restful-url {}'.format(url)) + sock.recv(CONFIG.service_socket_bufsize) + return True + except socket.error as err: + LOG.error('Stop error: reason=%s', err) + return False + + @staticmethod + def request_update_certificate(path): + try: + with contextlib.closing( + ServiceMonitor._make_client_socket()) as sock: + sock.send('certificate {}'.format(path)) + sock.recv(CONFIG.service_socket_bufsize) + return True + except socket.error as err: + LOG.error('Stop error: reason=%s', err) + return False + + +class InitWrapper(object): + + """Handle System V init script actions: start, stop, restart, etc. """ + + def __init__(self): + + """Dispatch command line action to the corresponding function. + + Candidate action functions are all class methods except ones + that start with an underscore. + """ + + parser = argparse.ArgumentParser() + actions = [m[0] + for m in inspect.getmembers(self) + if (inspect.ismethod(m[1]) + and not m[0].startswith('_'))] + parser.add_argument( + 'action', + choices=actions) + self.args = parser.parse_args() + getattr(self, self.args.action)() + + def start(self): + + """Start ServiceMonitor as a daemon unless one is already running. + + Use a pipe to report monitor status back to this process. + """ + + pipe = os.pipe() + child = os.fork() + if child == 0: + os.close(pipe[0]) + with daemon.DaemonContext(files_preserve=[pipe[1]]): + # prevent duplication of messages in log + global LOG + LOG = setup_logging(cleanup_handlers=True) + try: + monitor = ServiceMonitor() + status = 'OK' + except ServiceAlreadyStarted: + os.write(pipe[1], 'OK') + os.close(pipe[1]) + return + except Exception as err: + status = str(err) + os.write(pipe[1], status) + os.close(pipe[1]) + if status == 'OK': + try: + monitor.run() + except ServiceException as err: + LOG.warning(str(err)) + except Exception as err: + LOG.exception('Service monitor error: reason=%s', err) + else: + os.close(pipe[1]) + try: + status = os.read(pipe[0], CONFIG.service_socket_bufsize) + if status == 'OK': + sys.exit(0) + else: + LOG.warning('Service monitor failed to start: ' + 'status=%s', status) + except IOError as err: + LOG.warning('Failed to read monitor status: reason=%s', err) + os.close(pipe[0]) + os.waitpid(child, 0) + sys.exit(1) + + def stop(self): + + """Tell ServiceMonitor daemon to stop running. + + In case request fails stop ServiceMonitor and ceph_mgr proecsses + using SIGTERM followed by SIGKILL. + """ + + result = ServiceMonitor.request_stop() + if not result: + ceph_mgr = os.path.basename(CONFIG.ceph_mgr_service) + procs = [] + for proc in psutil.process_iter(): + name = proc.name() + if name == CONFIG.service_name: + procs.append(proc) + if name == ceph_mgr: + procs.append(proc) + psutil_terminate_kill(procs, CONFIG.ceph_mgr_kill_delay_sec) + + def restart(self): + self.stop() + self.start() + + def force_reload(self): + self.stop() + self.start() + + def reload(self): + self.stop() + self.start() + + def status(self): + + """Report status from ServiceMonitor. + + We don't just try to access REST API here because ServiceMonitor may + be in the process of starting/configuring ceph-mgr and restful + plugin in which case we report OK to avoid being restarted by SM. + """ + + status = ServiceMonitor.request_status() + sys.exit(0 if status is True else 1) + + +if __name__ == '__main__': + InitWrapper() diff --git a/recipes-extended/ceph/files/ceph/mgr-restful-plugin.service b/recipes-extended/ceph/files/ceph/mgr-restful-plugin.service new file mode 100644 index 0000000..b3e61f0 --- /dev/null +++ b/recipes-extended/ceph/files/ceph/mgr-restful-plugin.service @@ -0,0 +1,15 @@ +[Unit] +Description=Ceph MGR RESTful API Plugin +After=network-online.target sw-patch.service + +[Service] +Type=forking +Restart=no +KillMode=process +RemainAfterExit=yes +ExecStart=/etc/rc.d/init.d/mgr-restful-plugin start +ExecStop=/etc/rc.d/init.d/mgr-restul-plugin stop +ExecReload=/etc/rc.d/init.d/mgr-restful-plugin reload + +[Install] +WantedBy=multi-user.target diff --git a/recipes-extended/ceph/files/ceph/starlingx-docker-override.conf b/recipes-extended/ceph/files/ceph/starlingx-docker-override.conf new file mode 100644 index 0000000..5ffd859 --- /dev/null +++ b/recipes-extended/ceph/files/ceph/starlingx-docker-override.conf @@ -0,0 +1,3 @@ +[Service] +ExecStopPost=/usr/sbin/ceph-preshutdown.sh + diff --git a/recipes-extended/docker-distribution/docker-distribution_git.bbappend b/recipes-extended/docker-distribution/docker-distribution_git.bbappend new file mode 100644 index 0000000..576948c --- /dev/null +++ b/recipes-extended/docker-distribution/docker-distribution_git.bbappend @@ -0,0 +1,8 @@ +PKG_NAME = "github.com/docker/distribution" + +SYSROOT_PREPROCESS_FUNCS += "docker_distribution_sysroot_preprocess" + +docker_distribution_sysroot_preprocess () { + install -d ${SYSROOT_DESTDIR}${prefix}/local/go/src/${PKG_NAME} + cp -r ${S} ${SYSROOT_DESTDIR}${prefix}/local/go/src/$(dirname ${PKG_NAME}) +} diff --git a/recipes-extended/kubernetes/kubernetes_git.bbappend b/recipes-extended/kubernetes/kubernetes_git.bbappend new file mode 100644 index 0000000..7d2f682 --- /dev/null +++ b/recipes-extended/kubernetes/kubernetes_git.bbappend @@ -0,0 +1,11 @@ +PV = "1.13.5+git${SRCREV_kubernetes}" +SRCREV_kubernetes = "2166946f41b36dea2c4626f90a77706f426cdea2" + +SRC_URI = "git://github.com/kubernetes/kubernetes.git;branch=release-1.13;name=kubernetes \ + file://0001-hack-lib-golang.sh-use-CC-from-environment.patch \ + file://0001-cross-don-t-build-tests-by-default.patch \ + " + +INSANE_SKIP_${PN} += "textrel" +INSANE_SKIP_${PN}-misc += "textrel" +INSANE_SKIP_kubelet += "textrel" diff --git a/recipes-extended/postgresql/postgresql_%.bbappend b/recipes-extended/postgresql/postgresql_%.bbappend new file mode 100644 index 0000000..589765c --- /dev/null +++ b/recipes-extended/postgresql/postgresql_%.bbappend @@ -0,0 +1,2 @@ +COMPUTE_IP="127.0.0.1" +CONTROLLER_IP="127.0.0.1" diff --git a/recipes-extended/registry-token-server/files/registry-token-server-1.0.0.tar.gz b/recipes-extended/registry-token-server/files/registry-token-server-1.0.0.tar.gz new file mode 100644 index 0000000..ba20c20 Binary files /dev/null and b/recipes-extended/registry-token-server/files/registry-token-server-1.0.0.tar.gz differ diff --git a/recipes-extended/registry-token-server/files/registry-token-server.service b/recipes-extended/registry-token-server/files/registry-token-server.service new file mode 100644 index 0000000..477e85d --- /dev/null +++ b/recipes-extended/registry-token-server/files/registry-token-server.service @@ -0,0 +1,19 @@ +[Unit] +Description=v2 Registry token server for Docker + +[Service] +Type=simple +EnvironmentFile=/etc/docker-distribution/registry/token_server.conf +ExecStart=/usr/bin/registry-token-server -addr=${REGISTRY_TOKEN_SERVER_ADDR} \ + -issuer=${REGISTRY_TOKEN_SERVER_ISSUER} \ + -endpoint=${REGISTRY_TOKEN_SERVER_KS_ENDPOINT} \ + -tlscert=${REGISTRY_TOKEN_SERVER_TLSCERT} \ + -tlskey=${REGISTRY_TOKEN_SERVER_TLSKEY} \ + -realm=${REGISTRY_TOKEN_SERVER_REALM} \ + -key=${REGISTRY_TOKEN_SERVER_KEY} +Restart=on-failure +ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/registry-token-server.pid' +ExecStopPost=/bin/rm -f /var/run/registry-token-server.pid + +[Install] +WantedBy=multi-user.target diff --git a/recipes-extended/registry-token-server/files/token-server-certificate.pem b/recipes-extended/registry-token-server/files/token-server-certificate.pem new file mode 100644 index 0000000..c40df59 --- /dev/null +++ b/recipes-extended/registry-token-server/files/token-server-certificate.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDADCCAegCCQCSevkS4h7LQjANBgkqhkiG9w0BAQsFADBCMQswCQYDVQQGEwJY +WDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MRwwGgYDVQQKDBNEZWZhdWx0IENvbXBh +bnkgTHRkMB4XDTE4MDkyMTE0MTYwOFoXDTE5MDkyMTE0MTYwOFowQjELMAkGA1UE +BhMCWFgxFTATBgNVBAcMDERlZmF1bHQgQ2l0eTEcMBoGA1UECgwTRGVmYXVsdCBD +b21wYW55IEx0ZDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKtCbNJ/ +aPEMkZFEtMKRomOh9NgeOv0jYFY5i23fXghtTgdXu9//H3Huz5/KDJ+XEUp2DZgK +YQ2UHVR+cqj2sFjCllfAVrzmv9FFR0CQpQxqKcxChefVwsMh6XsqF+GzbqzFOx67 +bT39Xb5+spAmDHctFl3nrmyA1wM6e+OXcktC0chILeN+UEyq5Xeng6/BpVnI2UaY +J1OpfuUrffddy5t0oeuKGZ/xG2g9sL6GMGBeVslOmLg4CBOwq3knUGoOTFYSjHVx +rU/p4YgUotIUvb4GBsXqbiI7M2NakItTR6mxfcYiKkxfjadQlptFyGucI84mMYx8 +vO3o6TFLfcTYqZ8CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAHXZR0U0pyMkYIeO5 +Y/n0H9Onj/PtCJHBbYzMHZGMPlX2IbW+JAeE/0XNIYGHtAtFwlb825Tkg2p7wpa8 +8HmOBqkTyn2ywDdmPqdfjCiMu/Ge6tkLjqkmYWv2l/d4+qEMR4dUh9g8SrrtUdZg +DP7H22B+0knQ7s04JuiJ27hqi4nPOzdwdJNpz5Przgce8vN1ihk8194pR/uoNrjP +td3Po+DwmxFKigoKPQCHgQuD63mAFor4vVnht+IkNbB3/lQyXP6Qv7DnWVW9WDBL +nKxgXhRwyy5mYebYmwA//JX41O/Kdp1Q6oWgv4zSLd8M9FIMtESG8k4gSl0XfUBa +Y24p0Q== +-----END CERTIFICATE----- diff --git a/recipes-extended/registry-token-server/files/token-server-private-key.pem b/recipes-extended/registry-token-server/files/token-server-private-key.pem new file mode 100644 index 0000000..4332eb3 --- /dev/null +++ b/recipes-extended/registry-token-server/files/token-server-private-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAq0Js0n9o8QyRkUS0wpGiY6H02B46/SNgVjmLbd9eCG1OB1e7 +3/8fce7Pn8oMn5cRSnYNmAphDZQdVH5yqPawWMKWV8BWvOa/0UVHQJClDGopzEKF +59XCwyHpeyoX4bNurMU7HrttPf1dvn6ykCYMdy0WXeeubIDXAzp745dyS0LRyEgt +435QTKrld6eDr8GlWcjZRpgnU6l+5St9913Lm3Sh64oZn/EbaD2wvoYwYF5WyU6Y +uDgIE7CreSdQag5MVhKMdXGtT+nhiBSi0hS9vgYGxepuIjszY1qQi1NHqbF9xiIq +TF+Np1CWm0XIa5wjziYxjHy87ejpMUt9xNipnwIDAQABAoIBAFHCIV+QkdHZ9TiL +u1vT2NmFvPTb4b9tfxVK3YRziVmujPy2Zqu2CRYEMzyOYd5iaU/J8g1ujwzDdAkd +YLHHK0MEim+UFBSUeGh4kV6CbzjxCclIzNJz20n6y5MP8ly+o4x5kBLI2YsphPJn +W+mzMGpIrQ/hhgSosX0KE5EAgQDqOfJSlhZvSgSO5UF9nXvEn7Y9Zc8GK0XQdcwB +Pr8iFhuhEJmmb4LrCm+3Me/fhLxFjUAOAcLSkFnqfxo2vAuRqk99OOLxFEfPYZB8 +kLkKlQ+PwhkG3pjPg6w/rOmBHqW/ZEpd87972JWeHscXYpb/cLLVmcJbZI/claos +YOHS7CECgYEA4XKo7GzuqSkLskfaZM2pyNhHbxphqyNfk8GmW5NJnKavpmY8YiXh +7hNXXf4HCkcHvHMn4JUCHgHVavDNhNnrHNrQAzO3KwuUrrFiBP+yP1tRyQ4BP395 +KIBSUyeEOo9vM7d3yerI8WHboio5gaoqEfeNS1dakZ6ZiOpoP94CIxECgYEAwnfW +Drdcqkpj794gYDlXH4D279f7+qmq11eI4C0zkZzTFkExl8BGfqpy49kruaTm0e4t +L1B23TYfKC0ei4BQskyNCHUnl/eic/JHe9gJRd6BAZi2REfV0LI4ytYGgniCu50H +EJVvTVMXS/+wWcjZr037oV6/WiB9Wzr7Z1oFoa8CgYBlmqdG5lEpK7Z5wqhKheXe +/pozGFCsMGUC0mOHIfoq/3RqKelM0oXgJhdZ5QKHPzvdUojGTmGF5I2qhJwbI5sy +her5hnUmkTGRCaCDYDmVFDLnycgGNg0Ek9CGaWjOe5ZCWI1EEuw83T1++Eiyh14u +esLTEatftXq8megh4IxWAQKBgQCTNfox27ZnJrcuXn0tulpse8jy2RJjt0qfhyET +asRN52SXxTRQhvoWattcBgsmlmEw69cCqSvB23WMiVNFERaFUpO0olMdpBUzJmXc +pzal0IDh/4OCfsqqGDALxCbbX3S/p2gwsp617z+EhYMvBG9dWHAywTGjfVLH3Ady +PmBi+wKBgQCWJS/PmTpyO8LU4MYZk91mJmjHAsPlgi/9n8yEqdmins+X698IsoCr +s2FN8rol8+UP8c3m9o4kp62ouoby2QzAZw0y3UGWcxOb3ZpoozatKodsoETSLLoL +T//wVn2Z2MsS9tLOBLZzsZiYlHyYxTUm7UTOdxdjbSLWVdLbCpKEhg== +-----END RSA PRIVATE KEY----- diff --git a/recipes-extended/registry-token-server/files/token_server.conf b/recipes-extended/registry-token-server/files/token_server.conf new file mode 100644 index 0000000..4683478 --- /dev/null +++ b/recipes-extended/registry-token-server/files/token_server.conf @@ -0,0 +1 @@ +# This is a puppet managed config file diff --git a/recipes-extended/registry-token-server/registry-token-server_1.0.0.bb b/recipes-extended/registry-token-server/registry-token-server_1.0.0.bb new file mode 100644 index 0000000..dfb61e5 --- /dev/null +++ b/recipes-extended/registry-token-server/registry-token-server_1.0.0.bb @@ -0,0 +1,59 @@ +DESCRIPTION = " Token server for use with Docker registry with Openstack Keystone back end" +LICENSE = "Apache-2.0" +LIC_FILES_CHKSUM = "file://LICENSE;md5=d2794c0df5b907fdace235a619d80314" + +GO_IMPORT = "registry-token-server" + +SRC_URI = "file://registry-token-server-1.0.0.tar.gz \ + file://registry-token-server.service \ + file://token_server.conf \ + " + +RDEPENDS_${PN}-dev_append = "systemd" + +DEPENDS += "\ + go-logrus \ + docker-distribution \ + go-libtrust \ + go-patricia \ + go-mux \ + go-context \ + go-phercloud \ + " + +inherit go goarch ${@bb.utils.contains('VIRTUAL-RUNTIME_init_manager','systemd','systemd','', d)} + +do_compile() { + mkdir -p _build/src + ln -sfn ${WORKDIR}/${PN}-${PV} ./_build/src/registry-token-server + + # Pass the needed cflags/ldflags so that cgo + # can find the needed headers files and libraries + export GOARCH=${TARGET_GOARCH} + export CGO_ENABLED="1" + export CGO_CFLAGS="${CFLAGS} --sysroot=${STAGING_DIR_TARGET}" + export CGO_LDFLAGS="${LDFLAGS} --sysroot=${STAGING_DIR_TARGET}" + + export GOPATH="${WORKDIR}/build/_build:${STAGING_DIR_TARGET}/${prefix}/local/go" + cd _build/src/${GO_IMPORT} + #oe_runmake registry-token-server + export GOROOT=${STAGING_DIR_TARGET}/${prefix}/local/go + go build -o ${WORKDIR}/build/bin/registry-token-server registry-token-server +} + +SYSTEMD_PACKAGES = "${PN}" +SYSTEMD_SERVICE_${PN} = "registry-token-server.service" +SYSTEMD_AUTO_ENABLE = "enable" + +do_install() { + install -d ${D}/${sbindir} + install -m 0755 bin/registry-token-server ${D}/${sbindir} + install -d ${D}/${sysconfdir}/registry-token-server/registry + install -m 0644 ${WORKDIR}/token_server.conf ${D}/${sysconfdir}/registry-token-server/registry + + if ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then + install -d ${D}${systemd_unitdir}/system + install -m 0644 ${WORKDIR}/registry-token-server.service ${D}${systemd_unitdir}/system/ + fi +} + diff --git a/recipes-extended/sudo/files/sysadmin b/recipes-extended/sudo/files/sysadmin new file mode 100644 index 0000000..0e3f513 --- /dev/null +++ b/recipes-extended/sudo/files/sysadmin @@ -0,0 +1,12 @@ +## +## User privilege specification +## +sysadmin ALL=(ALL) ALL +sysadmin ALL=(root) NOPASSWD: /usr/bin/config_controller +sysadmin ALL=(root) NOPASSWD: /usr/bin/config_region +sysadmin ALL=(root) NOPASSWD: /usr/bin/config_subcloud +sysadmin ALL=(root) NOPASSWD: /usr/bin/config_management +sysadmin ALL=(root) NOPASSWD: /usr/local/sbin/collect + +Defaults lecture=never, secure_path=/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin +Defaults passprompt="Password: " diff --git a/recipes-extended/sudo/sudo_%.bbappend b/recipes-extended/sudo/sudo_%.bbappend new file mode 100644 index 0000000..aec69c0 --- /dev/null +++ b/recipes-extended/sudo/sudo_%.bbappend @@ -0,0 +1,18 @@ +FILESEXTRAPATHS_prepend := "${THISDIR}/files:" + +# This bbappend replaces stx sudo-config + +SRC_URI += "file://sysadmin" + +do_install_append () { + install -m644 ${S}/../sysadmin ${D}/${sysconfdir}/sudoers.d/sysadmin +} + +pkg_postinst_ontarget_sudo_append () { +SYSADMIN_P="4SuW8cnXFyxsk" +/usr/sbin/groupadd sys_protected +/usr/sbin/useradd -m -g sys_protected -G root \ + -d /home/sysadmin -p $SYSADMIN_P \ + -s /bin/sh sysadmin 2> /dev/null || : +/usr/bin/chage -d0 sysadmin +} diff --git a/recipes-kernel/linux/linux-yocto-rt_%.bbappend b/recipes-kernel/linux/linux-yocto-rt_%.bbappend new file mode 100644 index 0000000..9619428 --- /dev/null +++ b/recipes-kernel/linux/linux-yocto-rt_%.bbappend @@ -0,0 +1,11 @@ +# Enable: +# drbd +# IMA +# intel-e1000e +# intel-i40e +# intel-i40evf +# intel-ixgbe +# intel-ixgbevf +# qat17 +# tpmdd +# Preempt_rt diff --git a/recipes-kernel/linux/linux-yocto_%.bbappend b/recipes-kernel/linux/linux-yocto_%.bbappend new file mode 100644 index 0000000..be89926 --- /dev/null +++ b/recipes-kernel/linux/linux-yocto_%.bbappend @@ -0,0 +1,13 @@ +FILESEXTRAPATHS_append:= ":${THISDIR}/linux:" +# Enable: +# drbd +# IMA +# intel-e1000e +# intel-i40e +# intel-i40evf +# intel-ixgbe +# intel-ixgbevf +# qat17 +# tpmdd + +SRC_URI += "file://stx-kconfig.cfg" diff --git a/recipes-kernel/linux/linux/stx-kconfig.cfg b/recipes-kernel/linux/linux/stx-kconfig.cfg new file mode 100644 index 0000000..a7ec013 --- /dev/null +++ b/recipes-kernel/linux/linux/stx-kconfig.cfg @@ -0,0 +1,5 @@ +CONFIG_NFSD=y +CONFIG_NFSD_V3=y +CONFIG_NFSD_V3_ACL=y +CONFIG_NF_TABLES=m +CONFIG_NFT_COMPAT=m