From f5965a81bd52aa0285963ed2432f68f0254b6ac8 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 14 Apr 2022 13:48:32 +0200 Subject: [PATCH] Bypass charm going into blocked due to a bug After Octopus, the reporting about image states fails because of a permission issue in Ceph. This change disables that status reporting to allow a deployment to be healthy even when some tools cannot query status. Also modernize build and func testing: remove python 3.9 and xena from tests Related-Bug: #1879749 func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/1063 Change-Id: Id86fc043495b89609cf6873ec58aee1e2e388578 --- bindep.txt | 3 + charmcraft.yaml | 31 +-- osci.yaml | 9 +- requirements.txt | 19 +- src/lib/charm/openstack/ceph_rbd_mirror.py | 8 +- src/metadata.yaml | 2 +- src/tests/bundles/focal-xena.yaml | 187 ------------------ src/tests/bundles/focal-yoga.yaml | 4 +- src/tests/bundles/impish-xena.yaml | 186 ----------------- src/tests/bundles/jammy-yoga.yaml | 4 +- src/tox.ini | 26 +-- tox.ini | 66 ++----- ...est_lib_charm_openstack_ceph_rbd_mirror.py | 5 +- 13 files changed, 76 insertions(+), 474 deletions(-) create mode 100644 bindep.txt delete mode 100644 src/tests/bundles/focal-xena.yaml delete mode 100644 src/tests/bundles/impish-xena.yaml diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 0000000..bdbe8d5 --- /dev/null +++ b/bindep.txt @@ -0,0 +1,3 @@ +libffi-dev [platform:dpkg] +libxml2-dev [platform:dpkg] +libxslt1-dev [platform:dpkg] diff --git a/charmcraft.yaml b/charmcraft.yaml index 4968216..0b70679 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -2,24 +2,25 @@ type: charm parts: charm: + source: src/ + plugin: reactive + build-snaps: + - charm build-packages: - tox - git - python3-dev - override-build: | - apt-get install ca-certificates -y - tox -e build-reactive - override-stage: | - echo "Copying charm to staging area: $CHARMCRAFT_STAGE" - NAME=$(ls $CHARMCRAFT_PART_BUILD/build/builds) - cp -r $CHARMCRAFT_PART_BUILD/build/builds/$NAME/* $CHARMCRAFT_STAGE/ - override-prime: | - # For some reason, the normal priming chokes on the fact that there's a - # hooks directory. - cp -r $CHARMCRAFT_STAGE/* . + build-environment: + - CHARM_INTERFACES_DIR: /root/project/interfaces/ + - CHARM_LAYERS_DIR: /root/project/layers/ bases: - - name: ubuntu - channel: "20.04" - architectures: - - amd64 + - build-on: + - name: ubuntu + channel: "22.04" + architectures: + - amd64 + run-on: + - name: ubuntu + channel: "22.04" + architectures: [amd64, s390x, ppc64el, arm64] diff --git a/osci.yaml b/osci.yaml index 4f03970..d196797 100644 --- a/osci.yaml +++ b/osci.yaml @@ -1,17 +1,14 @@ - project: templates: - charm-unit-jobs-py38 - - charm-unit-jobs-py39 + - charm-unit-jobs-py310 check: jobs: - - focal-xena - - focal-yoga: - voting: false - - impish-xena: - voting: false + - focal-yoga - jammy-yoga: voting: false vars: needs_charm_build: true charm_build_name: ceph-rbd-mirror build_type: charmcraft + charmcraft_channel: 2.1/stable diff --git a/requirements.txt b/requirements.txt index a68620f..b3dc23f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,16 +8,13 @@ # requirements.txt setuptools<50.0.0 # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85 -# Build requirements -cffi==1.14.6; python_version < '3.6' # cffi 1.15.0 drops support for py35. -charm-tools==2.8.3 +# NOTE: newer versions of cryptography require a Rust compiler to build, +# see +# * https://github.com/openstack-charmers/zaza/issues/421 +# * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html +# +cryptography<3.4 + +git+https://github.com/juju/charm-tools.git simplejson - -# Newer versions use keywords that didn't exist in python 3.5 yet (e.g. -# "ModuleNotFoundError") -# NOTE(lourot): This might look like a duplication of test-requirements.txt but -# some tox targets use only test-requirements.txt whereas charm-build uses only -# requirements.txt -importlib-metadata<3.0.0; python_version < '3.6' -importlib-resources<3.0.0; python_version < '3.6' diff --git a/src/lib/charm/openstack/ceph_rbd_mirror.py b/src/lib/charm/openstack/ceph_rbd_mirror.py index c20ea48..3446c61 100644 --- a/src/lib/charm/openstack/ceph_rbd_mirror.py +++ b/src/lib/charm/openstack/ceph_rbd_mirror.py @@ -97,8 +97,12 @@ class CephRBDMirrorCharm(charms_openstack.plugins.CephCharm): if not pool_msg: pool_msg = 'Pools ' pool_msg += '{} ({}) '.format(health, count) - if health != 'OK': - status = 'blocked' + + # Disabling blocked state until + # https://bugs.launchpad.net/charm-ceph-rbd-mirror/+bug/1879749 + # is resolved + # if health != 'OK': + # status = 'blocked' for state, count in stats['image_states'].items(): if not image_msg: image_msg = 'Images ' diff --git a/src/metadata.yaml b/src/metadata.yaml index a07cf96..51dfde7 100644 --- a/src/metadata.yaml +++ b/src/metadata.yaml @@ -16,7 +16,7 @@ tags: - misc series: - focal -- impish +- jammy extra-bindings: public: cluster: diff --git a/src/tests/bundles/focal-xena.yaml b/src/tests/bundles/focal-xena.yaml deleted file mode 100644 index 3460aae..0000000 --- a/src/tests/bundles/focal-xena.yaml +++ /dev/null @@ -1,187 +0,0 @@ -variables: - openstack-origin: &openstack-origin cloud:focal-xena - -local_overlay_enabled: False - -series: &series focal - -machines: - '0': - constraints: "mem=3072M" - '1': - constraints: "mem=3072M" - '2': - constraints: "mem=3072M" - -applications: - - keystone-mysql-router: - charm: ch:mysql-router - channel: latest/edge - glance-mysql-router: - charm: ch:mysql-router - channel: latest/edge - cinder-mysql-router: - charm: ch:mysql-router - channel: latest/edge - - mysql-innodb-cluster: - charm: ch:mysql-innodb-cluster - num_units: 3 - options: - source: *openstack-origin - to: - - '0' - - '1' - - '2' - channel: latest/edge - - keystone: - charm: ch:keystone - num_units: 1 - options: - openstack-origin: *openstack-origin - channel: yoga/edge - - rabbitmq-server: - charm: ch:rabbitmq-server - num_units: 1 - options: - source: *openstack-origin - channel: latest/edge - - cinder: - charm: ch:cinder - num_units: 1 - options: - block-device: None - glance-api-version: 2 - openstack-origin: *openstack-origin - channel: yoga/edge - - cinder-ceph: - charm: ch:cinder-ceph - num_units: 0 - channel: yoga/edge - - glance: - charm: ch:glance - num_units: 1 - options: - openstack-origin: *openstack-origin - channel: yoga/edge - - nova-compute: - charm: ch:nova-compute - num_units: 1 - options: - openstack-origin: *openstack-origin - channel: yoga/edge - - ceph-mon: - charm: ch:ceph-mon - num_units: 3 - options: - expected-osd-count: 3 - source: *openstack-origin - channel: quincy/edge - - ceph-osd: - charm: ch:ceph-osd - num_units: 3 - storage: - osd-devices: 'cinder,10G' - options: - source: *openstack-origin - bluestore: False - use-direct-io: False - osd-devices: '/dev/test-non-existent' - channel: quincy/edge - - ceph-rbd-mirror: - series: *series - charm: ../../../ceph-rbd-mirror.charm - num_units: 1 - options: - source: *openstack-origin - - ceph-mon-b: - charm: ch:ceph-mon - num_units: 3 - options: - expected-osd-count: 3 - source: *openstack-origin - channel: quincy/edge - - ceph-osd-b: - charm: ch:ceph-osd - num_units: 3 - storage: - osd-devices: 'cinder,10G' - options: - source: *openstack-origin - bluestore: False - use-direct-io: False - osd-devices: '/dev/test-non-existent' - channel: quincy/edge - - ceph-rbd-mirror-b: - series: *series - charm: ../../../ceph-rbd-mirror.charm - num_units: 1 - options: - source: *openstack-origin - -relations: - -- - keystone:shared-db - - keystone-mysql-router:shared-db -- - keystone-mysql-router:db-router - - mysql-innodb-cluster:db-router - -- - glance:shared-db - - glance-mysql-router:shared-db -- - glance-mysql-router:db-router - - mysql-innodb-cluster:db-router - -- - cinder:shared-db - - cinder-mysql-router:shared-db -- - cinder-mysql-router:db-router - - mysql-innodb-cluster:db-router - -- - rabbitmq-server - - cinder - -- - keystone - - cinder -- - keystone - - glance - -- - cinder - - cinder-ceph -- - cinder-ceph:ceph - - ceph-mon:client - -- - nova-compute:ceph-access - - cinder-ceph:ceph-access -- - nova-compute:amqp - - rabbitmq-server:amqp - -- - glance:image-service - - nova-compute:image-service -- - glance - - ceph-mon - -- - ceph-mon:osd - - ceph-osd:mon -- - ceph-mon - - ceph-rbd-mirror:ceph-local -- - ceph-mon - - ceph-rbd-mirror-b:ceph-remote - -- - ceph-mon-b:osd - - ceph-osd-b:mon -- - ceph-mon-b - - ceph-rbd-mirror-b:ceph-local -- - ceph-mon-b - - ceph-rbd-mirror:ceph-remote diff --git a/src/tests/bundles/focal-yoga.yaml b/src/tests/bundles/focal-yoga.yaml index 2564d9c..9b8f51b 100644 --- a/src/tests/bundles/focal-yoga.yaml +++ b/src/tests/bundles/focal-yoga.yaml @@ -152,9 +152,9 @@ relations: - - rabbitmq-server - cinder -- - keystone +- - 'keystone:identity-service' - cinder -- - keystone +- - 'keystone:identity-service' - glance - - cinder diff --git a/src/tests/bundles/impish-xena.yaml b/src/tests/bundles/impish-xena.yaml deleted file mode 100644 index ddf2861..0000000 --- a/src/tests/bundles/impish-xena.yaml +++ /dev/null @@ -1,186 +0,0 @@ -variables: - openstack-origin: &openstack-origin distro - -local_overlay_enabled: False - -series: &series impish - -machines: - '0': - constraints: "mem=3072M" - '1': - constraints: "mem=3072M" - '2': - constraints: "mem=3072M" - -applications: - - keystone-mysql-router: - charm: ch:mysql-router - channel: latest/edge - glance-mysql-router: - charm: ch:mysql-router - channel: latest/edge - cinder-mysql-router: - charm: ch:mysql-router - channel: latest/edge - - mysql-innodb-cluster: - charm: ch:mysql-innodb-cluster - num_units: 3 - options: - source: *openstack-origin - to: - - '0' - - '1' - - '2' - channel: latest/edge - - keystone: - charm: ch:keystone - num_units: 1 - options: - openstack-origin: *openstack-origin - channel: yoga/edge - - rabbitmq-server: - charm: ch:rabbitmq-server - num_units: 1 - options: - source: *openstack-origin - channel: latest/edge - - cinder: - charm: ch:cinder - num_units: 1 - options: - block-device: None - glance-api-version: 2 - channel: yoga/edge - - cinder-ceph: - charm: ch:cinder-ceph - num_units: 0 - channel: yoga/edge - - glance: - charm: ch:glance - num_units: 1 - options: - openstack-origin: *openstack-origin - channel: yoga/edge - - nova-compute: - charm: ch:nova-compute - num_units: 1 - options: - openstack-origin: *openstack-origin - channel: yoga/edge - - ceph-mon: - charm: ch:ceph-mon - num_units: 3 - options: - expected-osd-count: 3 - source: *openstack-origin - channel: quincy/edge - - ceph-osd: - charm: ch:ceph-osd - num_units: 3 - storage: - osd-devices: 'cinder,10G' - options: - source: *openstack-origin - bluestore: False - use-direct-io: False - osd-devices: '/dev/test-non-existent' - channel: quincy/edge - - ceph-rbd-mirror: - series: *series - charm: ../../../ceph-rbd-mirror.charm - num_units: 1 - options: - source: *openstack-origin - - ceph-mon-b: - charm: ch:ceph-mon - num_units: 3 - options: - expected-osd-count: 3 - source: *openstack-origin - channel: quincy/edge - - ceph-osd-b: - charm: ch:ceph-osd - num_units: 3 - storage: - osd-devices: 'cinder,10G' - options: - source: *openstack-origin - bluestore: False - use-direct-io: False - osd-devices: '/dev/test-non-existent' - channel: quincy/edge - - ceph-rbd-mirror-b: - series: *series - charm: ../../../ceph-rbd-mirror.charm - num_units: 1 - options: - source: *openstack-origin - -relations: - -- - keystone:shared-db - - keystone-mysql-router:shared-db -- - keystone-mysql-router:db-router - - mysql-innodb-cluster:db-router - -- - glance:shared-db - - glance-mysql-router:shared-db -- - glance-mysql-router:db-router - - mysql-innodb-cluster:db-router - -- - cinder:shared-db - - cinder-mysql-router:shared-db -- - cinder-mysql-router:db-router - - mysql-innodb-cluster:db-router - -- - rabbitmq-server - - cinder - -- - keystone - - cinder -- - keystone - - glance - -- - cinder - - cinder-ceph -- - cinder-ceph:ceph - - ceph-mon:client - -- - nova-compute:ceph-access - - cinder-ceph:ceph-access -- - nova-compute:amqp - - rabbitmq-server:amqp - -- - glance:image-service - - nova-compute:image-service -- - glance - - ceph-mon - -- - ceph-mon:osd - - ceph-osd:mon -- - ceph-mon - - ceph-rbd-mirror:ceph-local -- - ceph-mon - - ceph-rbd-mirror-b:ceph-remote - -- - ceph-mon-b:osd - - ceph-osd-b:mon -- - ceph-mon-b - - ceph-rbd-mirror-b:ceph-local -- - ceph-mon-b - - ceph-rbd-mirror:ceph-remote diff --git a/src/tests/bundles/jammy-yoga.yaml b/src/tests/bundles/jammy-yoga.yaml index a3fd3f8..8e85d4a 100644 --- a/src/tests/bundles/jammy-yoga.yaml +++ b/src/tests/bundles/jammy-yoga.yaml @@ -151,9 +151,9 @@ relations: - - rabbitmq-server - cinder -- - keystone +- - 'keystone:identity-service' - cinder -- - keystone +- - 'keystone:identity-service' - glance - - cinder diff --git a/src/tox.ini b/src/tox.ini index b40d295..8ffff0f 100644 --- a/src/tox.ini +++ b/src/tox.ini @@ -6,32 +6,26 @@ [tox] envlist = pep8 -skipsdist = True # NOTE: Avoid build/test env pollution by not enabling sitepackages. sitepackages = False # NOTE: Avoid false positives by not skipping missing interpreters. skip_missing_interpreters = False -# NOTES: -# * We avoid the new dependency resolver by pinning pip < 20.3, see -# https://github.com/pypa/pip/issues/9187 -# * Pinning dependencies requires tox >= 3.2.0, see -# https://tox.readthedocs.io/en/latest/config.html#conf-requires -# * It is also necessary to pin virtualenv as a newer virtualenv would still -# lead to fetching the latest pip in the func* tox targets, see -# https://stackoverflow.com/a/38133283 -requires = pip < 20.3 - virtualenv < 20.0 -# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci -minversion = 3.18.0 [testenv] +# We use tox mainly for virtual environment management for test requirements +# and do not install the charm code as a Python package into that environment. +# Ref: https://tox.wiki/en/latest/config.html#skip_install +skip_install = True setenv = VIRTUAL_ENV={envdir} PYTHONHASHSEED=0 allowlist_externals = juju -passenv = HOME TERM CS_* OS_* TEST_* +passenv = + HOME + TERM + CS_* + OS_* + TEST_* deps = -r{toxinidir}/test-requirements.txt -install_command = - pip install {opts} {packages} [testenv:pep8] basepython = python3 diff --git a/tox.ini b/tox.ini index 2d60b8a..c028e9a 100644 --- a/tox.ini +++ b/tox.ini @@ -5,53 +5,51 @@ # https://github.com/openstack-charmers/release-tools [tox] -skipsdist = True envlist = pep8,py3 # NOTE: Avoid build/test env pollution by not enabling sitepackages. sitepackages = False # NOTE: Avoid false positives by not skipping missing interpreters. skip_missing_interpreters = False -# NOTES: -# * We avoid the new dependency resolver by pinning pip < 20.3, see -# https://github.com/pypa/pip/issues/9187 -# * Pinning dependencies requires tox >= 3.2.0, see -# https://tox.readthedocs.io/en/latest/config.html#conf-requires -# * It is also necessary to pin virtualenv as a newer virtualenv would still -# lead to fetching the latest pip in the func* tox targets, see -# https://stackoverflow.com/a/38133283 -requires = - pip < 20.3 - virtualenv < 20.0 - setuptools<50.0.0 - -# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci -minversion = 3.18.0 [testenv] +# We use tox mainly for virtual environment management for test requirements +# and do not install the charm code as a Python package into that environment. +# Ref: https://tox.wiki/en/latest/config.html#skip_install +skip_install = True setenv = VIRTUAL_ENV={envdir} PYTHONHASHSEED=0 TERM=linux LAYER_PATH={toxinidir}/layers INTERFACE_PATH={toxinidir}/interfaces JUJU_REPOSITORY={toxinidir}/build -passenv = http_proxy https_proxy INTERFACE_PATH LAYER_PATH JUJU_REPOSITORY -install_command = - {toxinidir}/pip.sh install {opts} {packages} + CHARM_LAYERS_DIR={toxinidir}/layers + CHARM_INTERFACES_DIR={toxinidir}/interfaces +passenv = + no_proxy + http_proxy + https_proxy + CHARM_INTERFACES_DIR + CHARM_LAYERS_DIR + JUJU_REPOSITORY allowlist_externals = charmcraft bash tox - rename.sh + {toxinidir}/rename.sh deps = -r{toxinidir}/requirements.txt [testenv:build] basepython = python3 -deps = -r{toxinidir}/build-requirements.txt +# charmcraft clean is done to ensure that +# `tox -e build` always performs a clean, repeatable build. +# For faster rebuilds during development, +# directly run `charmcraft -v pack && ./rename.sh`. commands = charmcraft clean - charmcraft -v build + charmcraft -v pack {toxinidir}/rename.sh + charmcraft clean [testenv:build-reactive] basepython = python3 @@ -68,11 +66,6 @@ basepython = python3 deps = -r{toxinidir}/test-requirements.txt commands = stestr run --slowest {posargs} -[testenv:py35] -basepython = python3.5 -deps = -r{toxinidir}/test-requirements.txt -commands = stestr run --slowest {posargs} - [testenv:py36] basepython = python3.6 deps = -r{toxinidir}/test-requirements.txt @@ -88,29 +81,12 @@ basepython = python3.8 deps = -r{toxinidir}/test-requirements.txt commands = stestr run --slowest {posargs} -[testenv:py39] -basepython = python3.9 -deps = -r{toxinidir}/test-requirements.txt -commands = stestr run --slowest {posargs} - [testenv:pep8] basepython = python3 deps = flake8==3.9.2 - charm-tools==2.8.3 + charm-tools==2.8.4 commands = flake8 {posargs} src unit_tests -[testenv:func-target] -# Hack to get functional tests working in the charmcraft -# world. We should fix this. -basepython = python3 -passenv = HOME TERM CS_* OS_* TEST_* -deps = -r{toxinidir}/src/test-requirements.txt -changedir = {toxinidir}/src -commands = - bash -c "if [ ! -f ../*.charm ]; then echo 'Charm does not exist. Run tox -e build'; exit 1; fi" - tox --version - tox -e func-target {posargs} - [testenv:cover] # Technique based heavily upon # https://github.com/openstack/nova/blob/master/tox.ini diff --git a/unit_tests/test_lib_charm_openstack_ceph_rbd_mirror.py b/unit_tests/test_lib_charm_openstack_ceph_rbd_mirror.py index e0c1645..49039fa 100644 --- a/unit_tests/test_lib_charm_openstack_ceph_rbd_mirror.py +++ b/unit_tests/test_lib_charm_openstack_ceph_rbd_mirror.py @@ -51,7 +51,10 @@ class TestCephRBDMirrorCharm(Helper): {'stopped': 2, 'replaying': 2}), }) result = crmc.custom_assess_status_check() - self.assertTrue('blocked' in result[0]) + # Disabling blocked state until + # https://bugs.launchpad.net/charm-ceph-rbd-mirror/+bug/1879749 + # is resolved + # self.assertTrue('blocked' in result[0]) # the order of which the statuses appear in the string is undefined self.assertTrue('OK (1)' in result[1]) self.assertTrue('WARN (1)' in result[1])