Airflow and Openstack modules upgrade

This PS upgrades Airflow to 2.10.4 and Openstack
modules to 2024.1 Caracal versions

Change-Id: I577520cddaa73f9e1441922ccd67f75a5f1c3908

.zuul.yaml

@@ -313,7 +313,7 @@
           commit: true
         static:
           - latest
-          - airflow_2.10.2
+          - airflow_2.10.4
 
 - job:
     name: deckhand-docker-tag-ubuntu_jammy

ChangeLog

@@ -1,6 +1,11 @@
 CHANGES
 =======
 
+* Airflow and Openstack modules upgrade
+* Remove ubuntu\_focal as image alias
+* Bump up docker Pypi module version
+* Airflow 2.10.2 fixes
+* Airflow 2.10.2 + ubuntu\_jammy
 * Update deploy-env parameters
 * Kubeadm based Airskiff gate
 * Fix deckhand-api dependences

charts/deckhand/Chart.yaml

@@ -15,7 +15,7 @@
 apiVersion: v1
 description: A Helm chart for Deckhand
 name: deckhand
-version: 0.2.2
+version: 0.2.3
 appVersion: 1.1.0
 keywords:
 - deckhand

deckhand/common/utils.py

@@ -17,6 +17,7 @@ import copy
 import re
 import six
 import string
+import yaml
 
 from beaker.cache import CacheManager
 from beaker.util import parse_cache_config_options
@@ -41,6 +42,18 @@ _CACHE = CacheManager(**parse_cache_config_options(_CACHE_OPTS))
 _ARRAY_RE = re.compile(r'.*\[\d+\].*')
 
 
+def safe_yaml_dump(data):
+    """
+    Automatically handle the difference between
+    yaml.safe_dump and yaml.safe_dump_all
+    to avoid errors when dumping YAML.
+    """
+    if isinstance(data, (list, tuple)):  # Multiple documents
+        return yaml.safe_dump_all(data)
+    else:  # Single document
+        return yaml.safe_dump(data)
+
+
 def to_camel_case(s):
     """Convert string to camel case."""
     return (s[0].lower() + string.capwords(s, sep='_')

deckhand/control/buckets.py

@@ -13,9 +13,11 @@
 # limitations under the License.
 
 import falcon
+
 from oslo_log import log as logging
 from oslo_utils import excutils
 
+from deckhand.common import utils
 from deckhand.common import document as document_wrapper
 from deckhand.control import base as api_base
 from deckhand.control.views import document as document_view
@@ -65,7 +67,8 @@ class BucketsResource(api_base.BaseResource):
         created_documents = self._create_revision_documents(
             bucket_name, documents)
 
-        resp.text = self.view_builder.list(created_documents)
+        resp.text = utils.safe_yaml_dump(
+            self.view_builder.list(created_documents))
         resp.status = falcon.HTTP_200
 
     def _encrypt_secret_documents(self, documents):

deckhand/control/revision_deepdiffing.py

@@ -18,6 +18,7 @@ import six
 from oslo_log import log as logging
 from oslo_utils import excutils
 
+from deckhand.common import utils
 from deckhand.control import base as api_base
 from deckhand.engine.revision_diff import revision_diff
 from deckhand import errors
@@ -51,4 +52,4 @@ class RevisionDeepDiffingResource(api_base.BaseResource):
                 LOG.exception(message)
 
         resp.status = falcon.HTTP_200
-        resp.text = resp_body
+        resp.text = utils.safe_yaml_dump(resp_body)

deckhand/control/revision_diffing.py

@@ -13,10 +13,12 @@
 # limitations under the License.
 
 import falcon
+
 import six
 from oslo_log import log as logging
 from oslo_utils import excutils
 
+from deckhand.common import utils
 from deckhand.control import base as api_base
 from deckhand.engine.revision_diff import revision_diff
 from deckhand import errors
@@ -50,4 +52,4 @@ class RevisionDiffingResource(api_base.BaseResource):
                 LOG.exception(message)
 
         resp.status = falcon.HTTP_200
-        resp.text = resp_body
+        resp.text = utils.safe_yaml_dump(resp_body)

deckhand/control/revision_documents.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import falcon
+
 from oslo_log import log as logging
 import six
 
@@ -81,7 +82,7 @@ class RevisionDocumentsResource(api_base.BaseResource):
             documents = documents[:limit]
 
         resp.status = falcon.HTTP_200
-        resp.text = self.view_builder.list(documents)
+        resp.text = utils.safe_yaml_dump(self.view_builder.list(documents))
 
 
 class RenderedDocumentsResource(api_base.BaseResource):
@@ -156,4 +157,5 @@ class RenderedDocumentsResource(api_base.BaseResource):
             rendered_documents = rendered_documents[:limit]
 
         resp.status = falcon.HTTP_200
-        resp.text = self.view_builder.list(rendered_documents)
+        resp.text = utils.safe_yaml_dump(
+            self.view_builder.list(rendered_documents))

deckhand/control/revision_tags.py

@@ -13,9 +13,11 @@
 # limitations under the License.
 
 import falcon
+
 from oslo_log import log as logging
 from oslo_utils import excutils
 
+from deckhand.common import utils
 from deckhand.control import base as api_base
 from deckhand.control.views import revision_tag as revision_tag_view
 from deckhand.db.sqlalchemy import api as db_api
@@ -44,7 +46,7 @@ class RevisionTagsResource(api_base.BaseResource):
 
         resp_body = revision_tag_view.ViewBuilder().show(resp_tag)
         resp.status = falcon.HTTP_201
-        resp.text = resp_body
+        resp.text = utils.safe_yaml_dump(resp_body)
 
     def on_get(self, req, resp, revision_id, tag=None):
         """Show tag details or list all tags for a revision."""
@@ -66,7 +68,7 @@ class RevisionTagsResource(api_base.BaseResource):
 
         resp_body = revision_tag_view.ViewBuilder().show(resp_tag)
         resp.status = falcon.HTTP_200
-        resp.text = resp_body
+        resp.text = utils.safe_yaml_dump(resp_body)
 
     @policy.authorize('deckhand:list_tags')
     def _list_all_tags(self, req, resp, revision_id):
@@ -80,7 +82,7 @@ class RevisionTagsResource(api_base.BaseResource):
 
         resp_body = revision_tag_view.ViewBuilder().list(resp_tags)
         resp.status = falcon.HTTP_200
-        resp.text = resp_body
+        resp.text = utils.safe_yaml_dump(resp_body)
 
     def on_delete(self, req, resp, revision_id, tag=None):
         """Deletes a single tag or deletes all tags for a revision."""

deckhand/control/revisions.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import falcon
+
 from oslo_log import log as logging
 from oslo_utils import excutils
 
@@ -61,7 +62,7 @@ class RevisionsResource(api_base.BaseResource):
 
         revision_resp = self.view_builder.show(revision)
         resp.status = falcon.HTTP_200
-        resp.text = revision_resp
+        resp.text = utils.safe_yaml_dump(revision_resp)
 
     @policy.authorize('deckhand:list_revisions')
     @common.sanitize_params(['tag', 'order', 'sort'])
@@ -74,7 +75,7 @@ class RevisionsResource(api_base.BaseResource):
             revisions = utils.multisort(revisions, sort_by, order_by)
 
         resp.status = falcon.HTTP_200
-        resp.text = self.view_builder.list(revisions)
+        resp.text = utils.safe_yaml_dump(self.view_builder.list(revisions))
 
     def _delete_all_barbican_secrets(self):
         filters = {'metadata.storagePolicy': 'encrypted'}

deckhand/control/rollback.py

@@ -13,9 +13,11 @@
 # limitations under the License.
 
 import falcon
+
 from oslo_log import log as logging
 from oslo_utils import excutils
 
+from deckhand.common import utils
 from deckhand.control import base as api_base
 from deckhand.control.views import revision as revision_view
 from deckhand.db.sqlalchemy import api as db_api
@@ -55,4 +57,4 @@ class RollbackResource(api_base.BaseResource):
 
         revision_resp = self.view_builder.show(rollback_revision)
         resp.status = falcon.HTTP_201
-        resp.text = revision_resp
+        resp.text = utils.safe_yaml_dump(revision_resp)

deckhand/control/validations.py

@@ -13,9 +13,11 @@
 # limitations under the License.
 
 import falcon
+
 from oslo_log import log as logging
 from oslo_utils import excutils
 
+from deckhand.common import utils
 from deckhand.control import base as api_base
 from deckhand.control.views import validation as validation_view
 from deckhand.db.sqlalchemy import api as db_api
@@ -50,7 +52,7 @@ class ValidationsResource(api_base.BaseResource):
                 LOG.exception(message)
 
         resp.status = falcon.HTTP_201
-        resp.text = self.view_builder.show(resp_body)
+        resp.text = utils.safe_yaml_dump(self.view_builder.show(resp_body))
 
     def on_get(self, req, resp, revision_id, validation_name=None,
                entry_id=None):
@@ -64,7 +66,7 @@ class ValidationsResource(api_base.BaseResource):
             resp_body = self._list_all_validations(req, resp, revision_id)
 
         resp.status = falcon.HTTP_200
-        resp.text = resp_body
+        resp.text = utils.safe_yaml_dump(resp_body)
 
     @policy.authorize('deckhand:show_validation')
     def _show_validation_entry(self, req, resp, revision_id, validation_name,
@@ -128,4 +130,4 @@ class ValidationsDetailsResource(api_base.BaseResource):
             raise falcon.HTTPNotFound(description=e.format_message())
 
         resp.status = falcon.HTTP_200
-        resp.text = self.view_builder.detail(entries)
+        resp.text = utils.safe_yaml_dump(self.view_builder.detail(entries))

deckhand/control/versions.py

@@ -14,6 +14,7 @@
 
 import falcon
 
+from deckhand.common import utils
 from deckhand.control import base as api_base
 
 
@@ -26,10 +27,11 @@ class VersionsResource(api_base.BaseResource):
     no_authentication_methods = ['GET']
 
     def on_get(self, req, resp):
-        resp.text = {
+        resp_body = {
             'v1.0': {
                 'path': '/api/v1.0',
                 'status': 'stable'
             }
         }
+        resp.text = utils.safe_yaml_dump(resp_body)
         resp.status = falcon.HTTP_200

deckhand/errors.py

@@ -15,7 +15,7 @@
 import falcon
 from oslo_log import log as logging
 import six
-import yaml
+from deckhand.common import utils
 
 LOG = logging.getLogger(__name__)
 
@@ -102,7 +102,7 @@ def format_error_resp(req,
     }
 
     resp.status = status_code
-    resp.text = yaml.safe_dump(error_response)
+    resp.text = utils.safe_yaml_dump(error_response)
 
 
 def default_exception_handler(req, resp, ex, params):

requirements-direct.txt

@@ -18,29 +18,29 @@ psycopg2-binary
 uwsgi
 xattr==0.10.1
 
-# Openstack Antelope 2023.1
-# https://releases.openstack.org/antelope/index.html
-barbican==16.0.0
+# Openstack Caracal 2024.1
+# https://releases.openstack.org/caracal/index.html
+barbican==18.0.0
 
-python-barbicanclient==5.5.0
-python-keystoneclient==5.1.0
+python-barbicanclient==5.7.0
+python-keystoneclient==5.4.0
 
-keystoneauth1==5.1.2
-keystonemiddleware==10.2.0
+keystoneauth1==5.6.0
+keystonemiddleware==10.6.0
 
-oslo.cache==3.3.1
-oslo.concurrency==5.1.1
-oslo.config==9.1.1
-oslo.context==5.1.1
-oslo.db==12.3.1
-oslo.i18n==6.0.0
-oslo.log==5.2.0
-oslo.messaging==14.2.4
-oslo.metrics==0.6.0
-oslo.middleware==5.1.1
-oslo.policy==4.1.1
-oslo.serialization==5.1.1
-oslo.service==3.1.1
-oslo.upgradecheck==2.1.1
-oslo.utils==6.1.0
-oslo.versionedobjects==3.1.0
+oslo.cache==3.7.0
+oslo.concurrency==6.0.0
+oslo.config==9.4.0
+oslo.context==5.5.0
+oslo.db==15.0.0
+oslo.i18n==6.3.0
+oslo.log==5.5.1
+oslo.messaging==14.7.2
+oslo.metrics==0.8.0
+oslo.middleware==6.1.0
+oslo.policy==4.3.0
+oslo.serialization==5.4.1
+oslo.service==3.4.1
+oslo.upgradecheck==2.3.0
+oslo.utils==7.1.0
+oslo.versionedobjects==3.3.0

requirements-frozen.txt

@@ -1,69 +1,69 @@
-alembic==1.13.2
-amqp==5.2.0
+alembic==1.14.0
+amqp==5.3.1
 attrs==24.2.0
 autopage==0.5.2
-barbican==16.0.0
-bcrypt==4.2.0
+barbican==18.0.0
+bcrypt==4.2.1
 Beaker==1.13.0
 cachetools==5.5.0
-castellan==5.1.1
+castellan==5.2.0
 certifi==2024.8.30
 cffi==1.17.1
-charset-normalizer==3.3.2
-cliff==4.7.0
-cmd2==2.4.3
+charset-normalizer==3.4.0
+cliff==4.8.0
+cmd2==2.5.8
 cryptography==42.0.8
 debtcollector==3.0.0
 decorator==5.1.1
-deepdiff==8.0.1
-dnspython==2.6.1
+deepdiff==8.1.1
+dnspython==2.7.0
 dogpile.cache==1.3.3
-eventlet==0.37.0
-falcon==3.1.3
+eventlet==0.38.1
+falcon==4.0.2
 fasteners==0.19
 fixtures==4.1.0
 futurist==3.0.0
-greenlet==3.1.0
+greenlet==3.1.1
 html5lib==0.9999999
 httpexceptor==1.4.0
 idna==3.10
 iso8601==2.1.0
 Jinja2==3.1.4
-jsonpath-ng==1.6.1
-jsonpickle==3.3.0
+jsonpath-ng==1.7.0
+jsonpickle==3.4.2
 jsonschema==4.23.0
 jsonschema-specifications==2023.12.1
-keystoneauth1==5.1.2
-keystonemiddleware==10.2.0
-kombu==5.4.1
+keystoneauth1==5.6.0
+keystonemiddleware==10.6.0
+kombu==5.4.2
 ldap3==2.9.1
 logutils==0.3.5
-Mako==1.3.5
-MarkupSafe==2.1.5
+Mako==1.3.8
+MarkupSafe==3.0.2
 microversion-parse==2.0.0
 msgpack==1.1.0
 netaddr==1.3.0
 netifaces==0.11.0
-networkx==3.3
-orderly-set==5.2.2
+networkx==3.4.2
+orderly-set==5.2.3
 os-service-types==1.7.0
-oslo.cache==3.3.1
-oslo.concurrency==5.1.1
-oslo.config==9.1.1
-oslo.context==5.1.1
-oslo.db==12.3.1
-oslo.i18n==6.0.0
-oslo.log==5.2.0
-oslo.messaging==14.2.4
-oslo.metrics==0.6.0
-oslo.middleware==5.1.1
-oslo.policy==4.1.1
-oslo.serialization==5.1.1
-oslo.service==3.1.1
-oslo.upgradecheck==2.1.1
-oslo.utils==6.1.0
-oslo.versionedobjects==3.1.0
-packaging==24.1
+oslo.cache==3.7.0
+oslo.concurrency==6.0.0
+oslo.config==9.4.0
+oslo.context==5.5.0
+oslo.db==15.0.0
+oslo.i18n==6.3.0
+oslo.log==5.5.1
+oslo.messaging==14.7.2
+oslo.metrics==0.8.0
+oslo.middleware==6.1.0
+oslo.policy==4.3.0
+oslo.serialization==5.4.1
+oslo.service==3.4.1
+oslo.upgradecheck==2.3.0
+oslo.utils==7.1.0
+oslo.versionedobjects==3.3.0
+packaging==24.2
 Paste==3.10.1
 PasteDeploy==3.1.0
 PasteScript==3.6.0
@@ -71,19 +71,20 @@ pbr==6.1.0
 pecan==1.5.1
 pip==24.1
 ply==3.11
-prettytable==3.11.0
-prometheus_client==0.20.0
-psycopg2-binary==2.9.9
+prettytable==3.12.0
+prometheus_client==0.21.1
+psycopg2-binary==2.9.10
 pyasn1==0.6.1
-pycadf==3.1.1
+pycadf==4.0.0
 pycparser==2.22
+PyJWT==2.10.1
 pylibyaml==0.1.0
-pyOpenSSL==24.2.1
-pyparsing==3.1.4
+pyOpenSSL==24.3.0
+pyparsing==3.2.0
 pyperclip==1.9.0
-python-barbicanclient==5.5.0
+python-barbicanclient==5.7.0
 python-dateutil==2.9.0.post0
-python-keystoneclient==5.1.0
+python-keystoneclient==5.4.0
 python-memcached==1.62
 python-mimeparse==2.0.0
 pytz==2024.2
@@ -94,29 +95,27 @@ requests==2.32.3
 resolver==0.2.1
 rfc3986==2.0.0
 Routes==2.5.1
-rpds-py==0.20.0
+rpds-py==0.22.3
 selector==0.10.1
 setuptools==70.1.0
 simplejson==3.19.3
-six==1.16.0
+six==1.17.0
 SQLAlchemy==1.4.54
-sqlalchemy-migrate==0.13.0
-sqlparse==0.5.1
 statsd==4.0.1
-stevedore==5.3.0
-Tempita==0.5.2
+stevedore==5.4.0
 testresources==2.0.1
 testscenarios==0.5.0
 testtools==2.7.2
 tiddlyweb==2.4.3
 typing_extensions==4.12.2
-urllib3==2.2.2
-uWSGI==2.0.27
+tzdata==2024.2
+urllib3==2.2.3
+uWSGI==2.0.28
 vine==5.1.0
 wcwidth==0.2.13
-WebOb==1.8.8
+WebOb==1.8.9
 Werkzeug==2.2.3
 wheel==0.43.0
-wrapt==1.16.0
+wrapt==1.17.0
 xattr==0.10.1
-yappi==1.6.0
+yappi==1.6.10

tools/whitespace-linter.sh

@@ -15,6 +15,7 @@ RES=$(find . \
   -not -name "*.html" \
   -not -name "favicon_32.png" \
   -not -name "*.pyc" \
+  -not -path "*/cover/*" \
   -type f -exec egrep -l " +$" {} \;)
 
 if [[ -n $RES ]]; then

tox.ini

@@ -34,7 +34,7 @@ passenv =
 deps =
   -r{toxinidir}/requirements-frozen.txt
   -r{toxinidir}/test-requirements.txt
-  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt
+  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.4/constraints-3.10.txt
 commands =
   find . -type f -name "*.pyc" -delete
   rm -Rf .testrepository/times.dbm
@@ -54,7 +54,7 @@ allowlist_externals=
   sh
 deps=
   -r{toxinidir}/requirements-direct.txt
-  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt
+  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.4/constraints-3.10.txt
 commands=
   rm -f requirements-frozen.txt
   sh -c "pip freeze --all | grep -vE 'deckhand|pyinotify|pkg-resources==0.0.0' > requirements-frozen.txt"