Airflow stable 2.6.2

This PS updates python modules and code to match Airflow 2.6.2: - bionic py36 gates were removed - python code corrected to match new modules versions - selection of python modules versions was perfrmed based on airflow-2.6.2 constraints - postgresql image updated to 14.8 Change-Id: Ibdcc75e600166c20b842508aa5539587cca466f0
2023-07-03 20:35:08 +00:00 · 2023-07-03 20:35:08 +00:00 · f99abfa433
commit f99abfa433
parent 25456d677b
13 changed files with 89 additions and 270 deletions
--- a/.zuul.yaml
+++ b/.zuul.yaml
@ -14,7 +14,6 @@
    templates:
      - docs-on-readthedocs
      - openstack-python38-jobs
-      - openstack-python36-jobs
    vars:
      rtd_webhook_id: '38809'
      rtd_project_name: 'airship-drydock'
@ -26,7 +25,6 @@
        - airship-drydock-chart-build-gate
        - airship-drydock-chart-build-latest-htk
        - airship-drydock-docker-build-gate-ubuntu_focal
-        - airship-drydock-docker-build-gate-ubuntu_bionic

    gate:
      jobs:
@ -35,11 +33,10 @@
        - airship-drydock-baclient-test
        - airship-drydock-chart-build-gate
        - airship-drydock-docker-build-gate-ubuntu_focal
-        - airship-drydock-docker-build-gate-ubuntu_bionic
+
    post:
      jobs:
        - airship-drydock-docker-publish-ubuntu_focal
-        - airship-drydock-docker-publish-ubuntu_bionic
        - drydock-upload-git-mirror

 - nodeset:
@ -84,7 +81,7 @@
    run: tools/gate/playbooks/build-charts.yaml
    nodeset: airship-drydock-single-node-focal
    vars:
-      HTK_COMMIT: a7cd689280cdbc0acd04a7a1b745941260e8700b
+      HTK_COMMIT: 97ce6d7d8e9a090c748800d69a57bbd9af698b60

 - job:
    name: airship-drydock-chart-build-latest-htk
@ -113,21 +110,6 @@
          patch_set: true


- job:
-    name: airship-drydock-docker-build-gate-ubuntu_bionic
-    timeout: 3600
-    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: airship-drydock-single-node
-    irrelevant-files:
-      - '^doc/.*'
-      - '^charts/.*'
-    vars:
-      publish: false
-      distro: ubuntu_bionic
-      tags:
-        dynamic:
-          patch_set: true
-

 - job:
    name: airship-drydock-docker-publish-ubuntu_focal
@ -148,26 +130,7 @@
          commit: true
        static:
          - latest
-
- job:
-    name: airship-drydock-docker-publish-ubuntu_bionic
-    timeout: 3600
-    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: airship-drydock-single-node-focal
-    secrets:
-      - airship_drydock_quay_creds
-    irrelevant-files:
-      - '^doc/.*'
-      - '^charts/.*'
-    vars:
-      publish: true
-      distro: ubuntu_bionic
-      tags:
-        dynamic:
-          branch: true
-          commit: true
-        static:
-          - latest
+          - airflow_2.6.2


 - secret:
--- a/charts/drydock/values.yaml
+++ b/charts/drydock/values.yaml
@ -35,7 +35,7 @@ images:
    ks_user: docker.io/openstackhelm/heat:newton
    ks_service: docker.io/openstackhelm/heat:newton
    ks_endpoints: docker.io/openstackhelm/heat:newton
-    drydock_db_init: docker.io/postgres:14.6
+    drydock_db_init: docker.io/postgres:14.8
    drydock_db_cleanup: quay.io/airshipit/drydock:master
    drydock_db_sync: quay.io/airshipit/drydock:master
  pull_policy: "IfNotPresent"
--- a/doc/source/_static/drydock.conf.sample
+++ b/doc/source/_static/drydock.conf.sample
@ -78,7 +78,7 @@
 # Domain name containing project (string value)
 #project_domain_name = <None>

-# ID of the trust to use as a trustee use (string value)
+# Trust ID (string value)
 #trust_id = <None>

 # Optional domain ID to use with v3 and v2 parameters. It will be used for both
--- a/etc/drydock/drydock.conf.sample
+++ b/etc/drydock/drydock.conf.sample
@ -78,7 +78,7 @@
 # Domain name containing project (string value)
 #project_domain_name = <None>

-# ID of the trust to use as a trustee use (string value)
+# Trust ID (string value)
 #trust_id = <None>

 # Optional domain ID to use with v3 and v2 parameters. It will be used for both
--- a/python/drydock_provisioner/drivers/node/maasdriver/actions/node.py
+++ b/python/drydock_provisioner/drivers/node/maasdriver/actions/node.py
@ -296,7 +296,7 @@ class DestroyNode(BaseMaasAction):
                                             ctx_type='node')
                    self.task.success(focus=n.get_id())
                    continue
-                elif type(machine) == maas_rack.RackController:
+                elif type(machine) is maas_rack.RackController:
                    msg = "Cannot delete rack controller {}.".format(n.name)
                    self.logger.info(msg)
                    self.task.add_status_msg(msg=msg,
@ -1130,7 +1130,7 @@ class IdentifyNode(BaseMaasAction):
                                             error=True,
                                             ctx=n.name,
                                             ctx_type='node')
-                elif type(machine) == maas_machine.Machine:
+                elif type(machine) is maas_machine.Machine:
                    machine.update_identity(
                        n,
                        domain=n.get_domain(site_design),
@ -1144,7 +1144,7 @@ class IdentifyNode(BaseMaasAction):
                                             ctx=n.name,
                                             ctx_type='node')
                    self.task.success(focus=n.get_id())
-                elif type(machine) == maas_rack.RackController:
+                elif type(machine) is maas_rack.RackController:
                    msg = "Rack controller %s identified in MaaS" % n.name
                    self.logger.debug(msg)
                    self.task.add_status_msg(msg=msg,
@ -1219,7 +1219,7 @@ class ConfigureHardware(BaseMaasAction):
                self.logger.debug("Locating node %s for commissioning" %
                                  (n.name))
                machine = find_node_in_maas(self.maas_client, n)
-                if type(machine) == maas_rack.RackController:
+                if type(machine) is maas_rack.RackController:
                    msg = "Located node %s in MaaS as rack controller. Skipping." % (
                        n.name)
                    self.logger.info(msg)
--- a/python/requirements-direct.txt
+++ b/python/requirements-direct.txt
@ -7,13 +7,19 @@ falcon
 iso8601
 jinja2
 jsonschema
-keystoneauth1
+keystoneauth1==4.3.1
 keystonemiddleware
 libvirt-python
 netaddr
+mock
 oauthlib
-oslo.config
-oslo.policy
+oslo.config==8.6.0
+oslo.context==4.0.0
+oslo.log==4.8.0
+oslo.policy==3.10.1
+oslo.serialization==4.1.0
+oslo.utils==4.12.3
+oslo.db==11.2.0
 oslo.versionedobjects
 Paste
 PasteDeploy
@ -22,9 +28,11 @@ PTable
 pyghmi
 pylibyaml
 pymongo
+python-memcached
 PyYAML
 redfish
 requests
-SQLAlchemy<=1.3.20
+responses
+SQLAlchemy
 ulid2
 uWSGI
--- a/python/requirements-frozen-bionic.txt
+++ b/python/requirements-frozen-bionic.txt
@ -1,99 +0,0 @@
-alembic==1.7.7
-amqp==5.1.1
-attrs==22.2.0
-bcrypt==4.0.1
-Beaker==1.12.1
-cached-property==1.5.2
-cachetools==4.2.4
-certifi==2023.5.7
-cffi==1.15.1
-charset-normalizer==2.0.12
-click==8.0.4
-cryptography==40.0.2
-debtcollector==2.5.0
-decorator==5.1.1
-defusedxml==0.7.1
-dnspython==2.2.1
-dogpile.cache==1.2.1
-eventlet==0.33.3
-falcon==3.1.1
-fasteners==0.18
-fixtures==4.0.1
-futurist==2.4.1
-greenlet==2.0.2
-idna==3.4
-importlib-metadata==4.8.3
-importlib-resources==5.4.0
-iso8601==1.1.0
-Jinja2==3.0.3
-jsonpatch==1.32
-jsonpath-rw==1.4.0
-jsonpointer==2.3
-jsonschema==3.2.0
-keystoneauth1==5.1.2
-keystonemiddleware==9.5.0
-kombu==5.1.0
-libvirt-python==9.3.0
-Mako==1.1.6
-MarkupSafe==2.0.1
-msgpack==1.0.5
-netaddr==0.8.0
-netifaces==0.11.0
-oauthlib==3.2.2
-os-service-types==1.7.0
-oslo.cache==2.11.0
-oslo.concurrency==4.5.1
-oslo.config==8.8.0
-oslo.context==4.1.0
-oslo.i18n==5.1.0
-oslo.log==4.8.0
-oslo.messaging==12.14.0
-oslo.metrics==0.4.0
-oslo.middleware==4.5.1
-oslo.policy==3.12.1
-oslo.serialization==4.3.0
-oslo.service==2.8.0
-oslo.utils==4.13.0
-oslo.versionedobjects==2.6.0
-packaging==21.3
-Paste==3.5.3
-PasteDeploy==2.1.1
-pbr==5.11.1
-pip==21.3.1
-ply==3.11
-prometheus-client==0.16.0
-psycopg2-binary==2.9.6
-PTable==0.9.2
-pycadf==3.1.1
-pycparser==2.21
-pyghmi==1.5.61
-pylibyaml==0.1.0
-pymongo==4.1.1
-pyparsing==3.0.9
-pyrsistent==0.18.0
-python-dateutil==2.8.2
-python-keystoneclient==4.5.0
-pytz==2023.3
-PyYAML==6.0
-redfish==3.1.9
-repoze.lru==0.7
-requests==2.27.1
-requests-toolbelt==1.0.0
-requests-unixsocket==0.3.0
-rfc3986==1.5.0
-Routes==2.5.1
-setuptools==59.6.0
-six==1.16.0
-SQLAlchemy==1.3.20
-statsd==4.0.1
-stevedore==3.5.2
-typing_extensions==4.1.1
-ulid2==0.3.0
-urllib3==1.26.16
-uWSGI==2.0.21
-vine==5.0.0
-WebOb==1.8.7
-wheel==0.37.1
-wrapt==1.15.0
-yappi==1.4.0
-zipp==3.6.0
--- a/python/requirements-frozen.txt
+++ b/python/requirements-frozen.txt
@ -1,131 +1,112 @@
-alabaster==0.7.13
-alembic==1.10.4
+alembic==1.11.1
 amqp==5.1.1
 attrs==23.1.0
-Babel==2.12.1
-bandit==1.7.5
+backports.zoneinfo==0.2.1
 bcrypt==4.0.1
 Beaker==1.12.1
-cachetools==5.3.0
-certifi==2022.12.7
+cachetools==5.3.1
+certifi==2023.5.7
 cffi==1.15.1
 charset-normalizer==3.1.0
-click==6.7
-coverage==7.2.3
+click==8.1.3
 cryptography==40.0.2
 debtcollector==2.5.0
 decorator==5.1.1
 defusedxml==0.7.1
 dnspython==2.3.0
-docutils==0.19
-dogpile.cache==1.1.8
+dogpile.cache==1.2.2
 eventlet==0.33.3
-exceptiongroup==1.1.1
 falcon==3.1.1
 fasteners==0.18
-fixtures==4.0.1
-flake8==6.0.0
+fixtures==4.1.0
 futurist==2.4.1
-gitdb==4.0.10
-GitPython==3.1.31
 greenlet==2.0.2
 idna==3.4
-imagesize==1.4.1
-iniconfig==2.0.0
-iso8601==1.1.0
+importlib-metadata==4.13.0
+importlib-resources==5.12.0
+iso8601==2.0.0
 Jinja2==3.1.2
 jsonpatch==1.32
 jsonpath-rw==1.4.0
 jsonpointer==2.3
-jsonschema==3.2.0
-keystoneauth1==5.1.2
-keystonemiddleware==10.2.0
-kombu==5.2.4
-libvirt-python==9.2.0
+jsonschema==4.17.3
+keystoneauth1==4.3.1
+keystonemiddleware==10.4.0
+kombu==5.3.0
+libvirt-python==9.6.0
 Mako==1.2.4
-markdown-it-py==2.2.0
-MarkupSafe==2.1.2
-mccabe==0.7.0
-mdurl==0.1.2
-mock==5.0.1
+MarkupSafe==2.1.3
+mock==5.1.0
 msgpack==1.0.5
 netaddr==0.8.0
 netifaces==0.11.0
 oauthlib==3.2.2
 os-service-types==1.7.0
-oslo.cache==3.3.1
+oslo.cache==3.4.0
 oslo.concurrency==5.1.1
-oslo.config==9.1.1
-oslo.context==5.1.1
+oslo.config==8.6.0
+oslo.context==4.0.0
+oslo.db==11.2.0
 oslo.i18n==6.0.0
-oslo.log==5.2.0
-oslo.messaging==14.2.0
+oslo.log==4.8.0
+oslo.messaging==14.3.1
 oslo.metrics==0.6.0
 oslo.middleware==5.1.1
-oslo.policy==4.1.1
-oslo.serialization==5.1.1
+oslo.policy==3.10.1
+oslo.serialization==4.1.0
 oslo.service==3.1.1
-oslo.utils==6.1.0
+oslo.utils==4.12.3
 oslo.versionedobjects==3.1.0
-packaging==23.1
-Paste==3.5.2
+packaging==21.3
+Paste==3.5.3
 PasteDeploy==3.0.1
 pbr==5.11.1
-pip==23.0.1
-pluggy==1.0.0
+pip==23.1.2
+pkgutil_resolve_name==1.3.10
 ply==3.11
-prometheus-client==0.16.0
+prometheus-client==0.17.0
 psycopg2-binary==2.9.6
 PTable==0.9.2
 pycadf==3.1.1
-pycodestyle==2.10.0
 pycparser==2.21
-pyflakes==3.0.1
-pyghmi==1.5.61
-Pygments==2.15.1
+pyghmi==1.5.62
 pylibyaml==0.1.0
-pymongo==3.10.1
+pymongo==4.3.3
 pyparsing==3.0.9
 pyrsistent==0.19.3
-pytest==7.3.1
-pytest-cov==4.0.0
-pytest-mock==3.10.0
 python-dateutil==2.8.2
 python-keystoneclient==5.1.0
+python-memcached==1.59
 pytz==2023.3
 PyYAML==6.0
-redfish==3.1.9
+redfish==3.2.1
 repoze.lru==0.7
-requests==2.29.0
-requests-toolbelt==0.10.1
+requests==2.31.0
+requests-toolbelt==1.0.0
 requests-unixsocket==0.3.0
 responses==0.23.1
-rfc3986==2.0.0
-rich==13.3.4
+rfc3986==1.5.0
 Routes==2.5.1
-setuptools==56.0.0
+setuptools==67.7.2
 six==1.16.0
-smmap==5.0.0
-snowballstemmer==2.2.0
-Sphinx==6.2.1
-sphinxcontrib-applehelp==1.0.4
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.1
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
-SQLAlchemy==1.3.21
+SQLAlchemy==1.4.48
+sqlalchemy-migrate==0.13.0
+sqlparse==0.4.4
 statsd==4.0.1
-stevedore==5.0.0
-tomli==2.0.1
-types-PyYAML==6.0.12.9
-typing_extensions==4.5.0
-ulid2==0.1.1
-urllib3==1.26.15
-uWSGI==2.0.21
+stevedore==5.1.0
+Tempita==0.5.2
+testresources==2.0.1
+testscenarios==0.5.0
+testtools==2.6.0
+types-PyYAML==6.0.12.10
+typing_extensions==4.6.3
+tzdata==2023.3
+ulid2==0.3.0
+urllib3==1.26.16
+uWSGI==2.0.22
 vine==5.0.0
 WebOb==1.8.7
 wheel==0.40.0
 wrapt==1.15.0
-yapf==0.33.0
 yappi==1.4.0
+zipp==3.15.0
--- a/python/test-requirements.txt
+++ b/python/test-requirements.txt
@ -1,15 +1,7 @@
-# tests
-click
-falcon==3.1.1
-jsonschema==3.2.0
-mock==5.0.1
-pylibyaml==0.1.0
-pymongo
 pytest >= 3.0
 pytest-cov==4.0.0
 pytest-mock
-responses
-ulid2
+

 # tools
 yapf
--- a/python/tests/postgres/start_postgres.sh
+++ b/python/tests/postgres/start_postgres.sh
@ -22,11 +22,11 @@ then
  docker stop 'psql_integration'
 fi

-docker run --rm -dp 5432:5432 --name 'psql_integration' -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.6
+docker run --rm -dp 5432:5432 --name 'psql_integration' -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.8
 sleep 15

-docker run --rm --net host postgres:14.6 psql -h localhost -c "create user drydock with password 'drydock';" postgres postgres
-docker run --rm --net host postgres:14.6 psql -h localhost -c "create database drydock;" postgres postgres
+docker run --rm --net host postgres:14.8 psql -h localhost -c "create user drydock with password 'drydock';" postgres postgres
+docker run --rm --net host postgres:14.8 psql -h localhost -c "create database drydock;" postgres postgres

 export DRYDOCK_DB_URL="postgresql+psycopg2://drydock:drydock@localhost:5432/drydock"

--- a/tools/drydock_image_run.sh
+++ b/tools/drydock_image_run.sh
@ -11,11 +11,11 @@ function start_db {
      sudo docker stop "${PSQL_CONTAINER_NAME}"
    fi

-    docker run --rm -dp 5432:5432 --name "${PSQL_CONTAINER_NAME}" -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.6
+    docker run --rm -dp 5432:5432 --name "${PSQL_CONTAINER_NAME}" -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.8
    sleep 15

-    docker run --rm --net host postgres:14.6 psql -h localhost -c "create user drydock with password 'drydock';" postgres postgres
-    docker run --rm --net host postgres:14.6 psql -h localhost -c "create database drydock;" postgres postgres
+    docker run --rm --net host postgres:14.8 psql -h localhost -c "create user drydock with password 'drydock';" postgres postgres
+    docker run --rm --net host postgres:14.8 psql -h localhost -c "create database drydock;" postgres postgres
 }

 function customize_conf {
--- a/tools/helm_install.sh
+++ b/tools/helm_install.sh
@ -17,7 +17,7 @@
 set -x

 HELM=$1
-HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz"}
+HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz"}


 function install_helm_binary {
--- a/tox.ini
+++ b/tox.ini
@ -8,6 +8,7 @@ setenv = YAMLDIR = {toxinidir}/python/tests/yaml_samples/
 passenv = http_proxy,HTTP_PROXY,https_proxy,HTTPS_PROXY,no_proxy,NO_PROXY
 deps=
  -r{toxinidir}/python/requirements-frozen.txt
+  -r{toxinidir}/python/test-requirements.txt

 [testenv:venv]
 commands = {posargs}
@ -19,26 +20,11 @@ allowlist_externals=
  sh
 deps=
  -r{toxinidir}/python/requirements-direct.txt
-  -r{toxinidir}/python/test-requirements.txt
+  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt
 commands=
  rm -f python/requirements-frozen.txt
  sh -c "pip freeze --all | grep -vE 'drydock_provisioner|pyinotify|pkg-resources==0.0.0' > python/requirements-frozen.txt"

-
-[testenv:freeze-bionic]
-basepython=python3
-recreate = True
-allowlist_externals=
-  rm
-  sh
-deps=
-  -r{toxinidir}/python/requirements-direct.txt
-commands=
-  rm -f python/requirements-frozen-bionic.txt
-  sh -c "pip freeze --all | grep -vE 'drydock_provisioner|pyinotify|pkg-resources==0.0.0' > python/requirements-frozen-bionic.txt"
-
-
-
 [testenv:yapf]
 allowlist_externals=find
 commands=
@ -50,6 +36,8 @@ commands=

 [testenv:py38]
 usedevelop=True
+allowlist_externals=
+  py.test
 setenv=
  PYTHONWARNING=all
  YAMLDIR={toxinidir}/python/tests/yaml_samples/
@ -58,21 +46,6 @@ commands=
    --cov=drydock_provisioner \
    {toxinidir}/python/tests/unit/{posargs}

-[testenv:py36]
-usedevelop=True
-setenv=
-  PYTHONWARNING=all
-  YAMLDIR={toxinidir}/python/tests/yaml_samples/
-deps=
-  -r{toxinidir}/python/requirements-frozen-bionic.txt
-  -r{toxinidir}/python/test-requirements.txt
-  tox
-commands=
-  py.test \
-    --cov=drydock_provisioner \
-    {toxinidir}/python/tests/unit/{posargs}
-
-
 [testenv:integration]
 passenv=DOCKER_REGISTRY,IMAGE_NAME,IMAGE_PREFIX,IMAGE_TAG,DISTRO
 setenv=
@ -81,6 +54,7 @@ setenv=
 allowlist_externals=
  sh
  pip3
+  py.test
 commands=
  pip3 install ./python/ --use-pep517
  sh -c {toxinidir}/python/tests/postgres/start_postgres.sh