Airflow 2.10.2 + ubuntu_jammy

Change-Id: Ic7bbe548418f4b3f113e092042c316bf66e6b8d3

.zuul.yaml

@@ -13,27 +13,28 @@
 - project:
     templates:
       - openstack-cover-jobs
-      - openstack-python38-jobs
 
     check:
       jobs:
+        - openstack-tox-py310
         - openstack-tox-docs
         - openstack-tox-pep8
         - armada-chart-build-gate
         - armada-chart-build-latest-htk
-        - armada-docker-build-gate-ubuntu_focal
+        - armada-docker-build-gate-ubuntu_jammy
-        - armada-airskiff-deployment-focal-kubeadm
+        - armada-airskiff-deployment-jammy-kubeadm
 
     gate:
       jobs:
+        - openstack-tox-py310
         - openstack-tox-docs
         - openstack-tox-pep8
         - armada-chart-build-gate
-        - armada-docker-build-gate-ubuntu_focal
+        - armada-docker-build-gate-ubuntu_jammy
 
     post:
       jobs:
-        - armada-docker-publish-ubuntu_focal
+        - armada-docker-publish-ubuntu_jammy
         - armada-upload-git-mirror
 
     promote:
@@ -47,10 +48,10 @@
         label: ubuntu-bionic
 
 - nodeset:
-    name: armada-single-node-focal
+    name: armada-single-node-jammy
     nodes:
       - name: primary
-        label: ubuntu-focal
+        label: ubuntu-jammy
 
 - job:
     name: armada-chart-build-gate
@@ -58,9 +59,9 @@
       Builds charts using pinned Helm toolkit.
     timeout: 900
     run: tools/gate/playbooks/build-charts.yaml
-    nodeset: armada-single-node-focal
+    nodeset: armada-single-node-jammy
     vars:
-      HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+      HTK_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b
 
 
 - job:
@@ -70,18 +71,18 @@
     timeout: 900
     voting: true
     run: tools/gate/playbooks/build-charts.yaml
-    nodeset: armada-single-node-focal
+    nodeset: armada-single-node-jammy
     vars:
       HTK_COMMIT: master
 
 - job:
-    name: armada-docker-build-gate-ubuntu_focal
+    name: armada-docker-build-gate-ubuntu_jammy
     timeout: 3600
     run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: armada-single-node-focal
+    nodeset: armada-single-node-jammy
     vars:
       publish: false
-      distro: ubuntu_focal
+      distro: ubuntu_jammy
       tags:
         dynamic:
           patch_set: true
@@ -107,6 +108,7 @@
       - ^swagger/.*$
     timeout: 10800
     pre-run:
+      - tools/gate/playbooks/checkout-treasuremap-ref.yaml
       - tools/gate/playbooks/prepare-hosts.yaml
       - tools/gate/playbooks/mount-volumes.yaml
       - tools/gate/playbooks/deploy-env.yaml
@@ -115,6 +117,7 @@
     post-run:
       - tools/gate/playbooks/osh-infra-collect-logs.yaml
     vars:
+      treasuremap_ref: refs/changes/43/927643/4
       extra_volume:
         size: 80G
         type: Linux
@@ -131,15 +134,16 @@
       loopback_device: /dev/loop100
       loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img"
       ceph_osd_data_device: /dev/loop100
-      kube_version_repo: "v1.29"
+      kube_version_repo: "v1.30"
-      kube_version: "1.29.5-1.1"
+      kube_version: "1.30.3-1.1"
       calico_setup: true
-      calico_version: "v3.27.0"
+      calico_version: "v3.27.4"
       cilium_setup: false
-      cilium_version: "1.15.6"
+      cilium_version: "1.16.0"
-      helm_version: "v3.6.3"
+      flannel_setup: false
-      yq_version: "v4.6.0"
+      flannel_version: v0.25.4
-      crictl_version: "v1.26.1"
+      helm_version: "v3.15.4"
+      crictl_version: "v1.30.1"
       zuul_osh_relative_path: ../../openstack/openstack-helm
       zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra
       zuul_treasuremap_relative_path: ../../airship/treasuremap
@@ -147,19 +151,20 @@
       run_helm_tests: "no"
 
 - job:
-    name: armada-airskiff-deployment-focal-kubeadm
+    name: armada-airskiff-deployment-jammy-kubeadm
-    nodeset: treasuremap-airskiff-1node-ubuntu_focal
+    nodeset: treasuremap-airskiff-1node-ubuntu_jammy
     description: |
       Deploy Memcached using Airskiff and submitted Armada changes.
     parent: armada-base
     vars:
       site: airskiff
       HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz
-      HTK_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8
+      HTK_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b
-      OSH_INFRA_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8
+      OSH_INFRA_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b
-      OSH_COMMIT: 049e679939fbd3b0c659dd0977911b8dc3b5a015
+      OSH_COMMIT: 540df5cb0dbdaed63c202e2d6f2b7891062f8203
+      TREASUREMAP_COMMIT: refs/changes/43/927643/4
       CLONE_ARMADA: false
-      DISTRO: ubuntu_focal
+      DISTRO: ubuntu_jammy
       DOCKER_REGISTRY: localhost:5000
       MAKE_ARMADA_IMAGES: true
       gate_scripts_relative_path: ../../airship/treasuremap
@@ -178,51 +183,24 @@
         - ./tools/gate/wait-for-shipyard.sh
     irrelevant-files: *irrelevant-files
 
-# - job:
-#     name: armada-airskiff-deploy-focal
-#     nodeset: armada-single-node-focal
-#     description: |
-#       Deploy Memcached using Airskiff and submitted Armada changes.
-#     timeout: 9600
-#     voting: true
-#     pre-run:
-#       - tools/gate/playbooks/airskiff-reduce-site.yaml
-#       - tools/gate/playbooks/git-config.yaml
-#     run: tools/gate/playbooks/airskiff-deploy.yaml
-#     post-run: tools/gate/playbooks/debug-report.yaml
-#     required-projects:
-#       - airship/treasuremap
-#     vars:
-#       CLONE_ARMADA: false
-#       ARMADA_IMAGE_DISTRO: ubuntu_focal
-#       HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz
-#       HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
-#       OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
-#       OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
-#     irrelevant-files:
-#       - ^.*\.rst$
-#       - ^doc/.*$
-#       - ^examples/.*$
-#       - ^releasenotes/.*$
-#       - ^swagger/.*$
 
 - job:
-    name: armada-docker-publish-ubuntu_focal
+    name: armada-docker-publish-ubuntu_jammy
     timeout: 3600
     run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: armada-single-node-focal
+    nodeset: armada-single-node-jammy
     secrets:
       - airship_armada_quay_creds
     vars:
       publish: true
-      distro: ubuntu_focal
+      distro: ubuntu_jammy
       tags:
         dynamic:
           branch: true
           commit: true
         static:
           - latest
-          - airflow_2.8.2
+          - airflow_2.10.0
 
 
 - secret:

Makefile

@@ -28,8 +28,10 @@ LABEL             ?= org.airshipit.build=community
 COMMIT            ?= $(shell git rev-parse HEAD)
 PYTHON            = python3
 CHARTS            := $(filter-out deps, $(patsubst charts/%/.,%,$(wildcard charts/*/.)))
-DISTRO            ?= ubuntu_focal
+DISTRO            ?= ubuntu_jammy
+DISTRO_ALIAS	   ?= ubuntu_focal
 IMAGE             := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}
+IMAGE_ALIAS              := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO_ALIAS}
 UBUNTU_BASE_IMAGE ?=
 
 # Helm binary download url
@@ -116,6 +118,13 @@ else
 		$(_BASE_IMAGE_ARG) \
 		--build-arg HELM_ARTIFACT_URL=$(HELM_ARTIFACT_URL) .
 endif
+ifneq ($(DISTRO), $(DISTRO_ALIAS))
+	docker tag $(IMAGE) $(IMAGE_ALIAS)
+endif
+ifeq ($(DOCKER_REGISTRY), localhost:5000)
+	docker push $(IMAGE)
+	docker push $(IMAGE_ALIAS)
+endif
 ifeq ($(PUSH_IMAGE), true)
 	docker push $(IMAGE)
 endif

etc/armada/armada.conf.sample

@@ -52,6 +52,14 @@
 # Minimum value: 0
 #lock_expiration = 600
 
+# Determines whether the operator has to be enabled         to apply charts
+# instead of armada-api itself (boolean value)
+#enable_operator = false
+
+# Determines whether the wait process has to be done         via armada-go
+# using client-go library (boolean value)
+#go_wait = false
+
 #
 # From oslo.log
 #
@@ -321,7 +329,7 @@
 # Domain name containing project (string value)
 #project_domain_name = <None>
 
-# Trust ID (string value)
+# ID of the trust to use as a trustee use (string value)
 #trust_id = <None>
 
 # Optional domain ID to use with v3 and v2 parameters. It will be used for both
@@ -532,6 +540,9 @@
 # middleware should parse the headers or not. (boolean value)
 #enable_proxy_headers_parsing = false
 
+# HTTP basic auth password file. (string value)
+#http_basic_auth_user_file = /etc/htpasswd
+
 
 [oslo_policy]
 

images/armada/Dockerfile.ubuntu_jammy

@@ -0,0 +1,104 @@
+ARG FROM=ubuntu:jammy
+ARG ARMADA_GO=quay.io/airshipit/armada-go:latest-ubuntu_focal
+FROM ${ARMADA_GO} AS armada_go
+FROM ${FROM}
+
+LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
+      org.opencontainers.image.url='https://airshipit.org' \
+      org.opencontainers.image.documentation='https://docs.airshipit.org/armada' \
+      org.opencontainers.image.source='https://opendev.org/airship/armada' \
+      org.opencontainers.image.vendor='The Airship Authors' \
+      org.opencontainers.image.licenses='Apache-2.0'
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
+
+EXPOSE 8000
+
+RUN set -ex && \
+    apt-get update && apt-get upgrade -y && \
+    apt-get -y install \
+    ca-certificates \
+    curl \
+    git \
+    netbase \
+    python3-dev \
+    python3-setuptools \
+    --no-install-recommends \
+    && apt-get autoremove -yqq --purge \
+    && apt-get clean \
+    && rm -rf \
+        /var/lib/apt/lists/* \
+        /tmp/* \
+        /var/tmp/* \
+        /usr/share/man \
+        /usr/share/doc \
+        /usr/share/doc-base
+
+WORKDIR /armada
+
+# Add armada user
+RUN useradd -u 1000 -g users -d $(pwd) armada
+
+ENTRYPOINT ["./entrypoint.sh"]
+CMD ["server"]
+
+COPY requirements-frozen.txt ./
+
+ENV LD_LIBRARY_PATH=/usr/local/lib
+
+ARG HELM_ARTIFACT_URL
+ARG LIBYAML_VERSION=0.2.5
+
+# Build
+RUN set -ex \
+    && buildDeps=' \
+      automake \
+      gcc \
+      libssl-dev \
+      libtool \
+      make \
+      python3-pip \
+    ' \
+    && apt-get -qq update \
+    # Keep git separate so it's not removed below
+    && apt-get install -y $buildDeps git --no-install-recommends \
+    && git clone https://github.com/yaml/libyaml.git \
+    && cd libyaml \
+    && git checkout $LIBYAML_VERSION \
+    && ./bootstrap \
+    && ./configure \
+    && make \
+    && make install \
+    && cd .. \
+    && rm -fr libyaml \
+    && python3 -m pip install -U pip \
+    && pip3 install -r requirements-frozen.txt --no-cache-dir \
+    && curl -fSSL -O ${HELM_ARTIFACT_URL} \
+    && tar -xvf $(basename ${HELM_ARTIFACT_URL}) \
+    && mv linux-amd64/helm /usr/local/bin \
+    && apt-get purge -y --auto-remove $buildDeps \
+    && apt-get autoremove -yqq --purge \
+    && apt-get clean \
+    && rm -rf \
+        /var/lib/apt/lists/* \
+        /tmp/* \
+        /var/tmp/* \
+        /usr/share/man \
+        /usr/share/doc \
+        /usr/share/doc-base
+
+COPY . ./
+COPY --from=armada_go /usr/local/bin/armada /usr/local/bin/armada-go
+
+# Setting the version explicitly for PBR
+ENV PBR_VERSION 0.8.0
+
+RUN \
+    chown -R armada:users . && \
+    pip3 install -e . --verbose --use-pep517 \
+    && echo "/armada" \
+        > /usr/local/lib/python3.10/dist-packages/armada.pth
+
+USER armada

requirements-frozen.txt

@@ -1,37 +1,35 @@
-alembic==1.13.1
+alembic==1.13.2
-attrs==23.2.0
+attrs==24.2.0
-bcrypt==4.1.2
+bcrypt==4.2.0
-cachetools==5.3.2
+cachetools==5.5.0
-certifi==2024.2.2
+certifi==2024.8.30
 charset-normalizer==3.3.2
 click==8.1.7
 debtcollector==3.0.0
 decorator==5.1.1
-deepdiff==6.7.1
+deepdiff==8.0.1
-dogpile.cache==1.3.2
+dogpile.cache==1.3.3
 falcon==3.1.3
 gitdb==4.0.11
-GitPython==3.1.42
+GitPython==3.1.43
-google-auth==2.28.1
+google-auth==2.34.0
-greenlet==3.0.3
+greenlet==3.1.0
-idna==3.6
+idna==3.10
-importlib-metadata==6.11.0
-importlib-resources==5.13.0
 iso8601==2.1.0
-Jinja2==3.1.3
+Jinja2==3.1.4
-jsonschema==4.21.1
+jsonschema==4.23.0
 jsonschema-specifications==2023.12.1
 keystoneauth1==5.1.2
 keystonemiddleware==10.2.0
-kubernetes==29.0.0
+kubernetes==30.1.0
-Mako==1.3.2
+Mako==1.3.5
 MarkupSafe==2.1.5
 mock==5.1.0
-msgpack==1.0.7
+msgpack==1.1.0
-netaddr==1.2.1
+netaddr==1.3.0
 netifaces==0.11.0
 oauthlib==3.2.2
-ordered-set==4.1.0
+orderly-set==5.2.2
 os-service-types==1.7.0
 oslo.cache==3.3.1
 oslo.config==9.1.1
@@ -43,49 +41,47 @@ oslo.middleware==5.1.1
 oslo.policy==4.1.1
 oslo.serialization==5.1.1
 oslo.utils==6.1.0
-packaging==23.2
+packaging==24.1
-Paste==3.7.1
+Paste==3.10.1
 PasteDeploy==3.1.0
-pbr==6.0.0
+pbr==6.1.0
-pip==23.2.1
+pip==24.1
-pkgutil_resolve_name==1.3.10
 prometheus_client==0.20.0
 py==1.11.0
-pyasn1==0.5.1
+pyasn1==0.6.1
-pyasn1-modules==0.3.0
+pyasn1_modules==0.4.0
 pycadf==3.1.1
 pylibyaml==0.1.0
-pyparsing==3.1.1
+pyparsing==3.1.4
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
 python-keystoneclient==5.1.0
 python-memcached==1.62
-pytz==2024.1
+pytz==2024.2
-PyYAML==6.0.1
+PyYAML==6.0.2
-referencing==0.33.0
+referencing==0.35.1
-requests==2.31.0
+requests==2.32.3
 requests-oauthlib==1.3.1
-responses==0.25.0
+responses==0.25.3
 retry==0.9.2
-rfc3986==1.5.0
+rfc3986==2.0.0
-rpds-py==0.18.0
+rpds-py==0.20.0
 rsa==4.9
-setuptools==68.2.2
+setuptools==70.1.0
 six==1.16.0
 smmap==5.0.1
-SQLAlchemy==1.4.51
+SQLAlchemy==1.4.54
 sqlalchemy-migrate==0.13.0
-sqlparse==0.4.4
+sqlparse==0.5.1
 statsd==4.0.1
-stevedore==5.2.0
+stevedore==5.3.0
 Tempita==0.5.2
 testresources==2.0.1
 testscenarios==0.5.0
-testtools==2.7.1
+testtools==2.7.2
-typing_extensions==4.9.0
+typing_extensions==4.12.2
-urllib3==1.26.18
+urllib3==2.2.2
-uWSGI==2.0.24
+uWSGI==2.0.27
-WebOb==1.8.7
+WebOb==1.8.8
-websocket-client==1.7.0
+websocket-client==1.8.0
-wheel==0.41.2
+wheel==0.43.0
 wrapt==1.16.0
-zipp==3.17.0

tools/gate/playbooks/checkout-treasuremap-ref.yaml

@@ -0,0 +1,31 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+
+  tasks:
+    - name: Checkout treasuremap ref
+      shell: |
+        set -xe;
+
+        : "${TREASUREMAP_REF:=v1.9}"
+
+        cd ../treasuremap
+        git fetch https://review.opendev.org/airship/treasuremap ${TREASUREMAP_REF} && git checkout FETCH_HEAD
+
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+      environment:
+        TREASUREMAP_REF: "{{ treasuremap_ref }}"
+
+...

tools/gate/playbooks/prepare-hosts.yaml

@@ -15,4 +15,7 @@
   roles:
     - bindep
     - start-zuul-console
+    - role: add-authorized-keys
+      public_keys:
+        - public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA7eM8WFJrqQmki8rR0O3QBHyl8xq42jb1RduwuRwjWoGYJI5cX7Fx+7VR4A9ITCoiqxKS8DMfgKbt5jKC6SmvMALULZsnYlthB34KywurgxsW6fgp68DHWQ7J4CCBhoIpl0W3JW7s6b0vHLhab59r0E+AYemBVuWUqbFEy8nDAHcQv1S/2o1udhmljIN7c2ogO4KAJ7Lge0BoIP9ps4u6AVwyQZixp4anU9DHGNA/UQj4M5UyuALj5buEAuATBe9Vqj4sOvZjObPJAGPUrNRrGEWAFk+lSZHRzKXo0eeWtPqoh5UN9UDb5Pocg1krncMIZwjHKovlD1z/O1y91aY5LM1wxm/7aaIiX8eCihyVZaOuDCLF7WDT2SMs7ABcotX2MDtVQTrNNV3MmMAScFNDflzPKszd7cdjLl6PBq8bvPxmCkLmnitPTGOoh9d8i+JlbINvgx1pguYrpeciIyreCO1rjTW3MgB0tyoMEa31V+7HrauBMeNnE68YTqLTIB0= smarkin@mirantis.com
 ...

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 skipsdist = True
 minversion = 3.18.0
-envlist = py38, pep8, cover, bandit
+envlist = py310, pep8, cover, bandit
 
 [testenv]
 deps=
@@ -38,13 +38,13 @@ allowlist_externals=
   sh
 deps=
   -r{toxinidir}/requirements-direct.txt
-  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.2/constraints-3.8.txt
+  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt
 commands=
   rm -f requirements-frozen.txt
   sh -c "pip freeze --all | grep -vE 'armada|pyinotify|pkg-resources==0.0.0' > requirements-frozen.txt"
 
 
-[testenv:py38]
+[testenv:py310]
 commands =
     {[testenv]commands}
     stestr run {posargs}