Airflow 2.10.2 + ubuntu_jammy

Change-Id: Ic7bbe548418f4b3f113e092042c316bf66e6b8d3
2024-09-04 16:09:22 +00:00 · 2024-09-04 16:09:22 +00:00 · edf8b5b88b
commit edf8b5b88b
parent ca9a16790f
8 changed files with 242 additions and 110 deletions
--- a/.zuul.yaml
+++ b/.zuul.yaml
@ -13,27 +13,28 @@
 - project:
    templates:
      - openstack-cover-jobs
-      - openstack-python38-jobs

    check:
      jobs:
+        - openstack-tox-py310
        - openstack-tox-docs
        - openstack-tox-pep8
        - armada-chart-build-gate
        - armada-chart-build-latest-htk
-        - armada-docker-build-gate-ubuntu_focal
-        - armada-airskiff-deployment-focal-kubeadm
+        - armada-docker-build-gate-ubuntu_jammy
+        - armada-airskiff-deployment-jammy-kubeadm

    gate:
      jobs:
+        - openstack-tox-py310
        - openstack-tox-docs
        - openstack-tox-pep8
        - armada-chart-build-gate
-        - armada-docker-build-gate-ubuntu_focal
+        - armada-docker-build-gate-ubuntu_jammy

    post:
      jobs:
-        - armada-docker-publish-ubuntu_focal
+        - armada-docker-publish-ubuntu_jammy
        - armada-upload-git-mirror

    promote:
@ -47,10 +48,10 @@
        label: ubuntu-bionic

 - nodeset:
-    name: armada-single-node-focal
+    name: armada-single-node-jammy
    nodes:
      - name: primary
-        label: ubuntu-focal
+        label: ubuntu-jammy

 - job:
    name: armada-chart-build-gate
@ -58,9 +59,9 @@
      Builds charts using pinned Helm toolkit.
    timeout: 900
    run: tools/gate/playbooks/build-charts.yaml
-    nodeset: armada-single-node-focal
+    nodeset: armada-single-node-jammy
    vars:
-      HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+      HTK_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b


 - job:
@ -70,18 +71,18 @@
    timeout: 900
    voting: true
    run: tools/gate/playbooks/build-charts.yaml
-    nodeset: armada-single-node-focal
+    nodeset: armada-single-node-jammy
    vars:
      HTK_COMMIT: master

 - job:
-    name: armada-docker-build-gate-ubuntu_focal
+    name: armada-docker-build-gate-ubuntu_jammy
    timeout: 3600
    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: armada-single-node-focal
+    nodeset: armada-single-node-jammy
    vars:
      publish: false
-      distro: ubuntu_focal
+      distro: ubuntu_jammy
      tags:
        dynamic:
          patch_set: true
@ -107,6 +108,7 @@
      - ^swagger/.*$
    timeout: 10800
    pre-run:
+      - tools/gate/playbooks/checkout-treasuremap-ref.yaml
      - tools/gate/playbooks/prepare-hosts.yaml
      - tools/gate/playbooks/mount-volumes.yaml
      - tools/gate/playbooks/deploy-env.yaml
@ -115,6 +117,7 @@
    post-run:
      - tools/gate/playbooks/osh-infra-collect-logs.yaml
    vars:
+      treasuremap_ref: refs/changes/43/927643/4
      extra_volume:
        size: 80G
        type: Linux
@ -131,15 +134,16 @@
      loopback_device: /dev/loop100
      loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img"
      ceph_osd_data_device: /dev/loop100
-      kube_version_repo: "v1.29"
-      kube_version: "1.29.5-1.1"
+      kube_version_repo: "v1.30"
+      kube_version: "1.30.3-1.1"
      calico_setup: true
-      calico_version: "v3.27.0"
+      calico_version: "v3.27.4"
      cilium_setup: false
-      cilium_version: "1.15.6"
-      helm_version: "v3.6.3"
-      yq_version: "v4.6.0"
-      crictl_version: "v1.26.1"
+      cilium_version: "1.16.0"
+      flannel_setup: false
+      flannel_version: v0.25.4
+      helm_version: "v3.15.4"
+      crictl_version: "v1.30.1"
      zuul_osh_relative_path: ../../openstack/openstack-helm
      zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra
      zuul_treasuremap_relative_path: ../../airship/treasuremap
@ -147,19 +151,20 @@
      run_helm_tests: "no"

 - job:
-    name: armada-airskiff-deployment-focal-kubeadm
-    nodeset: treasuremap-airskiff-1node-ubuntu_focal
+    name: armada-airskiff-deployment-jammy-kubeadm
+    nodeset: treasuremap-airskiff-1node-ubuntu_jammy
    description: |
      Deploy Memcached using Airskiff and submitted Armada changes.
    parent: armada-base
    vars:
      site: airskiff
      HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz
-      HTK_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8
-      OSH_INFRA_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8
-      OSH_COMMIT: 049e679939fbd3b0c659dd0977911b8dc3b5a015
+      HTK_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b
+      OSH_INFRA_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b
+      OSH_COMMIT: 540df5cb0dbdaed63c202e2d6f2b7891062f8203
+      TREASUREMAP_COMMIT: refs/changes/43/927643/4
      CLONE_ARMADA: false
-      DISTRO: ubuntu_focal
+      DISTRO: ubuntu_jammy
      DOCKER_REGISTRY: localhost:5000
      MAKE_ARMADA_IMAGES: true
      gate_scripts_relative_path: ../../airship/treasuremap
@ -178,51 +183,24 @@
        - ./tools/gate/wait-for-shipyard.sh
    irrelevant-files: *irrelevant-files

-# - job:
-#     name: armada-airskiff-deploy-focal
-#     nodeset: armada-single-node-focal
-#     description: |
-#       Deploy Memcached using Airskiff and submitted Armada changes.
-#     timeout: 9600
-#     voting: true
-#     pre-run:
-#       - tools/gate/playbooks/airskiff-reduce-site.yaml
-#       - tools/gate/playbooks/git-config.yaml
-#     run: tools/gate/playbooks/airskiff-deploy.yaml
-#     post-run: tools/gate/playbooks/debug-report.yaml
-#     required-projects:
-#       - airship/treasuremap
-#     vars:
-#       CLONE_ARMADA: false
-#       ARMADA_IMAGE_DISTRO: ubuntu_focal
-#       HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz
-#       HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
-#       OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
-#       OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
-#     irrelevant-files:
-#       - ^.*\.rst$
-#       - ^doc/.*$
-#       - ^examples/.*$
-#       - ^releasenotes/.*$
-#       - ^swagger/.*$

 - job:
-    name: armada-docker-publish-ubuntu_focal
+    name: armada-docker-publish-ubuntu_jammy
    timeout: 3600
    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: armada-single-node-focal
+    nodeset: armada-single-node-jammy
    secrets:
      - airship_armada_quay_creds
    vars:
      publish: true
-      distro: ubuntu_focal
+      distro: ubuntu_jammy
      tags:
        dynamic:
          branch: true
          commit: true
        static:
          - latest
-          - airflow_2.8.2
+          - airflow_2.10.0


 - secret:
--- a/11
+++ b/11
@ -28,8 +28,10 @@ LABEL             ?= org.airshipit.build=community
 COMMIT            ?= $(shell git rev-parse HEAD)
 PYTHON            = python3
 CHARTS            := $(filter-out deps, $(patsubst charts/%/.,%,$(wildcard charts/*/.)))
-DISTRO            ?= ubuntu_focal
+DISTRO            ?= ubuntu_jammy
+DISTRO_ALIAS	   ?= ubuntu_focal
 IMAGE             := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}
+IMAGE_ALIAS              := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO_ALIAS}
 UBUNTU_BASE_IMAGE ?=

 # Helm binary download url
@ -116,6 +118,13 @@ else
 		$(_BASE_IMAGE_ARG) \
 		--build-arg HELM_ARTIFACT_URL=$(HELM_ARTIFACT_URL) .
 endif
+ifneq ($(DISTRO), $(DISTRO_ALIAS))
+	docker tag $(IMAGE) $(IMAGE_ALIAS)
+endif
+ifeq ($(DOCKER_REGISTRY), localhost:5000)
+	docker push $(IMAGE)
+	docker push $(IMAGE_ALIAS)
+endif
 ifeq ($(PUSH_IMAGE), true)
 	docker push $(IMAGE)
 endif
--- a/etc/armada/armada.conf.sample
+++ b/etc/armada/armada.conf.sample
@ -52,6 +52,14 @@
 # Minimum value: 0
 #lock_expiration = 600

+# Determines whether the operator has to be enabled         to apply charts
+# instead of armada-api itself (boolean value)
+#enable_operator = false
+
+# Determines whether the wait process has to be done         via armada-go
+# using client-go library (boolean value)
+#go_wait = false
+
 #
 # From oslo.log
 #
@ -321,7 +329,7 @@
 # Domain name containing project (string value)
 #project_domain_name = <None>

-# Trust ID (string value)
+# ID of the trust to use as a trustee use (string value)
 #trust_id = <None>

 # Optional domain ID to use with v3 and v2 parameters. It will be used for both
@ -532,6 +540,9 @@
 # middleware should parse the headers or not. (boolean value)
 #enable_proxy_headers_parsing = false

+# HTTP basic auth password file. (string value)
+#http_basic_auth_user_file = /etc/htpasswd
+

 [oslo_policy]

--- a/images/armada/Dockerfile.ubuntu_jammy
+++ b/images/armada/Dockerfile.ubuntu_jammy
@ -0,0 +1,104 @@
+ARG FROM=ubuntu:jammy
+ARG ARMADA_GO=quay.io/airshipit/armada-go:latest-ubuntu_focal
+FROM ${ARMADA_GO} AS armada_go
+FROM ${FROM}
+
+LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
+      org.opencontainers.image.url='https://airshipit.org' \
+      org.opencontainers.image.documentation='https://docs.airshipit.org/armada' \
+      org.opencontainers.image.source='https://opendev.org/airship/armada' \
+      org.opencontainers.image.vendor='The Airship Authors' \
+      org.opencontainers.image.licenses='Apache-2.0'
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
+
+EXPOSE 8000
+
+RUN set -ex && \
+    apt-get update && apt-get upgrade -y && \
+    apt-get -y install \
+    ca-certificates \
+    curl \
+    git \
+    netbase \
+    python3-dev \
+    python3-setuptools \
+    --no-install-recommends \
+    && apt-get autoremove -yqq --purge \
+    && apt-get clean \
+    && rm -rf \
+        /var/lib/apt/lists/* \
+        /tmp/* \
+        /var/tmp/* \
+        /usr/share/man \
+        /usr/share/doc \
+        /usr/share/doc-base
+
+WORKDIR /armada
+
+# Add armada user
+RUN useradd -u 1000 -g users -d $(pwd) armada
+
+ENTRYPOINT ["./entrypoint.sh"]
+CMD ["server"]
+
+COPY requirements-frozen.txt ./
+
+ENV LD_LIBRARY_PATH=/usr/local/lib
+
+ARG HELM_ARTIFACT_URL
+ARG LIBYAML_VERSION=0.2.5
+
+# Build
+RUN set -ex \
+    && buildDeps=' \
+      automake \
+      gcc \
+      libssl-dev \
+      libtool \
+      make \
+      python3-pip \
+    ' \
+    && apt-get -qq update \
+    # Keep git separate so it's not removed below
+    && apt-get install -y $buildDeps git --no-install-recommends \
+    && git clone https://github.com/yaml/libyaml.git \
+    && cd libyaml \
+    && git checkout $LIBYAML_VERSION \
+    && ./bootstrap \
+    && ./configure \
+    && make \
+    && make install \
+    && cd .. \
+    && rm -fr libyaml \
+    && python3 -m pip install -U pip \
+    && pip3 install -r requirements-frozen.txt --no-cache-dir \
+    && curl -fSSL -O ${HELM_ARTIFACT_URL} \
+    && tar -xvf $(basename ${HELM_ARTIFACT_URL}) \
+    && mv linux-amd64/helm /usr/local/bin \
+    && apt-get purge -y --auto-remove $buildDeps \
+    && apt-get autoremove -yqq --purge \
+    && apt-get clean \
+    && rm -rf \
+        /var/lib/apt/lists/* \
+        /tmp/* \
+        /var/tmp/* \
+        /usr/share/man \
+        /usr/share/doc \
+        /usr/share/doc-base
+
+COPY . ./
+COPY --from=armada_go /usr/local/bin/armada /usr/local/bin/armada-go
+
+# Setting the version explicitly for PBR
+ENV PBR_VERSION 0.8.0
+
+RUN \
+    chown -R armada:users . && \
+    pip3 install -e . --verbose --use-pep517 \
+    && echo "/armada" \
+        > /usr/local/lib/python3.10/dist-packages/armada.pth
+
+USER armada
--- a/requirements-frozen.txt
+++ b/requirements-frozen.txt
@ -1,37 +1,35 @@
-alembic==1.13.1
-attrs==23.2.0
-bcrypt==4.1.2
-cachetools==5.3.2
-certifi==2024.2.2
+alembic==1.13.2
+attrs==24.2.0
+bcrypt==4.2.0
+cachetools==5.5.0
+certifi==2024.8.30
 charset-normalizer==3.3.2
 click==8.1.7
 debtcollector==3.0.0
 decorator==5.1.1
-deepdiff==6.7.1
-dogpile.cache==1.3.2
+deepdiff==8.0.1
+dogpile.cache==1.3.3
 falcon==3.1.3
 gitdb==4.0.11
-GitPython==3.1.42
-google-auth==2.28.1
-greenlet==3.0.3
-idna==3.6
-importlib-metadata==6.11.0
-importlib-resources==5.13.0
+GitPython==3.1.43
+google-auth==2.34.0
+greenlet==3.1.0
+idna==3.10
 iso8601==2.1.0
-Jinja2==3.1.3
-jsonschema==4.21.1
+Jinja2==3.1.4
+jsonschema==4.23.0
 jsonschema-specifications==2023.12.1
 keystoneauth1==5.1.2
 keystonemiddleware==10.2.0
-kubernetes==29.0.0
-Mako==1.3.2
+kubernetes==30.1.0
+Mako==1.3.5
 MarkupSafe==2.1.5
 mock==5.1.0
-msgpack==1.0.7
-netaddr==1.2.1
+msgpack==1.1.0
+netaddr==1.3.0
 netifaces==0.11.0
 oauthlib==3.2.2
-ordered-set==4.1.0
+orderly-set==5.2.2
 os-service-types==1.7.0
 oslo.cache==3.3.1
 oslo.config==9.1.1
@ -43,49 +41,47 @@ oslo.middleware==5.1.1
 oslo.policy==4.1.1
 oslo.serialization==5.1.1
 oslo.utils==6.1.0
-packaging==23.2
-Paste==3.7.1
+packaging==24.1
+Paste==3.10.1
 PasteDeploy==3.1.0
-pbr==6.0.0
-pip==23.2.1
-pkgutil_resolve_name==1.3.10
+pbr==6.1.0
+pip==24.1
 prometheus_client==0.20.0
 py==1.11.0
-pyasn1==0.5.1
-pyasn1-modules==0.3.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.0
 pycadf==3.1.1
 pylibyaml==0.1.0
-pyparsing==3.1.1
-python-dateutil==2.8.2
+pyparsing==3.1.4
+python-dateutil==2.9.0.post0
 python-keystoneclient==5.1.0
 python-memcached==1.62
-pytz==2024.1
-PyYAML==6.0.1
-referencing==0.33.0
-requests==2.31.0
+pytz==2024.2
+PyYAML==6.0.2
+referencing==0.35.1
+requests==2.32.3
 requests-oauthlib==1.3.1
-responses==0.25.0
+responses==0.25.3
 retry==0.9.2
-rfc3986==1.5.0
-rpds-py==0.18.0
+rfc3986==2.0.0
+rpds-py==0.20.0
 rsa==4.9
-setuptools==68.2.2
+setuptools==70.1.0
 six==1.16.0
 smmap==5.0.1
-SQLAlchemy==1.4.51
+SQLAlchemy==1.4.54
 sqlalchemy-migrate==0.13.0
-sqlparse==0.4.4
+sqlparse==0.5.1
 statsd==4.0.1
-stevedore==5.2.0
+stevedore==5.3.0
 Tempita==0.5.2
 testresources==2.0.1
 testscenarios==0.5.0
-testtools==2.7.1
-typing_extensions==4.9.0
-urllib3==1.26.18
-uWSGI==2.0.24
-WebOb==1.8.7
-websocket-client==1.7.0
-wheel==0.41.2
+testtools==2.7.2
+typing_extensions==4.12.2
+urllib3==2.2.2
+uWSGI==2.0.27
+WebOb==1.8.8
+websocket-client==1.8.0
+wheel==0.43.0
 wrapt==1.16.0
-zipp==3.17.0
--- a/tools/gate/playbooks/checkout-treasuremap-ref.yaml
+++ b/tools/gate/playbooks/checkout-treasuremap-ref.yaml
@ -0,0 +1,31 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+
+  tasks:
+    - name: Checkout treasuremap ref
+      shell: |
+        set -xe;
+
+        : "${TREASUREMAP_REF:=v1.9}"
+
+        cd ../treasuremap
+        git fetch https://review.opendev.org/airship/treasuremap ${TREASUREMAP_REF} && git checkout FETCH_HEAD
+
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+      environment:
+        TREASUREMAP_REF: "{{ treasuremap_ref }}"
+
+...
--- a/tools/gate/playbooks/prepare-hosts.yaml
+++ b/tools/gate/playbooks/prepare-hosts.yaml
@ -15,4 +15,7 @@
  roles:
    - bindep
    - start-zuul-console
+    - role: add-authorized-keys
+      public_keys:
+        - public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA7eM8WFJrqQmki8rR0O3QBHyl8xq42jb1RduwuRwjWoGYJI5cX7Fx+7VR4A9ITCoiqxKS8DMfgKbt5jKC6SmvMALULZsnYlthB34KywurgxsW6fgp68DHWQ7J4CCBhoIpl0W3JW7s6b0vHLhab59r0E+AYemBVuWUqbFEy8nDAHcQv1S/2o1udhmljIN7c2ogO4KAJ7Lge0BoIP9ps4u6AVwyQZixp4anU9DHGNA/UQj4M5UyuALj5buEAuATBe9Vqj4sOvZjObPJAGPUrNRrGEWAFk+lSZHRzKXo0eeWtPqoh5UN9UDb5Pocg1krncMIZwjHKovlD1z/O1y91aY5LM1wxm/7aaIiX8eCihyVZaOuDCLF7WDT2SMs7ABcotX2MDtVQTrNNV3MmMAScFNDflzPKszd7cdjLl6PBq8bvPxmCkLmnitPTGOoh9d8i+JlbINvgx1pguYrpeciIyreCO1rjTW3MgB0tyoMEa31V+7HrauBMeNnE68YTqLTIB0= smarkin@mirantis.com
 ...
--- a/tox.ini
+++ b/tox.ini
@ -1,7 +1,7 @@
 [tox]
 skipsdist = True
 minversion = 3.18.0
-envlist = py38, pep8, cover, bandit
+envlist = py310, pep8, cover, bandit

 [testenv]
 deps=
@ -38,13 +38,13 @@ allowlist_externals=
  sh
 deps=
  -r{toxinidir}/requirements-direct.txt
-  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.2/constraints-3.8.txt
+  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt
 commands=
  rm -f requirements-frozen.txt
  sh -c "pip freeze --all | grep -vE 'armada|pyinotify|pkg-resources==0.0.0' > requirements-frozen.txt"


-[testenv:py38]
+[testenv:py310]
 commands =
    {[testenv]commands}
    stestr run {posargs}