diff --git a/.zuul.yaml b/.zuul.yaml index 0c94ca3e..5daf4421 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -13,27 +13,28 @@ - project: templates: - openstack-cover-jobs - - openstack-python38-jobs check: jobs: + - openstack-tox-py310 - openstack-tox-docs - openstack-tox-pep8 - armada-chart-build-gate - armada-chart-build-latest-htk - - armada-docker-build-gate-ubuntu_focal - - armada-airskiff-deployment-focal-kubeadm + - armada-docker-build-gate-ubuntu_jammy + - armada-airskiff-deployment-jammy-kubeadm gate: jobs: + - openstack-tox-py310 - openstack-tox-docs - openstack-tox-pep8 - armada-chart-build-gate - - armada-docker-build-gate-ubuntu_focal + - armada-docker-build-gate-ubuntu_jammy post: jobs: - - armada-docker-publish-ubuntu_focal + - armada-docker-publish-ubuntu_jammy - armada-upload-git-mirror promote: @@ -47,10 +48,10 @@ label: ubuntu-bionic - nodeset: - name: armada-single-node-focal + name: armada-single-node-jammy nodes: - name: primary - label: ubuntu-focal + label: ubuntu-jammy - job: name: armada-chart-build-gate @@ -58,9 +59,9 @@ Builds charts using pinned Helm toolkit. timeout: 900 run: tools/gate/playbooks/build-charts.yaml - nodeset: armada-single-node-focal + nodeset: armada-single-node-jammy vars: - HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f + HTK_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b - job: @@ -70,18 +71,18 @@ timeout: 900 voting: true run: tools/gate/playbooks/build-charts.yaml - nodeset: armada-single-node-focal + nodeset: armada-single-node-jammy vars: HTK_COMMIT: master - job: - name: armada-docker-build-gate-ubuntu_focal + name: armada-docker-build-gate-ubuntu_jammy timeout: 3600 run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node-focal + nodeset: armada-single-node-jammy vars: publish: false - distro: ubuntu_focal + distro: ubuntu_jammy tags: dynamic: patch_set: true @@ -107,6 +108,7 @@ - ^swagger/.*$ timeout: 10800 pre-run: + - tools/gate/playbooks/checkout-treasuremap-ref.yaml - tools/gate/playbooks/prepare-hosts.yaml - tools/gate/playbooks/mount-volumes.yaml - tools/gate/playbooks/deploy-env.yaml @@ -115,6 +117,7 @@ post-run: - tools/gate/playbooks/osh-infra-collect-logs.yaml vars: + treasuremap_ref: refs/changes/43/927643/4 extra_volume: size: 80G type: Linux @@ -131,15 +134,16 @@ loopback_device: /dev/loop100 loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img" ceph_osd_data_device: /dev/loop100 - kube_version_repo: "v1.29" - kube_version: "1.29.5-1.1" + kube_version_repo: "v1.30" + kube_version: "1.30.3-1.1" calico_setup: true - calico_version: "v3.27.0" + calico_version: "v3.27.4" cilium_setup: false - cilium_version: "1.15.6" - helm_version: "v3.6.3" - yq_version: "v4.6.0" - crictl_version: "v1.26.1" + cilium_version: "1.16.0" + flannel_setup: false + flannel_version: v0.25.4 + helm_version: "v3.15.4" + crictl_version: "v1.30.1" zuul_osh_relative_path: ../../openstack/openstack-helm zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra zuul_treasuremap_relative_path: ../../airship/treasuremap @@ -147,19 +151,20 @@ run_helm_tests: "no" - job: - name: armada-airskiff-deployment-focal-kubeadm - nodeset: treasuremap-airskiff-1node-ubuntu_focal + name: armada-airskiff-deployment-jammy-kubeadm + nodeset: treasuremap-airskiff-1node-ubuntu_jammy description: | Deploy Memcached using Airskiff and submitted Armada changes. parent: armada-base vars: site: airskiff HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz - HTK_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8 - OSH_INFRA_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8 - OSH_COMMIT: 049e679939fbd3b0c659dd0977911b8dc3b5a015 + HTK_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b + OSH_INFRA_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b + OSH_COMMIT: 540df5cb0dbdaed63c202e2d6f2b7891062f8203 + TREASUREMAP_COMMIT: refs/changes/43/927643/4 CLONE_ARMADA: false - DISTRO: ubuntu_focal + DISTRO: ubuntu_jammy DOCKER_REGISTRY: localhost:5000 MAKE_ARMADA_IMAGES: true gate_scripts_relative_path: ../../airship/treasuremap @@ -178,51 +183,24 @@ - ./tools/gate/wait-for-shipyard.sh irrelevant-files: *irrelevant-files -# - job: -# name: armada-airskiff-deploy-focal -# nodeset: armada-single-node-focal -# description: | -# Deploy Memcached using Airskiff and submitted Armada changes. -# timeout: 9600 -# voting: true -# pre-run: -# - tools/gate/playbooks/airskiff-reduce-site.yaml -# - tools/gate/playbooks/git-config.yaml -# run: tools/gate/playbooks/airskiff-deploy.yaml -# post-run: tools/gate/playbooks/debug-report.yaml -# required-projects: -# - airship/treasuremap -# vars: -# CLONE_ARMADA: false -# ARMADA_IMAGE_DISTRO: ubuntu_focal -# HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz -# HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f -# OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f -# OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7 -# irrelevant-files: -# - ^.*\.rst$ -# - ^doc/.*$ -# - ^examples/.*$ -# - ^releasenotes/.*$ -# - ^swagger/.*$ - job: - name: armada-docker-publish-ubuntu_focal + name: armada-docker-publish-ubuntu_jammy timeout: 3600 run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node-focal + nodeset: armada-single-node-jammy secrets: - airship_armada_quay_creds vars: publish: true - distro: ubuntu_focal + distro: ubuntu_jammy tags: dynamic: branch: true commit: true static: - latest - - airflow_2.8.2 + - airflow_2.10.0 - secret: diff --git a/Makefile b/Makefile index 5a2168f1..5515be93 100644 --- a/Makefile +++ b/Makefile @@ -28,8 +28,10 @@ LABEL ?= org.airshipit.build=community COMMIT ?= $(shell git rev-parse HEAD) PYTHON = python3 CHARTS := $(filter-out deps, $(patsubst charts/%/.,%,$(wildcard charts/*/.))) -DISTRO ?= ubuntu_focal +DISTRO ?= ubuntu_jammy +DISTRO_ALIAS ?= ubuntu_focal IMAGE := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO} +IMAGE_ALIAS := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO_ALIAS} UBUNTU_BASE_IMAGE ?= # Helm binary download url @@ -116,6 +118,13 @@ else $(_BASE_IMAGE_ARG) \ --build-arg HELM_ARTIFACT_URL=$(HELM_ARTIFACT_URL) . endif +ifneq ($(DISTRO), $(DISTRO_ALIAS)) + docker tag $(IMAGE) $(IMAGE_ALIAS) +endif +ifeq ($(DOCKER_REGISTRY), localhost:5000) + docker push $(IMAGE) + docker push $(IMAGE_ALIAS) +endif ifeq ($(PUSH_IMAGE), true) docker push $(IMAGE) endif diff --git a/etc/armada/armada.conf.sample b/etc/armada/armada.conf.sample index bf7e3802..fc983069 100644 --- a/etc/armada/armada.conf.sample +++ b/etc/armada/armada.conf.sample @@ -52,6 +52,14 @@ # Minimum value: 0 #lock_expiration = 600 +# Determines whether the operator has to be enabled to apply charts +# instead of armada-api itself (boolean value) +#enable_operator = false + +# Determines whether the wait process has to be done via armada-go +# using client-go library (boolean value) +#go_wait = false + # # From oslo.log # @@ -321,7 +329,7 @@ # Domain name containing project (string value) #project_domain_name = -# Trust ID (string value) +# ID of the trust to use as a trustee use (string value) #trust_id = # Optional domain ID to use with v3 and v2 parameters. It will be used for both @@ -532,6 +540,9 @@ # middleware should parse the headers or not. (boolean value) #enable_proxy_headers_parsing = false +# HTTP basic auth password file. (string value) +#http_basic_auth_user_file = /etc/htpasswd + [oslo_policy] diff --git a/images/armada/Dockerfile.ubuntu_jammy b/images/armada/Dockerfile.ubuntu_jammy new file mode 100644 index 00000000..b11dc075 --- /dev/null +++ b/images/armada/Dockerfile.ubuntu_jammy @@ -0,0 +1,104 @@ +ARG FROM=ubuntu:jammy +ARG ARMADA_GO=quay.io/airshipit/armada-go:latest-ubuntu_focal +FROM ${ARMADA_GO} AS armada_go +FROM ${FROM} + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ + org.opencontainers.image.url='https://airshipit.org' \ + org.opencontainers.image.documentation='https://docs.airshipit.org/armada' \ + org.opencontainers.image.source='https://opendev.org/airship/armada' \ + org.opencontainers.image.vendor='The Airship Authors' \ + org.opencontainers.image.licenses='Apache-2.0' + +ENV DEBIAN_FRONTEND noninteractive +ENV LANG=C.UTF-8 +ENV LC_ALL=C.UTF-8 + +EXPOSE 8000 + +RUN set -ex && \ + apt-get update && apt-get upgrade -y && \ + apt-get -y install \ + ca-certificates \ + curl \ + git \ + netbase \ + python3-dev \ + python3-setuptools \ + --no-install-recommends \ + && apt-get autoremove -yqq --purge \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + /usr/share/man \ + /usr/share/doc \ + /usr/share/doc-base + +WORKDIR /armada + +# Add armada user +RUN useradd -u 1000 -g users -d $(pwd) armada + +ENTRYPOINT ["./entrypoint.sh"] +CMD ["server"] + +COPY requirements-frozen.txt ./ + +ENV LD_LIBRARY_PATH=/usr/local/lib + +ARG HELM_ARTIFACT_URL +ARG LIBYAML_VERSION=0.2.5 + +# Build +RUN set -ex \ + && buildDeps=' \ + automake \ + gcc \ + libssl-dev \ + libtool \ + make \ + python3-pip \ + ' \ + && apt-get -qq update \ + # Keep git separate so it's not removed below + && apt-get install -y $buildDeps git --no-install-recommends \ + && git clone https://github.com/yaml/libyaml.git \ + && cd libyaml \ + && git checkout $LIBYAML_VERSION \ + && ./bootstrap \ + && ./configure \ + && make \ + && make install \ + && cd .. \ + && rm -fr libyaml \ + && python3 -m pip install -U pip \ + && pip3 install -r requirements-frozen.txt --no-cache-dir \ + && curl -fSSL -O ${HELM_ARTIFACT_URL} \ + && tar -xvf $(basename ${HELM_ARTIFACT_URL}) \ + && mv linux-amd64/helm /usr/local/bin \ + && apt-get purge -y --auto-remove $buildDeps \ + && apt-get autoremove -yqq --purge \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + /usr/share/man \ + /usr/share/doc \ + /usr/share/doc-base + +COPY . ./ +COPY --from=armada_go /usr/local/bin/armada /usr/local/bin/armada-go + +# Setting the version explicitly for PBR +ENV PBR_VERSION 0.8.0 + +RUN \ + chown -R armada:users . && \ + pip3 install -e . --verbose --use-pep517 \ + && echo "/armada" \ + > /usr/local/lib/python3.10/dist-packages/armada.pth + +USER armada diff --git a/requirements-frozen.txt b/requirements-frozen.txt index f8b8afdc..5c4b7498 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -1,37 +1,35 @@ -alembic==1.13.1 -attrs==23.2.0 -bcrypt==4.1.2 -cachetools==5.3.2 -certifi==2024.2.2 +alembic==1.13.2 +attrs==24.2.0 +bcrypt==4.2.0 +cachetools==5.5.0 +certifi==2024.8.30 charset-normalizer==3.3.2 click==8.1.7 debtcollector==3.0.0 decorator==5.1.1 -deepdiff==6.7.1 -dogpile.cache==1.3.2 +deepdiff==8.0.1 +dogpile.cache==1.3.3 falcon==3.1.3 gitdb==4.0.11 -GitPython==3.1.42 -google-auth==2.28.1 -greenlet==3.0.3 -idna==3.6 -importlib-metadata==6.11.0 -importlib-resources==5.13.0 +GitPython==3.1.43 +google-auth==2.34.0 +greenlet==3.1.0 +idna==3.10 iso8601==2.1.0 -Jinja2==3.1.3 -jsonschema==4.21.1 +Jinja2==3.1.4 +jsonschema==4.23.0 jsonschema-specifications==2023.12.1 keystoneauth1==5.1.2 keystonemiddleware==10.2.0 -kubernetes==29.0.0 -Mako==1.3.2 +kubernetes==30.1.0 +Mako==1.3.5 MarkupSafe==2.1.5 mock==5.1.0 -msgpack==1.0.7 -netaddr==1.2.1 +msgpack==1.1.0 +netaddr==1.3.0 netifaces==0.11.0 oauthlib==3.2.2 -ordered-set==4.1.0 +orderly-set==5.2.2 os-service-types==1.7.0 oslo.cache==3.3.1 oslo.config==9.1.1 @@ -43,49 +41,47 @@ oslo.middleware==5.1.1 oslo.policy==4.1.1 oslo.serialization==5.1.1 oslo.utils==6.1.0 -packaging==23.2 -Paste==3.7.1 +packaging==24.1 +Paste==3.10.1 PasteDeploy==3.1.0 -pbr==6.0.0 -pip==23.2.1 -pkgutil_resolve_name==1.3.10 +pbr==6.1.0 +pip==24.1 prometheus_client==0.20.0 py==1.11.0 -pyasn1==0.5.1 -pyasn1-modules==0.3.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.0 pycadf==3.1.1 pylibyaml==0.1.0 -pyparsing==3.1.1 -python-dateutil==2.8.2 +pyparsing==3.1.4 +python-dateutil==2.9.0.post0 python-keystoneclient==5.1.0 python-memcached==1.62 -pytz==2024.1 -PyYAML==6.0.1 -referencing==0.33.0 -requests==2.31.0 +pytz==2024.2 +PyYAML==6.0.2 +referencing==0.35.1 +requests==2.32.3 requests-oauthlib==1.3.1 -responses==0.25.0 +responses==0.25.3 retry==0.9.2 -rfc3986==1.5.0 -rpds-py==0.18.0 +rfc3986==2.0.0 +rpds-py==0.20.0 rsa==4.9 -setuptools==68.2.2 +setuptools==70.1.0 six==1.16.0 smmap==5.0.1 -SQLAlchemy==1.4.51 +SQLAlchemy==1.4.54 sqlalchemy-migrate==0.13.0 -sqlparse==0.4.4 +sqlparse==0.5.1 statsd==4.0.1 -stevedore==5.2.0 +stevedore==5.3.0 Tempita==0.5.2 testresources==2.0.1 testscenarios==0.5.0 -testtools==2.7.1 -typing_extensions==4.9.0 -urllib3==1.26.18 -uWSGI==2.0.24 -WebOb==1.8.7 -websocket-client==1.7.0 -wheel==0.41.2 +testtools==2.7.2 +typing_extensions==4.12.2 +urllib3==2.2.2 +uWSGI==2.0.27 +WebOb==1.8.8 +websocket-client==1.8.0 +wheel==0.43.0 wrapt==1.16.0 -zipp==3.17.0 diff --git a/tools/gate/playbooks/checkout-treasuremap-ref.yaml b/tools/gate/playbooks/checkout-treasuremap-ref.yaml new file mode 100644 index 00000000..46a24051 --- /dev/null +++ b/tools/gate/playbooks/checkout-treasuremap-ref.yaml @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: all + + tasks: + - name: Checkout treasuremap ref + shell: | + set -xe; + + : "${TREASUREMAP_REF:=v1.9}" + + cd ../treasuremap + git fetch https://review.opendev.org/airship/treasuremap ${TREASUREMAP_REF} && git checkout FETCH_HEAD + + args: + chdir: "{{ zuul.project.src_dir }}" + environment: + TREASUREMAP_REF: "{{ treasuremap_ref }}" + +... \ No newline at end of file diff --git a/tools/gate/playbooks/prepare-hosts.yaml b/tools/gate/playbooks/prepare-hosts.yaml index e7e3d9c9..76087fde 100644 --- a/tools/gate/playbooks/prepare-hosts.yaml +++ b/tools/gate/playbooks/prepare-hosts.yaml @@ -15,4 +15,7 @@ roles: - bindep - start-zuul-console + - role: add-authorized-keys + public_keys: + - public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA7eM8WFJrqQmki8rR0O3QBHyl8xq42jb1RduwuRwjWoGYJI5cX7Fx+7VR4A9ITCoiqxKS8DMfgKbt5jKC6SmvMALULZsnYlthB34KywurgxsW6fgp68DHWQ7J4CCBhoIpl0W3JW7s6b0vHLhab59r0E+AYemBVuWUqbFEy8nDAHcQv1S/2o1udhmljIN7c2ogO4KAJ7Lge0BoIP9ps4u6AVwyQZixp4anU9DHGNA/UQj4M5UyuALj5buEAuATBe9Vqj4sOvZjObPJAGPUrNRrGEWAFk+lSZHRzKXo0eeWtPqoh5UN9UDb5Pocg1krncMIZwjHKovlD1z/O1y91aY5LM1wxm/7aaIiX8eCihyVZaOuDCLF7WDT2SMs7ABcotX2MDtVQTrNNV3MmMAScFNDflzPKszd7cdjLl6PBq8bvPxmCkLmnitPTGOoh9d8i+JlbINvgx1pguYrpeciIyreCO1rjTW3MgB0tyoMEa31V+7HrauBMeNnE68YTqLTIB0= smarkin@mirantis.com ... diff --git a/tox.ini b/tox.ini index 47bbd278..997a949a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] skipsdist = True minversion = 3.18.0 -envlist = py38, pep8, cover, bandit +envlist = py310, pep8, cover, bandit [testenv] deps= @@ -38,13 +38,13 @@ allowlist_externals= sh deps= -r{toxinidir}/requirements-direct.txt - -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.2/constraints-3.8.txt + -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt commands= rm -f requirements-frozen.txt sh -c "pip freeze --all | grep -vE 'armada|pyinotify|pkg-resources==0.0.0' > requirements-frozen.txt" -[testenv:py38] +[testenv:py310] commands = {[testenv]commands} stestr run {posargs}