Restored ubuntu_bionic image build

This PS restores image build for ubuntu_bionic and adds appropriate
gates to keep it tested by appropriate functional and integrational
tests.

Change-Id: I8ef524a1f66f1a88593a18334b8c37c4db0175c6

.gitignore

@@ -39,3 +39,16 @@ ENV/
 
 # vscode ide
 .vscode
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.stestr
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+cover
+.hypothesis/

images/promenade/Dockerfile.ubuntu_bionic

@@ -49,6 +49,7 @@ RUN set -ex \
         gpg-agent \
         libpcre3-dev \
         libtool \
+        libpq-dev \
         make \
         python3-dev \
         python3-pip \
@@ -78,8 +79,8 @@ RUN set -ex \
     && cd .. \
     && rm -fr libyaml
 
-COPY requirements-frozen.txt /opt/promenade
+COPY requirements-frozen-bionic.txt /opt/promenade
-RUN pip3 install --no-cache-dir -r requirements-frozen.txt
+RUN pip3 install --no-cache-dir -r requirements-frozen-bionic.txt
 
 # Setting promenade version for BPR
 ENV PBR_VERSION 0.9.0

images/promenade/Dockerfile.ubuntu_focal

@@ -52,6 +52,7 @@ RUN set -ex \
         gpg-agent \
         libpcre3-dev \
         libtool \
+        libpq-dev \
         make \
         python3-dev \
         python3-pip \

images/promenade/Dockerfile.ubuntu_xenial

@@ -1,89 +0,0 @@
-# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ARG FROM=ubuntu:16.04
-FROM ${FROM}
-
-LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
-      org.opencontainers.image.url='https://airshipit.org' \
-      org.opencontainers.image.documentation='https://airship-promenade.readthedocs.org' \
-      org.opencontainers.image.source='https://opendev.org/airship/promenade' \
-      org.opencontainers.image.vendor='The Airship Authors' \
-      org.opencontainers.image.licenses='Apache-2.0'
-
-VOLUME /etc/promenade
-VOLUME /target
-
-RUN mkdir /opt/promenade
-WORKDIR /opt/promenade
-
-ENV PORT 9000
-EXPOSE $PORT
-
-ENV LANG=C.UTF-8
-ENV LC_ALL=C.UTF-8
-
-ENTRYPOINT ["/opt/promenade/entrypoint.sh"]
-
-RUN set -ex \
-    && apt-get update -qq \
-    && apt-get install --no-install-recommends -y \
-        automake \
-        ca-certificates \
-        curl \
-        dnsutils \
-        gcc \
-        git \
-        libpcre3-dev \
-        libssl-dev \
-        libtool \
-        make \
-        python3-dev \
-        python3-pip \
-        python3-setuptools \
-        rsync \
-    && ln -s /usr/bin/python3 /usr/bin/python \
-    && curl -Lo /usr/local/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 \
-    && chmod 555 /usr/local/bin/cfssl \
-    && python3 -m pip install -U 'pip<21.0' \
-    && apt-get autoremove -yqq --purge \
-    && apt-get clean \
-    && useradd -u 1000 -g users -d /opt/promenade promenade \
-    && rm -rf /var/lib/apt/lists/*
-
-# Install LibYAML
-ENV LD_LIBRARY_PATH=/usr/local/lib
-
-ARG LIBYAML_VERSION=0.2.5
-RUN set -ex \
-    && git clone https://github.com/yaml/libyaml.git \
-    && cd libyaml \
-    && git checkout $LIBYAML_VERSION \
-    && ./bootstrap \
-    && ./configure \
-    && make \
-    && make install \
-    && cd .. \
-    && rm -fr libyaml
-
-COPY requirements-frozen.txt /opt/promenade
-RUN cat requirements-frozen.txt | xargs -d '\n' -l1 pip3 install --no-cache-dir
-
-# Setting promenade version for BPR
-ENV PBR_VERSION 0.8.0
-
-COPY . /opt/promenade
-RUN pip3 install -e /opt/promenade
-
-USER promenade

requirements-direct.txt

@@ -2,7 +2,7 @@
 Beaker<=1.12.0
 click
 ConfigParser
-Deckhand @ git+https://opendev.org/airship/deckhand.git@3a06b1b6040865483a09514e2bf7dc5fae24560e#egg=deckhand
+Deckhand @ git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5#egg=deckhand
 docutils<=0.17
 falcon
 fixtures

requirements-frozen-bionic.txt

@@ -0,0 +1,167 @@
+alabaster==0.7.13
+alembic==1.4.3
+amqp==2.6.1
+argcomplete==3.0.8
+attrs==22.2.0
+autopage==0.5.1
+Babel==2.11.0
+bcrypt==4.0.1
+Beaker==1.12.0
+cachetools==4.2.4
+certifi==2023.5.7
+cffi==1.15.1
+chardet==3.0.4
+charset-normalizer==3.0.1
+click==8.0.4
+cliff==3.10.1
+cmd2==2.4.3
+configparser==5.2.0
+coverage==6.2
+cryptography==3.4.8
+dataclasses==0.8
+debtcollector==2.5.0
+Deckhand @ git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5
+decorator==4.4.2
+deepdiff==5.7.0
+distlib==0.3.6
+dnspython==2.2.1
+docutils==0.17
+dogpile.cache==1.2.1
+dulwich==0.20.50
+eventlet==0.33.3
+extras==1.0.0
+falcon==3.1.1
+fasteners==0.18
+filelock==3.4.1
+fixtures==3.0.0
+flake8==3.8.4
+futurist==2.4.1
+google-auth==2.18.1
+greenlet==2.0.2
+hacking==4.1.0
+html5lib==0.9999999
+httpexceptor==1.4.0
+idna==2.10
+imagesize==1.4.1
+importlib-metadata==2.1.3
+importlib-resources==5.4.0
+iso8601==1.1.0
+Jinja2==3.0.3
+jsonpath-ng==1.5.3
+jsonpath-rw==1.4.0
+jsonpath-rw-ext==1.2.2
+jsonpickle==1.4.1
+jsonschema==3.2.0
+keystoneauth1==5.1.1
+keystonemiddleware==9.5.0
+kombu==4.6.11
+kubernetes==26.1.0
+Mako==1.1.6
+MarkupSafe==2.0.1
+mccabe==0.6.1
+mock==5.0.2
+msgpack==1.0.5
+netaddr==0.8.0
+netifaces==0.11.0
+networkx==2.5.1
+nose==1.3.7
+oauthlib==3.2.2
+ordered-set==4.0.2
+os-service-types==1.7.0
+oslo.cache==2.10.1
+oslo.concurrency==4.5.1
+oslo.config==8.7.1
+oslo.context==4.1.0
+oslo.db==10.0.0
+oslo.i18n==5.1.0
+oslo.log==4.6.0
+oslo.messaging==12.13.0
+oslo.metrics==0.4.0
+oslo.middleware==4.4.0
+oslo.policy==3.10.1
+oslo.serialization==4.2.0
+oslo.service==2.8.0
+oslo.utils==4.12.3
+packaging==21.3
+Paste==3.5.0
+PasteDeploy==2.1.1
+PasteScript==3.3.0
+pbr==5.5.1
+pip==21.3.1
+platformdirs==2.4.0
+ply==3.11
+prettytable==2.5.0
+prometheus-client==0.17.0
+psycopg2-binary==2.9.6
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pycadf==3.1.1
+pycodestyle==2.6.0
+pycparser==2.21
+pyflakes==2.2.0
+Pygments==2.14.0
+pylibyaml==0.1.0
+pymongo==4.1.1
+pyparsing==2.4.7
+pyperclip==1.8.2
+pyproject-api==0.1.0
+pyrsistent==0.18.0
+python-barbicanclient==5.2.0
+python-dateutil==2.8.1
+python-editor==1.0.4
+python-keystoneclient==4.5.0
+python-memcached==1.59
+python-mimeparse==1.6.0
+python-subunit==1.4.0
+pytz==2023.3
+PyYAML==5.4.1
+reno==4.0.0
+repoze.lru==0.7
+requests==2.23.0
+requests-oauthlib==1.3.1
+resolver==0.2.1
+rfc3986==1.5.0
+Routes==2.5.1
+rsa==4.9
+selector==0.10.1
+setuptools==45.2.0
+simplejson==3.19.1
+six==1.16.0
+snowballstemmer==2.2.0
+Sphinx==3.3.1
+sphinx-rtd-theme==0.5.0
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==2.0.0
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.5
+SQLAlchemy==1.3.20
+sqlalchemy-migrate==0.13.0
+sqlparse==0.4.4
+statsd==4.0.1
+stevedore==3.5.2
+Tempita==0.5.2
+testrepository==0.0.20
+testresources==2.0.1
+testscenarios==0.5.0
+testtools==2.5.0
+tiddlyweb==2.4.3
+tomli==1.2.3
+tomlkit==0.11.6
+typing-extensions==3.7.4.3
+urllib3==1.25.11
+uWSGI==2.0.21
+vine==1.3.0
+virtualenv==20.16.2
+wcwidth==0.2.6
+WebOb==1.8.7
+websocket-client==1.3.1
+Werkzeug==2.0.3
+wheel==0.37.1
+wrapt==1.15.0
+wsgi-intercept==1.11.0
+xmltodict==0.13.0
+yappi==1.4.0
+yq==3.2.1
+zipp==3.6.0

setup.cfg

@@ -13,6 +13,7 @@ classifier =
     License :: OSI Approved :: Apache Software License
     Operating System :: POSIX :: Linux
     Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.10
 

test-requirements.txt

@@ -1,5 +1,8 @@
-pytest >= 3.0
+pytest
 pytest-cov==4.0.0
 flake8==3.8.4
 bandit==1.6.0
 yapf
+stestr==3.2.0
+testrepository==0.0.20
+testtools==2.5.0

tools/install-external-deps.sh

@@ -4,7 +4,7 @@
 set -ex
 
 export DEBIAN_FRONTEND=noninteractive
-echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections
+echo 'debconf debconf/frontend select Noninteractive' | sudo -En debconf-set-selections
 
 CFSSL_URL=${CFSSL_URL:-https://pkg.cfssl.org/R1.2/cfssl_linux-amd64}
 
@@ -13,7 +13,7 @@ if [[ ! $(command -v cfssl) ]]; then
     pushd "${TMP_DIR}"
     curl -Lo cfssl "${CFSSL_URL}"
     chmod 755 cfssl
-    sudo mv cfssl /usr/local/bin/
+    sudo -En mv cfssl /usr/local/bin/
     popd
     rm -rf "${TMP_DIR}"
     cfssl version

tools/zuul/playbooks/docker-image-build.yaml

@@ -4,6 +4,7 @@
     - ensure-docker
     - ensure-python
     - ensure-pip
+    - ensure-tox
 
   tasks:
     - include_vars: vars.yaml
@@ -42,6 +43,16 @@
             executable: pip3
       become: True
 
+    - name: Make images - verbosive
+      when: not publish
+      shell: |
+        make images
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+        executable: /bin/bash
+      become: True
+
+
     - name: Make images
       when: not publish
       block:

tools/zuul/playbooks/install-deps.yaml

@@ -0,0 +1,27 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Combine several test tasks into a single playbook
+# to minimize Zuul node consumption
+
+- hosts: primary
+  roles:
+    - bindep
+    - ensure-tox
+  tasks:
+    - name: Install external deps
+      shell: |
+        ./tools/install-external-deps.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+        executable: /bin/bash
+      become: True

tox.ini

@@ -1,14 +1,28 @@
 [tox]
-envlist = pep8,py38,bandit,docs
+envlist = pep8,py38,docs
 
 [testenv]
-pass_env =
+basepython=python3
-    http_proxy
+deps=
+    -r{toxinidir}/requirements-frozen.txt
+    -r{toxinidir}/test-requirements.txt
+passenv=
     HTTP_PROXY
-    https_proxy
     HTTPS_PROXY
-    no_proxy
+    http_proxy
+    https_proxy
     NO_PROXY
+    no_proxy
+install_command = pip install {opts} {packages}
+allowlist_externals =
+    bash
+    find
+    rm
+commands =
+    find . -type f -name "*.pyc" -delete
+    rm -Rf .testrepository/times.dbm
+
+
 
 [testenv:venv]
 commands = {posargs}
@@ -16,16 +30,36 @@ commands = {posargs}
 [testenv:py38]
 allowlist_externals =
     pytest
-setenv =
+basepython=python3.8
-  PYTHONWARNING=all
-deps =
-    -r{toxinidir}/requirements-frozen.txt
 commands =
     pytest {posargs}
 
-[testenv:bandit]
+[testenv:py36]
+allowlist_externals =
+    pytest
+basepython=python3.6
 deps =
-    -r{toxinidir}/requirements-frozen.txt
+    -r{toxinidir}/requirements-frozen-bionic.txt
+    -r{toxinidir}/test-requirements.txt
+commands =
+    pytest {posargs}
+
+[testenv:cover]
+allowlist_externals =
+  find
+  stestr
+  coverage
+setenv =
+    PYTHON=coverage run --source promenade --parallel-mode
+commands =
+    py.test \
+        --cov=promenade \
+        --cov-report html:cover \
+        --cov-report xml:cover/coverage.xml \
+        --cov-report term \
+        tests/
+
+[testenv:bandit]
 commands =
     bandit  --skip B324 -r promenade
 
@@ -41,8 +75,6 @@ commands =
     sphinx-build -W -b html doc/source doc/build/html
 
 [testenv:fmt]
-deps =
-    -r{toxinidir}/requirements-frozen.txt
 allowlist_externals =
     yapf
 commands =
@@ -61,6 +93,18 @@ commands=
   rm -f requirements-tree.txt
   sh -c "pip freeze --all | grep -vE 'promenade|pyinotify|pkg-resources' > requirements-frozen.txt"
 
+[testenv:freeze-bionic]
+recreate = True
+allowlist_externals=
+  rm
+  sh
+deps=
+  -r{toxinidir}/requirements-direct.txt
+commands=
+  rm -f requirements-frozen-bionic.txt
+  sh -c "pip freeze --all | grep -vE 'promenade|pyinotify|pkg-resources' > requirements-frozen-bionic.txt"
+
+
 [testenv:gate-lint]
 deps =
     -r{toxinidir}/requirements-frozen.txt
@@ -75,7 +119,9 @@ allowlist_externals =
     yapf
     bandit
     flake8
+    find
 commands =
+    find . -not -path "*/\.*" -not -path "*/doc/build/*" -not -name "*.tgz"  -type f -exec egrep -l " +$" {:} \;
     yapf -rd {toxinidir}/promenade {toxinidir}/tests {toxinidir}/tools/image_tags.py
     flake8 {toxinidir}/promenade
     bandit  --skip B324 -r promenade

zuul.d/jobs.yaml

@@ -13,6 +13,33 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- job:
+    name: airship-promenade-openstack-tox-cover-focal
+    parent: openstack-tox-cover
+    description: Runs cover job on focal
+    nodeset: airship-promenade-single-node-focal
+    pre-run: tools/zuul/playbooks/install-deps.yaml
+
+- job:
+    name: airship-promenade-openstack-tox-py38-focal
+    parent: openstack-tox-py38
+    description: Runs cover job on focal
+    nodeset: airship-promenade-single-node-focal
+    pre-run: tools/zuul/playbooks/install-deps.yaml
+
+- job:
+    name: airship-promenade-openstack-tox-py36-bionic
+    parent: openstack-tox-py36
+    description: Runs cover job on bionic
+    nodeset: airship-promenade-single-node-bionic
+    pre-run: tools/zuul/playbooks/install-deps.yaml
+
+- job:
+    name: airship-promenade-openstack-tox-pep8-focal
+    parent: openstack-tox-pep8
+    description: Runs pep8 job on focal
+    nodeset: airship-promenade-single-node-focal
+
 - job:
     name: airship-promenade-genesis-containerd-gate
     description: |
@@ -26,14 +53,6 @@
     vars:
       distro: ubuntu_focal
 
-- job:
-    name: airship-promenade-lint-ws
-    description: |
-      Lints all files for trailing whitespace
-    run: tools/zuul/playbooks/zuul-linter.yaml
-    timeout: 300
-    nodeset: airship-promenade-single-node-focal
-
 - job:
     name: airship-promenade-chart-build-gate
     description: |
@@ -41,6 +60,18 @@
     run: tools/zuul/playbooks/helm-linter.yaml
     timeout: 300
     nodeset: airship-promenade-single-node-focal
+    vars:
+      HTK_COMMIT: a7cd689280cdbc0acd04a7a1b745941260e8700b
+
+- job:
+    name: airship-promenade-chart-build-gate-latest-htk
+    description: |
+      Lints charts using latest HTK
+    run: tools/zuul/playbooks/helm-linter.yaml
+    timeout: 300
+    nodeset: airship-promenade-single-node-focal
+    vars:
+      HTK_COMMIT: master
 
 - job:
     name: airship-promenade-chart-build-latest-htk
@@ -54,22 +85,7 @@
       HTK_COMMIT: master
 
 - job:
-    name: airship-promenade-unit-py38
+    name: airship-promenade-docker-build-gate-ubuntu_focal
-    description: |
-      Executes unit tests under Python 3.8
-    run: tools/zuul/playbooks/make-tests.yaml
-    timeout: 1500
-    nodeset: airship-promenade-single-node-focal
-    files:
-      - ^.*\.py$
-
-- job:
-    name: airship-promenade-linter
-    run: tools/zuul/playbooks/zuul-linter.yaml
-    nodeset: airship-promenade-single-node-focal
-
-- job:
-    name: airship-promenade-docker-build-gate
     voting: true
     run: tools/zuul/playbooks/docker-image-build.yaml
     nodeset: airship-promenade-single-node-focal
@@ -84,8 +100,27 @@
       - ^etc/.*$
       - ^tests/.*$
       - ^tools/.*$
+
 - job:
-    name: airship-promenade-docker-build-post
+    name: airship-promenade-docker-build-gate-ubuntu_bionic
+    voting: true
+    run: tools/zuul/playbooks/docker-image-build.yaml
+    nodeset: airship-promenade-single-node
+    vars:
+      publish: false
+      distro: ubuntu_bionic
+      tags:
+        dynamic:
+          patch_set: true
+    irrelevant-files:
+      - ^charts/.*$
+      - ^etc/.*$
+      - ^tests/.*$
+      - ^tools/.*$
+
+
+- job:
+    name: airship-promenade-docker-publish_ubuntu_focal
     voting: false
     run: tools/zuul/playbooks/docker-image-build.yaml
     nodeset: airship-promenade-single-node-focal
@@ -106,6 +141,29 @@
       - ^tests/.*$
       - ^tools/.*$
 
+- job:
+    name: airship-promenade-docker-publish_ubuntu_bionic
+    voting: false
+    run: tools/zuul/playbooks/docker-image-build.yaml
+    nodeset: airship-promenade-single-node
+    secrets:
+      - airship_promenade_quay_creds
+    vars:
+      publish: true
+      distro: ubuntu_bionic
+      tags:
+        dynamic:
+          branch: true
+          commit: true
+        static:
+          - latest
+    irrelevant-files:
+      - ^charts/.*$
+      - ^etc/.*$
+      - ^tests/.*$
+      - ^tools/.*$
+
+
 - secret:
     name: airship_promenade_quay_creds
     data:

zuul.d/nodesets.yaml

@@ -20,3 +20,9 @@
       - name: primary
         nodes:
           - primary
+
+- nodeset:
+    name: airship-promenade-single-node-bionic
+    nodes:
+      - name: primary
+        label: ubuntu-bionic

zuul.d/project.yaml

@@ -21,25 +21,31 @@
       rtd_project_name: 'airship-promenade'
     check:
       jobs:
-        - openstack-tox-pep8
+        - airship-promenade-openstack-tox-cover-focal
+        - airship-promenade-openstack-tox-pep8-focal
+        - airship-promenade-openstack-tox-py38-focal
+        - airship-promenade-openstack-tox-py36-bionic
         - openstack-tox-docs
-        - airship-promenade-lint-ws
+        - airship-promenade-docker-build-gate-ubuntu_focal
-        - airship-promenade-docker-build-gate
+        - airship-promenade-docker-build-gate-ubuntu_bionic
         - airship-promenade-chart-build-gate
         - airship-promenade-chart-build-latest-htk
-        - airship-promenade-unit-py38
         - airship-promenade-genesis-containerd-gate
     gate:
       jobs:
-        - openstack-tox-pep8
+        - airship-promenade-openstack-tox-cover-focal
+        - airship-promenade-openstack-tox-pep8-focal
+        - airship-promenade-openstack-tox-py38-focal
+        - airship-promenade-openstack-tox-py36-bionic
         - openstack-tox-docs
-        - airship-promenade-lint-ws
+        - airship-promenade-docker-build-gate-ubuntu_focal
-        - airship-promenade-docker-build-gate
+        - airship-promenade-docker-build-gate-ubuntu_bionic
         - airship-promenade-chart-build-gate
-        - airship-promenade-unit-py38
+        - airship-promenade-chart-build-latest-htk
         - airship-promenade-genesis-containerd-gate
     post:
       jobs:
-        - airship-promenade-docker-build-post
+        - airship-promenade-docker-publish_ubuntu_focal
+        - airship-promenade-docker-publish_ubuntu_bionic
         - promenade-upload-git-mirror
 ...