diff --git a/.gitignore b/.gitignore index 722ef2f2..f999d334 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,16 @@ ENV/ # vscode ide .vscode + +# Unit test / coverage reports +htmlcov/ +.tox/ +.stestr +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +cover +.hypothesis/ diff --git a/images/promenade/Dockerfile.ubuntu_bionic b/images/promenade/Dockerfile.ubuntu_bionic index 485a5f17..6cedc105 100644 --- a/images/promenade/Dockerfile.ubuntu_bionic +++ b/images/promenade/Dockerfile.ubuntu_bionic @@ -49,6 +49,7 @@ RUN set -ex \ gpg-agent \ libpcre3-dev \ libtool \ + libpq-dev \ make \ python3-dev \ python3-pip \ @@ -78,8 +79,8 @@ RUN set -ex \ && cd .. \ && rm -fr libyaml -COPY requirements-frozen.txt /opt/promenade -RUN pip3 install --no-cache-dir -r requirements-frozen.txt +COPY requirements-frozen-bionic.txt /opt/promenade +RUN pip3 install --no-cache-dir -r requirements-frozen-bionic.txt # Setting promenade version for BPR ENV PBR_VERSION 0.9.0 diff --git a/images/promenade/Dockerfile.ubuntu_focal b/images/promenade/Dockerfile.ubuntu_focal index 3f8d0727..d1121536 100644 --- a/images/promenade/Dockerfile.ubuntu_focal +++ b/images/promenade/Dockerfile.ubuntu_focal @@ -52,6 +52,7 @@ RUN set -ex \ gpg-agent \ libpcre3-dev \ libtool \ + libpq-dev \ make \ python3-dev \ python3-pip \ diff --git a/images/promenade/Dockerfile.ubuntu_xenial b/images/promenade/Dockerfile.ubuntu_xenial deleted file mode 100644 index ca9b0864..00000000 --- a/images/promenade/Dockerfile.ubuntu_xenial +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG FROM=ubuntu:16.04 -FROM ${FROM} - -LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ - org.opencontainers.image.url='https://airshipit.org' \ - org.opencontainers.image.documentation='https://airship-promenade.readthedocs.org' \ - org.opencontainers.image.source='https://opendev.org/airship/promenade' \ - org.opencontainers.image.vendor='The Airship Authors' \ - org.opencontainers.image.licenses='Apache-2.0' - -VOLUME /etc/promenade -VOLUME /target - -RUN mkdir /opt/promenade -WORKDIR /opt/promenade - -ENV PORT 9000 -EXPOSE $PORT - -ENV LANG=C.UTF-8 -ENV LC_ALL=C.UTF-8 - -ENTRYPOINT ["/opt/promenade/entrypoint.sh"] - -RUN set -ex \ - && apt-get update -qq \ - && apt-get install --no-install-recommends -y \ - automake \ - ca-certificates \ - curl \ - dnsutils \ - gcc \ - git \ - libpcre3-dev \ - libssl-dev \ - libtool \ - make \ - python3-dev \ - python3-pip \ - python3-setuptools \ - rsync \ - && ln -s /usr/bin/python3 /usr/bin/python \ - && curl -Lo /usr/local/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 \ - && chmod 555 /usr/local/bin/cfssl \ - && python3 -m pip install -U 'pip<21.0' \ - && apt-get autoremove -yqq --purge \ - && apt-get clean \ - && useradd -u 1000 -g users -d /opt/promenade promenade \ - && rm -rf /var/lib/apt/lists/* - -# Install LibYAML -ENV LD_LIBRARY_PATH=/usr/local/lib - -ARG LIBYAML_VERSION=0.2.5 -RUN set -ex \ - && git clone https://github.com/yaml/libyaml.git \ - && cd libyaml \ - && git checkout $LIBYAML_VERSION \ - && ./bootstrap \ - && ./configure \ - && make \ - && make install \ - && cd .. \ - && rm -fr libyaml - -COPY requirements-frozen.txt /opt/promenade -RUN cat requirements-frozen.txt | xargs -d '\n' -l1 pip3 install --no-cache-dir - -# Setting promenade version for BPR -ENV PBR_VERSION 0.8.0 - -COPY . /opt/promenade -RUN pip3 install -e /opt/promenade - -USER promenade diff --git a/requirements-direct.txt b/requirements-direct.txt index cb451fc6..ba4b3e6d 100644 --- a/requirements-direct.txt +++ b/requirements-direct.txt @@ -2,7 +2,7 @@ Beaker<=1.12.0 click ConfigParser -Deckhand @ git+https://opendev.org/airship/deckhand.git@3a06b1b6040865483a09514e2bf7dc5fae24560e#egg=deckhand +Deckhand @ git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5#egg=deckhand docutils<=0.17 falcon fixtures diff --git a/requirements-frozen-bionic.txt b/requirements-frozen-bionic.txt new file mode 100644 index 00000000..623823bb --- /dev/null +++ b/requirements-frozen-bionic.txt @@ -0,0 +1,167 @@ +alabaster==0.7.13 +alembic==1.4.3 +amqp==2.6.1 +argcomplete==3.0.8 +attrs==22.2.0 +autopage==0.5.1 +Babel==2.11.0 +bcrypt==4.0.1 +Beaker==1.12.0 +cachetools==4.2.4 +certifi==2023.5.7 +cffi==1.15.1 +chardet==3.0.4 +charset-normalizer==3.0.1 +click==8.0.4 +cliff==3.10.1 +cmd2==2.4.3 +configparser==5.2.0 +coverage==6.2 +cryptography==3.4.8 +dataclasses==0.8 +debtcollector==2.5.0 +Deckhand @ git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5 +decorator==4.4.2 +deepdiff==5.7.0 +distlib==0.3.6 +dnspython==2.2.1 +docutils==0.17 +dogpile.cache==1.2.1 +dulwich==0.20.50 +eventlet==0.33.3 +extras==1.0.0 +falcon==3.1.1 +fasteners==0.18 +filelock==3.4.1 +fixtures==3.0.0 +flake8==3.8.4 +futurist==2.4.1 +google-auth==2.18.1 +greenlet==2.0.2 +hacking==4.1.0 +html5lib==0.9999999 +httpexceptor==1.4.0 +idna==2.10 +imagesize==1.4.1 +importlib-metadata==2.1.3 +importlib-resources==5.4.0 +iso8601==1.1.0 +Jinja2==3.0.3 +jsonpath-ng==1.5.3 +jsonpath-rw==1.4.0 +jsonpath-rw-ext==1.2.2 +jsonpickle==1.4.1 +jsonschema==3.2.0 +keystoneauth1==5.1.1 +keystonemiddleware==9.5.0 +kombu==4.6.11 +kubernetes==26.1.0 +Mako==1.1.6 +MarkupSafe==2.0.1 +mccabe==0.6.1 +mock==5.0.2 +msgpack==1.0.5 +netaddr==0.8.0 +netifaces==0.11.0 +networkx==2.5.1 +nose==1.3.7 +oauthlib==3.2.2 +ordered-set==4.0.2 +os-service-types==1.7.0 +oslo.cache==2.10.1 +oslo.concurrency==4.5.1 +oslo.config==8.7.1 +oslo.context==4.1.0 +oslo.db==10.0.0 +oslo.i18n==5.1.0 +oslo.log==4.6.0 +oslo.messaging==12.13.0 +oslo.metrics==0.4.0 +oslo.middleware==4.4.0 +oslo.policy==3.10.1 +oslo.serialization==4.2.0 +oslo.service==2.8.0 +oslo.utils==4.12.3 +packaging==21.3 +Paste==3.5.0 +PasteDeploy==2.1.1 +PasteScript==3.3.0 +pbr==5.5.1 +pip==21.3.1 +platformdirs==2.4.0 +ply==3.11 +prettytable==2.5.0 +prometheus-client==0.17.0 +psycopg2-binary==2.9.6 +pyasn1==0.5.0 +pyasn1-modules==0.3.0 +pycadf==3.1.1 +pycodestyle==2.6.0 +pycparser==2.21 +pyflakes==2.2.0 +Pygments==2.14.0 +pylibyaml==0.1.0 +pymongo==4.1.1 +pyparsing==2.4.7 +pyperclip==1.8.2 +pyproject-api==0.1.0 +pyrsistent==0.18.0 +python-barbicanclient==5.2.0 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-keystoneclient==4.5.0 +python-memcached==1.59 +python-mimeparse==1.6.0 +python-subunit==1.4.0 +pytz==2023.3 +PyYAML==5.4.1 +reno==4.0.0 +repoze.lru==0.7 +requests==2.23.0 +requests-oauthlib==1.3.1 +resolver==0.2.1 +rfc3986==1.5.0 +Routes==2.5.1 +rsa==4.9 +selector==0.10.1 +setuptools==45.2.0 +simplejson==3.19.1 +six==1.16.0 +snowballstemmer==2.2.0 +Sphinx==3.3.1 +sphinx-rtd-theme==0.5.0 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 +SQLAlchemy==1.3.20 +sqlalchemy-migrate==0.13.0 +sqlparse==0.4.4 +statsd==4.0.1 +stevedore==3.5.2 +Tempita==0.5.2 +testrepository==0.0.20 +testresources==2.0.1 +testscenarios==0.5.0 +testtools==2.5.0 +tiddlyweb==2.4.3 +tomli==1.2.3 +tomlkit==0.11.6 +typing-extensions==3.7.4.3 +urllib3==1.25.11 +uWSGI==2.0.21 +vine==1.3.0 +virtualenv==20.16.2 +wcwidth==0.2.6 +WebOb==1.8.7 +websocket-client==1.3.1 +Werkzeug==2.0.3 +wheel==0.37.1 +wrapt==1.15.0 +wsgi-intercept==1.11.0 +xmltodict==0.13.0 +yappi==1.4.0 +yq==3.2.1 +zipp==3.6.0 diff --git a/setup.cfg b/setup.cfg index 3feb30e2..e77c8ffc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,6 +13,7 @@ classifier = License :: OSI Approved :: Apache Software License Operating System :: POSIX :: Linux Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.10 diff --git a/test-requirements.txt b/test-requirements.txt index cbd9baf7..1cb242a3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,8 @@ -pytest >= 3.0 +pytest pytest-cov==4.0.0 flake8==3.8.4 bandit==1.6.0 -yapf \ No newline at end of file +yapf +stestr==3.2.0 +testrepository==0.0.20 +testtools==2.5.0 \ No newline at end of file diff --git a/tools/install-external-deps.sh b/tools/install-external-deps.sh index fba71124..c8822d38 100755 --- a/tools/install-external-deps.sh +++ b/tools/install-external-deps.sh @@ -4,7 +4,7 @@ set -ex export DEBIAN_FRONTEND=noninteractive -echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections +echo 'debconf debconf/frontend select Noninteractive' | sudo -En debconf-set-selections CFSSL_URL=${CFSSL_URL:-https://pkg.cfssl.org/R1.2/cfssl_linux-amd64} @@ -13,7 +13,7 @@ if [[ ! $(command -v cfssl) ]]; then pushd "${TMP_DIR}" curl -Lo cfssl "${CFSSL_URL}" chmod 755 cfssl - sudo mv cfssl /usr/local/bin/ + sudo -En mv cfssl /usr/local/bin/ popd rm -rf "${TMP_DIR}" cfssl version diff --git a/tools/zuul/playbooks/docker-image-build.yaml b/tools/zuul/playbooks/docker-image-build.yaml index 7a4850df..89508e7c 100644 --- a/tools/zuul/playbooks/docker-image-build.yaml +++ b/tools/zuul/playbooks/docker-image-build.yaml @@ -4,6 +4,7 @@ - ensure-docker - ensure-python - ensure-pip + - ensure-tox tasks: - include_vars: vars.yaml @@ -42,6 +43,16 @@ executable: pip3 become: True + - name: Make images - verbosive + when: not publish + shell: | + make images + args: + chdir: "{{ zuul.project.src_dir }}" + executable: /bin/bash + become: True + + - name: Make images when: not publish block: diff --git a/tools/zuul/playbooks/install-deps.yaml b/tools/zuul/playbooks/install-deps.yaml new file mode 100644 index 00000000..e05d2bd7 --- /dev/null +++ b/tools/zuul/playbooks/install-deps.yaml @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Combine several test tasks into a single playbook +# to minimize Zuul node consumption + +- hosts: primary + roles: + - bindep + - ensure-tox + tasks: + - name: Install external deps + shell: | + ./tools/install-external-deps.sh + args: + chdir: "{{ zuul.project.src_dir }}" + executable: /bin/bash + become: True \ No newline at end of file diff --git a/tox.ini b/tox.ini index c4ce6689..ce66e5c8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,28 @@ [tox] -envlist = pep8,py38,bandit,docs +envlist = pep8,py38,docs [testenv] -pass_env = - http_proxy +basepython=python3 +deps= + -r{toxinidir}/requirements-frozen.txt + -r{toxinidir}/test-requirements.txt +passenv= HTTP_PROXY - https_proxy HTTPS_PROXY - no_proxy + http_proxy + https_proxy NO_PROXY + no_proxy +install_command = pip install {opts} {packages} +allowlist_externals = + bash + find + rm +commands = + find . -type f -name "*.pyc" -delete + rm -Rf .testrepository/times.dbm + + [testenv:venv] commands = {posargs} @@ -16,16 +30,36 @@ commands = {posargs} [testenv:py38] allowlist_externals = pytest -setenv = - PYTHONWARNING=all -deps = - -r{toxinidir}/requirements-frozen.txt +basepython=python3.8 commands = pytest {posargs} -[testenv:bandit] +[testenv:py36] +allowlist_externals = + pytest +basepython=python3.6 deps = - -r{toxinidir}/requirements-frozen.txt + -r{toxinidir}/requirements-frozen-bionic.txt + -r{toxinidir}/test-requirements.txt +commands = + pytest {posargs} + +[testenv:cover] +allowlist_externals = + find + stestr + coverage +setenv = + PYTHON=coverage run --source promenade --parallel-mode +commands = + py.test \ + --cov=promenade \ + --cov-report html:cover \ + --cov-report xml:cover/coverage.xml \ + --cov-report term \ + tests/ + +[testenv:bandit] commands = bandit --skip B324 -r promenade @@ -41,8 +75,6 @@ commands = sphinx-build -W -b html doc/source doc/build/html [testenv:fmt] -deps = - -r{toxinidir}/requirements-frozen.txt allowlist_externals = yapf commands = @@ -61,6 +93,18 @@ commands= rm -f requirements-tree.txt sh -c "pip freeze --all | grep -vE 'promenade|pyinotify|pkg-resources' > requirements-frozen.txt" +[testenv:freeze-bionic] +recreate = True +allowlist_externals= + rm + sh +deps= + -r{toxinidir}/requirements-direct.txt +commands= + rm -f requirements-frozen-bionic.txt + sh -c "pip freeze --all | grep -vE 'promenade|pyinotify|pkg-resources' > requirements-frozen-bionic.txt" + + [testenv:gate-lint] deps = -r{toxinidir}/requirements-frozen.txt @@ -75,7 +119,9 @@ allowlist_externals = yapf bandit flake8 + find commands = + find . -not -path "*/\.*" -not -path "*/doc/build/*" -not -name "*.tgz" -type f -exec egrep -l " +$" {:} \; yapf -rd {toxinidir}/promenade {toxinidir}/tests {toxinidir}/tools/image_tags.py flake8 {toxinidir}/promenade bandit --skip B324 -r promenade diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 1772bce2..f89e3e42 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -13,6 +13,33 @@ # See the License for the specific language governing permissions and # limitations under the License. +- job: + name: airship-promenade-openstack-tox-cover-focal + parent: openstack-tox-cover + description: Runs cover job on focal + nodeset: airship-promenade-single-node-focal + pre-run: tools/zuul/playbooks/install-deps.yaml + +- job: + name: airship-promenade-openstack-tox-py38-focal + parent: openstack-tox-py38 + description: Runs cover job on focal + nodeset: airship-promenade-single-node-focal + pre-run: tools/zuul/playbooks/install-deps.yaml + +- job: + name: airship-promenade-openstack-tox-py36-bionic + parent: openstack-tox-py36 + description: Runs cover job on bionic + nodeset: airship-promenade-single-node-bionic + pre-run: tools/zuul/playbooks/install-deps.yaml + +- job: + name: airship-promenade-openstack-tox-pep8-focal + parent: openstack-tox-pep8 + description: Runs pep8 job on focal + nodeset: airship-promenade-single-node-focal + - job: name: airship-promenade-genesis-containerd-gate description: | @@ -26,14 +53,6 @@ vars: distro: ubuntu_focal -- job: - name: airship-promenade-lint-ws - description: | - Lints all files for trailing whitespace - run: tools/zuul/playbooks/zuul-linter.yaml - timeout: 300 - nodeset: airship-promenade-single-node-focal - - job: name: airship-promenade-chart-build-gate description: | @@ -41,6 +60,18 @@ run: tools/zuul/playbooks/helm-linter.yaml timeout: 300 nodeset: airship-promenade-single-node-focal + vars: + HTK_COMMIT: a7cd689280cdbc0acd04a7a1b745941260e8700b + +- job: + name: airship-promenade-chart-build-gate-latest-htk + description: | + Lints charts using latest HTK + run: tools/zuul/playbooks/helm-linter.yaml + timeout: 300 + nodeset: airship-promenade-single-node-focal + vars: + HTK_COMMIT: master - job: name: airship-promenade-chart-build-latest-htk @@ -54,22 +85,7 @@ HTK_COMMIT: master - job: - name: airship-promenade-unit-py38 - description: | - Executes unit tests under Python 3.8 - run: tools/zuul/playbooks/make-tests.yaml - timeout: 1500 - nodeset: airship-promenade-single-node-focal - files: - - ^.*\.py$ - -- job: - name: airship-promenade-linter - run: tools/zuul/playbooks/zuul-linter.yaml - nodeset: airship-promenade-single-node-focal - -- job: - name: airship-promenade-docker-build-gate + name: airship-promenade-docker-build-gate-ubuntu_focal voting: true run: tools/zuul/playbooks/docker-image-build.yaml nodeset: airship-promenade-single-node-focal @@ -84,8 +100,27 @@ - ^etc/.*$ - ^tests/.*$ - ^tools/.*$ + - job: - name: airship-promenade-docker-build-post + name: airship-promenade-docker-build-gate-ubuntu_bionic + voting: true + run: tools/zuul/playbooks/docker-image-build.yaml + nodeset: airship-promenade-single-node + vars: + publish: false + distro: ubuntu_bionic + tags: + dynamic: + patch_set: true + irrelevant-files: + - ^charts/.*$ + - ^etc/.*$ + - ^tests/.*$ + - ^tools/.*$ + + +- job: + name: airship-promenade-docker-publish_ubuntu_focal voting: false run: tools/zuul/playbooks/docker-image-build.yaml nodeset: airship-promenade-single-node-focal @@ -106,6 +141,29 @@ - ^tests/.*$ - ^tools/.*$ +- job: + name: airship-promenade-docker-publish_ubuntu_bionic + voting: false + run: tools/zuul/playbooks/docker-image-build.yaml + nodeset: airship-promenade-single-node + secrets: + - airship_promenade_quay_creds + vars: + publish: true + distro: ubuntu_bionic + tags: + dynamic: + branch: true + commit: true + static: + - latest + irrelevant-files: + - ^charts/.*$ + - ^etc/.*$ + - ^tests/.*$ + - ^tools/.*$ + + - secret: name: airship_promenade_quay_creds data: diff --git a/zuul.d/nodesets.yaml b/zuul.d/nodesets.yaml index e2bbe6ac..713e69a2 100644 --- a/zuul.d/nodesets.yaml +++ b/zuul.d/nodesets.yaml @@ -19,4 +19,10 @@ groups: - name: primary nodes: - - primary \ No newline at end of file + - primary + +- nodeset: + name: airship-promenade-single-node-bionic + nodes: + - name: primary + label: ubuntu-bionic \ No newline at end of file diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 7fd19434..3b2bfa30 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -21,25 +21,31 @@ rtd_project_name: 'airship-promenade' check: jobs: - - openstack-tox-pep8 + - airship-promenade-openstack-tox-cover-focal + - airship-promenade-openstack-tox-pep8-focal + - airship-promenade-openstack-tox-py38-focal + - airship-promenade-openstack-tox-py36-bionic - openstack-tox-docs - - airship-promenade-lint-ws - - airship-promenade-docker-build-gate + - airship-promenade-docker-build-gate-ubuntu_focal + - airship-promenade-docker-build-gate-ubuntu_bionic - airship-promenade-chart-build-gate - airship-promenade-chart-build-latest-htk - - airship-promenade-unit-py38 - airship-promenade-genesis-containerd-gate gate: jobs: - - openstack-tox-pep8 + - airship-promenade-openstack-tox-cover-focal + - airship-promenade-openstack-tox-pep8-focal + - airship-promenade-openstack-tox-py38-focal + - airship-promenade-openstack-tox-py36-bionic - openstack-tox-docs - - airship-promenade-lint-ws - - airship-promenade-docker-build-gate + - airship-promenade-docker-build-gate-ubuntu_focal + - airship-promenade-docker-build-gate-ubuntu_bionic - airship-promenade-chart-build-gate - - airship-promenade-unit-py38 + - airship-promenade-chart-build-latest-htk - airship-promenade-genesis-containerd-gate post: jobs: - - airship-promenade-docker-build-post + - airship-promenade-docker-publish_ubuntu_focal + - airship-promenade-docker-publish_ubuntu_bionic - promenade-upload-git-mirror ...