Add check requirements job

Change-Id: I06e035a5570ffd25563f0483e6859ea55825b6f1

doc/requirements.txt

@@ -5,10 +5,5 @@
 # As openstack job build-reno-releasenotes and Read The Docs page support only
 # one single requirements file for a project, we need to have one such file
 # which will include all project requirements
--r ../requirements.txt
+reno>=3.2.0                   # Apache-2.0
--r ../extra-requirements.txt
+sphinx>=3.4.2                 # BSD
-oslo.log>=3.36.0              # Apache-2.0
-oslotest>=3.2.0               # Apache-2.0
-reno>=2.5.0                   # Apache-2.0
-sphinx!=1.6.6,!=1.6.7,>=1.6.2;python_version>='3.4'         # BSD
-sphinx_rtd_theme>=0.4.2,< 1   # Apache-2.0

doc/source/conf.py

@@ -23,14 +23,23 @@
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#
+
 import os
 import sys
+
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 TOBIKO_DIR = os.path.abspath(os.path.join(BASE_DIR, "..", ".."))
 sys.path.insert(0, TOBIKO_DIR)
 
 
+# -- Python logging ----------------------------------------------------------
+
+import logging
+from tools import common
+
+common.setup_logging(level=logging.INFO)
+
+
 # -- Project information -----------------------------------------------------
 
 project = 'Tobiko'
@@ -41,11 +50,17 @@ author = "Tobiko's Team"
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
-# Version info
+
-from tobiko import version
+from tools import get_version
-release = version.release
+release = get_version.get_version()
-# The short X.Y version.
+version = '.'.join(release.split('.', 2)[:2])
-version = version.version
+
+
+# -- Install requirements ----------------------------------------------------
+
+from tools import install
+install.pip_install('sphinx_rtd_theme>=0.5.1,<1')   # Apache-2.0
+install.install_tobiko()
 
 
 # -- General configuration ---------------------------------------------------
@@ -126,7 +141,7 @@ def _get_config_generator_config_definition(conf):
     # oslo_config.sphinxconfiggen appends '.conf.sample' to the filename,
     # strip file extentension (.conf or .ini).
     output_file_path = '_static/config-samples/%s' % conf.rsplit('.', 1)[0]
-    return (config_file_path, output_file_path)
+    return config_file_path, output_file_path
 
 
 config_generator_config_file = [

extra-requirements.txt

@@ -1,2 +1,5 @@
+ansi2html                # LGPLv3+
 pandas                   # BSD
+podman                   # Apache-2.0
+pytest-rerunfailures     # MPL-2.0
 validations-libs         # APACHE-2.0

lower-constraints.txt

@@ -1,4 +1,4 @@
-# Tobiko framework requirements
+# from requirements.txt
 
 docker==4.4.1
 fixtures==3.0.0
@@ -21,3 +21,14 @@ six==1.15.0
 sshtunnel==0.3.1
 stestr==3.1.0
 testtools==2.4.0
+
+
+# from test-requirements.txt
+
+ansi2html==1.6.0
+coverage==5.3.1
+mock==3.0.5
+psutil==5.8.0
+pytest==6.2.1
+pytest-html==3.1.1
+pytest-xdist==2.2.0

releasenotes/source/conf.py

@@ -23,13 +23,21 @@
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#
+
 import os
 import sys
+
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 TOBIKO_DIR = os.path.abspath(os.path.join(BASE_DIR, "..", ".."))
 sys.path.insert(0, TOBIKO_DIR)
 
+# -- Python logging ----------------------------------------------------------
+
+import logging
+from tools import common
+
+common.setup_logging(level=logging.INFO)
+
 
 # -- Project information -----------------------------------------------------
 
@@ -41,11 +49,16 @@ author = "Tobiko's Team"
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
-# Version info
+
-from tobiko import version
+from tools import get_version
-release = version.release
+release = get_version.get_version()
-# The short X.Y version.
+version = '.'.join(release.split('.', 2)[:2])
-version = version.version
+
+
+# -- Install requirements ----------------------------------------------------
+
+from tools import install
+install.pip_install('sphinx_rtd_theme>=0.5.1,<1')   # Apache-2.0
 
 
 # -- General configuration ---------------------------------------------------

requirements.txt

@@ -10,7 +10,6 @@ oslo.config>=8.4.0              # Apache-2.0
 oslo.log>=4.4.0                 # Apache-2.0
 paramiko>=2.7.2                 # LGPLv2.1
 pbr>=5.5.1                      # Apache-2.0
-podman>=1.6.0                   # Apache-2.0
 python-glanceclient>=3.2.2      # Apache-2.0
 python-heatclient>=2.3.0        # Apache-2.0
 python-neutronclient>=7.2.1     # Apache-2.0

test-requirements.txt

@@ -1,10 +1,8 @@
 # Unit tests requirements
 
-ansi2html>=1.6.0               # LGPLv3+
+coverage>=5.3.1                # Apache-2.0
-coverage>=4.5.0                # Apache-2.0
+mock>=3.0.5                    # BSD
-mock>=2.0.0                    # BSD
+psutil>=5.8.0                  # BSD
-psutil>=5.7.2                  # BSD
+pytest>=6.2.1                  # MIT
-pytest>=6.0.0                  # MIT
+pytest-html>=3.1.1             # MPL-2.0
-pytest-html>=3.0.0             # MPL-2.0
+pytest-xdist[psutil]>=2.2.0    # MIT
-pytest-rerunfailures>=9.0.0    # MPL-2.0
-pytest-xdist[psutil]>=2.0.0    # MIT

tobiko/openstack/topology/_topology.py

@@ -23,8 +23,6 @@ import six
 from six.moves.urllib import parse
 
 import tobiko
-from tobiko import docker
-from tobiko import podman
 from tobiko.shell import files
 from tobiko.shell import ip
 from tobiko.shell import sh
@@ -157,6 +155,7 @@ class OpenStackTopologyNode(object):
     def docker_client(self):
         docker_client = self._docker_client
         if not docker_client:
+            from tobiko import docker
             self._docker_client = docker_client = docker.get_docker_client(
                 ssh_client=self.ssh_client)
         return docker_client
@@ -165,6 +164,7 @@ class OpenStackTopologyNode(object):
     def podman_client(self):
         podman_client = self._podman_client
         if not podman_client:
+            from tobiko import podman
             self._podman_client = podman_client = podman.get_podman_client(
                 ssh_client=self.ssh_client)
         return podman_client

tobiko/tripleo/_overcloud.py

@@ -17,7 +17,6 @@ import io
 import os
 
 from oslo_log import log
-import pandas
 import six
 
 import tobiko
@@ -184,6 +183,7 @@ def get_overcloud_nodes_dataframe(oc_node_df_function):
 
     :return: dataframe of all overcloud nodes processes
     """
+    import pandas
     oc_nodes_selection = list_overcloud_nodes()
     oc_nodes_names = [node.name for node in oc_nodes_selection]
     oc_nodes_dfs = [oc_node_df_function(node_name) for

tools/common.py

@@ -33,7 +33,7 @@ def get_logger(name):
     return logging.getLogger(name)
 
 
-def setup_logging(main_script=None, level=logging.DEBUG):
+def setup_logging(main_script=None, level=logging.DEBUG) -> object:
     main_script = main_script or sys.modules['__main__'].__file__
     logging.basicConfig(
         level=level,

tools/get_version.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+# Copyright 2018 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+import os
+import sys
+
+TOP_DIR = os.path.realpath(os.path.dirname(os.path.dirname(__file__)))
+
+if TOP_DIR not in sys.path:
+    sys.path.insert(0, TOP_DIR)
+
+from tools import common  # noqa
+
+
+def main():
+    version = get_version()
+    sys.stdout.write(f'{version}\n')
+
+
+CACHE = {}
+
+
+def get_version():
+    version = CACHE.get('version')
+    if not version:
+        CACHE['version'] = version = common.execute(
+            f"git -C '{TOP_DIR}' describe").splitlines()[0]
+    return version
+
+
+if __name__ == '__main__':
+    main()

tools/install.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# Copyright 2018 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+import os
+import sys
+
+TOP_DIR = os.path.realpath(os.path.dirname(os.path.dirname(__file__)))
+
+if TOP_DIR not in sys.path:
+    sys.path.insert(0, TOP_DIR)
+
+from tools import common  # noqa
+from tools import get_version  # noqa
+
+LOG = common.get_logger(__name__)
+
+TOX_VERSION = os.environ.get('TOX_VERSION') or '>=3.8.0'
+
+TOX_CONSTRAINTS_FILE = (
+    os.environ.get('TOX_CONSTRAINTS_FILE') or
+    'https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt')
+
+TOX_CONSTRAINTS = (
+    os.environ.get('TOX_CONSTRAINTS') or f"'-c{TOX_CONSTRAINTS_FILE}'")
+
+
+def main():
+    common.setup_logging()
+    install_tox()
+    install_bindeps()
+    install_tobiko()
+
+
+def install_tox(version=TOX_VERSION):
+    LOG.info(f"Installing Tox... (version: {version})")
+    pip_install(f"'tox{version}'")
+
+
+def install_bindeps():
+    LOG.info(f"Installing Tobiko binary dependencies...")
+    common.execute(os.path.join(TOP_DIR, 'tools', 'install-bindeps.sh'),
+                   capture_stdout=False)
+
+
+def install_tobiko():
+    version = get_version.get_version()
+    LOG.info(f"Installing Tobiko version {version}...")
+    pip_install(f"-e '{TOP_DIR}'")
+
+
+def pip_install(args):
+    LOG.debug(f"Installing packages: {args}...")
+    common.execute_python(f"-m pip install {TOX_CONSTRAINTS} {args}",
+                          capture_stdout=False)
+
+
+if __name__ == '__main__':
+    main()

tox.ini

@@ -275,30 +275,55 @@ deps =
     -r{toxinidir}/doc/requirements.txt
 
 
+[testenv:linkcheck]
+
+allowlist_externals = sh
+basepython = {[docs]basepython}
+envdir = {[docs]envdir}
+deps = {[docs]deps}
+commands =
+    sh -c 'cd {toxinidir}/doc/source && sphinx-build -W -b linkcheck . ../build/linkcheck'
+# Do not install any requirements. We want this to be fast and work even if
+# system dependencies are missing, since it's used to tell you what system
+# dependencies are missing! This also means that bindep must be installed
+# separately, outside of the requirements files.
+usedevelop = false
+skipdist = true
+skip_install = true
+
+
 [testenv:docs]
 
+allowlist_externals = sh
 basepython = {[docs]basepython}
 envdir = {[docs]envdir}
 deps = {[docs]deps}
 changedir = doc/source
-commands = sphinx-build -W -b html . ../build/html
+commands =
-
+    {[testenv:linkcheck]commands}
-
+    sphinx-build -W -b html . ../build/html
-[testenv:linkcheck]
+# Do not install any requirements. We want this to be fast and work even if
-
+# system dependencies are missing, since it's used to tell you what system
-basepython = {[docs]basepython}
+# dependencies are missing! This also means that bindep must be installed
-envdir = {[docs]envdir}
+# separately, outside of the requirements files.
-deps = {[docs]deps}
+usedevelop = false
-commands = sphinx-build -W -b linkcheck doc/source doc/build/linkcheck
+skipdist = true
+skip_install = true
 
 
 [testenv:releasenotes]
 
 basepython = {[docs]basepython}
-envdir = {[docs]envdir}
 deps = {[docs]deps}
 commands =
     sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
+# Do not install any requirements. We want this to be fast and work even if
+# system dependencies are missing, since it's used to tell you what system
+# dependencies are missing! This also means that bindep must be installed
+# separately, outside of the requirements files.
+usedevelop = false
+skipdist = true
+skip_install = true
 
 
 # --- CI report environments --------------------------------------------------

zuul.d/project.yaml

@@ -2,12 +2,13 @@
     templates:
       - ansible-role-jobs
       - build-openstack-docs-pti
+      - build-release-notes-jobs-python3
+      - check-requirements
       - docs-on-readthedocs
       - openstack-cover-jobs
       - openstack-lower-constraints-jobs
       - openstack-python3-wallaby-jobs
       - publish-to-pypi
-      - build-release-notes-jobs-python3
     vars:
         rtd_webhook_id: '84947'