Add sphinx_python variable to sphinx role and job

Some projects, such as Zuul itself, need to use python3 to build docs.

This adds a new parameter, sphinx_python, which defines the version
of python to build documentation with. The default is python2 and
projects can choose python3 on a need basis.

Remove the comment about the need for a leading comment, since there
is now a real need for the leading comment anyway.

Replace a few direct references to {{ ansible_user_dir }}/.venv that
should have been using zuul_work_virtualenv.

Update sphinx invocation to source the virtualenv activate first.
Some sphinx modules, such as sphinxcontrib.programoutput, attempt to
execute programs and only invoking sphinx-build with the relative
path causes the paths to not be set up properly.

Co-Authored-By: David Moreau Simard <dmsimard@redhat.com>
Change-Id: Ie5e2c93f88465f4aa746827ff88a585dbaa44fd5

roles/ensure-sphinx/README.rst

@@ -16,6 +16,11 @@ All pip installs are done with a provided constraints file, if given.
 
    List of python packages to install for building docs.
 
+.. zuul:rolevar:: sphinx_python
+   :default: python2
+
+   Version of python to use, either ``python2`` or ``python3``.
+
 .. zuul:rolevar:: zuul_work_virtualenv
    :default: ~/.venv
 

roles/ensure-sphinx/defaults/main.yaml

@@ -1,4 +1,5 @@
 zuul_work_dir: "{{ zuul.project.src_dir }}"
 zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"
+sphinx_python: python2
 doc_building_packages:
   - sphinx

roles/ensure-sphinx/tasks/main.yaml

@@ -9,36 +9,27 @@
   include_role:
     name: find-constraints
 
-- name: Install virtualenv and doc requirements files if found
-  shell:
-    executable: /bin/bash
+# Note (dmsimard): This installs doc/requirements.txt or test-requirements.txt
+# if the former is not found. If neither are found, this task is skipped.
+# TODO(dmsimard) Don't assume virtualenv is installed
+- name: Initialize virtual environment
+  pip:
+    requirements: "{{ item }}"
     chdir: "{{ zuul_work_dir }}"
-    # NOTE(mordred) There is a bug in ansible-lint that mistakenly detects
-    # setting the VENV variable below as an error if it occurs on the fist
-    # line. Work around that by putting a comment as the first line until we
-    # can get a fix upstream.
-    cmd: |
-      # Create virtualenv is it does not already exist
-      VENV={{ zuul_work_virtualenv }}
-      if [ ! -d $VENV ] ; then
-          virtualenv $VENV
-      fi
-      source $VENV/bin/activate
-      # skipping requirements.txt as it gets picked up by installing the
-      # python package itself
-      for f in doc/requirements.txt test-requirements.txt ; do
-          if [ -f $f ] ; then
-              pip install $CONSTRAINTS -r $f
-              break
-          fi
-      done
-  environment:
-    CONSTRAINTS: "{{ upper_constraints|default('') }}"
+    virtualenv: "{{ zuul_work_virtualenv }}"
+    virtualenv_python: "{{ sphinx_python }}"
+    extra_args: "{{ upper_constraints | default(omit) }}"
+  with_first_found:
+    - files:
+        - "{{ zuul_work_dir }}/doc/requirements.txt"
+        - "{{ zuul_work_dir }}/test-requirements.txt"
+      skip: true
 
 - name: Install doc building packages
   pip:
     name: "{{ item }}"
     chdir: "{{ zuul_work_dir }}"
     virtualenv: "{{ zuul_work_virtualenv }}"
-    extra_args: "{{ upper_constraints|default(omit) }}"
+    virtualenv_python: "{{ sphinx_python }}"
+    extra_args: "{{ upper_constraints | default(omit) }}"
   with_items: "{{ doc_building_packages }}"

roles/install-if-python/tasks/main.yaml

@@ -28,7 +28,7 @@
 - name: Install requirements if they exist
   pip:
     chdir: "{{ zuul_work_dir }}"
-    virtualenv: "{{ ansible_user_dir }}/.venv"
+    virtualenv: "{{ zuul_work_virtualenv }}"
     requirements: requirements.txt
     extra_args: "{{ upper_constraints|default(omit) }}"
   register: requirements_install
@@ -45,7 +45,7 @@
 # the ChangeLog to have been generated.
 - name: Make sdist to generate ChangeLog
   command:
-    cmd: "{{ ansible_user_dir }}/.venv/bin/python setup.py sdist"
+    cmd: "{{ zuul_work_virtualenv }}/bin/python setup.py sdist"
     chdir: "{{ zuul_work_dir }}"
   when:
     - install_package
@@ -61,7 +61,7 @@
 - name: Install the project if it is a Python project
   pip:
     chdir: "{{ zuul_work_dir }}"
-    virtualenv: "{{ ansible_user_dir }}/.venv"
+    virtualenv: "{{ zuul_work_virtualenv }}"
     name: .
     extra_args: --no-deps
   when:

roles/sphinx/tasks/main.yaml

@@ -10,11 +10,14 @@
     sphinx_warning_is_error: "{{ check_result.warning_is_error }}"
 
 - name: Run sphinx
-  command:
-    cmd: >
-      {{ zuul_work_virtualenv }}/bin/sphinx-build
-        -b {{ item }}
-        {% if sphinx_warning_is_error %} -W {% endif %}
+  shell:
+    executable: /bin/bash
+    cmd: |
+      # Source the activate file so that sphinx subcommands have the correct
+      # paths set.
+      source {{ zuul_work_virtualenv }}/bin/activate
+      sphinx-build -b {{ item }} \
+        {% if sphinx_warning_is_error %} -W {% endif %} \
         {{ sphinx_source_dir }} {{ sphinx_build_dir }}/{{ item }}
     chdir: "{{ zuul_work_dir }}"
   with_items: "{{ sphinx_builders }}"

zuul.yaml

@@ -174,6 +174,11 @@
          Optional path to a pip constraints file for installing python
          libraries.
 
+      .. zuul:jobvar:: sphinx_python
+          :default: python2
+
+         Version of python to use, either ``python2`` or ``python3``.
+
       .. zuul:jobvar:: zuul_work_dir
          :default: {{ zuul.project.src_dir }}