Update dockerfiles and gates

Introduces image build for bionic with py3 and nautilus.
Switches centos to py3 and fixes related issues.
Xenial is now run with stable/ocata.

Adds ability to skip specific build for particular component specifying
"supported_releases" in zuul component configuration. Sets
supported_releases for placement not available for releases prior to
Stein.

Removes redundant parameters for project reference using defaults
instead, as they relate to loci project, not to project we build
with loci. Also hard-codes parameters in playbooks/vars.yaml, making
it a definitive build configuration.

Change-Id: I7290203d80461f46ed70d5e3bea55c9765f9e746

.zuul.d/placement.yaml

@@ -15,6 +15,7 @@
     parent: loci-base
     vars:
       project: placement
+      supported_releases: ["stein", "train", "master"]
     required-projects:
       - openstack/loci
       - openstack/requirements

Dockerfile

@@ -1,4 +1,4 @@
-ARG FROM=ubuntu:xenial
+ARG FROM=ubuntu:bionic
 FROM ${FROM}
 
 ENV PATH=/var/lib/openstack/bin:$PATH

README.md

@@ -87,10 +87,10 @@ $ docker build https://opendev.org/openstack/loci.git \
 
 For more advanced building you can use docker build arguments to define:
   * `FROM` The base Docker image to build from. Currently supported are
-    `ubuntu:xenial`, `centos:7`, `opensuse/leap:15`, or a base image
-    derived from one of those distributions. Dockerfiles to bootstrap the
-    base images can be found in the `dockerfiles` directory, and are a good
-    starting point for customizing a base image.
+    `ubuntu:bionic`, `ubuntu:xenial`, `centos:7`, `opensuse/leap:15`, or
+    a base image derived from one of those distributions. Dockerfiles to
+    bootstrap the base images can be found in the `dockerfiles` directory,
+    and are a good starting point for customizing a base image.
   * `PROJECT` The name of the project to install.
   * `PROJECT_REPO` The git repo containing the OpenStack project the container
     should contain

dockerfiles/debian/Dockerfile

@@ -4,7 +4,7 @@ FROM ${FROM}
 ARG DEBIAN_URL=http://deb.debian.org/debian/
 ARG DEBIAN_SECURITY_URL=http://security.debian.org/
 ARG DEBIAN_SECURITY_DISTRIBUTION=stretch/updates
-ARG CEPH_URL=http://download.ceph.com/debian-luminous/
+ARG CEPH_URL=http://download.ceph.com/debian-nautilus/
 ARG ALLOW_UNAUTHENTICATED=false
 ARG PIP_INDEX_URL=https://pypi.python.org/simple/
 ARG PIP_TRUSTED_HOST=pypi.python.org

dockerfiles/ubuntu_bionic/Dockerfile

@@ -0,0 +1,23 @@
+ARG FROM=ubuntu:bionic
+FROM ${FROM}
+
+ARG UBUNTU_URL=http://archive.ubuntu.com/ubuntu/
+ARG CLOUD_ARCHIVE_URL=http://ubuntu-cloud.archive.canonical.com/ubuntu/
+ARG CEPH_URL=http://download.ceph.com/debian-nautilus/
+ARG ALLOW_UNAUTHENTICATED=false
+ARG PIP_INDEX_URL=https://pypi.python.org/simple/
+ARG PIP_TRUSTED_HOST=pypi.python.org
+ENV PIP_INDEX_URL=${PIP_INDEX_URL}
+ENV PIP_TRUSTED_HOST=${PIP_TRUSTED_HOST}
+
+COPY sources.list /etc/apt/
+COPY cloud-archive.gpg ceph.gpg /etc/apt/trusted.gpg.d/
+RUN sed -i \
+        -e "s|%%UBUNTU_URL%%|${UBUNTU_URL}|g" \
+        -e "s|%%CLOUD_ARCHIVE_URL%%|${CLOUD_ARCHIVE_URL}|g" \
+        -e "s|%%CEPH_URL%%|${CEPH_URL}|g" \
+        /etc/apt/sources.list
+RUN echo "APT::Get::AllowUnauthenticated \"${ALLOW_UNAUTHENTICATED}\";\n\
+Acquire::AllowInsecureRepositories \"${ALLOW_UNAUTHENTICATED}\";\n\
+Acquire::AllowDowngradeToInsecureRepositories \"${ALLOW_UNAUTHENTICATED}\";" \
+    >> /etc/apt/apt.conf.d/allow-unathenticated

dockerfiles/ubuntu_bionic/sources.list

@@ -0,0 +1,6 @@
+deb %%UBUNTU_URL%% bionic main universe
+deb %%UBUNTU_URL%% bionic-updates main universe
+deb %%UBUNTU_URL%% bionic-backports main universe
+deb %%UBUNTU_URL%% bionic-security main universe
+deb %%CEPH_URL%% bionic main
+deb %%CLOUD_ARCHIVE_URL%% bionic-updates/ussuri main

playbooks/loci-builder.yaml

@@ -13,10 +13,13 @@
       block:
         - docker_image:
             name: loci/requirements
-            tag: "{{ branch }}-{{ item.name }}"
+            tag: "{{ item.release }}-{{ item.name }}"
             repository: 172.17.0.1:5000/loci/requirements
             push: yes
           with_items: "{{ distros }}"
+          when: &condition >
+            supported_releases is undefined or
+            item.release in supported_releases
       when:
        - reuse_requirements | bool
        - project != 'requirements'
@@ -30,6 +33,7 @@
             tag: "{{ item.name }}"
             buildargs: "{{ item.buildargs.base }}"
           with_items: "{{ distros }}"
+          when: *condition
 
     - name: Build requirements image
       block:
@@ -37,12 +41,14 @@
           docker_image:
             path: "{{ loci_src_dir }}"
             name: loci/requirements
-            tag: "{{ branch }}-{{ item.name }}"
+            tag: "{{ item.release }}-{{ item.name }}"
             repository: 172.17.0.1:5000/loci/requirements
             push: yes
             pull: False
             buildargs: "{{ item.buildargs.requirements }}"
           with_items: "{{ distros }}"
+          when: *condition
+
       when:
        - (not reuse_requirements) | bool
        - project != 'requirements'
@@ -53,7 +59,8 @@
           docker_image:
             path: "{{ loci_src_dir }}"
             name: loci/{{ project }}
-            tag: "{{ branch }}-{{ item.name }}"
+            tag: "{{ item.release }}-{{ item.name }}"
             pull: False
             buildargs: "{{ item.buildargs.project }}"
           with_items: "{{ distros }}"
+          when: *condition

playbooks/push.yaml

@@ -8,6 +8,9 @@
           no_log: True
         - command: docker push loci/{{ project }}:{{ branch }}-{{ item.name }}
           with_items: "{{ distros }}"
+          when: &condition >
+            supported_releases is undefined or
+            item.release in supported_releases
 
     - name: Push project to quay.io
       block:
@@ -15,5 +18,7 @@
           no_log: True
         - command: docker tag loci/{{ project }}:{{ branch }}-{{ item.name }} quay.io/loci/{{ project }}:{{ branch }}-{{ item.name }}
           with_items: "{{ distros }}"
+          when: *condition
         - command: docker push quay.io/loci/{{ project }}:{{ branch }}-{{ item.name }}
           with_items: "{{ distros }}"
+          when: *condition

playbooks/vars.yaml

@@ -17,21 +17,11 @@ reuse_requirements: False
 # Override Zuul inference of source directory from project name to always
 # use "loci".
 loci_src_dir: "src/opendev.org/openstack/loci"
-# Branch name used for image building and publishing
-# If running in an environment with zuul and under a branched project
-# (like openstack/cinder, openstack/nova, ... but not openstack/loci)
-# this will be automatically be set to the appropriate branch name (rocky, queens, ...)
-branch: "{{ branchname | default(zuul_execution_branch.split('/')[-1]) }}"
-# Upstream code reference.
-# As LOCI is not building images on a per-commit basis on upstream openstack projects,
-# pointing to upstream zuul_branch for gating is enough.
-# If a project_ref is passed in a gating variable (like how you would do directly
-# in the command line), it will get consumed during image building.
-project_reference: "{% if project_ref is defined %}{{ project_ref }}{% elif zuul_branch is defined %}{{ zuul_branch }}{% else %}master{% endif %}"
 
 distros:
   - name: centos
     image: centos:7
+    release: master
     buildargs:
       base:
         PACKAGE_MIRROR: "{{ zuul_site_mirror_fqdn }}"
@@ -40,15 +30,39 @@ distros:
       project:
         PROJECT: "{{ project }}"
         PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }}
-        PROJECT_REF: "{{ project_reference }}"
-        WHEELS: 172.17.0.1:5000/loci/requirements:{{ branch }}-centos
+        WHEELS: 172.17.0.1:5000/loci/requirements:master-centos
         FROM: base:centos
+        PYTHON3: yes
       requirements:
         PROJECT: requirements
         PROJECT_REPO: http://172.17.0.1/git/openstack/requirements
         FROM: base:centos
-  - name: ubuntu
+        PYTHON3: yes
+  - name: ubuntu_bionic
+    image: ubuntu:bionic
+    release: master
+    buildargs:
+      base:
+        UBUNTU_URL: http://{{ zuul_site_mirror_fqdn }}/ubuntu/
+        CLOUD_ARCHIVE_URL: http://{{ zuul_site_mirror_fqdn }}/ubuntu-cloud-archive/
+        CEPH_URL: http://{{ zuul_site_mirror_fqdn }}/ceph-deb-nautilus/
+        ALLOW_UNAUTHENTICATED: "true"
+        PIP_INDEX_URL: http://{{ zuul_site_mirror_fqdn }}/pypi/simple
+        PIP_TRUSTED_HOST: "{{ zuul_site_mirror_fqdn }}"
+      project:
+        PROJECT: "{{ project }}"
+        PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }}
+        WHEELS: 172.17.0.1:5000/loci/requirements:master-ubuntu_bionic
+        FROM: base:ubuntu_bionic
+        PYTHON3: yes
+      requirements:
+        PROJECT: requirements
+        PROJECT_REPO: http://172.17.0.1/git/openstack/requirements
+        FROM: base:ubuntu_bionic
+        PYTHON3: yes
+  - name: ubuntu_xenial
     image: ubuntu:xenial
+    release: ocata
     buildargs:
       base:
         UBUNTU_URL: http://{{ zuul_site_mirror_fqdn }}/ubuntu/
@@ -60,14 +74,18 @@ distros:
       project:
         PROJECT: "{{ project }}"
         PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }}
-        PROJECT_REF: "{{ project_reference }}"
-        WHEELS: 172.17.0.1:5000/loci/requirements:{{ branch }}-ubuntu
-        FROM: base:ubuntu
+        PROJECT_REF: "stable/ocata"
+        PROJECT_RELEASE: "ocata"
+        WHEELS: 172.17.0.1:5000/loci/requirements:ocata-ubuntu_xenial
+        FROM: base:ubuntu_xenial
       requirements:
         PROJECT: requirements
         PROJECT_REPO: http://172.17.0.1/git/openstack/requirements
-        FROM: base:ubuntu
+        PROJECT_REF: "stable/ocata"
+        PROJECT_RELEASE: "ocata"
+        FROM: base:ubuntu_xenial
   - name: leap15
+    release: master
     buildargs:
       base:
         PACKAGE_MIRROR: "http://{{ zuul_site_mirror_fqdn }}/opensuse/"
@@ -76,7 +94,7 @@ distros:
       project:
         PROJECT: "{{ project }}"
         PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }}
-        WHEELS: 172.17.0.1:5000/loci/requirements:{{ branch }}-leap15
+        WHEELS: 172.17.0.1:5000/loci/requirements:master-leap15
         FROM: base:leap15
         PYTHON3: yes
       requirements:

scripts/install.sh

@@ -11,7 +11,7 @@ if [[ "${PYTHON3}" == "no" ]]; then
     python3=""
 else
     dpkg_python_packages=("python3" "python3-virtualenv" "python3-distutils")
-    rpm_python_packages=("python3" "python3-virtualenv")
+    rpm_python_packages=("python3")
     python3="python3"
 fi
 
@@ -36,12 +36,16 @@ case ${distro} in
             redhat-lsb-core \
             sudo \
             ${rpm_python_packages[@]}
+        if [[ "${PYTHON3}" != "no" ]]; then
+          pip3 install virtualenv
+        fi
         ;;
     opensuse|opensuse-leap|opensuse-tumbleweed|sles)
         if [[ "${PYTHON3}" == "no" ]]; then
            rpm_python_packages+=("python-devel" "python-setuptools")
         else
-           rpm_python_packages+=("python3-devel" "python3-setuptools")
+           rpm_python_packages+=("python3-devel" "python3-setuptools"
+                                 "python3-virtualenv")
         fi
         zypper --non-interactive --gpg-auto-import-keys refresh
         zypper --non-interactive install --no-recommends \