Update upper-constraints handling

For tinyIPA, migrate the generate_upper_constraints script in-tree.

For DIB, use a source repository with requirements to fetch
upper-constraints. As a nice side effect, it allows depending on
requirements patches in the CI.

Change-Id: I6be51b98adab641cfaa1613306cbdcc3b42de1c2

dib/ironic-python-agent-ramdisk/install.d/ironic-python-agent-ramdisk-source-install/60-ironic-python-agent-ramdisk-install

@@ -8,11 +8,9 @@ set -o pipefail
 
 SCRIPTDIR=$(dirname $0)
 IPADIR=/tmp/ironic-python-agent
+UPPER_CONSTRAINTS=/tmp/requirements/upper-constraints.txt
 VENVDIR=/opt/ironic-python-agent
 
-# Generate upper-constraints
-$IPADIR/imagebuild/common/generate_upper_constraints.sh $IPADIR/upper-constraints.txt
-
 # create the virtual environment using the default python
 $DIB_PYTHON -m virtualenv -- $VENVDIR
 
@@ -20,7 +18,7 @@ $DIB_PYTHON -m virtualenv -- $VENVDIR
 $VENVDIR/bin/pip install pip --upgrade
 
 # install IPA inside the virtual environment
-$VENVDIR/bin/pip install -c $IPADIR/upper-constraints.txt $IPADIR --install-option="--install-scripts=/usr/local/bin/"
+$VENVDIR/bin/pip install -c $UPPER_CONSTRAINTS $IPADIR --install-option="--install-scripts=/usr/local/bin/"
 
 case "$DIB_INIT_SYSTEM" in
     upstart)

dib/ironic-python-agent-ramdisk/source-repository-requirements

@@ -0,0 +1 @@
+requirements git /tmp/requirements https://opendev.org/openstack/requirements

roles/ipa-build-dib-image/defaults/main.yaml

@@ -2,5 +2,6 @@ image_distro: centos7
 image_release:
 ipa_branch_path: '{{ zuul.branch | replace("/", "-") }}'
 ipa_source_path: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent'
+requirements_path: '{{ ansible_user_dir }}/src/opendev.org/openstack/requirements'
 ipa_raw_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_RAW'
 ipa_tar_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_TAR'

roles/ipa-build-dib-image/tasks/main.yaml

@@ -13,6 +13,8 @@
     # Use repositories checked out by Zuul
     DIB_REPOLOCATION_ironic_python_agent: '{{ ipa_source_path }}'
     DIB_REPOREF_ironic_python_agent: HEAD
+    DIB_REPOLOCATION_requirements: '{{ requirements_path }}'
+    DIB_REPOREF_requirements: HEAD
 
 - name: Move the resulting files
   shell: |

tinyipa/build-tinyipa.sh

@@ -106,7 +106,7 @@ if [ -n "$PYTHON_EXTRA_SOURCES_DIR_LIST" ]; then
     done
 fi
 
-imagebuild/common/generate_upper_constraints.sh upper-constraints.txt
+$WORKDIR/generate_upper_constraints.sh upper-constraints.txt
 if [ -n "$IRONIC_LIB_SOURCE" ]; then
     sed -i '/ironic-lib/d' upper-constraints.txt $BUILDDIR/tmp/ipa-requirements.txt
 fi

tinyipa/generate_upper_constraints.sh

@@ -0,0 +1,96 @@
+#!/bin/bash -eu
+
+SCRIPT_NAME=$(basename $0)
+COMMON_ROOT=$(dirname $0)
+DESTINATION="$1"
+TOX_INI=${2:-tox.ini}
+
+copy() {
+    local src=$1
+    local destination=$2
+
+    if test -z "${src}"; then
+        return 1
+    fi
+
+    if test -e "${src}"; then
+        log "File '${src}' exists. Using as upper-constraints."
+        cp "${src}" "${destination}"
+    else
+        log "File '${src}' not found. Skipping local file strategy."
+        return 1
+    fi
+    return 0
+}
+
+download() {
+    local url=$1
+    local destination=$2
+
+    if test -z "${url}"; then
+        return 1
+    else
+        log "Downloading from '${url}'"
+        curl -L ${url} -o "${destination}"
+    fi
+    return 0
+}
+
+log() {
+    echo "${SCRIPT_NAME}: ${@}"
+}
+
+fail() {
+    log ${@}
+    exit 1
+}
+
+upper_constraints_is_not_null() {
+    test "${UPPER_CONSTRAINTS_FILE:-""}" != ""
+}
+
+copy_uc() {
+    copy "${UPPER_CONSTRAINTS_FILE:-""}" "${DESTINATION}"
+}
+
+download_uc() {
+    download "${UPPER_CONSTRAINTS_FILE:-""}" "${DESTINATION}"
+}
+
+copy_new_requirements_uc() {
+    if [ -e "/opt/stack/new/requirements" ]; then
+        copy "/opt/stack/new/requirements/upper-constraints.txt" "${DESTINATION}"
+    elif [ -e "/opt/stack/requirements" ]; then
+        copy "/opt/stack/requirements/upper-constraints.txt" "${DESTINATION}"
+    else
+        log "No local requirements repository, will download upper-constraints"
+        # Allow the caller to handle the failure
+        return 1
+    fi
+}
+
+download_from_tox_ini_url() {
+    local url
+    # NOTE(mmitchell): This extracts the URL defined as the default value for
+    #                  UPPER_CONSTRAINTS_FILE in tox.ini. This is used by image
+    #                  builders to avoid duplicating the default value in multiple
+    #                  scripts. This is specially done to leverage the release
+    #                  tools that automatically update the tox.ini when projects
+    #                  are released.
+    url=$(sed -n 's/^.*{env:UPPER_CONSTRAINTS_FILE\:\([^}]*\)}.*$/\1/p' $TOX_INI | head -n1)
+    log "tox.ini indicates '${url}' as fallback."
+    download "${url}" "${DESTINATION}"
+}
+
+log "Generating local constraints file..."
+
+if upper_constraints_is_not_null; then
+    log "UPPER_CONSTRAINTS_FILE is defined as '${UPPER_CONSTRAINTS_FILE:-""}'"
+    copy_uc || download_uc || fail "Failed to copy or download file indicated in UPPER_CONSTRAINTS_FILE."
+else
+    log "UPPER_CONSTRAINTS_FILE is not defined. Using fallback strategies."
+
+    copy_new_requirements_uc || \
+        download_from_tox_ini_url || \
+        fail "Failed to download upper-constraints.txt from either CI or tox.ini location."
+fi