From a4174a1e0a211e1629b841ecf26596b01c2464b1 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Thu, 25 Jul 2024 15:40:41 -0400 Subject: [PATCH] Remove CentOS/OpenSUSE build support StarlingX stopped supporting CentOS builds in the after release 7.0. This update will strip CentOS from our code base. It will also remove references to the failed OpenSUSE feature as well. There is still one CentOS based docker image (n3000), so this update will leave limited CentOS support in download and docker image building tools. Verified with a full Jenkins master branch build for Debian. - download full and incremental - package build full and incremental - iso build - build of base container image - build all flock container images - helm chart build. Story: 2011110 Task: 49939 Change-Id: I57939d2026d7df76091e8658750b4bd0fa8e4f5f Signed-off-by: Scott Little --- .gitignore | 3 - build-data/build_avoidance_source | 23 - build-data/unbuilt_rpm_patterns | 22 - build-tools/Cached_Data.txt | 78 - build-tools/audit-pkgs | 80 - build-tools/build-avoidance-utils.sh | 923 ----- build-tools/build-docker-images/README | 9 +- .../base-image-build-centos-dev.cfg | 2 - .../base-image-build-centos-stable.cfg | 2 - .../build-docker-images/build-stx-base.sh | 43 +- ...arlingx-enable-disable-package-repos.patch | 44 +- .../stx-centos/Dockerfile.dev | 16 - .../stx-centos/Dockerfile.stable | 31 - .../build-docker-images/update-stx-image.sh | 2 +- build-tools/build-guest | 412 -- build-tools/build-helm-charts.sh | 100 +- build-tools/build-img | 638 --- build-tools/build-iso | 853 ---- build-tools/build-pkg-srpm | 63 - build-tools/build-pkgs | 126 - build-tools/build-pkgs-parallel | 538 --- build-tools/build-pkgs-serial | 538 --- build-tools/build-rpms | 44 - build-tools/build-rpms-parallel | 2507 ------------ build-tools/build-rpms-serial | 2220 ---------- build-tools/build-srpms | 52 - build-tools/build-srpms-common.sh | 106 - build-tools/build-srpms-parallel | 1605 -------- build-tools/build-srpms-serial | 1424 ------- build-tools/build-wheels/FIXME | 5 +- build-tools/build-wheels/build-base-wheels.sh | 4 +- .../build-wheels/build-wheel-tarball.sh | 2 +- build-tools/build-wheels/centos/Dockerfile | 46 - .../build-wheels/centos/dev-wheels-py2.cfg | 20 - .../build-wheels/centos/dev-wheels.cfg | 21 - build-tools/build-wheels/centos/openstack.cfg | 13 - .../build-wheels/centos/stable-wheels-py2.cfg | 178 - .../build-wheels/centos/stable-wheels.cfg | 183 - build-tools/build-wheels/doc/wheels-cfg.md | 2 +- build-tools/build-wheels/get-stx-wheels.sh | 54 +- build-tools/build_guest/build-guest-image.py | 123 - build-tools/build_guest/image-rt.inc | 14 - build-tools/build_guest/image.inc | 14 - build-tools/build_guest/rootfs-exclude.txt | 13 - .../build_guest/rootfs-rt/boot/extlinux.conf | 7 - build-tools/build_guest/rootfs-setup.sh | 92 - .../build_guest/rootfs-std/boot/extlinux.conf | 7 - .../cloud/cloud.cfg.d/99_wrs-datasources.cfg | 18 - .../build_guest/rootfs/etc/dhcp/dhclient.conf | 21 - .../build_guest/rootfs/etc/iptables.rules | 12 - .../rootfs/etc/modprobe.d/floppy.conf | 1 - .../rootfs/etc/modprobe.d/wrs_avp.conf | 1 - .../rootfs/etc/modules-load.d/wrs_avp.conf | 1 - .../etc/sysconfig/network-scripts/ifcfg-eth0 | 8 - .../etc/udev/rules.d/65-renumber-net.rules | 4 - .../rootfs/usr/lib/udev/renumber_device | 12 - .../build_guest/rpm-install-list-rt.txt | 294 -- build-tools/build_guest/rpm-install-list.txt | 291 -- build-tools/build_guest/rpm-remove-list.txt | 7 - build-tools/build_iso/anaconda-ks.cfg | 40 - build-tools/build_iso/cgts_deps.sh | 352 -- build-tools/build_iso/comps.xml.gz | Bin 160726 -> 0 bytes build-tools/build_iso/gather_packages.pl | 122 - build-tools/build_iso/image-dev.inc | 6 - build-tools/build_iso/image.inc | 84 - build-tools/build_iso/isolinux.cfg | 125 - build-tools/build_iso/ks.cfg | 36 - build-tools/build_iso/minimal_rpm_list.txt | 256 -- build-tools/build_iso/openstack_kilo.txt | 2 - build-tools/build_minimal_iso/README | 112 - build-tools/build_minimal_iso/README.2 | 5 - build-tools/build_minimal_iso/build.cfg | 108 - build-tools/build_minimal_iso/build.sh | 64 - build-tools/build_minimal_iso/build_centos.sh | 62 - build-tools/build_minimal_iso/cgts_deps.sh | 265 -- build-tools/build_minimal_iso/yum.conf | 22 - build-tools/classify | 55 - build-tools/create-yum-conf | 87 - build-tools/create_dependancy_cache.py | 716 ---- build-tools/default_build_srpm | 277 -- build-tools/find_klm | 59 - .../find_patched_srpms_needing_upgrade | 54 - build-tools/helm_chart_modify.py | 8 +- build-tools/image-utils.sh | 4 +- build-tools/ip_report.py | 523 --- build-tools/make-installer-images.sh | 343 -- build-tools/mk/_sign_pkgs.mk | 31 - build-tools/mockchain-parallel | 114 - build-tools/mockchain-parallel-1.3.4 | 1219 ------ build-tools/mockchain-parallel-1.4.16 | 1226 ------ build-tools/mockchain-parallel-2.6 | 1221 ------ build-tools/mockchain-parallel-2.7 | 1221 ------ build-tools/modify-build-cfg | 205 - build-tools/patch-iso | 427 -- build-tools/patch_rebase_1 | 140 - build-tools/patch_rebase_2 | 158 - build-tools/patch_rebase_3 | 129 - build-tools/patch_rebase_4 | 413 -- build-tools/repo_files/comps.xml | 39 - build-tools/repo_files/mock.cfg.all.proto | 1 - .../repo_files/mock.cfg.centos7.all.proto | 62 - .../repo_files/mock.cfg.centos7.distro.proto | 62 - build-tools/repo_files/mock.cfg.centos7.proto | 61 - .../repo_files/mock.cfg.centos8.all.proto | 63 - .../repo_files/mock.cfg.centos8.distro.proto | 63 - build-tools/repo_files/mock.cfg.centos8.proto | 62 - build-tools/repo_files/mock.cfg.distro.proto | 1 - build-tools/repo_files/mock.cfg.proto | 1 - build-tools/sign-rpms | 293 -- build-tools/sign-secure-boot | 538 --- build-tools/source_lookup.txt | 85 - build-tools/spec-utils | 713 ---- build-tools/srpm-utils | 3630 ----------------- build-tools/stx/discovery.py | 2 +- build-tools/stxRpmUtils.py | 41 - build-tools/sync-jenkins | 154 - build-tools/sync_jenkins.sh | 145 - build-tools/tis.macros | 11 - build-tools/update-efiboot-image | 243 -- build-tools/update-pxe-network-installer | 197 - build-tools/url_utils.sh | 2 + build-tools/wheel-utils.sh | 4 +- 122 files changed, 64 insertions(+), 30672 deletions(-) delete mode 100644 build-data/build_avoidance_source delete mode 100644 build-data/unbuilt_rpm_patterns delete mode 100644 build-tools/Cached_Data.txt delete mode 100755 build-tools/audit-pkgs delete mode 100644 build-tools/build-avoidance-utils.sh delete mode 100644 build-tools/build-docker-images/base-image-build-centos-dev.cfg delete mode 100644 build-tools/build-docker-images/base-image-build-centos-stable.cfg delete mode 100644 build-tools/build-docker-images/stx-centos/Dockerfile.dev delete mode 100644 build-tools/build-docker-images/stx-centos/Dockerfile.stable delete mode 100755 build-tools/build-guest delete mode 100755 build-tools/build-img delete mode 100755 build-tools/build-iso delete mode 100644 build-tools/build-pkg-srpm delete mode 100755 build-tools/build-pkgs delete mode 100755 build-tools/build-pkgs-parallel delete mode 100755 build-tools/build-pkgs-serial delete mode 100755 build-tools/build-rpms delete mode 100755 build-tools/build-rpms-parallel delete mode 100755 build-tools/build-rpms-serial delete mode 100755 build-tools/build-srpms delete mode 100644 build-tools/build-srpms-common.sh delete mode 100755 build-tools/build-srpms-parallel delete mode 100755 build-tools/build-srpms-serial delete mode 100644 build-tools/build-wheels/centos/Dockerfile delete mode 100644 build-tools/build-wheels/centos/dev-wheels-py2.cfg delete mode 100644 build-tools/build-wheels/centos/dev-wheels.cfg delete mode 100644 build-tools/build-wheels/centos/openstack.cfg delete mode 100644 build-tools/build-wheels/centos/stable-wheels-py2.cfg delete mode 100644 build-tools/build-wheels/centos/stable-wheels.cfg delete mode 100755 build-tools/build_guest/build-guest-image.py delete mode 100644 build-tools/build_guest/image-rt.inc delete mode 100644 build-tools/build_guest/image.inc delete mode 100644 build-tools/build_guest/rootfs-exclude.txt delete mode 100644 build-tools/build_guest/rootfs-rt/boot/extlinux.conf delete mode 100755 build-tools/build_guest/rootfs-setup.sh delete mode 100644 build-tools/build_guest/rootfs-std/boot/extlinux.conf delete mode 100644 build-tools/build_guest/rootfs/etc/cloud/cloud.cfg.d/99_wrs-datasources.cfg delete mode 100644 build-tools/build_guest/rootfs/etc/dhcp/dhclient.conf delete mode 100644 build-tools/build_guest/rootfs/etc/iptables.rules delete mode 100644 build-tools/build_guest/rootfs/etc/modprobe.d/floppy.conf delete mode 100644 build-tools/build_guest/rootfs/etc/modprobe.d/wrs_avp.conf delete mode 100644 build-tools/build_guest/rootfs/etc/modules-load.d/wrs_avp.conf delete mode 100644 build-tools/build_guest/rootfs/etc/sysconfig/network-scripts/ifcfg-eth0 delete mode 100644 build-tools/build_guest/rootfs/etc/udev/rules.d/65-renumber-net.rules delete mode 100755 build-tools/build_guest/rootfs/usr/lib/udev/renumber_device delete mode 100644 build-tools/build_guest/rpm-install-list-rt.txt delete mode 100644 build-tools/build_guest/rpm-install-list.txt delete mode 100644 build-tools/build_guest/rpm-remove-list.txt delete mode 100644 build-tools/build_iso/anaconda-ks.cfg delete mode 100755 build-tools/build_iso/cgts_deps.sh delete mode 100644 build-tools/build_iso/comps.xml.gz delete mode 100755 build-tools/build_iso/gather_packages.pl delete mode 100644 build-tools/build_iso/image-dev.inc delete mode 100644 build-tools/build_iso/image.inc delete mode 100644 build-tools/build_iso/isolinux.cfg delete mode 100644 build-tools/build_iso/ks.cfg delete mode 100644 build-tools/build_iso/minimal_rpm_list.txt delete mode 100644 build-tools/build_iso/openstack_kilo.txt delete mode 100644 build-tools/build_minimal_iso/README delete mode 100644 build-tools/build_minimal_iso/README.2 delete mode 100644 build-tools/build_minimal_iso/build.cfg delete mode 100755 build-tools/build_minimal_iso/build.sh delete mode 100755 build-tools/build_minimal_iso/build_centos.sh delete mode 100755 build-tools/build_minimal_iso/cgts_deps.sh delete mode 100644 build-tools/build_minimal_iso/yum.conf delete mode 100644 build-tools/classify delete mode 100755 build-tools/create-yum-conf delete mode 100755 build-tools/create_dependancy_cache.py delete mode 100755 build-tools/default_build_srpm delete mode 100755 build-tools/find_klm delete mode 100755 build-tools/find_patched_srpms_needing_upgrade delete mode 100755 build-tools/ip_report.py delete mode 100755 build-tools/make-installer-images.sh delete mode 100644 build-tools/mk/_sign_pkgs.mk delete mode 100755 build-tools/mockchain-parallel delete mode 100755 build-tools/mockchain-parallel-1.3.4 delete mode 100755 build-tools/mockchain-parallel-1.4.16 delete mode 100755 build-tools/mockchain-parallel-2.6 delete mode 100755 build-tools/mockchain-parallel-2.7 delete mode 100755 build-tools/modify-build-cfg delete mode 100755 build-tools/patch-iso delete mode 100755 build-tools/patch_rebase_1 delete mode 100755 build-tools/patch_rebase_2 delete mode 100755 build-tools/patch_rebase_3 delete mode 100755 build-tools/patch_rebase_4 delete mode 100644 build-tools/repo_files/comps.xml delete mode 120000 build-tools/repo_files/mock.cfg.all.proto delete mode 100644 build-tools/repo_files/mock.cfg.centos7.all.proto delete mode 100644 build-tools/repo_files/mock.cfg.centos7.distro.proto delete mode 100644 build-tools/repo_files/mock.cfg.centos7.proto delete mode 100644 build-tools/repo_files/mock.cfg.centos8.all.proto delete mode 100644 build-tools/repo_files/mock.cfg.centos8.distro.proto delete mode 100644 build-tools/repo_files/mock.cfg.centos8.proto delete mode 120000 build-tools/repo_files/mock.cfg.distro.proto delete mode 120000 build-tools/repo_files/mock.cfg.proto delete mode 100755 build-tools/sign-rpms delete mode 100755 build-tools/sign-secure-boot delete mode 100644 build-tools/source_lookup.txt delete mode 100644 build-tools/spec-utils delete mode 100644 build-tools/srpm-utils delete mode 100644 build-tools/stxRpmUtils.py delete mode 100755 build-tools/sync-jenkins delete mode 100755 build-tools/sync_jenkins.sh delete mode 100644 build-tools/tis.macros delete mode 100755 build-tools/update-efiboot-image delete mode 100755 build-tools/update-pxe-network-installer diff --git a/.gitignore b/.gitignore index 56f0088c..72860aec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,6 @@ *.swp .tox __pycache__/ -/centos-repo -/cgcs-centos-repo -/cgcs-tis-repo /local-build-data /local-repo /public-keys/ diff --git a/build-data/build_avoidance_source b/build-data/build_avoidance_source deleted file mode 100644 index 1e371f24..00000000 --- a/build-data/build_avoidance_source +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2018 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# - - -# -# What files and directories need to be copied -# -BUILD_AVOIDANCE_SRPM_DIRECTORIES="inputs srpm_assemble rpmbuild/SRPMS rpmbuild/SOURCES" -BUILD_AVOIDANCE_SRPM_FILES="" -BUILD_AVOIDANCE_RPM_DIRECTORIES="results rpmbuild/RPMS rpmbuild/SPECS repo/local-repo/dependancy-cache" -BUILD_AVOIDANCE_RPM_FILES=".platform_release" - - -# -# Copy the lines below to $MY_REPO/local-build-data/build_avoidance_source, -# then uncomment and fill in the values giving the location of your local reference build. -# -# BUILD_AVOIDANCE_USR="jenkins" -# BUILD_AVOIDANCE_HOST="machine.corp.com" -# BUILD_AVOIDANCE_DIR="/localdisk/loadbuild/jenkins/StarlingX_Build" diff --git a/build-data/unbuilt_rpm_patterns b/build-data/unbuilt_rpm_patterns deleted file mode 100644 index 35df2a4f..00000000 --- a/build-data/unbuilt_rpm_patterns +++ /dev/null @@ -1,22 +0,0 @@ -[-]locale[-] -[-]doc[-] -[-]dbg[-] -vswitch-staticdev -vim-spell -openssh-server-sysvinit -openstack-neutron-linuxbridge -^libcacard- -^kernel-bootwrapper -^kernel-doc- -^kernel-abi-whitelists -^kernel-debug- -^kernel-kdump -^kernel-rt-bootwrapper -^kernel-rt-doc- -^kernel-rt-abi-whitelists -^kernel-rt-debug- -^kernel-rt-debuginfo -^kernel-rt-kdump -^kernel-rt-cross-headers -^kernel-rt-kvm-debuginfo -^kernel-rt-tools-debuginfo diff --git a/build-tools/Cached_Data.txt b/build-tools/Cached_Data.txt deleted file mode 100644 index e3a7d1fe..00000000 --- a/build-tools/Cached_Data.txt +++ /dev/null @@ -1,78 +0,0 @@ -Data on an source rpm: - - location: - ${MY_WORKSPACE}/${BUILD_TYPE}/rpmbuild/SPECS/${SRPM_FILE_NAME}/ - - files: - *.spec # spec file found in the source rpm - - subdirectories: - NAMES/ # Directory contains an emtpy file, where the file name - # is the name of the source rpm. - - SERVICES/ # Directory contains zero or more emtpy files, where the - # file name is the name of the service provided by one - # or more of the rpms. - - BUILDS/ # Directory contains emtpy files, where the file name is - # the name of a binary rpm built from the source rpm. - - BUILDS_VR/ # Directory contains emtpy files, where the file name is - # the name-verion-release of a binary rpm built from the - # source rpm. - - location: - ${MY_WORKSPACE}/${BUILD_TYPE}/rpmbuild/SOURCES/${SRPM_FILE_NAME}/ - - files: - BIG # if it exists, it contains one line, the numeric value - # extracted from build_srpms.data if the line - # BUILD_IS_BIG=### if present. - # This is the estimated filesystem size (GB) required to - # host a mock build of the package. - # Note: not all parallel build environments are the same - # size. The smallest build environmnet is 3 GB and this - # is sufficient for most packages. Don't bother adding a - # BUILD_IS_BIG=### directive unless 3 gb is proven to be - # insufficient. - - SLOW # if it exists, it contains one line, the numeric value i - # extracted from build_srpms.data if the line - # BUILD_IS_SLOW=### if present. - # This is the estimated build time (minutes) required to - # host perform a mock build of the package. - # Note: Currently we only use this value as a boolean. - # Non-zero and we try to start the build of this package - # earlier rather than later. Build times >= 3 minutes are - # worth anotating. Else don't bother adding a - # BUILD_IS_SLOW=### directive -e.g. - -cd $MY_WORKSPACE/std/rpmbuild/SPECS/openstack-cinder-9.1.1-0.tis.40.src.rpm -find . -./BUILDS -./BUILDS/openstack-cinder -./BUILDS/python-cinder -./BUILDS/python-cinder-tests -./NAMES -./NAMES/openstack-cinder -./SERVICES -./SERVICES/cinder -./BUILDS_VR -./BUILDS_VR/openstack-cinder-9.1.1-0.tis.40 -./BUILDS_VR/python-cinder-9.1.1-0.tis.40 -./BUILDS_VR/python-cinder-tests-9.1.1-0.tis.40 -./openstack-cinder.spec - - -e.g. -cd $MY_WORKSPACE/std/rpmbuild/SOURCES/kernel-3.10.0-514.16.1.el7.29.tis.src.rpm -find . -./BIG -./SLOW - -cat ./BIG -8 - -cat ./SLOW -12 diff --git a/build-tools/audit-pkgs b/build-tools/audit-pkgs deleted file mode 100755 index 370c9b0c..00000000 --- a/build-tools/audit-pkgs +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -rpm_compare () { - local r="$1" - local r2="$2" - local line - local f=$(basename $r) - local f2=$(basename $r2) - - rpm -q --dump --nosignature -p $r | awk ' { print $1 "\n" $1 " " $5 " " $6 " " $7 " " $8 " " $9 " " $10 " " $11 } ' > /tmp/dump.new - rpm -q --dump --nosignature -p $r2 | awk ' { print $1 "\n" $1 " " $5 " " $6 " " $7 " " $8 " " $9 " " $10 " " $11 } ' > /tmp/dump.old - first_line=1 - diff -y -W 200 --suppress-common-lines /tmp/dump.new /tmp/dump.old | grep '|' | - while read -r line; do - left=$(echo "$line" | awk -F '|' '{ print $1 }') - right=$(echo "$line" | awk -F '|' '{ print $2 }') - left_f=$(echo "$left" | awk '{ print $1 }') - right_f=$(echo "$right" | awk '{ print $1 }') - if [ "$left_f" != "$right_f" ];then - continue - fi - if [ $first_line -eq 1 ]; then - echo "" - echo "$f vs $f2" - first_line=0 - fi - echo "$line" - done -} - -# For backward compatibility. Old repo location or new? -CENTOS_REPO=${MY_REPO}/centos-repo -if [ ! -d ${CENTOS_REPO} ]; then - CENTOS_REPO=${MY_REPO}/cgcs-centos-repo - if [ ! -d ${CENTOS_REPO} ]; then - echo "ERROR: directory ${MY_REPO}/centos-repo not found." - exit 1 - fi -fi - -echo "" -echo "======================================================" -echo "Auditing built packages vs unpatched upstream packages" -echo "======================================================" -for r in $(find $MY_WORKSPACE/*/rpmbuild/RPMS -name '*.rpm' | grep -v '.src.rpm' | grep -v debuginfo); do - f=$(basename $r) - f2=$(echo $f | sed 's#[.]tis[.][0-9]*[.]#.#' | sed 's#[.]tis[.]#.#') - r2=$(find ${CENTOS_REPO}/Binary/ -name $f2) - if [ "$r2" == "" ]; then - # Probably one of our own - # echo "Couldn't find '$f2'" - continue - fi - rpm_compare "$r" "$r2" -done - -echo "" -echo "============================" -echo "Auditing built for conflicts" -echo "============================" -grep 'conflicts with file from package' -r --binary-files=without-match $MY_WORKSPACE/*/results/ | - -while read -r line; do - w=$(echo "$line" | awk '{ print $8 }')".rpm" - w2=$(echo "$line" | awk '{ print $14 }')".rpm" - echo "$w $w2" -done | sort --unique | sed 's#bash-completion-1:#bash-completion-#' | - -while read -r line2; do - f=$(echo "$line2" | awk '{ print $1 }') - f2=$(echo "$line2" | awk '{ print $2 }') - r=$(find ${CENTOS_REPO}/Binary/ $MY_WORKSPACE/*/rpmbuild/RPMS -name $f) - r2=$(find ${CENTOS_REPO}/Binary/ $MY_WORKSPACE/*/rpmbuild/RPMS -name $f2) - # echo "" - # echo "$f vs $f2" - # echo "$r vs $r2" - if [ "$r" != "" ] && [ "$r2" != "" ]; then - rpm_compare "$r" "$r2" - fi -done diff --git a/build-tools/build-avoidance-utils.sh b/build-tools/build-avoidance-utils.sh deleted file mode 100644 index 5ac5858d..00000000 --- a/build-tools/build-avoidance-utils.sh +++ /dev/null @@ -1,923 +0,0 @@ -# -# Copyright (c) 2018 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# - -# -# Functions related to build avoidance. -# -# Do not call directly. Used by build-pkgs. -# -# Build avoidance downloads rpm, src.rpm and other artifacts of -# build-pkgs for a local reference build. The reference would -# typically be an automated build run atleast daily. -# The MY_WORKSPACE directory for the reference build shall have -# a common root directory, and a leaf directory that is a time stamp -# in a sortable parsable format. Default YYYYMMDDThhmmssZ. -# e.g. /localdisk/loadbuild/jenkins/StarlingX/20180719T113021Z -# -# Other formats can be used by setting the following variables -# in $MY_REPO/local-build-data/build_avoidance_source. -# e.g. to allow format YYYY-MM-DD_hh-mm-ss -# BUILD_AVOIDANCE_DATE_FORMAT="%Y-%m-%d" -# BUILD_AVOIDANCE_TIME_FORMAT="%H-%M-%S" -# BUILD_AVOIDANCE_DATE_TIME_DELIM="_" -# BUILD_AVOIDANCE_DATE_TIME_POSTFIX="" -# -# Note: Must be able to rsync and ssh to the machine that holds the -# reference builds. -# -# In future alternative transfer protocols may be supported. -# Select the alternate protocol by setting the following variables -# in $MY_REPO/local-build-data/build_avoidance_source. -# e.g. -# BUILD_AVOIDANCE_FILE_TRANSFER="my-supported-prototcol" -# - -BUILD_AVOIDANCE_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" - -source "${BUILD_AVOIDANCE_UTILS_DIR}/git-utils.sh" - -BUILD_AVOIDANCE_USR="" -BUILD_AVOIDANCE_HOST="" -BUILD_AVOIDANCE_DIR="" -BUILD_AVOIDANCE_URL="" - -# Default date/time format, iso-8601 compact, 20180912T143913Z -# Syntax is a subset of that use by the unix 'date' command. -BUILD_AVOIDANCE_DATE_FORMAT="%Y%m%d" -BUILD_AVOIDANCE_TIME_FORMAT="%H%M%S" -BUILD_AVOIDANCE_DATE_TIME_DELIM="T" -BUILD_AVOIDANCE_DATE_TIME_POSTFIX="Z" - -# Default file transfer method -BUILD_AVOIDANCE_FILE_TRANSFER="rsync" - -# Default is to use timestamps and days in UTC -# -# If you prefer local time, then set 'BUILD_AVOIDANCE_DATE_UTC=0' -# in '$MY_REPO/local-build-data/build_avoidance_source' -BUILD_AVOIDANCE_DATE_UTC=1 - -BUILD_AVOIDANCE_DATA_DIR="$MY_WORKSPACE/build_avoidance_data" -BUILD_AVOIDANCE_SOURCE="$MY_REPO/build-data/build_avoidance_source" -BUILD_AVOIDANCE_LOCAL_SOURCE="$MY_REPO/local-build-data/build_avoidance_source" -BUILD_AVOIDANCE_TEST_CONTEXT="$BUILD_AVOIDANCE_DATA_DIR/test_context" - -if [ ! -f $BUILD_AVOIDANCE_SOURCE ]; then - echo "Couldn't read $BUILD_AVOIDANCE_SOURCE" - exit 1 -fi - -echo "Reading: $BUILD_AVOIDANCE_SOURCE" -source $BUILD_AVOIDANCE_SOURCE - -if [ -f $BUILD_AVOIDANCE_LOCAL_SOURCE ]; then - echo "Reading: $BUILD_AVOIDANCE_LOCAL_SOURCE" - source $BUILD_AVOIDANCE_LOCAL_SOURCE -fi - -UTC="" - -if [ $BUILD_AVOIDANCE_DATE_UTC -eq 1 ]; then - UTC="--utc" -fi - - -if [ "x$BUILD_AVOIDANCE_OVERRIDE_DIR" != "x" ]; then - BUILD_AVOIDANCE_DIR="$BUILD_AVOIDANCE_OVERRIDE_DIR" -fi - -if [ "x$BUILD_AVOIDANCE_OVERRIDE_HOST" != "x" ]; then - BUILD_AVOIDANCE_HOST="$BUILD_AVOIDANCE_OVERRIDE_HOST" -fi - -if [ "x$BUILD_AVOIDANCE_OVERRIDE_USR" != "x" ]; then - BUILD_AVOIDANCE_USR="$BUILD_AVOIDANCE_OVERRIDE_USR" -fi - -echo "BUILD_AVOIDANCE_DIR=$BUILD_AVOIDANCE_DIR" -echo "BUILD_AVOIDANCE_HOST=$BUILD_AVOIDANCE_HOST" -echo "BUILD_AVOIDANCE_USR=$BUILD_AVOIDANCE_USR" - -build_avoidance_last_sync_file () { - local BUILD_TYPE=$1 - - if [ -z "$BUILD_TYPE" ]; then - echo "build_avoidance_last_sync_file: Build type not set" - exit 1 - fi - echo "$BUILD_AVOIDANCE_DATA_DIR/$BUILD_TYPE/last_sync_context" -} - -build_avoidance_clean () { - local BUILD_TYPE=$1 - local lsf - - if [ "$BUILD_TYPE" == "" ]; then - for lsf in $(find $BUILD_AVOIDANCE_DATA_DIR -name last_sync_context); do - \rm -f -v "$lsf" - done - else - lsf="$(build_avoidance_last_sync_file $BUILD_TYPE)" - if [ -f $lsf ]; then - \rm -f -v "$lsf" - fi - fi -} - - -date_to_iso_8601 () { - local DATE="$1" - local CENTURY="" - local YEAR_IN_CENTURY="00" - local MONTH="01" - local DAY="01" - local DAY_OF_YEAR="" - - CENTURY="$(date '+%C')" - - for x in $(echo "${BUILD_AVOIDANCE_DATE_FORMAT}" | tr ' ' '#' | sed 's/%%/#/g' | tr '%' ' ' ); do - # Consume format case options - case ${x:0:1} in - ^) x=${x:1};; - \#) x=${x:1};; - *) ;; - esac - - # Process format - case $x in - Y*) CENTURY=${DATE:0:2}; YEAR_IN_CENTURY=${DATE:2:2}; DATE=${DATE:4}; x=${x:1};; - 0Y*) CENTURY=${DATE:0:2}; YEAR_IN_CENTURY=${DATE:2:2}; DATE=${DATE:4}; x=${x:2};; - _Y*) CENTURY=$(echo "${DATE:0:2}" | tr ' ' '0'); YEAR_IN_CENTURY=${DATE:2:2}; DATE=${DATE:4}; x=${x:2};; - - y*) YEAR_IN_CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};; - 0y*) YEAR_IN_CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};; - _y*) YEAR_IN_CENTURY=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};; - - C*) CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};; - 0C*) CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};; - _C*) CENTURY=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};; - - m*) MONTH=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};; - 0m*) MONTH=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};; - _m*) MONTH=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};; - e*) MONTH=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:1};; - 0e*) MONTH=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};; - _e*) MONTH=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};; - b*) MONTH="$(date -d "${DATE:0:3} 1 2000" '+%m')"; DATE=${DATE:3}; x=${x:1};; - h*) MONTH="$(date -d "${DATE:0:3} 1 2000" '+%m')"; DATE=${DATE:3}; x=${x:1};; - - d*) DAY=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};; - 0d*) DAY=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};; - _d*) DAY=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};; - - j*) DAY_OF_YEAR=${DATE:0:3}; DATE=${DATE:3}; x=${x:1};; - 0j*) DAY_OF_YEAR=${DATE:0:3}; DATE=${DATE:3}; x=${x:2};; - _j*) DAY_OF_YEAR=$(echo "${DATE:0:3}" | tr ' ' '0'); DATE=${DATE:3}; x=${x:2};; - - D*) MONTH=${DATE:0:2}; DAY=${DATE:3:2}; YEAR_IN_CENTURY=${DATE:6:2}; DATE=${DATE:8}; x=${x:1};; - F*) CENTURY=${DATE:0:2}; YEAR_IN_CENTURY=${DATE:2:2}; MONTH=${DATE:5:2}; DAY=${DATE:8:2}; DATE=${DATE:10}; x=${x:1};; - *) >&2 echo "$FUNCNAME (${LINENO}): Unsupported date format: ${BUILD_AVOIDANCE_DATE_FORMAT}"; return 1;; - esac - - # consume remaing non-interpreted content - if [ "$(echo "${DATE:0:${#x}}" | tr ' ' '#')" != "${x}" ]; then - >&2 echo "$FUNCNAME (${LINENO}): Unexpected content '${DATE:0:${#x}}' does not match expected '${x}': '$1' being parsed vs '${BUILD_AVOIDANCE_DATE_FORMAT}'" - return 1 - fi - DATE=${DATE:${#x}} - done - - if [ "${DAY_OF_YEAR}" != "" ]; then - local YEAR_SEC - local DOY_SEC - YEAR_SEC="$(date -d "${CENTURY}${YEAR_IN_CENTURY}-01-01" '+%s')" - DOY_SEC=$((YEAR_SEC+(DAY_OF_YEAR-1)*24*60*60)) - MONTH="$(date "@$DOY_SEC" "+%m")" - DAY="$(date "@$DOY_SEC" "+%d")" - fi - - echo "${CENTURY}${YEAR_IN_CENTURY}-${MONTH}-${DAY}" - return 0 -} - -time_to_iso_8601 () { - TIME="$1" - local HOUR="00" - local H12="" - local AMPM="" - local MINUTE="00" - local SECOND="00" - - CENTURY="$(date '+%C')" - - for x in $(echo "${BUILD_AVOIDANCE_TIME_FORMAT}" | tr ' ' '#' | sed 's/%%/#/g' | tr '%' ' ' ); do - # Consume format case options - case ${x:0:1} in - ^) x=${x:1};; - \#) x=${x:1};; - *) ;; - esac - - # Process format - case $x in - H*) HOUR=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};; - 0H*) HOUR=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; - _H*) HOUR="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; - k*) HOUR="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:1};; - 0k*) HOUR=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; - _k*) HOUR="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; - - I*) H12=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};; - 0I*) H12=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; - _I*) H12="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; - l*) H12="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:1};; - 0l*) H12=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; - _l*) H12="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; - p*) AMPM=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};; - - M*) MINUTE=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};; - 0M*) MINUTE=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; - _M*) MINUTE="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; - - S*) SECOND=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};; - 0S*) SECOND=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; - _S*) SECOND="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; - - R*) HOUR=${TIME:0:2}; MINUTE=${TIME:3:2} TIME=${TIME:5}; x=${x:1};; - r*) H12=${TIME:0:2}; MINUTE=${TIME:3:2}; SECOND=${TIME:6:2}; AMPM=${TIME:9:2}; TIME=${TIME:11}; x=${x:1};; - T*) HOUR=${TIME:0:2}; MINUTE=${TIME:3:2}; SECOND=${TIME:6:2}; TIME=${TIME:8}; x=${x:1};; - - *) >&2 echo "$FUNCNAME (${LINENO}): Unsupported time format: ${BUILD_AVOIDANCE_TIME_FORMAT}"; return 1;; - esac - - # consume remaing non-interpreted content - if [ "$(echo "${TIME:0:${#x}}" | tr ' ' '#')" != "${x}" ]; then - >&2 echo "$FUNCNAME (${LINENO}): Unexpected content '${TIME:0:${#x}}' does not match expected '${x}': '$1' being parsed vs '${BUILD_AVOIDANCE_TIME_FORMAT}'" - return 1 - fi - TIME=${TIME:${#x}} - done - - if [ "$H12" != "" ] && [ "$AMPM" != "" ]; then - HOUR="$(date "$H12:01:01 $AMPM" '+%H')" - else - if [ "$H12" != "" ] && [ "$AMPM" != "" ]; then - >&2 echo "$FUNCNAME (${LINENO}): Unsupported time format: ${BUILD_AVOIDANCE_TIME_FORMAT}" - return 1 - fi - fi - - echo "${HOUR}:${MINUTE}:${SECOND}" - return 0 -} - -date_time_to_iso_8601 () { - local DATE_TIME="$1" - local DATE - local TIME - local DECODED_DATE - local DECODED_TIME - DATE=$(echo "${DATE_TIME}" | cut -d ${BUILD_AVOIDANCE_DATE_TIME_DELIM} -f 1) - TIME=$(echo "${DATE_TIME}" | cut -d ${BUILD_AVOIDANCE_DATE_TIME_DELIM} -f 2 | sed "s#${BUILD_AVOIDANCE_DATE_TIME_POSTFIX}\$##") - DECODED_DATE=$(date_to_iso_8601 "${DATE}") - DECODED_TIME=$(time_to_iso_8601 "${TIME}") - echo "${DECODED_DATE}T${DECODED_TIME}$(date $UTC '+%:z')" -} - -# -# test_build_avoidance_context -# -# Is the provided context file compatible with the current -# state of all of our gits? A compatible context is one -# where every commit in the context file is visible in our -# current git history. -# -# Returns: Timestamp of context tested. -# Exit code: 0 = Compatible -# 1 = This context is older than the last applied -# build avoidance context. If you are searching -# newest to oldest, you might as well stop. -# 2 = Not compatible -# -test_build_avoidance_context () { - local context="$1" - local BA_LAST_SYNC_CONTEXT="$2" - local BA_CONTEXT="" - - BA_CONTEXT=$(basename $context | cut -d '.' -f 1) - >&2 echo "test: $BA_CONTEXT" - - if [ "$BA_CONTEXT" == "$BA_LAST_SYNC_CONTEXT" ]; then - # Stop the search. We've reached the last sync point - BA_CONTEXT="" - echo "$BA_CONTEXT" - return 1 - fi - - git_test_context "$context" - result=$? - if [ $result -eq 0 ]; then - # found a new context !!! - echo "$BA_CONTEXT" - return 0 - fi - - # Continue the search - BA_CONTEXT="" - echo "$BA_CONTEXT" - return 2 -} - - -# -# get_build_avoidance_context -# -# Return URL of the most recent jenkins build that is compatable with -# the current software context under $MY_REPO. -# -get_build_avoidance_context () { - ( - local BUILD_TYPE=$1 - local context - local BA_CONTEXT="" - local BA_LAST_SYNC_CONTEXT="" - - export BUILD_AVOIDANCE_LAST_SYNC_FILE="$(build_avoidance_last_sync_file $BUILD_TYPE)" - mkdir -p "$(dirname $BUILD_AVOIDANCE_LAST_SYNC_FILE)" - - # Load last synced context - if [ -f $BUILD_AVOIDANCE_LAST_SYNC_FILE ]; then - BA_LAST_SYNC_CONTEXT=$(head -n 1 $BUILD_AVOIDANCE_LAST_SYNC_FILE) - fi - - mkdir -p $BUILD_AVOIDANCE_DATA_DIR - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): mkdir -p $BUILD_AVOIDANCE_DATA_DIR" - return 1 - fi - - local REMOTE_CTX_DIR="context" - local LOCAL_CTX_DIR="$BUILD_AVOIDANCE_DATA_DIR/context" - - # First copy the directory containing all the context files for - # the reference builds. - >&2 echo "Download latest reference build contexts" - - # Must set this prior to build_avoidance_copy_dir. - # The setting is not exported outside of the subshell. - if [ -z "$BUILD_AVOIDANCE_HOST" ]; then - BUILD_AVOIDANCE_URL="$BUILD_AVOIDANCE_DIR" - else - BUILD_AVOIDANCE_URL="$BUILD_AVOIDANCE_HOST:$BUILD_AVOIDANCE_DIR" - fi - - - build_avoidance_copy_dir "$REMOTE_CTX_DIR" "$LOCAL_CTX_DIR" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy_dir '$REMOTE_CTX_DIR' '$LOCAL_CTX_DIR'" - return 1 - fi - - # Search for a new context to sync - cd $MY_REPO - - if [ "$BUILD_AVOIDANCE_DAY" == "" ]; then - # Normal case: - # Search all contexts, newest to oldest, for a good context. - for context in $(ls -1rd $LOCAL_CTX_DIR/*context); do - >&2 echo "context=$context" - BA_CONTEXT=$(test_build_avoidance_context $context $BA_LAST_SYNC_CONTEXT) - if [ $? -le 1 ]; then - # Stop search. Might or might not have found a good context. - break; - fi - done - else - # Special case when a target day is specified. Why would we do this? - # Reason is we might want the reference build to itself use build - # avoidance referencing prior builds of itself, except for one build - # a week when we use a full build rather than a build avoidance build. - # e.g. Sunday - full build - # Mon-Sat - avoidance builds that refernce Sunday build. - # - # Starting from last (e.g. "Sunday"), search newest to - # oldest for a good context. If none found, increment the target - # day (e.g. Monday) and search again. Keep incrementing until a - # good build is found, or target day + offset days would be a date - # in the furure. - # - local TARG_DAY=$BUILD_AVOIDANCE_DAY - local TODAY_DATE - local TODAY_DAY - local TARG_DATE="" - local TARG_TS - local TODAY_TS - - TODAY_DATE=$(date $UTC +%Y-%m-%d) - TODAY_DAY=$(date $UTC "+%A") - - for OFFSET_DAYS in 0 1 2 3 4 5 6; do - if [ "$TARG_DAY" != "" ]; then - # Convert TARG_DAY+OFFSET_DAYS to TARG_DATE - - if [ "$TODAY_DAY" == "$TARG_DAY" ]; then - TARG_DATE=$(date $UTC -d"$TARG_DAY+$OFFSET_DAYS days" +%Y-%m-%d) - else - TARG_DATE=$(date $UTC -d"last-$TARG_DAY+$OFFSET_DAYS days" +%Y-%m-%d) - fi - >&2 echo "TARG_DATE=$TARG_DATE" - - TARG_TS=$(date $UTC -d "$TARG_DATE" +%s) - TODAY_TS=$(date $UTC -d "$TODAY_DATE" +%s) - if [ $TARG_TS -gt $TODAY_TS ]; then - # Skip if offset has pushed us into future dates - continue; - fi - - if [ "$TARG_DATE" == "$TODAY_DATE" ]; then - TARG_DATE="" - fi - fi - - # Search build, newest to oldest, satisfying TARG_DATE - for f in $(ls -1rd $LOCAL_CTX_DIR/*context); do - DATE=$(date_to_iso_8601 $(basename "$f")) - if [ $? -ne 0 ]; then - >&2 echo "Failed to extract date from filename '$(basename "$f")', ignoring file" - continue - fi - - >&2 echo " DATE=$DATE, TARG_DATE=$TARG_DATE" - - if [ "$DATE" == "$TARG_DATE" ] || [ "$TARG_DATE" == "" ] ; then - context=$f; - else - continue - fi - - >&2 echo "context=$context" - - BA_CONTEXT=$(test_build_avoidance_context $context $BA_LAST_SYNC_CONTEXT) - - if [ $? -le 1 ]; then - # Stop search. Might or might not have found a good context. - break; - fi - done - - if [ "$BA_CONTEXT" != "" ]; then - # Found a good context. - break - fi - done - fi - - if [ "$BA_CONTEXT" == "" ]; then - # No new context found - return 1 - fi - - # test that the reference build context hasn't been deleted - local BA_CONTEXT_DIR="$BUILD_AVOIDANCE_DIR/$BA_CONTEXT" - - if [ -z "$BUILD_AVOIDANCE_HOST" ]; then - >&2 echo "[ -d $BA_CONTEXT_DIR ]" - if ! [ -d $BA_CONTEXT_DIR ] ; then - return 1 - fi - else - >&2 echo "ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]'" - if ! ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]' ; then - return 1 - fi - fi - - # Save the latest context - >&2 echo "BA_CONTEXT=$BA_CONTEXT" - >&2 echo "BUILD_AVOIDANCE_LAST_SYNC_FILE=$BUILD_AVOIDANCE_LAST_SYNC_FILE" - echo $BA_CONTEXT > $BUILD_AVOIDANCE_LAST_SYNC_FILE - - # The location of the load with the most compatable new context - if [ -z "$BUILD_AVOIDANCE_HOST" ]; then - URL=$BA_CONTEXT_DIR - else - URL=$BUILD_AVOIDANCE_HOST:$BA_CONTEXT_DIR - fi - - # return URL to caller. - echo $URL - return 0 - ) -} - - -# -# build_avoidance_pre_clean -# -# A place for any cleanup actions that must preceed a build avoidance build. -# -build_avoidance_pre_clean () { - local BUILD_TYPE="$1" - - if [ "$BUILD_TYPE" == "" ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required" - return 1 - fi - - # clean prior builds - if [ -d $MY_WORKSPACE/$BUILD_TYPE ]; then - build-pkgs --clean --$BUILD_TYPE --no-build-avoidance - if [ $? -ne 0 ]; then - return 1 - fi - fi - - for f in $BUILD_AVOIDANCE_SRPM_FILES $BUILD_AVOIDANCE_RPM_FILES; do - if [ -f $MY_WORKSPACE/$BUILD_TYPE/$f ]; then - \rm -f $MY_WORKSPACE/$BUILD_TYPE/$f - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): rm -f $MY_WORKSPACE/$BUILD_TYPE/$f" - return 1 - fi - fi - done - - for d in $BUILD_AVOIDANCE_SRPM_DIRECTORIES $BUILD_AVOIDANCE_RPM_DIRECTORIES; do - - if [ -d $MY_WORKSPACE/$BUILD_TYPE/$d ]; then - \rm -rf $MY_WORKSPACE/$BUILD_TYPE/$d - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): rm -rf $MY_WORKSPACE/$BUILD_TYPE/$d" - return 1 - fi - fi - done - - return 0 -} - - -# -# build_avoidance_copy_dir_rsync ['verbose'] -# -# Copy a file from $BUILD_AVOIDANCE_URL/ -# to using rsync. -# -build_avoidance_copy_dir_rsync () { - local FROM="$1" - local TO="$2" - local VERBOSE="$3" - local FLAGS="-a -u" - - if [ "$BUILD_AVOIDANCE_URL" == "" ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_AVOIDANCE_URL no set" - return 1 - fi - - if [ "$VERBOSE" != "" ]; then - FLAGS="$FLAGS -v" - echo "rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM/' '$TO/'" - fi - - rsync $FLAGS "$BUILD_AVOIDANCE_URL/$FROM/" "$TO/" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): command failed: rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM/' '$TO/'" - return 1 - fi - - chmod -R 'ug+w' "$TO/" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): command failed: chmod -R 'ug+w' '$TO/'" - return 1 - fi - return 0 -} - -# -# build_avoidance_copy_file_rsync ['verbose'] -# -# Copy a file from $BUILD_AVOIDANCE_URL/ -# to using rsync. -# -build_avoidance_copy_file_rsync () { - local FROM="$1" - local TO="$2" - local VERBOSE="$3" - local FLAGS="-a -u" - - if [ "$BUILD_AVOIDANCE_URL" == "" ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_AVOIDANCE_URL no set" - return 1 - fi - if [ "$VERBOSE" != "" ]; then - FLAGS="$FLAGS -v" - echo "rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM' '$TO'" - fi - - rsync $FLAGS "$BUILD_AVOIDANCE_URL/$FROM" "$TO" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): command failed: rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM' '$TO'" - return 1 - fi - - chmod -R 'ug+w' "$TO" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): command failed: chmod -R 'ug+w' '$TO'" - return 1 - fi - return $? -} - -# -# build_avoidance_copy_dir ['verbose'] -# -# Copy a file from $BUILD_AVOIDANCE_URL/ -# to . The copy method will be determined by -# BUILD_AVOIDANCE_FILE_TRANSFER. Only 'rsync' is supported at present. -# -# should be a directory, -# mkdir -p will be called on . -# -build_avoidance_copy_dir () { - local FROM="$1" - local TO="$2" - local VERBOSE="$3" - - if [ "$VERBOSE" != "" ]; then - echo "mkdir -p '$TO'" - fi - mkdir -p "$TO" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): mkdir -p $TO" - return 1 - fi - - case ${BUILD_AVOIDANCE_FILE_TRANSFER} in - rsync) - build_avoidance_copy_dir_rsync "$FROM" "$TO" "$VERBOSE" - return $? - ;; - *) - >&2 echo "Error: $FUNCNAME (${LINENO}): Unknown BUILD_AVOIDANCE_FILE_TRANSFER '${BUILD_AVOIDANCE_FILE_TRANSFER}'" - return 1 - ;; - esac - return 1 -} - -# -# build_avoidance_copy_file ['verbose'] -# -# Copy a file from $BUILD_AVOIDANCE_URL/ -# to . The copy method will be determined by -# BUILD_AVOIDANCE_FILE_TRANSFER. Only 'rsync' is supported at present. -# -# should be a file, not a directory, -# mkdir -p will be called on $(dirname ) -# -build_avoidance_copy_file () { - local FROM="$1" - local TO="$2" - local VERBOSE="$3" - - if [ "$VERBOSE" != "" ]; then - echo "mkdir -p $(dirname '$TO')" - fi - mkdir -p "$(dirname "$TO")" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): mkdir -p $(dirname "$TO")" - return 1 - fi - - case ${BUILD_AVOIDANCE_FILE_TRANSFER} in - rsync) - build_avoidance_copy_file_rsync "$FROM" "$TO" "$VERBOSE" - return $? - ;; - *) - >&2 echo "Error: $FUNCNAME (${LINENO}): Unknown BUILD_AVOIDANCE_FILE_TRANSFER '${BUILD_AVOIDANCE_FILE_TRANSFER}'" - return 1 - ;; - esac - return 1 -} - -# -# build_avoidance_copy ['verbose'] -# -# Copy the needed build artifacts for from $BUILD_AVOIDANCE_URL. -# -build_avoidance_copy () { - local BUILD_TYPE="$1" - local VERBOSE="$2" - - if [ "$BUILD_TYPE" == "" ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required" - return 1 - fi - - # Iterate through list of directories to copy - for d in $BUILD_AVOIDANCE_SRPM_DIRECTORIES $BUILD_AVOIDANCE_RPM_DIRECTORIES; do - build_avoidance_copy_dir "$BUILD_TYPE/$d" "$MY_WORKSPACE/$BUILD_TYPE/$d" "$VERBOSE" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy_dir '$BUILD_TYPE/$d' '$MY_WORKSPACE/$BUILD_TYPE/$d'" - return 1 - fi - done - - # Iterate through list of files to copy - for f in $BUILD_AVOIDANCE_SRPM_FILES $BUILD_AVOIDANCE_RPM_FILES; do - build_avoidance_copy_file "$BUILD_TYPE/$f" "$MY_WORKSPACE/$BUILD_TYPE/$f" "$VERBOSE" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy_file '$BUILD_TYPE/$f' '$MY_WORKSPACE/$BUILD_TYPE/$f'" - return 1 - fi - done - - return 0 -} - -# -# build_avoidance_fixups -# -# Fix paths in the build artifacts that we coppied that contain -# the user name. -# -# Also, our credentials may differ from the reference build, -# so substitute unsigned packages in place of signed packages. -# -build_avoidance_fixups () { - local BUILD_TYPE="$1" - - local BA_SOURCE_BUILD_ENVIRONMENT - BA_SOURCE_BUILD_ENVIRONMENT="${BUILD_AVOIDANCE_USR}-$(basename $(dirname $BUILD_AVOIDANCE_URL))-$(basename $BUILD_AVOIDANCE_URL)-${SRC_BUILD_ENVIRONMENT}" - local RESULT_DIR="" - local FROM_DIR="" - local TO_DIR="" - local rpm_path_post_signing - local rpm_path_pre_signing - local rpm_name - local md5sum_post_signing - local md5sum_pre_signing - - if [ "$BUILD_TYPE" == "" ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required" - return 1 - fi - - RESULT_DIR="$MY_WORKSPACE/$BUILD_TYPE/results" - FROM_DIR="${RESULT_DIR}/${BA_SOURCE_BUILD_ENVIRONMENT}-${BUILD_TYPE}" - TO_DIR="${RESULT_DIR}/${MY_BUILD_ENVIRONMENT}-${BUILD_TYPE}" - echo "$FUNCNAME: FROM_DIR=$FROM_DIR" - echo "$FUNCNAME: TO_DIR=$TO_DIR" - echo "$FUNCNAME: MY_BUILD_ENVIRONMENT=$MY_BUILD_ENVIRONMENT" - - # Fix patchs the use MY_BUILD_ENVIRONMENT - if [ ! -d "$FROM_DIR" ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): Expected directory '$FROM_DIR' is missing." - return 1 - fi - - echo "$FUNCNAME: mv '$FROM_DIR' '$TO_DIR'" - \mv "$FROM_DIR" "$TO_DIR" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): mv '$FROM_DIR' '$TO_DIR'" - return 1 - fi - - local MY_WS_BT="$MY_WORKSPACE/$BUILD_TYPE" - - # Replace signed rpms with non-signed copies .... we aren't a formal build - for rpm_path_post_signing in $(find $MY_WS_BT/rpmbuild/RPMS -type f -name '*.rpm' | grep -v src.rpm); do - - rpm_name=$(basename $rpm_path_post_signing) - rpm_path_pre_signing=$(find $MY_WS_BT/results -name $rpm_name | head -n1) - if [ "$rpm_path_pre_signing" != "" ]; then - md5sum_post_signing=$(md5sum ${rpm_path_post_signing} | cut -d ' ' -f 1) - md5sum_pre_signing=$(md5sum ${rpm_path_pre_signing} | cut -d ' ' -f 1) - if [ "${md5sum_post_signing}" != "${md5sum_pre_signing}" ]; then - echo "$FUNCNAME: fixing $rpm_name" - \rm -f ${rpm_path_post_signing} - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): rm -f ${rpm_path_post_signing}" - return 1 - fi - - \cp ${rpm_path_pre_signing} ${rpm_path_post_signing} - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): cp ${rpm_path_pre_signing} ${rpm_path_post_signing}" - return 1 - fi - fi - fi; - done - - return 0 -} - - -# -# build_avoidance -# -# Look for a reference build that is applicable to our current git context. -# and copy it to our local workspace, if we haven't already done so. -# -build_avoidance () { - local BUILD_TYPE="$1" - - echo "==== Build Avoidance Start ====" - - export BUILD_AVOIDANCE_LAST_SYNC_FILE="$(build_avoidance_last_sync_file $BUILD_TYPE)" - mkdir -p "$(dirname $BUILD_AVOIDANCE_LAST_SYNC_FILE)" - - if [ "$BUILD_TYPE" == "" ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required" - return 1 - fi - - if [ ! -d $MY_WORKSPACE/$BUILD_TYPE ]; then - mkdir -p $MY_WORKSPACE/$BUILD_TYPE - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): Failed to create directory $MY_WORKSPACE/$BUILD_TYPE" - return 1 - fi - fi - - if [ ! -L $MY_WORKSPACE/$BUILD_TYPE/repo ]; then - ln -s $MY_REPO $MY_WORKSPACE/$BUILD_TYPE/repo - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): Failed to create symlink $MY_WORKSPACE/$BUILD_TYPE/repo -> $MY_REPO" - return 1 - fi - fi - - build_avoidance_pre_clean $BUILD_TYPE - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_pre_clean $BUILD_TYPE" - return 1 - fi - - build_avoidance_copy $BUILD_TYPE 'verbose' - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy $BUILD_TYPE" - return 1 - fi - - build_avoidance_fixups $BUILD_TYPE - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_fixups $BUILD_TYPE" - return 1 - fi - - echo "==== Build Avoidance Complete ====" - return 0 -} - -# -# build_avoidance_save_reference_context -# -# For use by a reference build. Copy the 'CONTEXT' file -# from the build into a central directory where we save -# the context of old builds. -# -# Individual reference builds use: -# MY_WORKSPACE=/ -# and context files are collected in dir: -# DEST_CTX_DIR=/context -# using name: -# DEST_CTX=.context - -build_avoidance_save_reference_context () { - local DIR - DIR=$(dirname "${MY_WORKSPACE}") - - # Note: SUB_DIR should be a timestamp - local SUB_DIR - SUB_DIR=$(basename "${MY_WORKSPACE}") - - local SRC_CTX="${MY_WORKSPACE}/CONTEXT" - local DEST_CTX_DIR="${DIR}/context" - local DEST_CTX="${DEST_CTX_DIR}/${SUB_DIR}.context" - - if [ ! -f "${SRC_CTX}" ]; then - echo "Context file not found at '${SRC_CTX}'" - return 1 - fi - - mkdir -p "${DEST_CTX_DIR}" - if [ $? -ne 0 ]; then - echo "Error: $FUNCNAME (${LINENO}): Failed to create directory '${DEST_CTX_DIR}'" - return 1 - fi - - cp "${SRC_CTX}" "${DEST_CTX}" - if [ $? -ne 0 ]; then - echo "Error: $FUNCNAME (${LINENO}): Failed to copy ${SRC_CTX} -> ${DEST_CTX}" - return 1 - fi - - return 0 -} diff --git a/build-tools/build-docker-images/README b/build-tools/build-docker-images/README index e961232e..c5dbf6ed 100644 --- a/build-tools/build-docker-images/README +++ b/build-tools/build-docker-images/README @@ -3,21 +3,22 @@ PRIVATE_REGISTRY_USERID=myuser PRIVATE_REGISTRY=xxx.xxx.xxx.xxx:9001 VERSION=2018.11.13 -OS=centos +OS=debian OS_VERSION=7.5.1804 BUILD_STREAM=stable HOST_PORT=8088 +PUBLISH_URL=https://mirror.starlingx.windriver.com/mirror/starlingx/master/${OS}/monolithic/latest_build/ -## Step 1: Build stx-centos +## Step 1: Build stx-debian time $MY_REPO/build-tools/build-docker-images/build-stx-base.sh \ --os ${OS} \ --os-version ${OS_VERSION} \ --version ${VERSION} \ --user ${PRIVATE_REGISTRY_USERID} \ --registry ${PRIVATE_REGISTRY} \ + --repo 'deb [trusted=yes check-valid-until=0] $PUBLISH_URL/inputs/packages ./' + --repo 'deb [trusted=yes check-valid-until=0] $PUBLISH_URL/outputs/std/packages ./' --push \ - --repo stx-local-build,http://${HOSTNAME}:${HOST_PORT}/${MY_WORKSPACE}/std/rpmbuild/RPMS \ - --repo stx-mirror-distro,http://${HOSTNAME}:${HOST_PORT}/${MY_REPO}/cgcs-root/cgcs-${OS}-repo/Binary \ --clean diff --git a/build-tools/build-docker-images/base-image-build-centos-dev.cfg b/build-tools/build-docker-images/base-image-build-centos-dev.cfg deleted file mode 100644 index e4ea3cd6..00000000 --- a/build-tools/build-docker-images/base-image-build-centos-dev.cfg +++ /dev/null @@ -1,2 +0,0 @@ -# one option per line, option=value -repo=ussuri-wsgi,https://mirror.starlingx.windriver.com/mirror/centos/centos/mirror.centos.org/centos/7/sclo/x86_64/rh/ diff --git a/build-tools/build-docker-images/base-image-build-centos-stable.cfg b/build-tools/build-docker-images/base-image-build-centos-stable.cfg deleted file mode 100644 index e4ea3cd6..00000000 --- a/build-tools/build-docker-images/base-image-build-centos-stable.cfg +++ /dev/null @@ -1,2 +0,0 @@ -# one option per line, option=value -repo=ussuri-wsgi,https://mirror.starlingx.windriver.com/mirror/centos/centos/mirror.centos.org/centos/7/sclo/x86_64/rh/ diff --git a/build-tools/build-docker-images/build-stx-base.sh b/build-tools/build-docker-images/build-stx-base.sh index acb60a00..7acd62f5 100755 --- a/build-tools/build-docker-images/build-stx-base.sh +++ b/build-tools/build-docker-images/build-stx-base.sh @@ -18,7 +18,7 @@ if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then exit 1 fi -SUPPORTED_OS_ARGS=('centos' 'debian') +SUPPORTED_OS_ARGS=( 'debian' ) OS= # default: autodetect OS_VERSION= # default: lookup "ARG RELEASE" in Dockerfile BUILD_STREAM=stable @@ -52,7 +52,6 @@ Options: --version: Specify version for output image --stream: Build stream, stable or dev (default: stable) --repo: Software repository, can be specified multiple times - * CentOS format: "NAME,BASEURL" * Debian format: "TYPE [OPTION=VALUE...] URL DISTRO COMPONENTS..." This will be added to /etc/apt/sources.list as is, see also sources.list(5) manpage. @@ -281,13 +280,7 @@ fi if [ ${#REPO_LIST[@]} -eq 0 ]; then # Either --repo or --local must be specified - if [ "${LOCAL}" = "yes" ]; then - if [[ "$OS" == "centos" ]] ; then - REPO_LIST+=("local-std,http://${HOST}:8088${MY_WORKSPACE}/std/rpmbuild/RPMS") - REPO_LIST+=("stx-distro,http://${HOST}:8089${MY_REPO}/cgcs-${OS}-repo/Binary") - fi - # debian is handled down below - elif [ "${BUILD_STREAM}" != "dev" -a "${BUILD_STREAM}" != "master" ]; then + if [ "${LOCAL}" != "yes" -a "${BUILD_STREAM}" != "dev" -a "${BUILD_STREAM}" != "master" ]; then echo "Either --local or --repo must be specified" >&2 exit 1 fi @@ -314,33 +307,7 @@ fi cp ${SRC_DOCKERFILE} ${BUILDDIR}/Dockerfile # Generate the stx.repo file -if [[ "$OS" == "centos" ]] ; then - STX_REPO_FILE=${BUILDDIR}/stx.repo - for repo in ${REPO_LIST[@]}; do - repo_name=$(echo $repo | awk -F, '{print $1}') - repo_baseurl=$(echo $repo | awk -F, '{print $2}') - - if [ -z "${repo_name}" -o -z "${repo_baseurl}" ]; then - echo "Invalid repo specified: ${repo}" >&2 - echo "Expected format: name,baseurl" >&2 - exit 1 - fi - - cat >>${STX_REPO_FILE} <&2 + exit 1 diff --git a/build-tools/build-docker-images/stx-centos/Dockerfile.dev b/build-tools/build-docker-images/stx-centos/Dockerfile.dev deleted file mode 100644 index af30a6b6..00000000 --- a/build-tools/build-docker-images/stx-centos/Dockerfile.dev +++ /dev/null @@ -1,16 +0,0 @@ -# Expected build arguments: -# RELEASE: centos release -# -ARG RELEASE=7.5.1804 -FROM centos:${RELEASE} - -RUN set -ex ;\ - sed -i '/\[main\]/ atimeout=120' /etc/yum.conf ;\ - yum install -y centos-release-openstack-stein ;\ - rm -rf \ - /var/log/* \ - /tmp/* \ - /var/tmp/* - -# root CA cert expired on October 1st, 2021 -RUN yum update -y ca-certificates diff --git a/build-tools/build-docker-images/stx-centos/Dockerfile.stable b/build-tools/build-docker-images/stx-centos/Dockerfile.stable deleted file mode 100644 index b30f615a..00000000 --- a/build-tools/build-docker-images/stx-centos/Dockerfile.stable +++ /dev/null @@ -1,31 +0,0 @@ -# Expected build arguments: -# RELEASE: centos release -# REPO_OPTS: yum options to enable StarlingX repo -# -ARG RELEASE=7.5.1804 -FROM centos:${RELEASE} - -ARG REPO_OPTS - -# The stx.repo file must be generated by the build tool first -COPY stx.repo / - -RUN set -ex ;\ - sed -i '/\[main\]/ atimeout=120' /etc/yum.conf ;\ - mv /stx.repo /etc/yum.repos.d/ ;\ - yum upgrade --disablerepo=* ${REPO_OPTS} -y ;\ - yum install --disablerepo=* ${REPO_OPTS} -y \ - qemu-img \ - openssh-clients \ - python3 \ - python3-pip \ - python3-wheel \ - rh-python36-mod_wsgi \ - ;\ - rm -rf \ - /var/log/* \ - /tmp/* \ - /var/tmp/* - -# root CA cert expired on October 1st, 2021 -RUN yum update -y ca-certificates diff --git a/build-tools/build-docker-images/update-stx-image.sh b/build-tools/build-docker-images/update-stx-image.sh index da6cd3d2..80fa8d8a 100755 --- a/build-tools/build-docker-images/update-stx-image.sh +++ b/build-tools/build-docker-images/update-stx-image.sh @@ -58,7 +58,7 @@ Options: --module-src: Specify path to module source to install/update (dir or git repo) Formats: dir[|version] url[|branch][|version] - --pkg: Specify path to distro package to install/update (ie. rpm) + --pkg: Specify path to distro package to install/update (ie. deb) --customize: Customization script --extra: Extra file (to be accessible to customization script) --push: Push to docker repo diff --git a/build-tools/build-guest b/build-tools/build-guest deleted file mode 100755 index ab1d9d69..00000000 --- a/build-tools/build-guest +++ /dev/null @@ -1,412 +0,0 @@ -#!/bin/env bash - -# -# Copyright (c) 2018 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# - -# -# Build the tis-centos-image.img or tis-centos-image-rt.img file -# - -BUILD_GUEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" -source "${BUILD_GUEST_DIR}/image-utils.sh" - -PROGNAME=$(basename "$0") - -# NOTE: TMP_DIR must end in '/' -# NOTE: /tmp/ is now tmpfs like. Can't be trusted across multiple mock commands -# TMP_DIR=/tmp/ -TMP_DIR=/ - -# Use RPMs from the std build only, for now -export BUILD_TYPE=std -export MY_BUILD_DIR_TOP=$MY_BUILD_DIR - -function init_vars { - # Output path (current dir unless MY_WORKSPACE defined) - OUTPUT_DIR="$PWD/export" - if [ ! -z "$MY_WORKSPACE" ] && [ -d "$MY_WORKSPACE" ] ; then - OUTPUT_DIR="$MY_WORKSPACE/export" - CGCS_REPO_DIR="$MY_WORKSPACE/rpmbuild/RPMS" - fi - - if [ -n "$MY_GUEST_DIR" ]; then - GUEST_DIR=$MY_GUEST_DIR - else - GUEST_DIR=$MY_WORKSPACE/guest - fi - - MOCK=/usr/bin/mock - if [ $VERBOSE -eq 0 ]; then - MOCK="$MOCK -q" - fi - - # Path to guest configuration - GUEST_BUILD_DIR="${BUILD_GUEST_DIR}/build_guest" - GUEST_BUILD_CMD=$GUEST_BUILD_DIR/build-guest-image.py - if [ $VERBOSE -eq 1 ]; then - GUEST_BUILD_CMD="$GUEST_BUILD_CMD -x" - fi - - if [ $BUILD_MODE == 'std' ]; then - OUTPUT_FILE=$OUTPUT_DIR/tis-centos-guest.img - elif [ $BUILD_MODE == 'rt' ]; then - OUTPUT_FILE=$OUTPUT_DIR/tis-centos-guest-rt.img - else - printf " Error -- unknown BUILD_MODE '$BUILD_MODE'\n"; - exit 1 - fi -} - - -function check_vars { - # Where to store data - printf "Finding cgcs-root\n" - printf " Checking \$MY_REPO (value \"$MY_REPO\")\n" - - if [ ! -z "$MY_REPO" ] && [ -d "$MY_REPO" ] ; then - INTERNAL_REPO_ROOT=$MY_REPO - printf " Found!\n" - fi - - if [ -z "$INTERNAL_REPO_ROOT" ] ; then - printf " No joy -- checking \$MY_REPO_ROOT_DIR (value \"$MY_REPO_ROOT_DIR\")\n" - if [ ! -z "$MY_REPO_ROOT_DIR" ] && [ -d "$MY_REPO_ROOT_DIR/cgcs-root" ] ; then - INTERNAL_REPO_ROOT=$MY_REPO_ROOT_DIR/cgcs-root - printf " Found!\n" - fi - fi - - if [ -z "$INTERNAL_REPO_ROOT" ] ; then - printf " No joy -- checking for \$MY_WORKSPACE/cgcs-root\n" - if [ -d "$MY_WORKSPACE/cgcs-root" ] ; then - INTERNAL_REPO_ROOT=$MY_WORKSPACE/cgcs-root - printf " Found!\n" - fi - fi - - if [ -z "$INTERNAL_REPO_ROOT" ] ; then - printf " Error -- could not locate cgcs-root repo.\n" - exit 1 - fi - - STX_DIR=$INTERNAL_REPO_ROOT/stx - - if [ "x$MY_BUILD_CFG" == "x" ];then - printf " Error -- reqiure MY_BUILD_CFG to be defined.\n" - exit 1 - fi - - RELEASE_INFO="$(get_release_info)" - if [ $? -ne 0 ]; then - echo "WARNING: failed to find a release info file." - else - export PLATFORM_RELEASE=$(source "$RELEASE_INFO" && echo $PLATFORM_RELEASE) - fi - -} - - -function create_rootfs { - printf "\nCreating guest file system\n" - - mkdir -p $GUEST_DIR - if [ $? -ne 0 ]; then - printf " Error -- Could not create $GUEST_DIR\n"; - exit 1 - fi - - # Place build-time environment variables in mock configuration - GUEST_ENV="${MY_BUILD_ENVIRONMENT}-guest" - GUEST_CFG=$GUEST_DIR/$MY_BUILD_ENVIRONMENT_FILE - - MY_BUILD_ENVIRONMENT=$GUEST_ENV "${BUILD_GUEST_DIR}/modify-build-cfg" $GUEST_CFG - if [ $? -ne 0 ]; then - printf " Error -- Could not update $GUEST_CFG\n"; - exit 1 - fi - - # Setup mock directories for the guest - if [ -d /localdisk/loadbuild/mock ]; then - LNK=/localdisk/loadbuild/mock/$GUEST_ENV - if [ ! -L $LNK ]; then - ln -s $GUEST_DIR $LNK - fi - fi - - if [ -d /localdisk/loadbuild/mock-cache ]; then - mkdir -p $GUEST_DIR/cache - LNK=/localdisk/loadbuild/mock-cache/$GUEST_ENV - if [ ! -L $LNK ]; then - ln -s $GUEST_DIR/cache $LNK - fi - fi - - # Setup mock chroot environment - $MOCK -r $GUEST_CFG --clean && $MOCK -r $GUEST_CFG --init - if [ $? -ne 0 ]; then - printf " Error -- Failed to setup guest mock chroot\n"; - exit 1 - fi - - # Install the RPMs to the root filesystem - - # Note that the "rt" build needs access to both local-std and local-rt repos - local EXTRA_REPOS="" - - if [ $BUILD_MODE == 'std' ]; then - INC_RPM_LIST=$(grep -v '^#' ${GUEST_BUILD_DIR}/rpm-install-list.txt) - TIS_RPM_LIST=$(image_inc_list guest std centos) - elif [ $BUILD_MODE == 'rt' ]; then - INC_RPM_LIST=$(grep -v '^#' ${GUEST_BUILD_DIR}/rpm-install-list-rt.txt) - TIS_RPM_LIST=$(image_inc_list guest rt centos) - EXTRA_REPOS="--enablerepo local-rt" - else - printf " Error -- unknown BUILD_MODE '$BUILD_MODE'\n"; - exit 1 - fi - - $MOCK -r $GUEST_CFG ${EXTRA_REPOS} --install ${INC_RPM_LIST} ${TIS_RPM_LIST} "$@" - if [ $? -ne 0 ]; then - printf "=====\n" - cat $GUEST_DIR/mock/result/root.log | sed -n '/Error:/,$p' | sed '/Child return code was:/q' - printf "=====\n" - printf " Error -- Failed to install RPM packages\n"; - exit 1 - fi - - # Make sure all requested packages are installed - MISSING=$( - extra_rpm_names="$( - for p in "$@" ; do - # skip URLs - if [[ "$p" =~ :// ]] ; then - continue - fi - # if it contains a slash or ends with .rpm, assume its a local file - # and read its embedded package name - if [[ "$p" =~ / || "$p" =~ [.]rpm$ ]] ; then - rpm -q --qf '%{name}\n' -p "$p" - # otherwise assume its a package name already - else - echo "$p" - fi - done - )" - $MOCK -r $GUEST_CFG --chroot -- rpm -q --whatprovides ${INC_RPM_LIST} ${TIS_RPM_LIST} $extra_rpm_names \ - | sed -n 's/^no package provides //p' \ - | sort -u - ) - if [ -n "$MISSING" ]; then - printf "=====\n" - printf "WARNING: The following RPMs are missing or could not be installed:\n" - local p - for p in $MISSING ; do - echo " [$p]" - done - printf "=====\n" - fi - - # Remove RPMs that are not required in image (pruned package list) - # NOTE: these are automatically installed from the mock init not - # through dependencies. - EXC_RPM_LIST=$(grep -v '^#' ${GUEST_BUILD_DIR}/rpm-remove-list.txt) - - $MOCK -r $GUEST_CFG --remove ${EXC_RPM_LIST} - if [ $? -ne 0 ]; then - printf " Error -- Failed to remove RPM packages\n"; - exit 1 - fi - - printf " Done\n" -} - - -function update_rootfs { - printf "\nCustomizing guest file system\n" - - # Copy over skeleton configuration files - for GUEST_ROOTFS in $GUEST_BUILD_DIR/rootfs $GUEST_BUILD_DIR/rootfs-$BUILD_MODE; - do - for f in $(cd $GUEST_ROOTFS && find . -type f | cut -c3-); - do - echo "$MOCK -r $GUEST_CFG --copyin $GUEST_ROOTFS/$f $f" - $MOCK -r $GUEST_CFG --copyin $GUEST_ROOTFS/$f $f - if [ $? -ne 0 ]; then - printf " Error -- Failed to copyin file $f\n"; - exit 1 - fi - done - done - - # Run the root file system setup script inside the chroot - ROOTFS_SETUP=rootfs-setup.sh - $MOCK -r $GUEST_CFG --copyin $GUEST_BUILD_DIR/$ROOTFS_SETUP $TMP_DIR && \ - if [ $BUILD_MODE == 'rt' ]; then - ROOTFS_SETUP_CMD="$TMP_DIR$ROOTFS_SETUP --rt" - elif [ $BUILD_MODE == 'std' ]; then - ROOTFS_SETUP_CMD="$TMP_DIR$ROOTFS_SETUP --std" - else - ROOTFS_SETUP_CMD="$TMP_DIR$ROOTFS_SETUP" - fi - $MOCK -r $GUEST_CFG --chroot "$ROOTFS_SETUP_CMD" - if [ $? -ne 0 ]; then - printf " Error -- Failed to run guest $ROOTFS_SETUP\n"; - exit 1 - fi - $MOCK -r $GUEST_CFG --chroot "rm -f $TMP_DIR$ROOTFS_SETUP" - if [ $? -ne 0 ]; then - printf " Error -- Failed to delete $ROOTFS_SETUP from guest\n"; - exit 1 - fi - - printf " Done\n" -} - - -function build_image { - # Build the image - printf "\nBuilding guest image $OUTPUT_FILE\n" - - mkdir -p $OUTPUT_DIR - if [ $? -ne 0 ]; then - printf " Error -- Could not create $OUTPUT_DIR\n"; - exit 1 - fi - - # Build guest rootfs archive - ROOTFS_SPACE=$((500*1024*1024)) - ROOTFS_TAR=rootfs.tar - ROOTFS_EXCLUDE=rootfs-exclude.txt - - $MOCK -r $GUEST_CFG --copyin $GUEST_BUILD_DIR/$ROOTFS_EXCLUDE $TMP_DIR - $MOCK -r $GUEST_CFG --chroot -- tar -cf $TMP_DIR$ROOTFS_TAR -X $TMP_DIR$ROOTFS_EXCLUDE --exclude=$TMP_DIR$ROOTFS_TAR --numeric-owner / - $MOCK -r $GUEST_CFG --copyout $TMP_DIR$ROOTFS_TAR $GUEST_DIR - $MOCK -r $GUEST_CFG --chroot -- rm -f $TMP_DIR$ROOTFS_TAR - - $GUEST_BUILD_CMD -i $GUEST_DIR/$ROOTFS_TAR -o $OUTPUT_FILE -s $ROOTFS_SPACE - if [ $? -ne 0 ]; then - printf " Error -- Failed to build guest image\n"; - exit 1 - fi - - printf " Done\n" -} - - -function clean_guest { - printf "\nCleaning the guest $GUEST_DIR\n" - - if [ ! -e $GUEST_DIR ]; then - printf " Done...nothing to do\n"; - exit 0 - fi - - # Place build-time environment variables in mock configuration - GUEST_ENV="${MY_BUILD_ENVIRONMENT}-guest" - GUEST_CFG=$GUEST_DIR/$MY_BUILD_ENVIRONMENT_FILE - - if [ ! -e $GUEST_CFG ]; then - MY_BUILD_ENVIRONMENT=$GUEST_ENV "${BUILD_GUEST_DIR}/modify-build-cfg" $GUEST_CFG - if [ $? -ne 0 ]; then - printf " Error -- Could not update $GUEST_CFG\n"; - exit 1 - fi - fi - - $MOCK -r $GUEST_CFG --clean - $MOCK -r $GUEST_CFG --scrub=cache - - rm -rf $GUEST_DIR - if [ $? -ne 0 ]; then - printf " Error -- Failed to remove guest $GUEST_DIR\n"; - exit 1 - fi - - printf " Done\n" -} - -############################################# -# Main code -############################################# - -usage () { - echo "" - echo "Usage: " - echo " build-guest [--rt | --std] [--verbose] [EXTRA_RPMS...]" - echo " build-guest [--help]" - echo " build-guest [--clean]" - echo "" - echo "EXTRA_RPMS are either package names or full RPM file paths" -} - -# Default argument values -HELP=0 -CLEAN=0 -VERBOSE=0 -BUILD_MODE='std' - -# read the options -TEMP=`getopt -o h --long clean,rt,std,verbose,help -n "$PROGNAME" -- "$@"` || exit 1 -eval set -- "$TEMP" - -# extract options and their arguments into variables. -while true ; do - case "$1" in - -h|--help) HELP=1 ; shift ;; - --clean) CLEAN=1 ; shift ;; - --verbose) VERBOSE=1 ; shift ;; - --rt) BUILD_MODE='rt' ; shift ;; - --std) BUILD_MODE='std' ; shift ;; - --) shift ; break ;; - *) echo "Internal error!" ; exit 1 ;; - esac -done - -if [ $HELP -eq 1 ]; then - usage - exit 0 -fi - -if [[ $CLEAN -eq 1 && "$#" -gt 0 ]] ; then - echo "Too many arguments!" >&2 ; exit 1 -else - # make sure extra RPM files exist - for p in "$@" ; do - # skip URLs - if [[ "$p" =~ :// ]] ; then - continue - fi - # if it contains a slash or ends with .rpm assume its a local file name - if [[ "$p" =~ / || "$p" =~ [.]rpm$ ]] ; then - # make sure it exists and is an RPM file - true <"$p" || exit 1 - if ! file --brief --mime-type "$p" | grep -q "^application/x-rpm$" ; then - echo "$p: not an RPM file" >&2 - exit 1 - fi - fi - done - unset p -fi - -( -printf "\n*****************************\n" -printf "Create Titanium Cloud/CentOS Guest Image\n" -printf "*****************************\n\n" - -init_vars -check_vars - -if [ $CLEAN -eq 1 ]; then - clean_guest - exit 0 -fi - -create_rootfs "$@" -update_rootfs -build_image - -) 2>&1 | stdbuf -o0 awk '{ print strftime("%H:%M:%S"), $0; fflush(); }' ; exit ${PIPESTATUS[0]} diff --git a/build-tools/build-helm-charts.sh b/build-tools/build-helm-charts.sh index e17cc95d..7e38ba76 100755 --- a/build-tools/build-helm-charts.sh +++ b/build-tools/build-helm-charts.sh @@ -12,7 +12,7 @@ BUILD_HELM_CHARTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source $BUILD_HELM_CHARTS_DIR/utils.sh || exit 1 -SUPPORTED_OS_ARGS=('centos' 'debian') +SUPPORTED_OS_ARGS=('debian') OS= LABEL="" APP_NAME="stx-openstack" @@ -35,7 +35,7 @@ Usage: $(basename $0) [--os ] [-a, --app ] [-A, --app-version-file /path/to/$APP_VERSION_BASE] [-B, --app-version ] - [-r, --rpm ] [-i, --image-record ] [--label