airship/divingbell
Code Issues Proposed changes

[zuul] Zuul gates fixes and Helm version upgrade to 3.x

Browse Source 
Add firewall flush rules to zuul pre-update gates.

Wrap gate scripts by run-gates.sh script in order to preserve the scripts execution contexts.

Also migrated chart building process to Helm v3.x.

Fixed 020-test-divingbell.sh script.

Change-Id: I6295d55338a6a75ac43b54c092704670d61854d9
This commit is contained in:
Markin, Sergiy (sm515x) 2022-09-29 16:05:06 +00:00
parent 229bbe75b0
commit 0ba6181058
12 changed files with 115 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -76,3 +76,5 @@ logs/
*/requirements.lock
*/charts
*/charts/*
deps/

5
.zuul.yaml
View File

@ -74,10 +74,7 @@
zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra/
gate_scripts_relative_path: ""
gate_scripts:
- "{{ zuul_osh_infra_relative_path }}./tools/deployment/apparmor/001-setup-apparmor-profiles.sh"
- "{{ zuul_osh_infra_relative_path }}./tools/deployment/common/005-deploy-k8s.sh"
- ./tools/gate/scripts/010-build-charts.sh
- sudo ./tools/gate/scripts/020-test-divingbell.sh
- ./tools/gate/scripts/run-gates.sh
post-run: tools/gate/playbooks/osh-infra-collect-logs.yaml
required-projects:
- openstack/openstack-helm-infra

63
Makefile
View File

@ -12,52 +12,43 @@
# See the License for the specific language governing permissions and
# limitations under the License.
BUILD_DIR := $(shell mkdir -p build && mktemp -d -p build)
HELM := $(shell realpath $(BUILD_DIR))/helm
TASK := build
BUILD_DIR := $(shell mktemp -d)
HELM := $(BUILD_DIR)/helm
EXCLUDES := helm-toolkit docs tests tools logs
CHARTS := $(filter-out $(EXCLUDES), $(patsubst %/.,%,$(wildcard */.)))
CHART := divingbell
all: charts
export
all: $(CHART)
.PHONY: charts
charts: clean helm-install helm-toolkit
$(HELM) dependency update divingbell
$(HELM) package divingbell
$(CHART):
@echo
@echo "===== Processing [$@] chart ====="
@make $(TASK)-$@
init-%: clean helm-install
DEP_UP_LIST=$* tools/helm_tk.sh $(HELM)
# Perform Linting
.PHONY: lint
lint: helm_lint build_docs
lint-%: init-%
if [ -d $* ]; then $(HELM) lint $*; fi
dryrun-%: init-%
$(HELM) template $*
build-%: lint-%
if [ -d $* ]; then $(HELM) package $*; fi
# Dry run templating of chart
.PHONY: dry-run
dry-run: clean helm-toolkit
$(HELM) template divingbell
.PHONY: clean
clean:
rm -rf build
rm -rf docs/build
rm -rf deps
@echo "Removed .b64, _partials.tpl, and _globals.tpl files"
rm -rf helm-toolkit/secrets/*.b64
rm -rf */templates/_partials.tpl
rm -rf */templates/_globals.tpl
rm -rf doc/build
rm -f *.tgz
rm -f */charts/*.tgz
.PHONY: $(EXCLUDES) $(CHARTS)
.PHONY: charts
charts: clean build-$(CHART)
# TODO This needs to run the equivalent of what is in
# divingbell/tools/gate/setup.sh + test.sh . At present, this is
# being investigated on how to bring it up to date.
.PHONY: tests
tests: charts
.PHONY: helm_lint
helm_lint: clean helm-toolkit
$(HELM) dependency update divingbell
$(HELM) lint divingbell
.PHONY: docs
docs: clean build_docs
@ -66,6 +57,12 @@ docs: clean build_docs
build_docs:
tox -e docs
# Initialize local helm config
.PHONY: helm-toolkit
helm-toolkit: helm-install
tools/helm_tk.sh $(HELM)
# Install helm binary
.PHONY: helm-install
helm-install:
tools/helm_install.sh $(HELM)

2
divingbell/requirements.yaml
View File

@ -14,5 +14,5 @@
dependencies:
- name: helm-toolkit
repository: http://localhost:8879/charts
repository: file://../deps/helm-toolkit
version: ">= 0.1.0"

2
doc/source/conf.py
View File

@ -65,7 +65,7 @@ release = u'0.1.0'
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
language = 'en'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.

2
tools/gate/playbooks/osh-infra-upgrade-host.yaml
View File

@ -35,10 +35,12 @@
- upgrade-host
- start-zuul-console
- disable-local-nameserver
- setup-firewall
tags:
- upgrade-host
- start-zuul-console
- disable-local-nameserver
- setup-firewall
- hosts: all
vars_files:

29
tools/gate/playbooks/roles/setup-firewall/tasks/main.yaml Normal file
View File

@ -0,0 +1,29 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# NOTE(portdirect): This needs refinement but drops the firewall on zuul nodes
---
- name: deploy iptables packages
include_role:
name: deploy-package
tasks_from: dist
vars:
packages:
deb:
- iptables
rpm:
- iptables
- command: iptables -S
- command: iptables -F
- command: iptables -P INPUT ACCEPT
- command: iptables -S
...

16
tools/gate/scripts/020-test-divingbell.sh
View File

@ -29,6 +29,7 @@ fi
DEFAULT_IFS=$IFS
NAME=divingbell
kubectl create namespace "${NAME}"
: ${LOGS_DIR:=/tmp/artifacts}
: ${LOGS_SUBDIR:=${LOGS_DIR}/${NAME}/$(date +"%m-%d-%y_%H:%M:%S")}
mkdir -p "${LOGS_SUBDIR}"
@ -284,6 +285,7 @@ k2vFiMwcHdLpQ1IH8ORVRgPPsiBnBOJ/kIiXG2SxPUTjjEGOVgeA
EXEC_DIR=/var/${NAME}/exec
# this used in test_overrides to check amount of daemonsets defined
EXPECTED_NUMBER_OF_DAEMONSETS=17
apt update
type lshw || apt -y install lshw
type apparmor_parser || apt -y install apparmor
type ethtool || apt -y install ethtool
@ -323,7 +325,7 @@ wait_for_tiller_ready(){
while [ $retries -gt 0 ]; do
# the message typically returned before tiller is ready is
# 'Error: could not find a ready tiller pod'
helm_error="$(helm list ${NAME} 2>&1 | grep 'Error')"
helm_error="$(helm status ${NAME} 2>&1 | grep 'Error')"
if [ -z "${helm_error}" ]; then return 0; fi
sleep 10
(( retries-- ))
@ -334,9 +336,9 @@ wait_for_tiller_ready(){
purge_containers(){
local chart_status
chart_status="$(helm list ${NAME})"
chart_status="$(helm status ${NAME})"
if [ -n "${chart_status}" ]; then
helm delete --purge ${NAME}
helm uninstall ${NAME}
fi
}
@ -391,7 +393,7 @@ _reset_account(){
}
init_default_state(){
wait_for_tiller_ready
# wait_for_tiller_ready
purge_containers
clean_persistent_files
# set sysctl original vals
@ -413,15 +415,15 @@ init_default_state(){
install(){
purge_containers
helm install --name="${NAME}" --debug "${NAME}" --namespace="${NAME}" "$@"
helm upgrade --install "${NAME}" "${NAME}" --namespace="${NAME}" --debug "$@"
}
upgrade(){
helm upgrade --name="${NAME}" --debug "${NAME}" --namespace="${NAME}" "$@"
helm upgrade --install "${NAME}" "${NAME}" --namespace="${NAME}" --debug "$@"
}
dry_run(){
helm install --name="${NAME}" --dry-run --debug "${NAME}" --namespace="${NAME}" "$@"
helm upgrade --install "${NAME}" "${NAME}" --namespace="${NAME}" --dry-run --debug "$@"
}
# parameter 1 to get_container_status is the module name (e.g., "apt")

24
tools/gate/scripts/run-gates.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
../../openstack/openstack-helm-infra/tools/deployment/apparmor/001-setup-apparmor-profiles.sh
../../openstack/openstack-helm-infra/tools/deployment/common/005-deploy-k8s.sh
./tools/gate/scripts/010-build-charts.sh
sudo --preserve-env ./tools/gate/scripts/020-test-divingbell.sh

7
tools/helm_install.sh
View File

@ -17,7 +17,7 @@
set -x
HELM=$1
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v2.17.0-linux-amd64.tar.gz"}
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.6.3-linux-amd64.tar.gz"}
function install_helm_binary {
@ -31,10 +31,9 @@ function install_helm_binary {
then
TMP_DIR=${BUILD_DIR:-$(mktemp -d)}
curl -o "${TMP_DIR}/helm.tar.gz" "${HELM_ARTIFACT_URL}"
pushd ${TMP_DIR}
cd ${TMP_DIR}
tar -xvzf helm.tar.gz
cp "linux-amd64/helm" "helm"
popd
cp "${TMP_DIR}/linux-amd64/helm" "${HELM}"
else
echo "Cannot write to ${HELM}"
exit -1

67
tools/helm_tk.sh
View File

@ -1,5 +1,5 @@
#!/bin/bash
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -12,64 +12,21 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Script to setup helm-toolkit and helm dep up the shipyard chart
#
HELM=$1
HTK_REPO=${HTK_REPO:-"https://github.com/openstack/openstack-helm-infra"}
HTK_PATH=${HTK_PATH:-""}
HTK_STABLE_COMMIT=${HTK_COMMIT}
BUILD_DIR=${BUILD_DIR:-$(mktemp -d)}
DEP_UP_LIST=${DEP_UP_LIST:-"divingbell"}
if [[ ! -z $(echo $http_proxy) ]]
then
export no_proxy=$no_proxy,127.0.0.1
fi
set -x
set -eux
function helm_serve {
if [[ -d "$HOME/.helm" ]]; then
echo ".helm directory found"
else
${HELM} init --client-only --skip-refresh
fi
if [[ -z $(curl -s 127.0.0.1:8879 | grep 'Helm Repository') ]]; then
${HELM} serve & > /dev/null
while [[ -z $(curl -s 127.0.0.1:8879 | grep 'Helm Repository') ]]; do
sleep 1
echo "Waiting for Helm Repository"
done
else
echo "Helm serve already running"
fi
HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"}
HTK_STABLE_COMMIT=${HTK_COMMIT:-"5c4056ad341afcc577e63902b6ddbfb222d757e1"}
if ${HELM} repo list | grep -q "^stable" ; then
${HELM} repo remove stable
fi
TMP_DIR=$(mktemp -d)
${HELM} repo add local http://localhost:8879/charts
{
HTK_REPO_DIR=$TMP_DIR/htk
git clone "$HTK_REPO" "$HTK_REPO_DIR"
(cd "$HTK_REPO_DIR" && git reset --hard "${HTK_STABLE_COMMIT}")
mkdir -p deps
cp -r "${HTK_REPO_DIR}/helm-toolkit" deps/helm-toolkit
}
mkdir -p build
pushd build
git clone $HTK_REPO || true
pushd ${HTK_REPO##*/}/$HTK_PATH
if [[ -n ${HTK_STABLE_COMMIT} ]]; then
git reset --hard "${HTK_STABLE_COMMIT}"
fi
helm_serve
if [[ ${HELM} != "helm" ]]
then
export PATH=${PATH}:$(dirname ${HELM})
fi
make helm-toolkit
popd && popd
for c in $DEP_UP_LIST
do
${HELM} dep up $c
done
rm -rf "${TMP_DIR}"

2
tox.ini
View File

@ -8,7 +8,7 @@ usedevelop = True
whitelist_externals = rm
passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY PBR_VERSION
setenv = VIRTUAL_ENV={envdir}
install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
install_command = pip install {opts} {packages}
deps = -r{toxinidir}/test-requirements.txt
commands =