Add local file cache for Builder tarball fetch
Change-Id: I4741b50c4a4b505f4f70ff2cbc5e9689b9f28b7b
This commit is contained in:
Mark Burnett
parent
57093d29ce
commit
9cb2c5a5ca
@ -82,9 +82,13 @@ spec:
|
||||
mountPath: /etc/promenade/promenade.conf
|
||||
subPath: promenade.conf
|
||||
readOnly: true
|
||||
- name: cache
|
||||
mountPath: /tmp/cache
|
||||
volumes:
|
||||
- name: promenade-etc
|
||||
configMap:
|
||||
name: promenade-etc
|
||||
defaultMode: 0444
|
||||
- name: cache
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
|
||||
@ -1,4 +1,7 @@
|
||||
from . import logging, renderer
|
||||
from beaker.cache import CacheManager
|
||||
from beaker.util import parse_cache_config_options
|
||||
|
||||
import io
|
||||
import itertools
|
||||
import os
|
||||
@ -10,6 +13,18 @@ __all__ = ['Builder']
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
# Ignore bandit false positive:
|
||||
# B108:hardcoded_tmp_directory
|
||||
# This cache needs to be shared by all forks within the same container, and so
|
||||
# must be at a well-known location.
|
||||
CACHE_OPTS = {
|
||||
'cache.type': 'file',
|
||||
'cache.data_dir': '/tmp/cache/data', # nosec
|
||||
'cache.lock_dir': '/tmp/cache/lock', # nosec
|
||||
}
|
||||
|
||||
CACHE = CacheManager(**parse_cache_config_options(CACHE_OPTS))
|
||||
|
||||
|
||||
class Builder:
|
||||
def __init__(self, config, *, validators=False):
|
||||
@ -30,8 +45,8 @@ class Builder:
|
||||
if 'content' in file_spec:
|
||||
data = file_spec['content']
|
||||
elif 'tar_url' in file_spec:
|
||||
data = _fetch_tar_content(
|
||||
url=file_spec['tar_url'], path=file_spec['tar_path'])
|
||||
data = _fetch_tar_content(file_spec['tar_url'],
|
||||
file_spec['tar_path'])
|
||||
self._file_cache[path] = {
|
||||
'path': path,
|
||||
'data': data,
|
||||
@ -111,17 +126,24 @@ class Builder:
|
||||
sub_config, template='scripts/validate-join.sh')
|
||||
|
||||
|
||||
def _fetch_tar_content(*, url, path):
|
||||
LOG.debug('Fetching url=%s (tar path=%s)', url, path)
|
||||
response = requests.get(url)
|
||||
response.raise_for_status()
|
||||
LOG.debug('Finished downloading url=%s (tar path=%s)', url, path)
|
||||
f = io.BytesIO(response.content)
|
||||
@CACHE.cache('fetch_tarball_content', expire=72 * 3600)
|
||||
def _fetch_tar_content(url, path):
|
||||
content = _fetch_tar_url(url)
|
||||
f = io.BytesIO(content)
|
||||
tf = tarfile.open(fileobj=f, mode='r')
|
||||
buf_reader = tf.extractfile(path)
|
||||
return buf_reader.read()
|
||||
|
||||
|
||||
@CACHE.cache('fetch_tarball_url', expire=72 * 3600)
|
||||
def _fetch_tar_url(url):
|
||||
LOG.debug('Fetching url=%s', url)
|
||||
response = requests.get(url)
|
||||
response.raise_for_status()
|
||||
LOG.debug('Finished downloading url=%s', url)
|
||||
return response.content
|
||||
|
||||
|
||||
def _join_name(node_name):
|
||||
return 'join-%s.sh' % node_name
|
||||
|
||||
|
||||
@ -195,19 +195,19 @@ function wait_for_pod_termination {
|
||||
|
||||
end=$(($(date +%s) + $SEC))
|
||||
while true; do
|
||||
POD_PHASE=$(kubectl --request-timeout 10s --namespace $NAMESPACE get -o jsonpath="${POD_PHASE_JSONPATH}" pod $POD_NAME)
|
||||
POD_PHASE=$(kubectl --request-timeout 120s --namespace $NAMESPACE get -o jsonpath="${POD_PHASE_JSONPATH}" pod $POD_NAME)
|
||||
if [ "x$POD_PHASE" = "xSucceeded" ]; then
|
||||
log Pod $POD_NAME succeeded.
|
||||
break
|
||||
elif [ "x$POD_PHASE" = "xFailed" ]; then
|
||||
log Pod $POD_NAME failed.
|
||||
kubectl --request-timeout 10s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2
|
||||
kubectl --request-timeout 120s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2
|
||||
fail
|
||||
else
|
||||
now=$(date +%s)
|
||||
if [ $now -gt $end ]; then
|
||||
log Pod did not terminate before timeout.
|
||||
kubectl --request-timeout 10s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2
|
||||
kubectl --request-timeout 120s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2
|
||||
fail
|
||||
fi
|
||||
sleep 1
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
beaker==1.9.1
|
||||
click==6.7
|
||||
falcon==1.2.0
|
||||
jinja2==2.9.6
|
||||
|
||||
@ -1,12 +1,14 @@
|
||||
alembic==0.8.2
|
||||
amqp==2.2.2
|
||||
Babel==2.5.3
|
||||
amqp==2.3.2
|
||||
Babel==2.6.0
|
||||
Beaker==1.9.1
|
||||
cachetools==2.1.0
|
||||
certifi==2018.4.16
|
||||
chardet==3.0.4
|
||||
click==6.7
|
||||
cliff==2.11.0
|
||||
cmd2==0.8.5
|
||||
cliff==2.12.1
|
||||
cmd2==0.9.1
|
||||
colorama==0.3.9
|
||||
contextlib2==0.5.5
|
||||
debtcollector==1.19.0
|
||||
git+https://git.openstack.org/openstack/airship-deckhand@177675e96fffcda9799c68bbce831424c1167020#egg=deckhand
|
||||
@ -19,7 +21,7 @@ fasteners==0.14.1
|
||||
fixtures==3.0.0
|
||||
flake8==2.6.2
|
||||
futurist==1.7.0
|
||||
google-auth==1.4.1
|
||||
google-auth==1.5.0
|
||||
greenlet==0.4.13
|
||||
hacking==1.1.0
|
||||
idna==2.6
|
||||
@ -30,7 +32,7 @@ jsonpath-ng==1.4.3
|
||||
jsonschema==2.6.0
|
||||
keystoneauth1==3.2.0
|
||||
keystonemiddleware==4.17.0
|
||||
kombu==4.1.0
|
||||
kombu==4.2.1
|
||||
kubernetes==3.0.0
|
||||
linecache2==1.0.0
|
||||
Mako==1.0.7
|
||||
@ -45,15 +47,15 @@ oslo.cache==1.30.1
|
||||
oslo.concurrency==3.27.0
|
||||
oslo.config==6.2.1
|
||||
oslo.context==2.19.2
|
||||
oslo.db==4.38.0
|
||||
oslo.db==4.39.0
|
||||
oslo.i18n==3.20.0
|
||||
oslo.log==3.38.1
|
||||
oslo.messaging==6.2.0
|
||||
oslo.messaging==6.5.0
|
||||
oslo.middleware==3.35.0
|
||||
oslo.policy==1.22.1
|
||||
oslo.serialization==2.25.0
|
||||
oslo.serialization==2.27.0
|
||||
oslo.service==1.31.2
|
||||
oslo.utils==3.36.2
|
||||
oslo.utils==3.36.3
|
||||
Paste==2.0.3
|
||||
PasteDeploy==1.5.2
|
||||
pbr==3.0.1
|
||||
@ -61,18 +63,18 @@ ply==3.11
|
||||
positional==1.2.1
|
||||
prettytable==0.7.2
|
||||
psycopg2==2.7.4
|
||||
pyasn1==0.4.2
|
||||
pyasn1==0.4.3
|
||||
pyasn1-modules==0.2.1
|
||||
pycadf==2.7.0
|
||||
pycodestyle==2.0.0
|
||||
pyflakes==1.2.3
|
||||
pyinotify==0.9.6
|
||||
pyparsing==2.2.0
|
||||
pyperclip==1.6.0
|
||||
pyperclip==1.6.2
|
||||
python-barbicanclient==4.6.0
|
||||
python-dateutil==2.7.3
|
||||
python-editor==1.0.3
|
||||
python-keystoneclient==3.16.0
|
||||
python-keystoneclient==3.17.0
|
||||
python-memcached==1.59
|
||||
python-mimeparse==1.6.0
|
||||
pytz==2018.4
|
||||
@ -83,7 +85,7 @@ rfc3986==1.1.0
|
||||
Routes==2.4.1
|
||||
rsa==3.4.2
|
||||
six==1.11.0
|
||||
SQLAlchemy==1.2.7
|
||||
SQLAlchemy==1.2.8
|
||||
sqlalchemy-migrate==0.11.0
|
||||
sqlparse==0.2.4
|
||||
statsd==3.2.2
|
||||
@ -95,11 +97,11 @@ testscenarios==0.5.0
|
||||
testtools==2.3.0
|
||||
traceback2==1.4.0
|
||||
unittest2==1.1.0
|
||||
urllib3==1.22
|
||||
urllib3==1.23
|
||||
uWSGI==2.0.15
|
||||
vine==1.1.4
|
||||
wcwidth==0.1.7
|
||||
WebOb==1.8.1
|
||||
WebOb==1.8.2
|
||||
websocket-client==0.40.0
|
||||
Werkzeug==0.14.1
|
||||
wrapt==1.10.11
|
||||
|
||||
@ -109,7 +109,7 @@
|
||||
"name": "Check full etcd cluster",
|
||||
"script": "check-etcd-health.sh",
|
||||
"arguments": [
|
||||
"-w", "30",
|
||||
"-w", "60",
|
||||
"-e", "kubernetes n0 n0 n1 n2 n3",
|
||||
"-e", "calico n0 n0 n1 n2 n3"
|
||||
]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user