From 74d0185dde623a2948f25b28cd49ad4c044c163c Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Fri, 27 May 2016 05:29:39 +0000
Subject: [PATCH] Cleanup source-repositories output
This element takes up a *huge* part of the logs of openstack-infra
builds as we go and cache every git tree.
This silences most of the noise which will reduce the mess
in the logs considerably.
Note that we've had logging on this turned down since
I91c5e55814ba9840769357261d203f4850e2eba6 but it has been
ineffective in stopping the log-spew output, see the
dependency change.
Change-Id: I60f06f84d57087c82b3907575bff125015d35171
Depends-On: I1e39822f218dc0322e2490a770f3dc867a55802c
---
.../extra-data.d/98-source-repositories | 48 +++++++++++--------
1 file changed, 28 insertions(+), 20 deletions(-)
diff --git a/elements/source-repositories/extra-data.d/98-source-repositories b/elements/source-repositories/extra-data.d/98-source-repositories
index 4770bea60..6aef5689f 100755
--- a/elements/source-repositories/extra-data.d/98-source-repositories
+++ b/elements/source-repositories/extra-data.d/98-source-repositories
@@ -36,19 +36,24 @@ function get_repos_for_element(){
local REGEX="^([^ ]+) (git|tar|file|package) ?(/[^ ]+)? ?([^ ]+)? ?([^ ]*)$"
+ # this can be a rather long list (e.g. caching ~1000 openstack
+ # repos), so we make a rough count for progress display
+ local COUNT=1
+ local REPO_SOURCES_COUNT=$(grep -v '^$\|^\s*\#' $REPO_SOURCES | wc -l)
+
while read line; do
- # temporarily turn off globbing '*' (since it may be used as the REPOREF for tarballs)
- set -f
- # expand variables
- line=$(eval echo $line)
-
- # restore globbing
- set +f
+ # temporarily turn off globbing '*' (since it may be used as
+ # the REPOREF for tarballs) and expand variables
+ set -f; line=$(eval echo $line); set +f
# ignore blank lines and lines beginning in '#'
[[ "$line" == \#* ]] || [[ -z "$line" ]] && continue
+ # rough progress counter
+ printf "(%04d / %04d)\n" ${COUNT} ${REPO_SOURCES_COUNT}
+ COUNT=$(( COUNT + 1))
+
if [[ "$line" =~ $REGEX ]] ; then
local REPONAME=${BASH_REMATCH[1]}
local REPOTYPE=${BASH_REMATCH[2]}
@@ -114,7 +119,7 @@ function get_repos_for_element(){
if [ ! -e "$CACHE_PATH" ] ; then
echo "Caching $REPONAME from $REPOLOCATION in $CACHE_PATH"
- git clone $REPOLOCATION $CACHE_PATH.tmp
+ git clone -q $REPOLOCATION $CACHE_PATH.tmp
mv ${CACHE_PATH}{.tmp,}
fi
@@ -125,7 +130,7 @@ function get_repos_for_element(){
echo "Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF"
# Copy named refs (which might be outside the usual heads
# pattern) - e.g. gerrit
- if [ "$REPOREF" == "*" ] || ! git --git-dir=$CACHE_PATH/.git fetch --prune --update-head-ok $REPOLOCATION \
+ if [ "$REPOREF" == "*" ] || ! git --git-dir=$CACHE_PATH/.git fetch -q --prune --update-head-ok $REPOLOCATION \
+${REPOREF}:${REPOREF} ; then
# Copy all heads from the remote repository - this permits
# using a SHA1 object reference so long as the object
@@ -133,12 +138,15 @@ function get_repos_for_element(){
# not permit arbitrary sha fetching from remote servers.
# This is a separate fetch to the prior one as the prior
# one will fail when REPOREF is a SHA1.
- git --git-dir=$CACHE_PATH/.git fetch --prune --update-head-ok $REPOLOCATION \
+ git --git-dir=$CACHE_PATH/.git fetch -q --prune --update-head-ok $REPOLOCATION \
+refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*
fi
# Ensure that we have a reference to the revision.
if [ "$REPOREF" != "*" ] ; then
- git --git-dir=$CACHE_PATH/.git rev-parse -q --verify $REPOREF^{commit}
+ if ! git --git-dir=$CACHE_PATH/.git rev-parse -q --verify $REPOREF^{commit} > /dev/null; then
+ echo "Failed to find reference to $REPOREF"
+ exit 1
+ fi
fi
fi
@@ -150,7 +158,7 @@ function get_repos_for_element(){
echo "REPOLOCATIONS don't match ("$CACHE_PATH" != "$DESIRED")" >&2
exit 1
elif [[ "$REPOREF" != "*" ]]; then
- pushd $REPO_DEST
+ pushd $REPO_DEST > /dev/null
# When we first clone we create a branch naming what we fetched
# that must match, or we are asking for two different references from the
# same repo, which is an error
@@ -158,22 +166,22 @@ function get_repos_for_element(){
echo "REPOREFS don't match - failed to get sha1 of fetch_$REPOREF" >&2
exit 1
fi
- popd
+ popd > /dev/null
fi
else
- sudo git clone $CACHE_PATH $REPO_DEST
- pushd $REPO_DEST
+ sudo git clone -q $CACHE_PATH $REPO_DEST
+ pushd $REPO_DEST > /dev/null
if [[ "$REPOREF" == "*" ]]; then
- sudo git fetch --prune --update-head-ok $CACHE_PATH \
+ sudo git fetch -q --prune --update-head-ok $CACHE_PATH \
+refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*
git_sha=$(git rev-parse HEAD)
else
- sudo git fetch $CACHE_PATH $REPOREF:fetch_$REPOREF
+ sudo git fetch -q $CACHE_PATH $REPOREF:fetch_$REPOREF
sudo git reset --hard FETCH_HEAD
# Get the sha in use
git_sha=$(git rev-parse FETCH_HEAD)
fi
- popd
+ popd > /dev/null
# Write the sha being used into the source-repositories manifest
echo "$REPONAME git $REPOPATH $REPOLOCATION $git_sha" >> $GIT_MANIFEST
@@ -225,7 +233,7 @@ function get_repos_for_element(){
# Capture the in-instance repository path for later review / other
# elements (like a pypi dependency cache).
- echo "$REPOPATH" | sudo dd of=$TMP_MOUNT_PATH/etc/dib-source-repositories oflag=append conv=notrunc
+ echo "$REPOPATH" | sudo tee -a $TMP_MOUNT_PATH/etc/dib-source-repositories > /dev/null
else
echo "Couldn't parse '$line' as a source repository"
@@ -248,7 +256,7 @@ rm -f $GIT_MANIFEST
for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "source-repository-*" -not -name '*~'); do
repo_flock=$CACHE_BASE/repositories_flock
(
- echo "Getting $repo_flock: $(date)"
+ echo "Getting $repo_flock: $(date) for $_SOURCEREPO"
if ! flock -w 600 9 ; then
echo "Did not get $repo_flock: $(date)"
exit 1