Merge "Add GIT_TIMEOUT variable to watch git operations"

functions-common

@@ -527,16 +527,16 @@ function git_clone {
         if [[ ! -d $GIT_DEST ]]; then
             [[ "$ERROR_ON_CLONE" = "True" ]] && \
                 die $LINENO "Cloning not allowed in this configuration"
-            git clone $GIT_REMOTE $GIT_DEST
+            git_timed clone $GIT_REMOTE $GIT_DEST
         fi
         cd $GIT_DEST
-        git fetch $GIT_REMOTE $GIT_REF && git checkout FETCH_HEAD
+        git_timed fetch $GIT_REMOTE $GIT_REF && git checkout FETCH_HEAD
     else
         # do a full clone only if the directory doesn't exist
         if [[ ! -d $GIT_DEST ]]; then
             [[ "$ERROR_ON_CLONE" = "True" ]] && \
                 die $LINENO "Cloning not allowed in this configuration"
-            git clone $GIT_REMOTE $GIT_DEST
+            git_timed clone $GIT_REMOTE $GIT_DEST
             cd $GIT_DEST
             # This checkout syntax works for both branches and tags
             git checkout $GIT_REF
@@ -545,7 +545,7 @@ function git_clone {
             cd $GIT_DEST
             # set the url to pull from and fetch
             git remote set-url origin $GIT_REMOTE
-            git fetch origin
+            git_timed fetch origin
             # remove the existing ignored files (like pyc) as they cause breakage
             # (due to the py files having older timestamps than our pyc, so python
             # thinks the pyc files are correct using them)
@@ -570,6 +570,37 @@ function git_clone {
     git show --oneline | head -1
 }
 
+# git can sometimes get itself infinitely stuck with transient network
+# errors or other issues with the remote end.  This wraps git in a
+# timeout/retry loop and is intended to watch over non-local git
+# processes that might hang.  GIT_TIMEOUT, if set, is passed directly
+# to timeout(1); otherwise the default value of 0 maintains the status
+# quo of waiting forever.
+# usage: git_timed <git-command>
+function git_timed() {
+    local count=0
+    local timeout=0
+
+    if [[ -n "${GIT_TIMEOUT}" ]]; then
+        timeout=${GIT_TIMEOUT}
+    fi
+
+    until timeout -s SIGINT ${timeout} git "$@"; do
+        # 124 is timeout(1)'s special return code when it reached the
+        # timeout; otherwise assume fatal failure
+        if [[ $? -ne 124 ]]; then
+            die $LINENO "git call failed: [git $@]"
+        fi
+
+        count=$(($count + 1))
+        warn "timeout ${count} for git call: [git $@]"
+        if [ $count -eq 3 ]; then
+            die $LINENO "Maximum of 3 git retries reached"
+        fi
+        sleep 5
+    done
+}
+
 # git update using reference as a branch.
 # git_update_branch ref
 function git_update_branch() {
@@ -600,7 +631,7 @@ function git_update_tag() {
 
     git tag -d $GIT_TAG
     # fetching given tag only
-    git fetch origin tag $GIT_TAG
+    git_timed fetch origin tag $GIT_TAG
     git checkout -f $GIT_TAG
 }
 

stackrc

@@ -80,6 +80,17 @@ fi
 # (currently only implemented for MySQL backend)
 DATABASE_QUERY_LOGGING=$(trueorfalse True $DATABASE_QUERY_LOGGING)
 
+# Set a timeout for git operations.  If git is still running when the
+# timeout expires, the command will be retried up to 3 times.  This is
+# in the format for timeout(1);
+#
+#  DURATION is a floating point number with an optional suffix: 's'
+#  for seconds (the default), 'm' for minutes, 'h' for hours or 'd'
+#  for days.
+#
+# Zero disables timeouts
+GIT_TIMEOUT=${GIT_TIMEOUT:-0}
+
 # Repositories
 # ------------