fix(scm): add clean options to after and before

git-scm clean after and before extensions take an optional parameter to
remove untracked nested git repositories

Added support to specify such options while maintaining the previous
specification for after and before clean extensions, although a
deprecation message will be shown

Signed-off-by: sp-ricard-valverde <ricard.valverde@socialpoint.es>
Change-Id: I0ff101bbc68fc81f390b27fe5690f8821fdea1d0

jenkins_jobs/modules/project_githuborg.py

@@ -246,8 +246,14 @@ def github_org(xml_parent, data):
     :extensions:
 
         * **clean** (`dict`)
-            * **after** (`bool`) - Clean the workspace after checkout
-            * **before** (`bool`) - Clean the workspace before checkout
+            * **after** (`dict`) - Clean the workspace after checkout
+                * **remove-stale-nested-repos** (`bool`) - Deletes untracked
+                  submodules and any other subdirectories which contain .git directories
+                  (default false)
+            * **before** (`dict`) - Clean the workspace before checkout
+                * **remove-stale-nested-repos** (`bool`) - Deletes untracked
+                  submodules and any other subdirectories which contain .git directories
+                  (default false)
         * **depth** (`int`) - Set shallow clone depth (default 1)
         * **disable-pr-notifications** (`bool`) - Disable default github status
             notifications on pull requests (default false) (Requires the

jenkins_jobs/modules/project_multibranch.py

@@ -427,8 +427,14 @@ def bitbucket_scm(xml_parent, data):
     :extensions:
 
         * **clean** (`dict`)
-            * **after** (`bool`) - Clean the workspace after checkout
-            * **before** (`bool`) - Clean the workspace before checkout
+            * **after** (`dict`) - Clean the workspace after checkout
+                * **remove-stale-nested-repos** (`bool`) - Deletes untracked
+                  submodules and any other subdirectories which contain .git directories
+                  (default false)
+            * **before** (`dict`) - Clean the workspace before checkout
+                * **remove-stale-nested-repos** (`bool`) - Deletes untracked
+                  submodules and any other subdirectories which contain .git directories
+                  (default false)
         * **prune** (`bool`) - Prune remote branches (default false)
         * **shallow-clone** (`bool`) - Perform shallow clone (default false)
         * **sparse-checkout** (dict)
@@ -678,8 +684,14 @@ def gerrit_scm(xml_parent, data):
     :extensions:
 
         * **clean** (`dict`)
-            * **after** (`bool`) - Clean the workspace after checkout
-            * **before** (`bool`) - Clean the workspace before checkout
+            * **after** (`dict`) - Clean the workspace after checkout
+                * **remove-stale-nested-repos** (`bool`) - Deletes untracked
+                  submodules and any other subdirectories which contain .git directories
+                  (default false)
+            * **before** (`dict`) - Clean the workspace before checkout
+                * **remove-stale-nested-repos** (`bool`) - Deletes untracked
+                  submodules and any other subdirectories which contain .git directories
+                  (default false)
         * **prune** (`bool`) - Prune remote branches (default false)
         * **shallow-clone** (`bool`) - Perform shallow clone (default false)
         * **sparse-checkout** (dict)
@@ -848,8 +860,14 @@ def git_scm(xml_parent, data):
     :extensions:
 
         * **clean** (`dict`)
-            * **after** (`bool`) - Clean the workspace after checkout
-            * **before** (`bool`) - Clean the workspace before checkout
+            * **after** (`dict`) - Clean the workspace after checkout
+                * **remove-stale-nested-repos** (`bool`) - Deletes untracked
+                  submodules and any other subdirectories which contain .git directories
+                  (default false)
+            * **before** (`dict`) - Clean the workspace before checkout
+                * **remove-stale-nested-repos** (`bool`) - Deletes untracked
+                  submodules and any other subdirectories which contain .git directories
+                  (default false)
         * **prune** (`bool`) - Prune remote branches (default false)
         * **shallow-clone** (`bool`) - Perform shallow clone (default false)
         * **sparse-checkout** (dict)
@@ -1006,8 +1024,14 @@ def github_scm(xml_parent, data):
     :extensions:
 
         * **clean** (`dict`)
-            * **after** (`bool`) - Clean the workspace after checkout
-            * **before** (`bool`) - Clean the workspace before checkout
+            * **after** (`dict`) - Clean the workspace after checkout
+                * **remove-stale-nested-repos** (`bool`) - Deletes untracked
+                    submodules and any other subdirectories which contain .git directories
+                    (default false)
+            * **before** (`dict`) - Clean the workspace before checkout
+                * **remove-stale-nested-repos** (`bool`) - Deletes untracked
+                    submodules and any other subdirectories which contain .git directories
+                    (default false)
         * **prune** (`bool`) - Prune remote branches (default false)
         * **shallow-clone** (`bool`) - Perform shallow clone (default false)
         * **sparse-checkout** (dict)

jenkins_jobs/modules/scm.py

@@ -262,8 +262,14 @@ def git(registry, xml_parent, data):
             to build. Can be one of `default`,`inverse`, or `gerrit`
             (default 'default')
         * **clean** (`dict`)
-            * **after** (`bool`) - Clean the workspace after checkout
-            * **before** (`bool`) - Clean the workspace before checkout
+            * **after** (`dict`) - Clean the workspace after checkout
+                * **remove-stale-nested-repos** (`bool`) - Deletes untracked
+                  submodules and any other subdirectories which contain .git directories
+                  (default false)
+            * **before** (`dict`) - Clean the workspace before checkout
+                * **remove-stale-nested-repos** (`bool`) - Deletes untracked
+                  submodules and any other subdirectories which contain .git directories
+                  (default false)
         * **committer** (`dict`)
             * **name** (`str`) - Name to use as author of new commits
             * **email** (`str`) - E-mail address to use for new commits
@@ -518,7 +524,17 @@ def git_extensions(xml_parent, data):
         else:
             clean_after = data["clean"].get("after", False)
             clean_before = data["clean"].get("before", False)
-        if clean_after:
+        if clean_after is not False:
+            if isinstance(clean_after, bool):
+                clean_after_opts = {}
+                logger.warning(
+                    "'clean: after: bool' configuration format is deprecated, "
+                    "after should be an empty dict or filled with accepted "
+                    "options."
+                )
+            else:
+                clean_after_opts = clean_after
+
             ext_name = impl_prefix + "CleanCheckout"
             if trait:
                 trait_name = "CleanAfterCheckoutTrait"
@@ -526,7 +542,20 @@ def git_extensions(xml_parent, data):
                 ext = XML.SubElement(tr, "extension", {"class": ext_name})
             else:
                 ext = XML.SubElement(xml_parent, ext_name)
-        if clean_before:
+            if "remove-stale-nested-repos" in clean_after_opts:
+                elm = XML.SubElement(ext, "deleteUntrackedNestedRepositories")
+                elm.text = "true"
+        if clean_before is not False:
+            if isinstance(clean_before, bool):
+                clean_before_opts = {}
+                logger.warning(
+                    "'clean: before: bool' configuration format is deprecated, "
+                    "before should be an empty dict or filled with accepted "
+                    "options."
+                )
+            else:
+                clean_before_opts = clean_before
+
             ext_name = impl_prefix + "CleanBeforeCheckout"
             if trait:
                 trait_name = "CleanBeforeCheckoutTrait"
@@ -534,6 +563,9 @@ def git_extensions(xml_parent, data):
                 ext = XML.SubElement(tr, "extension", {"class": ext_name})
             else:
                 ext = XML.SubElement(xml_parent, ext_name)
+            if "remove-stale-nested-repos" in clean_before_opts:
+                elm = XML.SubElement(ext, "deleteUntrackedNestedRepositories")
+                elm.text = "true"
     committer = data.get("committer", {})
     if committer:
         ext_name = impl_prefix + "UserIdentity"

tests/scm/fixtures/git-clean02.xml

@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <scm class="hudson.plugins.git.GitSCM">
+    <configVersion>2</configVersion>
+    <userRemoteConfigs>
+      <hudson.plugins.git.UserRemoteConfig>
+        <name>origin</name>
+        <refspec>+refs/heads/*:refs/remotes/origin/*</refspec>
+        <url>https://github.com/openstack-infra/jenkins-job-builder.git</url>
+      </hudson.plugins.git.UserRemoteConfig>
+    </userRemoteConfigs>
+    <branches>
+      <hudson.plugins.git.BranchSpec>
+        <name>**</name>
+      </hudson.plugins.git.BranchSpec>
+    </branches>
+    <disableSubmodules>false</disableSubmodules>
+    <recursiveSubmodules>false</recursiveSubmodules>
+    <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+    <remotePoll>false</remotePoll>
+    <gitTool>Default</gitTool>
+    <submoduleCfg class="list"/>
+    <reference/>
+    <gitConfigName/>
+    <gitConfigEmail/>
+    <extensions>
+      <hudson.plugins.git.extensions.impl.CleanCheckout>
+        <deleteUntrackedNestedRepositories>true</deleteUntrackedNestedRepositories>
+      </hudson.plugins.git.extensions.impl.CleanCheckout>
+      <hudson.plugins.git.extensions.impl.CleanBeforeCheckout>
+        <deleteUntrackedNestedRepositories>true</deleteUntrackedNestedRepositories>
+      </hudson.plugins.git.extensions.impl.CleanBeforeCheckout>
+      <hudson.plugins.git.extensions.impl.WipeWorkspace/>
+    </extensions>
+  </scm>
+</project>

tests/scm/fixtures/git-clean02.yaml

@@ -0,0 +1,8 @@
+scm:
+    - git:
+        url: https://github.com/openstack-infra/jenkins-job-builder.git
+        clean:
+            after:
+                remove-stale-nested-repos: true
+            before:
+                remove-stale-nested-repos: true

tests/scm/fixtures/git-clean03.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <scm class="hudson.plugins.git.GitSCM">
+    <configVersion>2</configVersion>
+    <userRemoteConfigs>
+      <hudson.plugins.git.UserRemoteConfig>
+        <name>origin</name>
+        <refspec>+refs/heads/*:refs/remotes/origin/*</refspec>
+        <url>https://github.com/openstack-infra/jenkins-job-builder.git</url>
+      </hudson.plugins.git.UserRemoteConfig>
+    </userRemoteConfigs>
+    <branches>
+      <hudson.plugins.git.BranchSpec>
+        <name>**</name>
+      </hudson.plugins.git.BranchSpec>
+    </branches>
+    <disableSubmodules>false</disableSubmodules>
+    <recursiveSubmodules>false</recursiveSubmodules>
+    <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+    <remotePoll>false</remotePoll>
+    <gitTool>Default</gitTool>
+    <submoduleCfg class="list"/>
+    <reference/>
+    <gitConfigName/>
+    <gitConfigEmail/>
+    <extensions>
+      <hudson.plugins.git.extensions.impl.CleanCheckout/>
+      <hudson.plugins.git.extensions.impl.CleanBeforeCheckout/>
+      <hudson.plugins.git.extensions.impl.WipeWorkspace/>
+    </extensions>
+  </scm>
+</project>

tests/scm/fixtures/git-clean03.yaml

@@ -0,0 +1,6 @@
+scm:
+    - git:
+        url: https://github.com/openstack-infra/jenkins-job-builder.git
+        clean:
+            after: {}
+            before: {}