From 0c012177d3d93e44750e47cde142d3c5f619086d Mon Sep 17 00:00:00 2001
From: Victor Seva <linuxmaniac@torreviejawireless.org>
Date: Tue, 28 May 2024 15:38:08 +0200
Subject: [PATCH] fix expansion on `{obj:key}` variable on includes

* '{name}' param should be expanded first
* already expanded params should be reused

Change-Id: Iedc384932a6e72aebfa87da141812934e0625c03
---
 jenkins_jobs/expander.py                      |   7 +-
 tests/yamlparser/job_fixtures/include03.xml   | 166 ++++++++++++++++++
 tests/yamlparser/job_fixtures/include03.yaml  |  34 ++++
 .../job_fixtures/include03_gerrit.yaml.inc    |  17 ++
 .../include03_manage-docker.yaml.inc          |  22 +++
 .../include03_source-tests.yaml.inc           |  20 +++
 ...ude03_source-tests_trigger_builds.yaml.inc |   5 +
 ...ests_trigger_parameterized_builds.yaml.inc |   4 +
 8 files changed, 272 insertions(+), 3 deletions(-)
 create mode 100644 tests/yamlparser/job_fixtures/include03.xml
 create mode 100644 tests/yamlparser/job_fixtures/include03.yaml
 create mode 100644 tests/yamlparser/job_fixtures/include03_gerrit.yaml.inc
 create mode 100644 tests/yamlparser/job_fixtures/include03_manage-docker.yaml.inc
 create mode 100644 tests/yamlparser/job_fixtures/include03_source-tests.yaml.inc
 create mode 100644 tests/yamlparser/job_fixtures/include03_source-tests_trigger_builds.yaml.inc
 create mode 100644 tests/yamlparser/job_fixtures/include03_source-tests_trigger_parameterized_builds.yaml.inc

diff --git a/jenkins_jobs/expander.py b/jenkins_jobs/expander.py
index fed2f51f3..034f41dfe 100644
--- a/jenkins_jobs/expander.py
+++ b/jenkins_jobs/expander.py
@@ -11,7 +11,7 @@
 # under the License.
 
 from functools import partial
-
+from itertools import filterfalse
 from jinja2 import StrictUndefined
 
 from .errors import Context, JenkinsJobsException
@@ -255,7 +255,7 @@ def expand_parameters(expander, param_dict):
             required_params = list(enum_required_params(format, value_pos))
             deps[name] = (key_pos, value_pos)
             try:
-                params = LocDict()
+                params = LocDict.merge(expanded_params)
                 for n in required_params:
                     v, kp, vp = expand(n)
                     params.set_item(n, v, kp, vp)
@@ -272,6 +272,7 @@ def expand_parameters(expander, param_dict):
         expanded_params.set_item(name, value, key_pos, value_pos)
         return (value, key_pos, value_pos)
 
-    for name in param_dict:
+    expand("name")  # expand 'name' parameter first
+    for name in filterfalse(lambda x: x == "name", param_dict):
         expand(name)
     return expanded_params
diff --git a/tests/yamlparser/job_fixtures/include03.xml b/tests/yamlparser/job_fixtures/include03.xml
new file mode 100644
index 000000000..307402c3f
--- /dev/null
+++ b/tests/yamlparser/job_fixtures/include03.xml
@@ -0,0 +1,166 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>Gerrit Jenkins job listening for code review related events for acc-cdi project.
+&lt;!-- ['NGCP-project'] --&gt;
+&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <disabled>false</disabled>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>true</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders/>
+  <publishers>
+    <hudson.tasks.ArtifactArchiver>
+      <artifacts>*_parameters.txt</artifacts>
+      <allowEmptyArchive>true</allowEmptyArchive>
+      <onlyIfSuccessful>false</onlyIfSuccessful>
+      <fingerprint>true</fingerprint>
+      <defaultExcludes>true</defaultExcludes>
+      <caseSensitive>true</caseSensitive>
+      <latestOnly>false</latestOnly>
+      <followSymlinks>false</followSymlinks>
+    </hudson.tasks.ArtifactArchiver>
+    <hudson.plugins.ws__cleanup.WsCleanup plugin="ws-cleanup">
+      <deleteDirs>false</deleteDirs>
+      <cleanupMatrixParent>false</cleanupMatrixParent>
+      <externalDelete/>
+      <disableDeferredWipeout>true</disableDeferredWipeout>
+      <cleanWhenSuccess>true</cleanWhenSuccess>
+      <cleanWhenUnstable>true</cleanWhenUnstable>
+      <cleanWhenFailure>true</cleanWhenFailure>
+      <cleanWhenNotBuilt>true</cleanWhenNotBuilt>
+      <cleanWhenAborted>true</cleanWhenAborted>
+      <notFailBuild>true</notFailBuild>
+    </hudson.plugins.ws__cleanup.WsCleanup>
+  </publishers>
+  <buildWrappers>
+    <hudson.plugins.ws__cleanup.PreBuildCleanup plugin="ws-cleanup">
+      <deleteDirs>false</deleteDirs>
+      <cleanupParameter/>
+      <externalDelete/>
+      <disableDeferredWipeout>false</disableDeferredWipeout>
+    </hudson.plugins.ws__cleanup.PreBuildCleanup>
+    <hudson.plugins.timestamper.TimestamperBuildWrapper/>
+  </buildWrappers>
+</project>
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>Default tests job
+&lt;!-- ['NGCP-project'] --&gt;
+&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <disabled>false</disabled>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>true</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties>
+    <hudson.plugins.copyartifact.CopyArtifactPermissionProperty plugin="copyartifact">
+      <projectNameList>
+        <string>acc-cdi-*</string>
+      </projectNameList>
+    </hudson.plugins.copyartifact.CopyArtifactPermissionProperty>
+  </properties>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.plugins.parameterizedtrigger.TriggerBuilder>
+      <configs>
+        <hudson.plugins.parameterizedtrigger.BlockableBuildTriggerConfig>
+          <configs>
+            <hudson.plugins.parameterizedtrigger.CurrentBuildParameters/>
+            <hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
+              <properties>docker_file=$docker_file
+docker_tag=$docker_tag
+docker_tag_ppa=$docker_tag_ppa
+docker_push=$docker_push
+</properties>
+            </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
+          </configs>
+          <projects>acc-cdi-manage-docker</projects>
+          <condition>ALWAYS</condition>
+          <triggerWithNoParameters>false</triggerWithNoParameters>
+          <buildAllNodesWithLabel>false</buildAllNodesWithLabel>
+          <block>
+            <buildStepFailureThreshold>
+              <name>FAILURE</name>
+              <ordinal>2</ordinal>
+              <color>RED</color>
+              <completeBuild>true</completeBuild>
+            </buildStepFailureThreshold>
+            <unstableThreshold>
+              <name>UNSTABLE</name>
+              <ordinal>1</ordinal>
+              <color>YELLOW</color>
+              <completeBuild>true</completeBuild>
+            </unstableThreshold>
+            <failureThreshold>
+              <name>FAILURE</name>
+              <ordinal>2</ordinal>
+              <color>RED</color>
+              <completeBuild>true</completeBuild>
+            </failureThreshold>
+          </block>
+        </hudson.plugins.parameterizedtrigger.BlockableBuildTriggerConfig>
+      </configs>
+    </hudson.plugins.parameterizedtrigger.TriggerBuilder>
+  </builders>
+  <publishers>
+    <hudson.plugins.parameterizedtrigger.BuildTrigger>
+      <configs>
+        <hudson.plugins.parameterizedtrigger.BuildTriggerConfig>
+          <configs>
+            <hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
+              <properties>uuid=$uuid
+release=$release
+release_uuid=$release_uuid
+branch=$branch
+tag=$tag
+distribution=$distribution
+ppa=$ppa
+</properties>
+            </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
+            <hudson.plugins.parameterizedtrigger.CurrentBuildParameters/>
+          </configs>
+          <projects>acc-cdi-source</projects>
+          <condition>UNSTABLE_OR_BETTER</condition>
+          <triggerFromChildProjects>false</triggerFromChildProjects>
+          <triggerWithNoParameters>false</triggerWithNoParameters>
+        </hudson.plugins.parameterizedtrigger.BuildTriggerConfig>
+      </configs>
+    </hudson.plugins.parameterizedtrigger.BuildTrigger>
+    <hudson.plugins.ws__cleanup.WsCleanup plugin="ws-cleanup">
+      <deleteDirs>false</deleteDirs>
+      <cleanupMatrixParent>false</cleanupMatrixParent>
+      <externalDelete/>
+      <disableDeferredWipeout>true</disableDeferredWipeout>
+      <cleanWhenSuccess>true</cleanWhenSuccess>
+      <cleanWhenUnstable>true</cleanWhenUnstable>
+      <cleanWhenFailure>true</cleanWhenFailure>
+      <cleanWhenNotBuilt>true</cleanWhenNotBuilt>
+      <cleanWhenAborted>true</cleanWhenAborted>
+      <notFailBuild>true</notFailBuild>
+    </hudson.plugins.ws__cleanup.WsCleanup>
+  </publishers>
+  <buildWrappers>
+    <hudson.plugins.ws__cleanup.PreBuildCleanup plugin="ws-cleanup">
+      <deleteDirs>false</deleteDirs>
+      <cleanupParameter/>
+      <externalDelete/>
+      <disableDeferredWipeout>false</disableDeferredWipeout>
+    </hudson.plugins.ws__cleanup.PreBuildCleanup>
+    <hudson.plugins.timestamper.TimestamperBuildWrapper/>
+    <EnvInjectBuildWrapper>
+      <info>
+        <loadFilesFromMaster>false</loadFilesFromMaster>
+        <secureGroovyScript>
+          <sandbox>false</sandbox>
+        </secureGroovyScript>
+      </info>
+    </EnvInjectBuildWrapper>
+  </buildWrappers>
+</project>
diff --git a/tests/yamlparser/job_fixtures/include03.yaml b/tests/yamlparser/job_fixtures/include03.yaml
new file mode 100644
index 000000000..944d42157
--- /dev/null
+++ b/tests/yamlparser/job_fixtures/include03.yaml
@@ -0,0 +1,34 @@
+- defaults:
+    name: def
+    predefined-parameters: &predefined-parameters |
+      uuid=$uuid
+      release=$release
+      release_uuid=$release_uuid
+      branch=$branch
+      tag=$tag
+      distribution=$distribution
+      ppa=$ppa
+    predefined-docker-parameters: &predefined-docker-parameters |
+      docker_file=$docker_file
+      docker_tag=$docker_tag
+      docker_tag_ppa=$docker_tag_ppa
+      docker_push=$docker_push
+
+- _default_project: &default_project
+    name: default_project
+    jobs:
+      - '{name}-gerrit'
+      - '{name}-source-tests': &default_source-tests
+          tb: !include: include03_source-tests_trigger_builds.yaml.inc
+          tpb: !include: include03_source-tests_trigger_parameterized_builds.yaml.inc
+          lastjob: {}
+    labels:
+      - NGCP-project
+
+- job-template: !include: include03_gerrit.yaml.inc
+- job-template: !include: include03_source-tests.yaml.inc
+- job-template: !include: include03_manage-docker.yaml.inc
+
+- project:
+    name: acc-cdi
+    <<: *default_project
diff --git a/tests/yamlparser/job_fixtures/include03_gerrit.yaml.inc b/tests/yamlparser/job_fixtures/include03_gerrit.yaml.inc
new file mode 100644
index 000000000..d697dc79a
--- /dev/null
+++ b/tests/yamlparser/job_fixtures/include03_gerrit.yaml.inc
@@ -0,0 +1,17 @@
+name: '{name}-gerrit'
+project-type: freestyle
+description: |
+  Gerrit Jenkins job listening for code review related events for {name} project.
+  <!-- {labels} -->
+disabled: false
+concurrent: true
+publishers:
+  - archive:
+      artifacts: '*_parameters.txt'
+      allow-empty: true
+      fingerprint: true
+  - workspace-cleanup:
+      disable-deferred-wipeout: true
+wrappers:
+  - workspace-cleanup
+  - timestamps
diff --git a/tests/yamlparser/job_fixtures/include03_manage-docker.yaml.inc b/tests/yamlparser/job_fixtures/include03_manage-docker.yaml.inc
new file mode 100644
index 000000000..25ab05b9f
--- /dev/null
+++ b/tests/yamlparser/job_fixtures/include03_manage-docker.yaml.inc
@@ -0,0 +1,22 @@
+name: '{name}-manage-docker'
+project-type: freestyle
+description: |
+  build/tag/delete project docker image(s)
+  <!-- {labels} -->
+disabled: false
+concurrent: true
+builders:
+  - copyartifact:
+      project: '{name}-get-code'
+      filter: 'source.tar.gz'
+      which-build: upstream-build
+      fallback-to-last-successful: false
+publishers:
+  - workspace-cleanup:
+      disable-deferred-wipeout: true
+wrappers:
+  - workspace-cleanup
+  - timestamps
+  - inject:
+      properties-content: |
+        PROJECTNAME={name}
diff --git a/tests/yamlparser/job_fixtures/include03_source-tests.yaml.inc b/tests/yamlparser/job_fixtures/include03_source-tests.yaml.inc
new file mode 100644
index 000000000..01b3906c5
--- /dev/null
+++ b/tests/yamlparser/job_fixtures/include03_source-tests.yaml.inc
@@ -0,0 +1,20 @@
+name: '{name}-source-tests'
+project-type: freestyle
+description: |
+  Default tests job
+  <!-- {labels} -->
+properties:
+  - copyartifact:
+      projects: '{name}-*'
+disabled: false
+concurrent: true
+builders:
+  - trigger-builds: '{obj:tb}'
+publishers:
+  - trigger-parameterized-builds: '{obj:tpb}'
+  - workspace-cleanup:
+      disable-deferred-wipeout: true
+wrappers:
+  - workspace-cleanup
+  - timestamps
+  - inject: '{obj:lastjob}'
diff --git a/tests/yamlparser/job_fixtures/include03_source-tests_trigger_builds.yaml.inc b/tests/yamlparser/job_fixtures/include03_source-tests_trigger_builds.yaml.inc
new file mode 100644
index 000000000..d1c0b935b
--- /dev/null
+++ b/tests/yamlparser/job_fixtures/include03_source-tests_trigger_builds.yaml.inc
@@ -0,0 +1,5 @@
+- project: '{name}-manage-docker'
+  condition: UNSTABLE_OR_BETTER
+  block: true
+  current-parameters: CurrentBuildParameters
+  predefined-parameters: *predefined-docker-parameters
diff --git a/tests/yamlparser/job_fixtures/include03_source-tests_trigger_parameterized_builds.yaml.inc b/tests/yamlparser/job_fixtures/include03_source-tests_trigger_parameterized_builds.yaml.inc
new file mode 100644
index 000000000..5acedb9f3
--- /dev/null
+++ b/tests/yamlparser/job_fixtures/include03_source-tests_trigger_parameterized_builds.yaml.inc
@@ -0,0 +1,4 @@
+- project: '{name}-source'
+  condition: UNSTABLE_OR_BETTER
+  predefined-parameters: *predefined-parameters
+  current-parameters: CurrentBuildParameters