AIAP multinode deployment

It deploys 3 controlplane and 2 worker nodes using AIAP gate.

+ kubectl --context target-cluster get node
NAME     STATUS   ROLES                  AGE   VERSION
node01   Ready    control-plane,master   48m   v1.21.2
node03   Ready    worker                 43s   v1.21.2
node04   Ready    control-plane,master   16m   v1.21.2
node05   Ready    control-plane,master   26m   v1.21.2
node06   Ready    worker                 54s   v1.21.2

Tested with 32 GB node.
https://zuul.opendev.org/t/openstack/build/da1c2c440d3d4026ab454de73ee518e2/logs

Closes: #652
Closes: #228

Change-Id: Ie2267e15ed75b57e2e27f45b9be19ddcf7b0a0a8

manifests/function/k8scontrol/controlplane.yaml

@@ -3,7 +3,7 @@ apiVersion: controlplane.cluster.x-k8s.io/v1alpha4
 metadata:
   name: cluster-controlplane
 spec:
-  replicas: 1
+  replicas: 3
   version: v1.21.2
   machineTemplate:
     infrastructureRef:

manifests/function/phase-helpers/check_ingress_ctrl/kubectl_check_ingress_ctrl.sh

@@ -18,12 +18,11 @@ export TIMEOUT=${TIMEOUT:-60}
 
 end=$(($(date +%s) + $TIMEOUT))
 while true; do
-  export TARGET_IP="$(kubectl --request-timeout 10s \
+  export TARGET_IP="$(kubectl -n kube-system get po | \
-    --context $KCTL_CONTEXT \
+	  grep -i apiserver | \
-    --namespace ingress \
+	  cut -f1 -d ' ' | \
-    get pods \
+	  xargs kubectl -n kube-system \
-    -l app.kubernetes.io/component=controller \
+	  get po $1 -o=jsonpath='{.items[0].status.hostIP}')"
-    -o jsonpath='{.items[*].status.hostIP}')"
   if [ ! -z $TARGET_IP ]; then
     break
   else

manifests/function/workers-capm3/machinedeployment.yaml

@@ -6,7 +6,7 @@ metadata:
     cluster.x-k8s.io/cluster-name: target-cluster
 spec:
   clusterName: target-cluster
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       cluster.x-k8s.io/cluster-name: target-cluster

manifests/phases/executors.yaml

@@ -34,6 +34,7 @@ config:
         value: "provisioned"
   pruneOptions:
     prune: false
+  inventoryPolicy: force-adopt
 ---
 apiVersion: airshipit.org/v1alpha1
 kind: KubernetesApply

manifests/site/test-site/ephemeral/controlplane/kustomization.yaml

@@ -11,3 +11,14 @@ namespace: target-infra
 
 transformers:
   - ../../../../type/gating/ephemeral/controlplane/replacements
+
+patchesJson6902:
+  - target:
+      group: controlplane.cluster.x-k8s.io
+      version: v1alpha4
+      kind: KubeadmControlPlane
+      name: cluster-controlplane
+    path: patch_json6902_controlplane.yaml
+
+patchesStrategicMerge:
+  - versions-airshipctl.yaml

manifests/site/test-site/ephemeral/controlplane/patch_json6902_controlplane.yaml

@@ -0,0 +1,3 @@
+- op: replace
+  path: "/spec/replicas"
+  value: 1

manifests/site/test-site/ephemeral/controlplane/versions-airshipctl.yaml

@@ -0,0 +1,12 @@
+# Override default controlplane image location
+apiVersion: airshipit.org/v1alpha1
+kind: VersionsCatalogue
+metadata:
+  name: versions-airshipctl
+spec:
+  files:
+    k8scontrol:
+      # Host the image in a locally served location for CI
+      cluster_controlplane_image:
+        url: http://10.23.24.137:80/images/control-plane.qcow2
+        checksum: http://10.23.24.137:80/images/control-plane.qcow2.md5sum

manifests/site/test-site/host-inventory/hostgenerator/host-generation.yaml

@@ -13,3 +13,5 @@ hosts:
     - node02
     - node03
     - node04
+    - node05
+    - node06

manifests/site/test-site/target/catalogues/shareable/hosts.yaml

@@ -54,15 +54,45 @@ hosts:
         pxe: 52:54:00:b6:ed:23
       hardwareProfile: default # defined in the hardwareprofile-example function
     node04:
-      bootMode: UEFI
+      bootMode: legacy
-      macAddress: 52:54:00:36:5e:e3
+      macAddress: 52:54:00:01:71:d8
-      bmcAddress: redfish+http://10.23.25.2:8000/redfish/v1/Systems/air-target-2
+      bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-target-2
       bmcUsername: username
       bmcPassword: password
+      disableCertificateVerification: false
       ipAddresses:
         oam-ipv4: 10.23.25.104
         pxe-ipv4: 10.23.24.104
       macAddresses:
-        oam: 52:54:00:dc:ab:04
+        oam: 52:54:00:f0:57:e0
-        pxe: 52:54:00:51:0b:e4
+        pxe: 52:54:00:01:71:d8
+      hardwareProfile: default # defined in the hostgenerator-m3 function
+    node05:
+      bootMode: legacy
+      macAddress: 52:54:00:a3:ee:18
+      bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-target-3
+      bmcUsername: username
+      bmcPassword: password
+      disableCertificateVerification: false
+      ipAddresses:
+        oam-ipv4: 10.23.25.105
+        pxe-ipv4: 10.23.24.105
+      macAddresses:
+        oam: 52:54:00:91:36:49
+        pxe: 52:54:00:a3:ee:18
+      hardwareProfile: default # defined in the hostgenerator-m3 function
+    node06:
+      bootMode: legacy
+      macAddress: 52:54:00:b6:ed:19
+      bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-worker-2
+      bmcUsername: username
+      bmcPassword: password
+      disableCertificateVerification: false
+      ipAddresses:
+        oam-ipv4: 10.23.25.106
+        pxe-ipv4: 10.23.24.106
+      macAddresses:
+        oam: 52:54:00:9b:27:08
+        pxe: 52:54:00:b6:ed:19
       hardwareProfile: default # defined in the hardwareprofile-example function
+

manifests/site/test-site/target/controlplane/hostgenerator/host-generation.yaml

@@ -7,3 +7,5 @@ metadata:
   name: host-generation-catalogue
 hosts:
   m3:
+  - node04
+  - node05

manifests/site/test-site/target/controlplane/kustomization.yaml

@@ -7,6 +7,9 @@ resources:
   - ../catalogues
   - nodes
 
+patchesStrategicMerge:
+  - metal3machinetemplate.yaml
+
 namespace: target-infra
 
 commonLabels:

manifests/site/test-site/target/controlplane/metal3machinetemplate.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5
+kind: Metal3MachineTemplate
+metadata:
+  annotations:
+    config.kubernetes.io/path: metal3machinetemplate_cluster-controlplane.yaml
+  name: cluster-controlplane
+spec:
+  template:
+    spec:
+      hostSelector:
+        matchLabels:
+          airshipit.org/k8s-role: controlplane-host
+      image:
+        url: http://10.23.24.137:80/images/control-plane.qcow2
+        checksum: http://10.23.24.137:80/images/control-plane.qcow2.md5sum

manifests/site/test-site/target/workers/hostgenerator/host-generation.yaml

@@ -8,3 +8,4 @@ metadata:
 hosts:
   m3:
     - node03
+    - node06

manifests/type/gating/phases/plan.yaml

@@ -99,6 +99,8 @@ phases:
   - name: eject-virtual-media-ephemeral
   # Power off Ephemeral baremetal host avoid DHCP conflict
   - name: power-off-ephemeral
+  # Create target k8s cluster resources
+  - name: controlplane-target
   # List all nodes in target cluster
   # Scripts for this phase placed in manifests/function/phase-helpers/get_node/
   # To get ConfigMap for this phase, execute `airshipctl phase render --source config -k ConfigMap`
@@ -124,6 +126,7 @@ validation:
     - VariableCatalogue
   crdList:
     - airshipctl/manifests/function/airshipctl-schemas
+    - airshipctl/manifests/function/baremetal-operator/v0.5.0/upstream/crd
 ---
 apiVersion: airshipit.org/v1alpha1
 kind: PhasePlan

playbooks/airship-airshipctl-build-gate.yaml

@@ -69,6 +69,7 @@
         path: "{{ roleinputvar.path }}"
       loop:
         - { name: 'target', path: 'ephemeral/controlplane' }
+        - { name: 'target', path: 'target/controlplane' }
         - { name: 'ephemeral', path: 'ephemeral/bootstrap' }
         - { name: 'worker', path: 'target/workers' }
       loop_control:
@@ -96,12 +97,12 @@
             target_vm_cfg: "{{ target_vm_cfg }}"
             target_vm_memory_mb: 7168
             target_vm_vcpus: 2
-            target_vms_count: 1
+            target_vms_count: 3
             worker_disk_size: 30G
             worker_vm_cfg: "{{ worker_vm_cfg }}"
             worker_vm_memory_mb: 7168
             worker_vm_vcpus: 2
-            worker_vms_count: 1
+            worker_vms_count: 2
         airship_gate_file_exchanger:
           servername: "localhost"
           ip:

playbooks/get-vm-config.yaml

@@ -17,7 +17,6 @@
       {{ airship_config_manifest_directory }}/{{ airship_config_site_path }}/{{ path }} 2>/dev/null |
     kustomize cfg grep  "kind=BareMetalHost"
   register: bmh_command
-  failed_when: "bmh_command.stdout == ''"
   environment:
     KUSTOMIZE_ENABLE_ALPHA_COMMANDS: "true"
     SOPS_IMPORT_PGP: "{{ airship_config_pgp }}"
@@ -37,6 +36,7 @@
     KUSTOMIZE_ENABLE_ALPHA_COMMANDS: "true"
     SOPS_IMPORT_PGP: "{{ airship_config_pgp }}"
   with_items: "{{ bmh }}"
+  when: bmh|length > 0
 
 - name: get links from network data per BareMetalHost object
   set_fact:
@@ -47,6 +47,8 @@
           map(attribute='stringData.networkData') | map('from_yaml') |
           map(attribute='links') | list
       }}
+  when: bmh|length > 0
+
 
 - name: define list of VM mac addresses and VM boot mode
   set_fact:
@@ -64,6 +66,12 @@
   loop_control:
     index_var: idx
   register: vm_cfg_fact
+  when: bmh|length > 0
+
+- set_fact:
+    "{{ name }}_vm_cfg": "{{  (lookup('vars', name + '_vm_cfg') |list)  + (vm_cfg_fact.results | map(attribute='ansible_facts.vm_cfg') | list) }}"
+  when: "{{ name + '_vm_cfg' }} is defined and bmh|length > 0"
 
 - set_fact:
     "{{ name }}_vm_cfg": "{{ vm_cfg_fact.results | map(attribute='ansible_facts.vm_cfg') | list }}"
+  when: "{{ name + '_vm_cfg' }} is not defined and bmh|length > 0"

tools/airship-in-a-pod/examples/base/airship-in-a-pod.yaml

@@ -24,7 +24,6 @@ data:
   AIRSHIP_CONFIG_MANIFEST_REPO_AUTH_USERNAME: ""
   AIRSHIP_CONFIG_MANIFEST_REPO_AUTH_HTTP_PASSWORD: ""
   AIRSHIP_CONFIG_MANIFEST_REPO_AUTH_SSH_PASSWORD: ""
-  SOPS_IMPORT_PGP: ""
 ---
 
 apiVersion: v1

tools/airship-in-a-pod/infra-builder/assets/opt/ansible/playbooks/roles/build-infra/defaults/main.yaml

@@ -18,7 +18,7 @@ floorplan:
 
 ephemeral_node:
   name: air-ephemeral
-  cpu: 4
+  cpu: 2
   ram: 6144
   nat_mac_address: [ "52:54:00:9b:27:02" ]
   prov_mac_address: [ "52:54:00:b6:ed:02" ]
@@ -26,22 +26,22 @@ ephemeral_node:
     - 30G
 
 target_nodes:
-  count: 1
+  count: 3
   name: air-target
   cpu: 2
-  ram: 7168
+  ram: 6144
-  nat_mac_address: [ "52:54:00:9b:27:4c" ]
+  nat_mac_address: [ "52:54:00:9b:27:4c", "52:54:00:f0:57:e0", "52:54:00:91:36:49" ]
-  prov_mac_address: [ "52:54:00:b6:ed:31" ]
+  prov_mac_address: [ "52:54:00:b6:ed:31", "52:54:00:01:71:d8", "52:54:00:a3:ee:18" ]
   block:
     - 30G
 
 worker_nodes:
-  count: 1
+  count: 2
   name: air-worker
   cpu: 2
-  ram: 7168
+  ram: 6144
-  nat_mac_address: [ "52:54:00:9b:27:07" ]
+  nat_mac_address: [ "52:54:00:9b:27:07", "52:54:00:9b:27:08" ]
-  prov_mac_address: [ "52:54:00:b6:ed:23" ]
+  prov_mac_address: [ "52:54:00:b6:ed:23", "52:54:00:b6:ed:19"  ]
   block:
     - 30G