From 5d70de0f3d16be68c28c134966939df543de1dfc Mon Sep 17 00:00:00 2001 From: Kostiantyn Kalynovskyi Date: Thu, 3 Jun 2021 17:44:36 +0000 Subject: [PATCH] Generate bootMAC and change boot network to NAT Change-Id: Id8776e8153c57682ed02f4e6534d8dfda4ed25a1 --- .../bases/airship.airshipit.org_vinoes.yaml | 13 +- config/manager/flavor-templates.yaml | 1 + config/manager/network-templates.yaml | 2 +- config/samples/vino_cr.yaml | 2 +- config/samples/vino_cr_4_workers_1_cp.yaml | 2 - docs/api/vino.md | 48 ++------ pkg/api/v1/vino_builder.go | 18 +-- pkg/api/v1/vino_types.go | 10 +- pkg/managers/bmh.go | 113 ++++++++++++------ 9 files changed, 110 insertions(+), 99 deletions(-) diff --git a/config/crd/bases/airship.airshipit.org_vinoes.yaml b/config/crd/bases/airship.airshipit.org_vinoes.yaml index 7484af3..3764ea5 100644 --- a/config/crd/bases/airship.airshipit.org_vinoes.yaml +++ b/config/crd/bases/airship.airshipit.org_vinoes.yaml @@ -77,10 +77,6 @@ spec: vinoBuilderImage: type: string type: object - managementPhysicalInterfaceName: - description: ManagementPhysicalInterfaceName will be used to connect - to libvirt network - type: string networks: description: Define network parameters items: @@ -151,11 +147,6 @@ spec: that will be created These labels will override keys from k8s node, that are specified in vino.NodeLabelKeysToCopy type: object - bootInterfaceName: - description: BootInterfaceName references the interface name - in the list of NetworkInterfaces Vino will take this interface - find its mac address and use it as bootMACAddress for BMH - type: string count: type: integer diskDrives: @@ -216,6 +207,10 @@ spec: type: string type: object type: array + rootDeviceName: + description: RootDeviceName is the root device for underlying + VM, /dev/vda for example default is /dev/vda + type: string type: object type: array pxeBootImageHost: diff --git a/config/manager/flavor-templates.yaml b/config/manager/flavor-templates.yaml index d5e8c23..1be1f4d 100644 --- a/config/manager/flavor-templates.yaml +++ b/config/manager/flavor-templates.yaml @@ -70,6 +70,7 @@ flavorTemplates: + diff --git a/config/manager/network-templates.yaml b/config/manager/network-templates.yaml index fa81bdc..2ae0528 100644 --- a/config/manager/network-templates.yaml +++ b/config/manager/network-templates.yaml @@ -3,7 +3,7 @@ libvirtNetworks: libvirtTemplate: | pxe - + diff --git a/config/samples/vino_cr.yaml b/config/samples/vino_cr.yaml index 00f0636..4509a44 100644 --- a/config/samples/vino_cr.yaml +++ b/config/samples/vino_cr.yaml @@ -16,7 +16,7 @@ spec: networks: - name: vm-infra subnet: 192.168.2.0/20 - type: ipv4 + type: bridge allocationStart: 192.168.2.10 allocationStop: 192.168.2.14 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc) routes: diff --git a/config/samples/vino_cr_4_workers_1_cp.yaml b/config/samples/vino_cr_4_workers_1_cp.yaml index 0b174a6..dfe7be8 100644 --- a/config/samples/vino_cr_4_workers_1_cp.yaml +++ b/config/samples/vino_cr_4_workers_1_cp.yaml @@ -33,7 +33,6 @@ spec: networkDataTemplate: name: "test-template" namespace: "default" - bootInterfaceName: pxe networkInterfaces: - name: management type: network @@ -46,7 +45,6 @@ spec: networkDataTemplate: name: "test-template" namespace: "default" - bootInterfaceName: pxe networkInterfaces: - name: management type: network diff --git a/docs/api/vino.md b/docs/api/vino.md index f7673f0..4c554e0 100644 --- a/docs/api/vino.md +++ b/docs/api/vino.md @@ -127,16 +127,6 @@ string -managementPhysicalInterfaceName
- -string - - - - - - - pxeBootImageHost
string @@ -247,6 +237,16 @@ string +bootMACAddress
+ +string + + + + + + + interfaces
@@ -1087,14 +1087,14 @@ NamespacedName -bootInterfaceName
+rootDeviceName
string -

BootInterfaceName references the interface name in the list of NetworkInterfaces -Vino will take this interface find its mac address and use it as bootMACAddress for BMH

+

RootDeviceName is the root device for underlying VM, /dev/vda for example +default is /dev/vda

@@ -1337,17 +1337,6 @@ and place them on BMHs that correspond to this node

-managementPhysicalInterfaceName
- -string - - - -

ManagementPhysicalInterfaceName will be used to connect to libvirt network

- - - - pxeBootImageHost
string @@ -1508,17 +1497,6 @@ and place them on BMHs that correspond to this node

-managementPhysicalInterfaceName
- -string - - - -

ManagementPhysicalInterfaceName will be used to connect to libvirt network

- - - - pxeBootImageHost
string diff --git a/pkg/api/v1/vino_builder.go b/pkg/api/v1/vino_builder.go index 9b3cb8d..30b4a53 100644 --- a/pkg/api/v1/vino_builder.go +++ b/pkg/api/v1/vino_builder.go @@ -18,10 +18,9 @@ package v1 // TODO (kkalynovskyi) create an API object for this, and refactor vino-builder to read it from kubernetes. type Builder struct { - GWIPBridge string `json:"gwIPBridge,omitempty"` - ManagementPhysicalInterfaceName string `json:"managementPhysicalInterfaceName,omitempty"` - PXEBootImageHost string `json:"pxeBootImageHost,omitempty"` - PXEBootImageHostPort int `json:"pxeBootImageHostPort,omitempty"` + GWIPBridge string `json:"gwIPBridge,omitempty"` + PXEBootImageHost string `json:"pxeBootImageHost,omitempty"` + PXEBootImageHostPort int `json:"pxeBootImageHostPort,omitempty"` Networks []Network `json:"networks,omitempty"` Nodes []NodeSet `json:"nodes,omitempty"` @@ -31,15 +30,16 @@ type Builder struct { } type BuilderNetworkInterface struct { - IPAddress string `json:"ipAddress,omitempty"` - MACAddress string `json:"macAddress,omitempty"` - NetworkInterface + IPAddress string `json:"ipAddress,omitempty"` + MACAddress string `json:"macAddress,omitempty"` + NetworkInterface `json:",inline"` } // BuilderDomain represents a VINO libvirt domain type BuilderDomain struct { - Name string `json:"name,omitempty"` - Role string `json:"role,omitempty"` + Name string `json:"name,omitempty"` + Role string `json:"role,omitempty"` + BootMACAddress string `json:"bootMACAddress,omitempty"` Interfaces []BuilderNetworkInterface `json:"interfaces,omitempty"` } diff --git a/pkg/api/v1/vino_types.go b/pkg/api/v1/vino_types.go index 20ce2e3..fc15ee2 100644 --- a/pkg/api/v1/vino_types.go +++ b/pkg/api/v1/vino_types.go @@ -39,6 +39,8 @@ const ( VinoNodeNetworkValuesAnnotation = "airshipit.org/vino.network-values" // VinoNetworkDataTemplateDefaultKey expected template key networkdata template secret for vino node VinoNetworkDataTemplateDefaultKey = "template" + // VinoDefaultRootDeviceName is default root device for the underlying libvirt VM + VinoDefaultRootDeviceName = "/dev/sda" ) // Constants for BasicAuth @@ -69,8 +71,6 @@ type VinoSpec struct { // NodeLabelKeysToCopy vino controller will get these labels from k8s nodes // and place them on BMHs that correspond to this node NodeLabelKeysToCopy []string `json:"nodeLabelKeysToCopy,omitempty"` - // ManagementPhysicalInterfaceName will be used to connect to libvirt network - ManagementPhysicalInterfaceName string `json:"managementPhysicalInterfaceName,omitempty"` // PXEBootImageHost will be used to download the PXE boot image PXEBootImageHost string `json:"pxeBootImageHost,omitempty"` // PXEBootImageHostPort will be used to download the PXE boot image @@ -135,9 +135,9 @@ type NodeSet struct { DiskDrives []DiskDrivesTemplate `json:"diskDrives,omitempty"` // NetworkDataTemplate must have a template key NetworkDataTemplate NamespacedName `json:"networkDataTemplate,omitempty"` - // BootInterfaceName references the interface name in the list of NetworkInterfaces - // Vino will take this interface find its mac address and use it as bootMACAddress for BMH - BootInterfaceName string `json:"bootInterfaceName,omitempty"` + // RootDeviceName is the root device for underlying VM, /dev/vda for example + // default is /dev/vda + RootDeviceName string `json:"rootDeviceName,omitempty"` } // NamespacedName to be used to spawn VMs diff --git a/pkg/managers/bmh.go b/pkg/managers/bmh.go index ef61505..bef2f14 100644 --- a/pkg/managers/bmh.go +++ b/pkg/managers/bmh.go @@ -42,8 +42,7 @@ const ( ) type networkTemplateValues struct { - BMHName string - BootMACAddress string + BMHName string Node vinov1.NodeSet // the specific node type to be templated Networks []vinov1.Network @@ -54,9 +53,10 @@ type BMHManager struct { Namespace string client.Client - ViNO *vinov1.Vino - Ipam *ipam.Ipam - Logger logr.Logger + ViNO *vinov1.Vino + BootNetwork *vinov1.Network + Ipam *ipam.Ipam + Logger logr.Logger bmhList []*metal3.BareMetalHost networkSecrets []*corev1.Secret @@ -160,25 +160,45 @@ func (r *BMHManager) requestVMs(ctx context.Context) error { } func (r *BMHManager) createIpamNetworks(ctx context.Context, vino *vinov1.Vino) error { - for _, network := range vino.Spec.Networks { - subnetRange, err := ipam.NewRange(network.AllocationStart, network.AllocationStop) - if err != nil { - return err + // TODO (kkalynovskyi) these needs to be propagated into network template, and be configurable + // TODO (kkalynovskyi) develop generic network templates that would allow to handle all networks + // in single generic way. + // Bootnetwork needs to be handled spearately because it needs to be created by libvirt + // And have different configuration. + if r.BootNetwork == nil { + r.BootNetwork = &vinov1.Network{ + SubNet: "10.153.241.0/24", + AllocationStart: "10.153.241.2", + AllocationStop: "10.153.241.254", + Name: "pxe-boot", + MACPrefix: "52:54:00:32:00:00", } - macPrefix := network.MACPrefix - if macPrefix == "" { - r.Logger.Info("No MACPrefix provided; using default MACPrefix for network", - "default prefix", DefaultMACPrefix, "network name", network.Name) - macPrefix = DefaultMACPrefix - } - err = r.Ipam.AddSubnetRange(ctx, network.SubNet, subnetRange, macPrefix) - if err != nil { + } + networks := vino.Spec.Networks + // Append bootnetwork to be created in IPAM + networks = append(networks, *r.BootNetwork) + for _, network := range networks { + if err := r.createIpamNetwork(ctx, network); err != nil { return err } } return nil } +func (r *BMHManager) createIpamNetwork(ctx context.Context, network vinov1.Network) error { + subnetRange, err := ipam.NewRange(network.AllocationStart, network.AllocationStop) + if err != nil { + return err + } + macPrefix := network.MACPrefix + if macPrefix == "" { + r.Logger.Info("No MACPrefix provided; using default MACPrefix for network", + "default prefix", DefaultMACPrefix, "network name", network.Name) + macPrefix = DefaultMACPrefix + } + return r.Ipam.AddSubnetRange(ctx, network.SubNet, subnetRange, macPrefix) +} + func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod) error { domains := []vinov1.BuilderDomain{} @@ -223,6 +243,11 @@ func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod) error { labels[label] = value } + rootDeviceName := node.RootDeviceName + if rootDeviceName == "" { + rootDeviceName = vinov1.VinoDefaultRootDeviceName + } + credentialSecretName := r.setBMHCredentials(bmhName) bmh := &metal3.BareMetalHost{ ObjectMeta: metav1.ObjectMeta{ @@ -241,6 +266,9 @@ func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod) error { DisableCertificateVerification: true, }, BootMACAddress: domainValues.BootMACAddress, + RootDeviceHints: &metal3.RootDeviceHints{ + DeviceName: rootDeviceName, + }, }, } r.bmhList = append(r.bmhList, bmh) @@ -249,13 +277,12 @@ func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod) error { r.Logger.Info("annotating node", "node", k8sNode.Name) vinoBuilder := vinov1.Builder{ - PXEBootImageHost: r.ViNO.Spec.PXEBootImageHost, - PXEBootImageHostPort: r.ViNO.Spec.PXEBootImageHostPort, - ManagementPhysicalInterfaceName: r.ViNO.Spec.ManagementPhysicalInterfaceName, - Networks: r.ViNO.Spec.Networks, - Nodes: r.ViNO.Spec.Nodes, - CPUConfiguration: r.ViNO.Spec.CPUConfiguration, - Domains: domains, + PXEBootImageHost: r.ViNO.Spec.PXEBootImageHost, + PXEBootImageHostPort: r.ViNO.Spec.PXEBootImageHostPort, + Networks: r.ViNO.Spec.Networks, + Nodes: r.ViNO.Spec.Nodes, + CPUConfiguration: r.ViNO.Spec.CPUConfiguration, + Domains: domains, } return r.annotateNode(ctx, k8sNode, vinoBuilder) } @@ -287,8 +314,6 @@ func (r *BMHManager) domainSpecificNetValues( // Allocate an IP for each of this BMH's network interfaces domainInterfaces := []vinov1.BuilderNetworkInterface{} - - var bootMAC string for _, iface := range node.NetworkInterfaces { networkName := iface.NetworkName subnet := "" @@ -297,8 +322,7 @@ func (r *BMHManager) domainSpecificNetValues( for _, network := range networks { if network.Name == networkName { subnet = network.SubNet - subnetRange, err = ipam.NewRange(network.AllocationStart, - network.AllocationStop) + subnetRange, err = ipam.NewRange(network.AllocationStart, network.AllocationStop) if err != nil { return networkTemplateValues{}, err } @@ -318,23 +342,38 @@ func (r *BMHManager) domainSpecificNetValues( MACAddress: macAddress, NetworkInterface: iface, }) - if iface.Name == node.BootInterfaceName { - bootMAC = macAddress - } + r.Logger.Info("Got MAC and IP for the network and node", - "MAC", macAddress, "IP", ipAddress, "bmh name", bmhName, "bootMAC", bootMAC) + "MAC", macAddress, "IP", ipAddress, "bmh name", bmhName) } + // Handle bootMAC separately + bootMAC, err := r.generatePXEBootMAC(ctx, bmhName) + if err != nil { + return networkTemplateValues{}, err + } + r.Logger.Info("Got bootMAC address for BMH node", "bmh name", bmhName, "bootMAC", bootMAC) return networkTemplateValues{ - Node: node, - BMHName: bmhName, - Networks: networks, - BootMACAddress: bootMAC, + Node: node, + BMHName: bmhName, + Networks: networks, BuilderDomain: vinov1.BuilderDomain{ - Interfaces: domainInterfaces, + BootMACAddress: bootMAC, + Interfaces: domainInterfaces, }, }, nil } +func (r *BMHManager) generatePXEBootMAC(ctx context.Context, bmhName string) (string, error) { + subnetRange, err := ipam.NewRange(r.BootNetwork.AllocationStart, r.BootNetwork.AllocationStop) + if err != nil { + return "", err + } + + ipAllocatedTo := fmt.Sprintf("%s/%s", bmhName, "pxe-boot") + _, mac, err := r.Ipam.AllocateIP(ctx, r.BootNetwork.SubNet, subnetRange, ipAllocatedTo) + return mac, err +} + func (r *BMHManager) annotateNode(ctx context.Context, k8sNode *corev1.Node, vinoBuilder vinov1.Builder) error { b, err := yaml.Marshal(vinoBuilder) if err != nil {