airship/vino
Code Issues Proposed changes

Generate bootMAC and change boot network to NAT

Browse Source 
Change-Id: Id8776e8153c57682ed02f4e6534d8dfda4ed25a1
This commit is contained in:
Kostiantyn Kalynovskyi 2021-06-03 17:44:36 +00:00
parent b5b350090c
commit 5d70de0f3d
9 changed files with 110 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
config/crd/bases/airship.airshipit.org_vinoes.yaml
View File

@ -77,10 +77,6 @@ spec:
vinoBuilderImage:
type: string
type: object
managementPhysicalInterfaceName:
description: ManagementPhysicalInterfaceName will be used to connect
to libvirt network
type: string
networks:
description: Define network parameters
items:
@ -151,11 +147,6 @@ spec:
that will be created These labels will override keys from
k8s node, that are specified in vino.NodeLabelKeysToCopy
type: object
bootInterfaceName:
description: BootInterfaceName references the interface name
in the list of NetworkInterfaces Vino will take this interface
find its mac address and use it as bootMACAddress for BMH
type: string
count:
type: integer
diskDrives:
@ -216,6 +207,10 @@ spec:
type: string
type: object
type: array
rootDeviceName:
description: RootDeviceName is the root device for underlying
VM, /dev/vda for example default is /dev/vda
type: string
type: object
type: array
pxeBootImageHost:

1
config/manager/flavor-templates.yaml
View File

@ -70,6 +70,7 @@ flavorTemplates:
<interface type='network'>
<source network='pxe'/>
<mac address='{{ domain.bootMACAddress }}'/>
<model type='virtio'/>
</interface>

2
config/manager/network-templates.yaml
View File

@ -3,7 +3,7 @@ libvirtNetworks:
libvirtTemplate: |
<network>
<name>pxe</name>
<forward mode='route'/>
<forward mode='nat'/>
<bridge name='pxe' stp='off' delay='0'/>
<ip address='10.153.241.1' netmask='255.255.255.0'>
<!-- <tftp root='/srv/tftp'/> -->

2
config/samples/vino_cr.yaml
View File

@ -16,7 +16,7 @@ spec:
networks:
- name: vm-infra
subnet: 192.168.2.0/20
type: ipv4
type: bridge
allocationStart: 192.168.2.10
allocationStop: 192.168.2.14 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc)
routes:

2
config/samples/vino_cr_4_workers_1_cp.yaml
View File

@ -33,7 +33,6 @@ spec:
networkDataTemplate:
name: "test-template"
namespace: "default"
bootInterfaceName: pxe
networkInterfaces:
- name: management
type: network
@ -46,7 +45,6 @@ spec:
networkDataTemplate:
name: "test-template"
namespace: "default"
bootInterfaceName: pxe
networkInterfaces:
- name: management
type: network

48
docs/api/vino.md
View File

@ -127,16 +127,6 @@ string
</tr>
<tr>
<td>
<code>managementPhysicalInterfaceName</code><br>
<em>
string
</em>
</td>
<td>
</td>
</tr>
<tr>
<td>
<code>pxeBootImageHost</code><br>
<em>
string
@ -247,6 +237,16 @@ string
</tr>
<tr>
<td>
<code>bootMACAddress</code><br>
<em>
string
</em>
</td>
<td>
</td>
</tr>
<tr>
<td>
<code>interfaces</code><br>
<em>
<a href="#airship.airshipit.org/v1.BuilderNetworkInterface">
@ -1087,14 +1087,14 @@ NamespacedName
</tr>
<tr>
<td>
<code>bootInterfaceName</code><br>
<code>rootDeviceName</code><br>
<em>
string
</em>
</td>
<td>
<p>BootInterfaceName references the interface name in the list of NetworkInterfaces
Vino will take this interface find its mac address and use it as bootMACAddress for BMH</p>
<p>RootDeviceName is the root device for underlying VM, /dev/vda for example
default is /dev/vda</p>
</td>
</tr>
</tbody>
@ -1337,17 +1337,6 @@ and place them on BMHs that correspond to this node</p>
</tr>
<tr>
<td>
<code>managementPhysicalInterfaceName</code><br>
<em>
string
</em>
</td>
<td>
<p>ManagementPhysicalInterfaceName will be used to connect to libvirt network</p>
</td>
</tr>
<tr>
<td>
<code>pxeBootImageHost</code><br>
<em>
string
@ -1508,17 +1497,6 @@ and place them on BMHs that correspond to this node</p>
</tr>
<tr>
<td>
<code>managementPhysicalInterfaceName</code><br>
<em>
string
</em>
</td>
<td>
<p>ManagementPhysicalInterfaceName will be used to connect to libvirt network</p>
</td>
</tr>
<tr>
<td>
<code>pxeBootImageHost</code><br>
<em>
string

18
pkg/api/v1/vino_builder.go
View File

@ -18,10 +18,9 @@ package v1
// TODO (kkalynovskyi) create an API object for this, and refactor vino-builder to read it from kubernetes.
type Builder struct {
GWIPBridge string `json:"gwIPBridge,omitempty"`
ManagementPhysicalInterfaceName string `json:"managementPhysicalInterfaceName,omitempty"`
PXEBootImageHost string `json:"pxeBootImageHost,omitempty"`
PXEBootImageHostPort int `json:"pxeBootImageHostPort,omitempty"`
GWIPBridge string `json:"gwIPBridge,omitempty"`
PXEBootImageHost string `json:"pxeBootImageHost,omitempty"`
PXEBootImageHostPort int `json:"pxeBootImageHostPort,omitempty"`
Networks []Network `json:"networks,omitempty"`
Nodes []NodeSet `json:"nodes,omitempty"`
@ -31,15 +30,16 @@ type Builder struct {
}
type BuilderNetworkInterface struct {
IPAddress string `json:"ipAddress,omitempty"`
MACAddress string `json:"macAddress,omitempty"`
NetworkInterface
IPAddress string `json:"ipAddress,omitempty"`
MACAddress string `json:"macAddress,omitempty"`
NetworkInterface `json:",inline"`
}
// BuilderDomain represents a VINO libvirt domain
type BuilderDomain struct {
Name string `json:"name,omitempty"`
Role string `json:"role,omitempty"`
Name string `json:"name,omitempty"`
Role string `json:"role,omitempty"`
BootMACAddress string `json:"bootMACAddress,omitempty"`
Interfaces []BuilderNetworkInterface `json:"interfaces,omitempty"`
}

10
pkg/api/v1/vino_types.go
View File

@ -39,6 +39,8 @@ const (
VinoNodeNetworkValuesAnnotation = "airshipit.org/vino.network-values"
// VinoNetworkDataTemplateDefaultKey expected template key networkdata template secret for vino node
VinoNetworkDataTemplateDefaultKey = "template"
// VinoDefaultRootDeviceName is default root device for the underlying libvirt VM
VinoDefaultRootDeviceName = "/dev/sda"
)
// Constants for BasicAuth
@ -69,8 +71,6 @@ type VinoSpec struct {
// NodeLabelKeysToCopy vino controller will get these labels from k8s nodes
// and place them on BMHs that correspond to this node
NodeLabelKeysToCopy []string `json:"nodeLabelKeysToCopy,omitempty"`
// ManagementPhysicalInterfaceName will be used to connect to libvirt network
ManagementPhysicalInterfaceName string `json:"managementPhysicalInterfaceName,omitempty"`
// PXEBootImageHost will be used to download the PXE boot image
PXEBootImageHost string `json:"pxeBootImageHost,omitempty"`
// PXEBootImageHostPort will be used to download the PXE boot image
@ -135,9 +135,9 @@ type NodeSet struct {
DiskDrives []DiskDrivesTemplate `json:"diskDrives,omitempty"`
// NetworkDataTemplate must have a template key
NetworkDataTemplate NamespacedName `json:"networkDataTemplate,omitempty"`
// BootInterfaceName references the interface name in the list of NetworkInterfaces
// Vino will take this interface find its mac address and use it as bootMACAddress for BMH
BootInterfaceName string `json:"bootInterfaceName,omitempty"`
// RootDeviceName is the root device for underlying VM, /dev/vda for example
// default is /dev/vda
RootDeviceName string `json:"rootDeviceName,omitempty"`
}
// NamespacedName to be used to spawn VMs

113
pkg/managers/bmh.go
View File

@ -42,8 +42,7 @@ const (
)
type networkTemplateValues struct {
BMHName string
BootMACAddress string
BMHName string
Node vinov1.NodeSet // the specific node type to be templated
Networks []vinov1.Network
@ -54,9 +53,10 @@ type BMHManager struct {
Namespace string
client.Client
ViNO *vinov1.Vino
Ipam *ipam.Ipam
Logger logr.Logger
ViNO *vinov1.Vino
BootNetwork *vinov1.Network
Ipam *ipam.Ipam
Logger logr.Logger
bmhList []*metal3.BareMetalHost
networkSecrets []*corev1.Secret
@ -160,25 +160,45 @@ func (r *BMHManager) requestVMs(ctx context.Context) error {
}
func (r *BMHManager) createIpamNetworks(ctx context.Context, vino *vinov1.Vino) error {
for _, network := range vino.Spec.Networks {
subnetRange, err := ipam.NewRange(network.AllocationStart, network.AllocationStop)
if err != nil {
return err
// TODO (kkalynovskyi) these needs to be propagated into network template, and be configurable
// TODO (kkalynovskyi) develop generic network templates that would allow to handle all networks
// in single generic way.
// Bootnetwork needs to be handled spearately because it needs to be created by libvirt
// And have different configuration.
if r.BootNetwork == nil {
r.BootNetwork = &vinov1.Network{
SubNet: "10.153.241.0/24",
AllocationStart: "10.153.241.2",
AllocationStop: "10.153.241.254",
Name: "pxe-boot",
MACPrefix: "52:54:00:32:00:00",
}
macPrefix := network.MACPrefix
if macPrefix == "" {
r.Logger.Info("No MACPrefix provided; using default MACPrefix for network",
"default prefix", DefaultMACPrefix, "network name", network.Name)
macPrefix = DefaultMACPrefix
}
err = r.Ipam.AddSubnetRange(ctx, network.SubNet, subnetRange, macPrefix)
if err != nil {
}
networks := vino.Spec.Networks
// Append bootnetwork to be created in IPAM
networks = append(networks, *r.BootNetwork)
for _, network := range networks {
if err := r.createIpamNetwork(ctx, network); err != nil {
return err
}
}
return nil
}
func (r *BMHManager) createIpamNetwork(ctx context.Context, network vinov1.Network) error {
subnetRange, err := ipam.NewRange(network.AllocationStart, network.AllocationStop)
if err != nil {
return err
}
macPrefix := network.MACPrefix
if macPrefix == "" {
r.Logger.Info("No MACPrefix provided; using default MACPrefix for network",
"default prefix", DefaultMACPrefix, "network name", network.Name)
macPrefix = DefaultMACPrefix
}
return r.Ipam.AddSubnetRange(ctx, network.SubNet, subnetRange, macPrefix)
}
func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod) error {
domains := []vinov1.BuilderDomain{}
@ -223,6 +243,11 @@ func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod) error {
labels[label] = value
}
rootDeviceName := node.RootDeviceName
if rootDeviceName == "" {
rootDeviceName = vinov1.VinoDefaultRootDeviceName
}
credentialSecretName := r.setBMHCredentials(bmhName)
bmh := &metal3.BareMetalHost{
ObjectMeta: metav1.ObjectMeta{
@ -241,6 +266,9 @@ func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod) error {
DisableCertificateVerification: true,
},
BootMACAddress: domainValues.BootMACAddress,
RootDeviceHints: &metal3.RootDeviceHints{
DeviceName: rootDeviceName,
},
},
}
r.bmhList = append(r.bmhList, bmh)
@ -249,13 +277,12 @@ func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod) error {
r.Logger.Info("annotating node", "node", k8sNode.Name)
vinoBuilder := vinov1.Builder{
PXEBootImageHost: r.ViNO.Spec.PXEBootImageHost,
PXEBootImageHostPort: r.ViNO.Spec.PXEBootImageHostPort,
ManagementPhysicalInterfaceName: r.ViNO.Spec.ManagementPhysicalInterfaceName,
Networks: r.ViNO.Spec.Networks,
Nodes: r.ViNO.Spec.Nodes,
CPUConfiguration: r.ViNO.Spec.CPUConfiguration,
Domains: domains,
PXEBootImageHost: r.ViNO.Spec.PXEBootImageHost,
PXEBootImageHostPort: r.ViNO.Spec.PXEBootImageHostPort,
Networks: r.ViNO.Spec.Networks,
Nodes: r.ViNO.Spec.Nodes,
CPUConfiguration: r.ViNO.Spec.CPUConfiguration,
Domains: domains,
}
return r.annotateNode(ctx, k8sNode, vinoBuilder)
}
@ -287,8 +314,6 @@ func (r *BMHManager) domainSpecificNetValues(
// Allocate an IP for each of this BMH's network interfaces
domainInterfaces := []vinov1.BuilderNetworkInterface{}
var bootMAC string
for _, iface := range node.NetworkInterfaces {
networkName := iface.NetworkName
subnet := ""
@ -297,8 +322,7 @@ func (r *BMHManager) domainSpecificNetValues(
for _, network := range networks {
if network.Name == networkName {
subnet = network.SubNet
subnetRange, err = ipam.NewRange(network.AllocationStart,
network.AllocationStop)
subnetRange, err = ipam.NewRange(network.AllocationStart, network.AllocationStop)
if err != nil {
return networkTemplateValues{}, err
}
@ -318,23 +342,38 @@ func (r *BMHManager) domainSpecificNetValues(
MACAddress: macAddress,
NetworkInterface: iface,
})
if iface.Name == node.BootInterfaceName {
bootMAC = macAddress
}
r.Logger.Info("Got MAC and IP for the network and node",
"MAC", macAddress, "IP", ipAddress, "bmh name", bmhName, "bootMAC", bootMAC)
"MAC", macAddress, "IP", ipAddress, "bmh name", bmhName)
}
// Handle bootMAC separately
bootMAC, err := r.generatePXEBootMAC(ctx, bmhName)
if err != nil {
return networkTemplateValues{}, err
}
r.Logger.Info("Got bootMAC address for BMH node", "bmh name", bmhName, "bootMAC", bootMAC)
return networkTemplateValues{
Node: node,
BMHName: bmhName,
Networks: networks,
BootMACAddress: bootMAC,
Node: node,
BMHName: bmhName,
Networks: networks,
BuilderDomain: vinov1.BuilderDomain{
Interfaces: domainInterfaces,
BootMACAddress: bootMAC,
Interfaces: domainInterfaces,
},
}, nil
}
func (r *BMHManager) generatePXEBootMAC(ctx context.Context, bmhName string) (string, error) {
subnetRange, err := ipam.NewRange(r.BootNetwork.AllocationStart, r.BootNetwork.AllocationStop)
if err != nil {
return "", err
}
ipAllocatedTo := fmt.Sprintf("%s/%s", bmhName, "pxe-boot")
_, mac, err := r.Ipam.AllocateIP(ctx, r.BootNetwork.SubNet, subnetRange, ipAllocatedTo)
return mac, err
}
func (r *BMHManager) annotateNode(ctx context.Context, k8sNode *corev1.Node, vinoBuilder vinov1.Builder) error {
b, err := yaml.Marshal(vinoBuilder)
if err != nil {