airship/vino
Code Issues Proposed changes

Generate unique subranges for k8s nodes

Browse Source 
The ranges will be stored in IPPool object and will never change
once assigned.

Change-Id: Ie3184f2a9de405c00223367939f8c3e3112f0e68
This commit is contained in:
Kostiantyn Kalynovskyi 2021-06-14 22:51:46 +00:00
parent 9f95cbe4cb
commit 7602aa5324
12 changed files with 300 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
config/crd/bases/airship.airshipit.org_ippools.yaml
View File

@ -55,6 +55,21 @@ spec:
- mac
type: object
type: array
allocatedRanges:
items:
properties:
allocatedTo:
type: string
start:
type: string
stop:
type: string
required:
- allocatedTo
- start
- stop
type: object
type: array
macPrefix:
description: MACPrefix defines the MAC prefix to use for VM mac addresses
type: string

20
config/crd/bases/airship.airshipit.org_vinoes.yaml
View File

@ -82,20 +82,24 @@ spec:
items:
description: Network defines libvirt networks
properties:
allocationStart:
type: string
allocationStop:
type: string
bridgeName:
description: BridgeName is the name of the bridge to be created
as libvirt network. works if AllocateNodeIP is sepcified
type: string
dhcpAllocationStart:
description: DHCPAllocationStart must be inside the SubNet range
type: string
dhcpAllocationStop:
description: DHCPAllocationStop must be inside the SubNet range
type: string
dns_servers:
items:
type: string
type: array
instanceSubnet:
type: string
instanceSubnetBitStep:
description: InstanceSubnetBitStep indicates how many bites
to allocate for each node DHCP range
type: integer
libvirtTemplate:
description: LibvirtTemplate identifies which libvirt template
to be used to create a network
@ -126,6 +130,10 @@ spec:
type: string
type: object
type: array
staticAllocationStart:
type: string
staticAllocationStop:
type: string
subnet:
type: string
type:

4
config/manager/network-templates.yaml
View File

@ -5,10 +5,10 @@ libvirtNetworks:
<name>{{ network.name }}</name>
<forward mode='route'/>
<bridge name='{{ network.bridgeName | default('vm-infra-bridge') }}' stp='off' delay='0'/>
<ip address='{{ network.bridgeIP | default(omit) }}' netmask='{{ ipam.bridge_subnet_netmask }}'>
<ip address='{{ network.bridgeIP | default(omit) }}' netmask='{{ network.subnet | ansible.netcommon.ipaddr('netmask') }}'>
<!-- <tftp root='/srv/tftp'/> -->
<dhcp>
<range start='{{ ipam.instance_ips[0] }}' end='{{ ipam.instance_ips[-1] }}'/>
<range start='{{ network.range.start }}' end='{{ network.range.stop }}'/>
<bootp file='http://{{ pxeBootImageHost | default(ansible_default_ipv4.address) }}:{{ pxeBootImageHostPort | default(80) }}/dualboot.ipxe'/>
</dhcp>
</ip>

8
config/samples/vino_cr.yaml
View File

@ -16,10 +16,12 @@ spec:
- name: management
libvirtTemplate: management
subnet: 192.168.2.0/20
instanceSubnet: 192.168.4.0/22
dhcpAllocationStart: 192.168.4.0
dhcpAllocationStop: 192.168.7.255
instanceSubnetBitStep: 4
type: bridge
allocationStart: 192.168.2.10
allocationStop: 192.168.2.24 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc)
staticAllocationStart: 192.168.2.10
staticAllocationStop: 192.168.2.24
dns_servers: ["135.188.34.124"]
macPrefix: "52:54:00:06:00:00"
physicalInterface: enp3s7

12
config/samples/vino_cr_4_workers_1_cp.yaml
View File

@ -16,14 +16,16 @@ spec:
- name: management
libvirtTemplate: management
subnet: 192.168.2.0/20
instanceSubnet: 192.168.4.0/22
type: ipv4
allocationStart: 192.168.2.10
allocationStop: 192.168.2.25 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc)
dhcpAllocationStart: 192.168.4.0
dhcpAllocationStop: 192.168.7.255
instanceSubnetBitStep: 6
type: bridge
staticAllocationStart: 192.168.2.10
staticAllocationStop: 192.168.2.24
routes:
- network: 10.0.0.0
netmask: 255.255.255.0
gateway: $vinobridge # vino will need to populate this from the nodelabel value `airshipit.org/vino.nodebridgegw`
gateway: $vinobridge
dns_servers: ["135.188.34.124"]
macPrefix: "52:54:00:06:00:00"
physicalInterface: enp3s7

99
docs/api/vino.md
View File

@ -60,6 +60,51 @@ string
</table>
</div>
</div>
<h3 id="airship.airshipit.org/v1.AllocatedRange">AllocatedRange
</h3>
<p>
(<em>Appears on:</em>
<a href="#airship.airshipit.org/v1.IPPoolSpec">IPPoolSpec</a>)
</p>
<div class="md-typeset__scrollwrap">
<div class="md-typeset__table">
<table>
<thead>
<tr>
<th>Field</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<code>allocatedTo</code><br>
<em>
string
</em>
</td>
<td>
</td>
</tr>
<tr>
<td>
<code>Range</code><br>
<em>
<a href="#airship.airshipit.org/v1.Range">
Range
</a>
</em>
</td>
<td>
<p>
(Members of <code>Range</code> are embedded into this type.)
</p>
</td>
</tr>
</tbody>
</table>
</div>
</div>
<h3 id="airship.airshipit.org/v1.BMCCredentials">BMCCredentials
</h3>
<p>
@ -685,6 +730,18 @@ string
</tr>
<tr>
<td>
<code>allocatedRanges</code><br>
<em>
<a href="#airship.airshipit.org/v1.AllocatedRange">
[]AllocatedRange
</a>
</em>
</td>
<td>
</td>
</tr>
<tr>
<td>
<code>allocatedIPs</code><br>
<em>
<a href="#airship.airshipit.org/v1.AllocatedIP">
@ -781,6 +838,18 @@ string
</tr>
<tr>
<td>
<code>allocatedRanges</code><br>
<em>
<a href="#airship.airshipit.org/v1.AllocatedRange">
[]AllocatedRange
</a>
</em>
</td>
<td>
</td>
</tr>
<tr>
<td>
<code>allocatedIPs</code><br>
<em>
<a href="#airship.airshipit.org/v1.AllocatedIP">
@ -908,12 +977,35 @@ string
</tr>
<tr>
<td>
<code>instanceSubnet</code><br>
<code>dhcpAllocationStart</code><br>
<em>
string
</em>
</td>
<td>
<p>DHCPAllocationStart must be inside the SubNet range</p>
</td>
</tr>
<tr>
<td>
<code>dhcpAllocationStop</code><br>
<em>
string
</em>
</td>
<td>
<p>DHCPAllocationStop must be inside the SubNet range</p>
</td>
</tr>
<tr>
<td>
<code>instanceSubnetBitStep</code><br>
<em>
int
</em>
</td>
<td>
<p>InstanceSubnetBitStep indicates how many bites to allocate for each node DHCP range</p>
</td>
</tr>
<tr>
@ -928,7 +1020,7 @@ string
</tr>
<tr>
<td>
<code>allocationStart</code><br>
<code>staticAllocationStart</code><br>
<em>
string
</em>
@ -938,7 +1030,7 @@ string
</tr>
<tr>
<td>
<code>allocationStop</code><br>
<code>staticAllocationStop</code><br>
<em>
string
</em>
@ -1268,6 +1360,7 @@ bool
</h3>
<p>
(<em>Appears on:</em>
<a href="#airship.airshipit.org/v1.AllocatedRange">AllocatedRange</a>,
<a href="#airship.airshipit.org/v1.BuilderNetwork">BuilderNetwork</a>,
<a href="#airship.airshipit.org/v1.IPPoolSpec">IPPoolSpec</a>)
</p>

12
pkg/api/v1/ippool_types.go
View File

@ -26,9 +26,10 @@ import (
// within the subnet from which IPs can be allocated by IPAM,
// and a set of IPs that are currently allocated already.
type IPPoolSpec struct {
Subnet string `json:"subnet"`
Ranges []Range `json:"ranges"`
AllocatedIPs []AllocatedIP `json:"allocatedIPs"`
Subnet string `json:"subnet"`
Ranges []Range `json:"ranges"`
AllocatedRanges []AllocatedRange `json:"allocatedRanges,omitempty"`
AllocatedIPs []AllocatedIP `json:"allocatedIPs"`
// MACPrefix defines the MAC prefix to use for VM mac addresses
MACPrefix string `json:"macPrefix"`
// NextMAC indicates the next MAC address (in sequence) that
@ -43,6 +44,11 @@ type AllocatedIP struct {
AllocatedTo string `json:"allocatedTo"`
}
type AllocatedRange struct {
AllocatedTo string `json:"allocatedTo"`
Range `json:",inline"`
}
// Range has (inclusive) bounds within a subnet from which IPs can be allocated
type Range struct {
Start string `json:"start"`

23
pkg/api/v1/vino_types.go
View File

@ -41,6 +41,8 @@ const (
VinoNetworkDataTemplateDefaultKey = "template"
// VinoDefaultRootDeviceName is default root device for the underlying libvirt VM
VinoDefaultRootDeviceName = "/dev/vda"
// VinoDefaultInstanceSubnetBitStep is the value for InstanceSubnetBitStep
VinoDefaultInstanceSubnetBitStep = 4
)
// Constants for BasicAuth
@ -97,14 +99,19 @@ type CPUConfiguration struct {
// Network defines libvirt networks
type Network struct {
//Network Parameter defined
Name string `json:"name,omitempty"`
SubNet string `json:"subnet,omitempty"`
InstanceSubnet string `json:"instanceSubnet,omitempty"`
Type string `json:"type,omitempty"`
AllocationStart string `json:"allocationStart,omitempty"`
AllocationStop string `json:"allocationStop,omitempty"`
DNSServers []string `json:"dns_servers,omitempty"`
Routes []VMRoutes `json:"routes,omitempty"`
Name string `json:"name,omitempty"`
SubNet string `json:"subnet,omitempty"`
// DHCPAllocationStart must be inside the SubNet range
DHCPAllocationStart string `json:"dhcpAllocationStart,omitempty"`
// DHCPAllocationStop must be inside the SubNet range
DHCPAllocationStop string `json:"dhcpAllocationStop,omitempty"`
// InstanceSubnetBitStep indicates how many bites to allocate for each node DHCP range
InstanceSubnetBitStep int `json:"instanceSubnetBitStep,omitempty"`
Type string `json:"type,omitempty"`
StaticAllocationStart string `json:"staticAllocationStart,omitempty"`
StaticAllocationStop string `json:"staticAllocationStop,omitempty"`
DNSServers []string `json:"dns_servers,omitempty"`
Routes []VMRoutes `json:"routes,omitempty"`
// MACPrefix defines the zero-padded MAC prefix to use for
// VM mac addresses, and is the first address that will be
// allocated sequentially to VMs in this network.

21
pkg/api/v1/zz_generated.deepcopy.go
View File

@ -40,6 +40,22 @@ func (in *AllocatedIP) DeepCopy() *AllocatedIP {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *AllocatedRange) DeepCopyInto(out *AllocatedRange) {
*out = *in
out.Range = in.Range
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllocatedRange.
func (in *AllocatedRange) DeepCopy() *AllocatedRange {
if in == nil {
return nil
}
out := new(AllocatedRange)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *BMCCredentials) DeepCopyInto(out *BMCCredentials) {
*out = *in
@ -273,6 +289,11 @@ func (in *IPPoolSpec) DeepCopyInto(out *IPPoolSpec) {
*out = make([]Range, len(*in))
copy(*out, *in)
}
if in.AllocatedRanges != nil {
in, out := &in.AllocatedRanges, &out.AllocatedRanges
*out = make([]AllocatedRange, len(*in))
copy(*out, *in)
}
if in.AllocatedIPs != nil {
in, out := &in.AllocatedIPs, &out.AllocatedIPs
*out = make([]AllocatedIP, len(*in))

98
pkg/ipam/ipam.go
View File

@ -17,6 +17,7 @@ package ipam
import (
"context"
"fmt"
"math"
"net"
"regexp"
"strings"
@ -378,3 +379,100 @@ func (i *Ipam) getIPPools(ctx context.Context) (map[string]*vinov1.IPPoolSpec, e
}
return ippools, nil
}
func (i *Ipam) AllocateRange(ctx context.Context,
bitStep int,
host, macPrefix, start, stop, subnet string) (vinov1.Range, error) {
ipPool, err := i.getIPPoolWithRanges(ctx, bitStep, macPrefix, start, stop, subnet)
if err != nil {
return vinov1.Range{}, err
}
result, err := chooseRange(host, ipPool)
if err != nil {
return vinov1.Range{}, err
}
return result, i.applyIPPool(ctx, *ipPool)
}
func chooseRange(host string, ipPool *vinov1.IPPoolSpec) (vinov1.Range, error) {
const unallocated = -1
freeIndex := unallocated
for i, r := range ipPool.AllocatedRanges {
if r.AllocatedTo == host {
return r.Range, nil
} else if r.AllocatedTo == "" && freeIndex == unallocated {
freeIndex = i
}
}
if freeIndex != unallocated {
ipPool.AllocatedRanges[freeIndex].AllocatedTo = host
} else {
return vinov1.Range{}, fmt.Errorf("No free ranges available for host %s", host)
}
return ipPool.AllocatedRanges[freeIndex].Range, nil
}
func (i *Ipam) getIPPoolWithRanges(ctx context.Context, bitStep int,
macPrefix, start, stop, subnet string) (*vinov1.IPPoolSpec, error) {
ippools, err := i.getIPPools(ctx)
if err != nil {
return &vinov1.IPPoolSpec{}, err
}
logger := i.Log.WithValues("subnet", subnet)
ippool, exists := ippools[subnet]
if !exists {
logger.Info("IPAM creating subnet")
_, err = macStringToInt(macPrefix) // mac format validation
if err != nil {
return &vinov1.IPPoolSpec{}, err
}
ippool = &vinov1.IPPoolSpec{
Subnet: subnet,
Ranges: []vinov1.Range{},
AllocatedIPs: []vinov1.AllocatedIP{},
MACPrefix: macPrefix,
NextMAC: macPrefix,
}
ippools[subnet] = ippool
}
if len(ippool.AllocatedRanges) != 0 {
return ippool, nil
}
ranges, err := generateRanges(start, stop, bitStep)
if err != nil {
return nil, err
}
ippool.AllocatedRanges = ranges
return ippool, nil
}
func generateRanges(start, stop string, bitStep int) ([]vinov1.AllocatedRange, error) {
firstNetIPInt, err := ipStringToInt(start)
if err != nil {
return nil, err
}
// support only IPv4, use 32 netmask
subnetEnd, err := ipStringToInt(stop)
if err != nil {
return nil, err
}
ranges := []vinov1.AllocatedRange{}
shift := uint64(math.Pow(2, float64(bitStep)))
for start, end := firstNetIPInt, firstNetIPInt+shift; end-1 <= subnetEnd; {
fmt.Printf("start is %s, end is %s\n", intToIPv4String(start), intToIPv4String(subnetEnd))
ranges = append(ranges, vinov1.AllocatedRange{
Range: vinov1.Range{
Start: intToIPv4String(start),
Stop: intToIPv4String(end - 1),
},
})
start += shift
end += shift
}
return ranges, nil
}

21
pkg/managers/bmh.go
View File

@ -169,7 +169,7 @@ func (r *BMHManager) createIpamNetworks(ctx context.Context, vino *vinov1.Vino)
}
func (r *BMHManager) createIpamNetwork(ctx context.Context, network vinov1.Network) error {
subnetRange, err := ipam.NewRange(network.AllocationStart, network.AllocationStop)
subnetRange, err := ipam.NewRange(network.StaticAllocationStart, network.StaticAllocationStop)
if err != nil {
return err
}
@ -295,6 +295,21 @@ func (r *BMHManager) nodeNetworks(ctx context.Context,
builderNetwork.Network.Routes[routeIndex].Gateway = bridgeIP
}
}
bitStep := network.InstanceSubnetBitStep
if bitStep == 0 {
bitStep = vinov1.VinoDefaultInstanceSubnetBitStep
}
r, err := r.Ipam.AllocateRange(ctx,
bitStep,
k8sNode.Name,
network.MACPrefix,
network.DHCPAllocationStart,
network.DHCPAllocationStop,
network.SubNet)
if err != nil {
return []vinov1.BuilderNetwork{}, err
}
builderNetwork.Range = r
builderNetworks = append(builderNetworks, builderNetwork)
}
return builderNetworks, nil
@ -316,7 +331,7 @@ func (r *BMHManager) domainSpecificNetValues(
for _, network := range networks {
if network.Name == networkName {
subnet = network.SubNet
subnetRange, err = ipam.NewRange(network.AllocationStart, network.AllocationStop)
subnetRange, err = ipam.NewRange(network.StaticAllocationStart, network.StaticAllocationStop)
if err != nil {
return networkTemplateValues{}, err
}
@ -381,7 +396,7 @@ func (r *BMHManager) annotateNode(ctx context.Context, k8sNode *corev1.Node, vin
func (r *BMHManager) getBridgeIPandMAC(ctx context.Context,
network vinov1.Network,
k8sNode *corev1.Node) (string, string, error) {
subnetRange, err := ipam.NewRange(network.AllocationStart, network.AllocationStop)
subnetRange, err := ipam.NewRange(network.StaticAllocationStart, network.StaticAllocationStop)
if err != nil {
return "", "", err
}

5
vino-builder/assets/playbooks/vino-builder.yaml
View File

@ -39,11 +39,6 @@
- hosts: localhost
tasks:
# generate libvirt definitions for storage, networks, and domains
- name: generate management network ip addresses
include_role:
name: ipam
# generate libvirt definitions for storage, networks, and domains
- name: process libvirt definitions
include_role: