From 55b479b54f8cd064144ba8d1e2e5be33b6a975c8 Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Mon, 15 Jan 2018 14:44:10 +1100
Subject: [PATCH] GPT partitioning support
This adds support for a GPT label type to the partitioning code. This
is relatively straight-forward translation of the partition config
into a sgparted command-line and subsequent call.
A unit test is added based on a working GPT/EFI configuration and the
fedora-minimal functional test is updated to build a single-partition
GPT based using the new block-device-gpt override element. See notes
in the sample configuration files about partition requirements and
types.
Documentation has been updated.
Co-Authored-By: Marcin Juszkiewicz <marcin.juszkiewicz@linaro.org>
Change-Id: I6b819a8071389e7e4eb4874ff7750bd192695ff2
---
.../block_device/level1/partition.py | 11 +-
.../block_device/level1/partitioning.py | 114 +++++++++++++-----
.../block_device/tests/config/gpt_efi.yaml | 32 +++++
.../block_device/tests/test_gpt.py | 85 +++++++++++++
.../elements/block-device-gpt/README.rst | 11 ++
.../block-device-default.yaml | 22 ++++
.../test-elements/build-succeeds/element-deps | 4 +-
doc/source/user_guide/building_an_image.rst | 60 ++++++---
.../bootloader-gpt-d1047f81f3a0631b.yaml | 7 ++
tests/install_test_deps.sh | 8 ++
10 files changed, 308 insertions(+), 46 deletions(-)
create mode 100644 diskimage_builder/block_device/tests/config/gpt_efi.yaml
create mode 100644 diskimage_builder/block_device/tests/test_gpt.py
create mode 100644 diskimage_builder/elements/block-device-gpt/README.rst
create mode 100644 diskimage_builder/elements/block-device-gpt/block-device-default.yaml
create mode 100644 releasenotes/notes/bootloader-gpt-d1047f81f3a0631b.yaml
diff --git a/diskimage_builder/block_device/level1/partition.py b/diskimage_builder/block_device/level1/partition.py
index e91e1bf98..183778607 100644
--- a/diskimage_builder/block_device/level1/partition.py
+++ b/diskimage_builder/block_device/level1/partition.py
@@ -33,6 +33,12 @@ class PartitionNode(NodeBase):
self.partitioning = parent
self.prev_partition = prev_partition
+ # filter out some MBR only options for clarity
+ if self.partitioning.label == 'gpt':
+ if 'flags' in config and 'primary' in config['flags']:
+ raise BlockDeviceSetupException(
+ "Primary flag not supported for GPT partitions")
+
self.flags = set()
if 'flags' in config:
for f in config['flags']:
@@ -47,7 +53,10 @@ class PartitionNode(NodeBase):
raise BlockDeviceSetupException("No size in partition" % self.name)
self.size = config['size']
- self.ptype = int(config['type'], 16) if 'type' in config else 0x83
+ if self.partitioning.label == 'gpt':
+ self.ptype = str(config['type']) if 'type' in config else '8300'
+ elif self.partitioning.label == 'mbr':
+ self.ptype = int(config['type'], 16) if 'type' in config else 83
def get_flags(self):
return self.flags
diff --git a/diskimage_builder/block_device/level1/partitioning.py b/diskimage_builder/block_device/level1/partitioning.py
index e8a559867..bc2a728ef 100644
--- a/diskimage_builder/block_device/level1/partitioning.py
+++ b/diskimage_builder/block_device/level1/partitioning.py
@@ -58,8 +58,8 @@ class Partitioning(PluginBase):
raise BlockDeviceSetupException(
"Partitioning config needs 'label'")
self.label = config['label']
- if self.label not in ("mbr", ):
- raise BlockDeviceSetupException("Label must be 'mbr'")
+ if self.label not in ("mbr", "gpt"):
+ raise BlockDeviceSetupException("Label must be 'mbr' or 'gpt'")
# It is VERY important to get the alignment correct. If this
# is not correct, the disk performance might be very poor.
@@ -93,29 +93,9 @@ class Partitioning(PluginBase):
fd.seek(0, 2)
return fd.tell()
- # not this is NOT a node and this is not called directly! The
- # create() calls in the partition nodes this plugin has
- # created are calling back into this.
- def create(self):
- # This is a bit of a hack. Each of the partitions is actually
- # in the graph, so for every partition we get a create() call
- # as the walk happens. But we only need to create the
- # partition table once...
- if self.already_created:
- logger.info("Not creating the partitions a second time.")
- return
- self.already_created = True
-
- # the raw file on disk
- image_path = self.state['blockdev'][self.base]['image']
- # the /dev/loopX device of the parent
- device_path = self.state['blockdev'][self.base]['device']
- logger.info("Creating partition on [%s] [%s]", self.base, image_path)
-
- assert self.label == 'mbr'
-
- disk_size = self._size_of_block_dev(image_path)
- with MBR(image_path, disk_size, self.align) as part_impl:
+ def _create_mbr(self):
+ """Create partitions with MBR"""
+ with MBR(self.image_path, self.disk_size, self.align) as part_impl:
for part_cfg in self.partitions:
part_name = part_cfg.get_name()
part_bootflag = PartitionNode.flag_boot \
@@ -137,24 +117,100 @@ class Partitioning(PluginBase):
# We're going to mount all partitions with kpartx
# below once we're done. So the device this partition
# will be seen at becomes "/dev/mapper/loop0pX"
- assert device_path[:5] == "/dev/"
+ assert self.device_path[:5] == "/dev/"
partition_device_name = "/dev/mapper/%sp%d" % \
- (device_path[5:], part_no)
+ (self.device_path[5:], part_no)
self.state['blockdev'][part_name] \
= {'device': partition_device_name}
+ def _create_gpt(self):
+ """Create partitions with GPT"""
+
+ cmd = ['sgdisk', self.image_path]
+
+ # This padding gives us a little room for rounding so we don't
+ # go over the end of the disk
+ disk_free = self.disk_size - (2048 * 1024)
+ pnum = 1
+
+ for p in self.partitions:
+ args = {}
+ args['pnum'] = pnum
+ args['name'] = '"%s"' % p.get_name()
+ args['type'] = '%s' % p.get_type()
+
+ # convert from a relative/string size to bytes
+ size = parse_rel_size_spec(p.get_size(), disk_free)[1]
+
+ # We keep track in bytes, but specify things to sgdisk in
+ # megabytes so it can align on sensible boundaries. And
+ # create partitions right after previous so no need to
+ # calculate start/end - just size.
+ assert size <= disk_free
+ args['size'] = size // (1024 * 1024)
+
+ new_cmd = ("-n {pnum}:0:+{size}M -t {pnum}:{type} "
+ "-c {pnum}:{name}".format(**args))
+ cmd.extend(new_cmd.strip().split(' '))
+
+ # Fill the state; we mount all partitions with kpartx
+ # below once we're done. So the device this partition
+ # will be seen at becomes "/dev/mapper/loop0pX"
+ assert self.device_path[:5] == "/dev/"
+ device_name = "/dev/mapper/%sp%d" % (self.device_path[5:], pnum)
+ self.state['blockdev'][p.get_name()] \
+ = {'device': device_name}
+
+ disk_free = disk_free - size
+ pnum = pnum + 1
+ logger.debug("Partition %s added, %s remaining in disk",
+ pnum, disk_free)
+
+ logger.debug("cmd: %s", ' '.join(cmd))
+ exec_sudo(cmd)
+
+ # not this is NOT a node and this is not called directly! The
+ # create() calls in the partition nodes this plugin has
+ # created are calling back into this.
+ def create(self):
+ # This is a bit of a hack. Each of the partitions is actually
+ # in the graph, so for every partition we get a create() call
+ # as the walk happens. But we only need to create the
+ # partition table once...
+ if self.already_created:
+ logger.info("Not creating the partitions a second time.")
+ return
+ self.already_created = True
+
+ # the raw file on disk
+ self.image_path = self.state['blockdev'][self.base]['image']
+ # the /dev/loopX device of the parent
+ self.device_path = self.state['blockdev'][self.base]['device']
+ # underlying size
+ self.disk_size = self._size_of_block_dev(self.image_path)
+
+ logger.info("Creating partition on [%s] [%s]",
+ self.base, self.image_path)
+
+ assert self.label in ('mbr', 'gpt')
+
+ if self.label == 'mbr':
+ self._create_mbr()
+ elif self.label == 'gpt':
+ self._create_gpt()
+
# "saftey sync" to make sure the partitions are written
exec_sudo(["sync"])
# now all the partitions are created, get device-mapper to
# mount them
if not os.path.exists("/.dockerenv"):
- exec_sudo(["kpartx", "-avs", device_path])
+ exec_sudo(["kpartx", "-avs", self.device_path])
else:
# If running inside Docker, make our nodes manually,
# because udev will not be working. kpartx cannot run in
# sync mode in docker.
- exec_sudo(["kpartx", "-av", device_path])
+ exec_sudo(["kpartx", "-av", self.device_path])
exec_sudo(["dmsetup", "--noudevsync", "mknodes"])
return
diff --git a/diskimage_builder/block_device/tests/config/gpt_efi.yaml b/diskimage_builder/block_device/tests/config/gpt_efi.yaml
new file mode 100644
index 000000000..9ef2f682f
--- /dev/null
+++ b/diskimage_builder/block_device/tests/config/gpt_efi.yaml
@@ -0,0 +1,32 @@
+# A sample config that has GPT/bios and EFI boot partitions
+
+- local_loop:
+ name: image0
+
+- partitioning:
+ base: image0
+ label: gpt
+ partitions:
+ - name: ESP
+ type: 'EF00'
+ size: 8MiB
+ mkfs:
+ type: vfat
+ mount:
+ mount_point: /boot/efi
+ fstab:
+ options: "defaults"
+ fsck-passno: 1
+ - name: BSP
+ type: 'EF02'
+ size: 8MiB
+ - name: root
+ type: '8300'
+ size: 100%
+ mkfs:
+ type: ext4
+ mount:
+ mount_point: /
+ fstab:
+ options: "defaults"
+ fsck-passno: 1
diff --git a/diskimage_builder/block_device/tests/test_gpt.py b/diskimage_builder/block_device/tests/test_gpt.py
new file mode 100644
index 000000000..bc2009800
--- /dev/null
+++ b/diskimage_builder/block_device/tests/test_gpt.py
@@ -0,0 +1,85 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import fixtures
+import logging
+import mock
+import os
+
+import diskimage_builder.block_device.tests.test_config as tc
+
+from diskimage_builder.block_device.blockdevice import BlockDeviceState
+from diskimage_builder.block_device.config import config_tree_to_graph
+from diskimage_builder.block_device.config import create_graph
+from diskimage_builder.block_device.level0.localloop import image_create
+from diskimage_builder.block_device.level1.partition import PartitionNode
+
+logger = logging.getLogger(__name__)
+
+
+class TestGPT(tc.TestGraphGeneration):
+
+ @mock.patch('diskimage_builder.block_device.level1.partitioning.exec_sudo')
+ def test_gpt_efi(self, mock_exec_sudo):
+ # Test the command-sequence for a GPT/EFI partition setup
+ tree = self.load_config_file('gpt_efi.yaml')
+ config = config_tree_to_graph(tree)
+
+ state = BlockDeviceState()
+
+ graph, call_order = create_graph(config, self.fake_default_config,
+ state)
+
+ # Create a fake temp backing file (we check the size of it,
+ # etc).
+ # TODO(ianw): exec_sudo is generically mocked out, thus the
+ # actual creation is mocked out ... but we could do this
+ # without root and use parted to create the partitions on this
+ # for slightly better testing. An exercise for another day...
+ self.tmp_dir = fixtures.TempDir()
+ self.useFixture(self.tmp_dir)
+ self.image_path = os.path.join(self.tmp_dir.path, "image.raw")
+ # should be sparse...
+ image_create(self.image_path, 1024 * 1024 * 1024)
+ logger.debug("Temp image in %s", self.image_path)
+
+ # Fake state for the loopback device
+ state['blockdev'] = {}
+ state['blockdev']['image0'] = {}
+ state['blockdev']['image0']['image'] = self.image_path
+ state['blockdev']['image0']['device'] = "/dev/loopX"
+
+ for node in call_order:
+ if isinstance(node, PartitionNode):
+ node.create()
+
+ # check the parted call looks right
+ parted_cmd = ('sgdisk %s '
+ '-n 1:0:+8M -t 1:EF00 -c 1:"ESP" '
+ '-n 2:0:+8M -t 2:EF02 -c 2:"BSP" '
+ '-n 3:0:+1006M -t 3:8300 -c 3:"root"'
+ % self.image_path)
+ cmd_sequence = [
+ mock.call(parted_cmd.split(' ')),
+ mock.call(['sync']),
+ mock.call(['kpartx', '-avs', '/dev/loopX'])
+ ]
+ self.assertEqual(mock_exec_sudo.call_count, len(cmd_sequence))
+ mock_exec_sudo.assert_has_calls(cmd_sequence)
+
+ # Check two new partitions appear in state correctly
+ self.assertDictEqual(state['blockdev']['ESP'],
+ {'device': '/dev/mapper/loopXp1'})
+ self.assertDictEqual(state['blockdev']['BSP'],
+ {'device': '/dev/mapper/loopXp2'})
+ self.assertDictEqual(state['blockdev']['root'],
+ {'device': '/dev/mapper/loopXp3'})
diff --git a/diskimage_builder/elements/block-device-gpt/README.rst b/diskimage_builder/elements/block-device-gpt/README.rst
new file mode 100644
index 000000000..64d0ffbd8
--- /dev/null
+++ b/diskimage_builder/elements/block-device-gpt/README.rst
@@ -0,0 +1,11 @@
+================
+Block Device GPT
+================
+
+This is an override for the default block-device configuration
+provided in the ``vm`` element to get a GPT based single-partition
+disk, rather than the default MBR.
+
+Note this provides the extra `BIOS boot partition
+<https://en.wikipedia.org/wiki/BIOS_boot_partition>`__ as required for
+non-EFI boot environments.
diff --git a/diskimage_builder/elements/block-device-gpt/block-device-default.yaml b/diskimage_builder/elements/block-device-gpt/block-device-default.yaml
new file mode 100644
index 000000000..01223dda2
--- /dev/null
+++ b/diskimage_builder/elements/block-device-gpt/block-device-default.yaml
@@ -0,0 +1,22 @@
+# Default single partition loopback using a GPT based partition table
+
+- local_loop:
+ name: image0
+
+- partitioning:
+ base: image0
+ label: gpt
+ partitions:
+ - name: BSP
+ type: 'EF02'
+ size: 8MiB
+ - name: root
+ flags: [ boot ]
+ size: 100%
+ mkfs:
+ type: ext4
+ mount:
+ mount_point: /
+ fstab:
+ options: "defaults"
+ fsck-passno: 1
diff --git a/diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/element-deps b/diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/element-deps
index 7791c84fc..20b98fe31 100644
--- a/diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/element-deps
+++ b/diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/element-deps
@@ -1 +1,3 @@
-openstack-ci-mirrors
\ No newline at end of file
+block-device-gpt
+openstack-ci-mirrors
+vm
diff --git a/doc/source/user_guide/building_an_image.rst b/doc/source/user_guide/building_an_image.rst
index a9edf5317..720240cd4 100644
--- a/doc/source/user_guide/building_an_image.rst
+++ b/doc/source/user_guide/building_an_image.rst
@@ -75,7 +75,8 @@ There are currently two defaults:
The user can overwrite the default handling by setting the environment
variable `DIB_BLOCK_DEVICE_CONFIG`. This variable must hold YAML
-structured configuration data.
+structured configuration data or be a ``file://`` URL reference to a
+on-disk configuration file.
The default when using the `vm` element is:
@@ -247,8 +248,8 @@ encrypted, ...) and create partition information in it.
The symbolic name for this module is `partitioning`.
-Currently the only supported partitioning layout is Master Boot Record
-`MBR`.
+MBR
+***
It is possible to create primary or logical partitions or a mix of
them. The numbering of the primary partitions will start at 1,
@@ -267,19 +268,27 @@ partitions.
Partitions are created in the order they are configured. Primary
partitions - if needed - must be first in the list.
+GPT
+***
+
+GPT partitioning requires the ``sgdisk`` tool to be available.
+
+Options
+*******
+
There are the following key / value pairs to define one partition
table:
base
- (mandatory) The base device where to create the partitions in.
+ (mandatory) The base device to create the partitions in.
label
- (mandatory) Possible values: 'mbr'
- This uses the Master Boot Record (MBR) layout for the disk.
- (There are currently plans to add GPT later on.)
+ (mandatory) Possible values: 'mbr', 'gpt'
+ Configure use of either the Master Boot Record (MBR) or GUID
+ Partition Table (GPT) formats
align
- (optional - default value '1MiB')
+ (optional - default value '1MiB'; MBR only)
Set the alignment of the partition. This must be a multiple of the
block size (i.e. 512 bytes). The default of 1MiB (~ 2048 * 512
bytes blocks) is the default for modern systems and known to
@@ -308,9 +317,9 @@ flags
(optional) List of flags for the partition. Default: empty.
Possible values:
- boot
+ boot (MBR only)
Sets the boot flag for the partition
- primary
+ primary (MBR only)
Partition should be a primary partition. If not set a logical
partition will be created.
@@ -321,10 +330,15 @@ size
based on the remaining free space.
type (optional)
- The partition type stored in the MBR partition table entry. The
- default value is '0x83' (Linux Default partition). Any valid one
+ The partition type stored in the MBR or GPT partition table entry.
+
+ For MBR the default value is '0x83' (Linux Default partition). Any valid one
byte hexadecimal value may be specified here.
+ For GPT the default value is '8300' (Linux Default partition). Any valid two
+ byte hexadecimal value may be specified here. Due to ``sgdisk`` leading '0x'
+ should not be used.
+
Example:
.. code-block:: yaml
@@ -350,12 +364,28 @@ Example:
- name: data2
size: 100%
+ - partitioning:
+ base: gpt_image
+ label: gpt
+ partitions:
+ - name: ESP
+ type: EF00
+ size: 16MiB
+ - name: data1
+ size: 1GiB
+ - name: lvmdata
+ type: 8E00
+ size: 100%
+
On the `image0` two partitions are created. The size of the first is
1GiB, the second uses the remaining free space. On the `data_image`
-three partitions are created: all are about 1/3 of the disk size.
+three partitions are created: all are about 1/3 of the disk size. On
+the `gpt_image` three partitions are created: 16MiB one for EFI
+bootloader, 1GiB Linux filesystem one and rest of disk will be used
+for LVM partition.
-Module: Lvm
-···········
+Module: LVM
+...........
This module generates volumes on existing block devices. This means that it is
possible to take any previous created partition, and create volumes information
diff --git a/releasenotes/notes/bootloader-gpt-d1047f81f3a0631b.yaml b/releasenotes/notes/bootloader-gpt-d1047f81f3a0631b.yaml
new file mode 100644
index 000000000..6941056be
--- /dev/null
+++ b/releasenotes/notes/bootloader-gpt-d1047f81f3a0631b.yaml
@@ -0,0 +1,7 @@
+---
+features:
+ - |
+ GPT support is added to the bootloader; see documentation for
+ configuration examples. This should be considered a technology
+ preview; there may be minor behaviour modifications as we enable
+ UEFI and support across more architectures.
\ No newline at end of file
diff --git a/tests/install_test_deps.sh b/tests/install_test_deps.sh
index 17175417a..befea9a01 100755
--- a/tests/install_test_deps.sh
+++ b/tests/install_test_deps.sh
@@ -9,6 +9,8 @@ sudo apt-get install -y --force-yes \
bzip2 \
debootstrap \
docker.io \
+ dosfstools \
+ gdisk \
inetutils-ping \
lsb-release \
kpartx \
@@ -22,6 +24,8 @@ sudo apt-get install -y --force-yes \
dpkg \
debootstrap \
docker \
+ dosfstools \
+ gdisk \
kpartx \
util-linux \
qemu-img \
@@ -30,6 +34,8 @@ sudo apt-get install -y --force-yes \
bzip2 \
debootstrap \
docker \
+ dosfstools \
+ gdisk \
kpartx \
util-linux \
python-pyliblzma \
@@ -40,6 +46,8 @@ sudo apt-get install -y --force-yes \
app-emulation/qemu \
dev-python/pyyaml \
sys-block/parted \
+ sys-apps/gptfdisk \
sys-fs/multipath-tools \
+ sys-fs/dosfstools \
qemu-img \
yum-utils