[411430] Validate bootaction pkg_list

- Add a validator for bootactions to warn if a node doesn't
  have at least one
- Add a validator for bootactions to error if a package
  version specifier is invalid
- Unit tests for the validation

Change-Id: I61d8aa3831791af0484498e6fe9f7c1c83dbf540

docs/source/index.rst

@@ -69,3 +69,4 @@ Topology Documentation
    :maxdepth: 1
 
    topology
+   troubleshooting/index

docs/source/troubleshooting/index.rst

@@ -0,0 +1,40 @@
+..
+      Copyright 2017 AT&T Intellectual Property.
+      All Rights Reserved.
+
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+=============================
+Dryodck Troubleshooting Guide
+=============================
+
+This is a guide for troubleshooting issues that can arise when either
+installing/deploying Drydock or using Drydock to deploy nodes.
+
+Deployment Troubleshooting
+--------------------------
+
+Under Construction
+
+Topology Validation
+-------------------
+
+.. toctree::
+   :maxdepth: 2
+
+   validations
+
+Node Deployment
+---------------
+
+Under Construction

docs/source/troubleshooting/validations.rst

@@ -0,0 +1,39 @@
+..
+      Copyright 2018 AT&T Intellectual Property.
+      All Rights Reserved.
+
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+=============================
+Dryodck Topology Validation
+=============================
+
+DD1XXX - Storage Validations
+=============================
+
+To be continued
+
+DD2XXX - Network Validations
+=============================
+
+To be continued
+
+DD3XXX - Platform Validations
+=============================
+
+To be continued
+
+DD4XXX - Bootaction Validations
+=============================
+
+To be continued

drydock_provisioner/objects/bootaction.py

@@ -50,6 +50,8 @@ class BootAction(base.DrydockPersistentObject, base.DrydockObject):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        if not self.target_nodes:
+            self.target_nodes = []
 
     # NetworkLink keyed by name
     def get_id(self):
@@ -123,9 +125,11 @@ class BootActionAsset(base.DrydockObject):
             mode = None
 
         ba_type = kwargs.get('type', None)
+
+        package_list = None
         if ba_type == 'pkg_list':
             if isinstance(kwargs.get('data'), dict):
-                self._extract_package_list(kwargs.pop('data'))
+                package_list = self._extract_package_list(kwargs.pop('data'))
             # If the data section doesn't parse as a dictionary
             # then the package data needs to be sourced dynamically
             # Otherwise the Bootaction is invalid
@@ -133,7 +137,7 @@ class BootActionAsset(base.DrydockObject):
                 raise errors.InvalidPackageListFormat(
                     "Requires a top-level mapping/object.")
 
-        super().__init__(permissions=mode, **kwargs)
+        super().__init__(package_list=package_list, permissions=mode, **kwargs)
         self.rendered_bytes = None
 
     def render(self, nodename, site_design, action_id, design_ref):
@@ -180,7 +184,7 @@ class BootActionAsset(base.DrydockObject):
             parsed_data = yaml.safe_load(data_string)
 
             if isinstance(parsed_data, dict):
-                self._extract_package_list(parsed_data)
+                self.package_list = self._extract_package_list(parsed_data)
             else:
                 raise errors.InvalidPackageListFormat(
                     "Package data should have a top-level mapping/object.")
@@ -193,13 +197,14 @@ class BootActionAsset(base.DrydockObject):
 
         :param pkg_dict: a dictionary of packages to install
         """
-        self.package_list = dict()
+        package_list = dict()
         for k, v in pkg_dict.items():
-            if isinstance(k, str) and isinstance(v, str):
+            if (isinstance(k, str) and (not v or isinstance(v, str))):
-                self.package_list[k] = v
+                package_list[k] = v
             else:
                 raise errors.InvalidPackageListFormat(
                     "Keys and values must be strings.")
+        return package_list
 
     def _get_template_context(self, nodename, site_design, action_id,
                               design_ref):

drydock_provisioner/orchestrator/validations/bootaction_validity.py

@@ -0,0 +1,88 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+
+from drydock_provisioner import error as errors
+from drydock_provisioner.orchestrator.validations.validators import Validators
+
+
+class BootactionDefined(Validators):
+    """Issue warnings if no bootactions are defined for a node."""
+
+    def __init__(self):
+        super().__init__('Bootaction Definition', 'DD4001')
+
+    def run_validation(self, site_design, orchestrator=None):
+        """Validate each node has at least one bootaction."""
+        node_list = site_design.baremetal_nodes or []
+        ba_list = site_design.bootactions or []
+
+        nodes_with_ba = [n for ba in ba_list for n in ba.target_nodes]
+        nodes_wo_ba = [n for n in node_list if n.name not in nodes_with_ba]
+
+        for n in nodes_wo_ba:
+            msg = "Node %s is not in scope for any bootactions." % n.name
+            self.report_warn(msg, [
+                n.doc_ref
+            ], "It is expected all nodes have at least one post-deploy action."
+                             )
+        return
+
+
+class BootactionPackageListValid(Validators):
+    """Check that bootactions with pkg_list assets are valid."""
+
+    def __init__(self):
+        super().__init__('Bootaction pkg_list Validation', 'DD4002')
+        version_fields = '(\d+:)?([a-zA-Z0-9.+~-]+)(-[a-zA-Z0-9.+~]+)'
+        self.version_fields = re.compile(version_fields)
+
+    def run_validation(self, site_design, orchestrator=None):
+        """Validate that each package list in bootaction assets is valid."""
+        ba_list = site_design.bootactions or []
+
+        for ba in ba_list:
+            for a in ba.asset_list:
+                if a.type == 'pkg_list':
+                    if not a.location and not a.package_list:
+                        msg = "Bootaction has asset of type 'pkg_list' but no valid package data"
+                        self.report_error(msg, [
+                            ba.doc_ref
+                        ], "pkg_list bootaction assets must specify a list of packages."
+                                          )
+                    elif a.package_list:
+                        for p, v in a.package_list.items():
+                            try:
+                                self.validate_package_version(v)
+                            except errors.InvalidPackageListFormat as ex:
+                                msg = str(ex)
+                                self.report_error(msg, [
+                                    ba.doc_ref
+                                ], "pkg_list version specifications must be in a valid format."
+                                                  )
+        return
+
+    def validate_package_version(self, v):
+        """Validate that a version specification is valid.
+
+        :param v: a string package specification.
+        """
+        if not v:
+            return
+
+        spec_match = self.version_fields.fullmatch(v)
+
+        if not spec_match:
+            raise errors.InvalidPackageListFormat(
+                "Version specifier %s is not a valid format." % v)

drydock_provisioner/orchestrator/validations/validator.py

@@ -30,6 +30,8 @@ from drydock_provisioner.orchestrator.validations.unique_network_check import Un
 from drydock_provisioner.orchestrator.validations.hostname_validity import HostnameValidity
 from drydock_provisioner.orchestrator.validations.oob_valid_ipmi import IpmiValidity
 from drydock_provisioner.orchestrator.validations.oob_valid_libvirt import LibvirtValidity
+from drydock_provisioner.orchestrator.validations.bootaction_validity import BootactionDefined
+from drydock_provisioner.orchestrator.validations.bootaction_validity import BootactionPackageListValid
 
 
 class Validator():
@@ -91,5 +93,7 @@ rule_set = [
     UniqueNetworkCheck(),
     HostnameValidity(),
     IpmiValidity(),
-    LibvirtValidity()
+    LibvirtValidity(),
+    BootactionDefined(),
+    BootactionPackageListValid(),
 ]

tests/unit/test_validation_rule_bootactions.py

@@ -0,0 +1,92 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Test Validation Rule Rational Boot Storage"""
+
+import logging
+
+from drydock_provisioner.orchestrator.orchestrator import Orchestrator
+from drydock_provisioner import objects
+from drydock_provisioner.objects import fields as hd_fields
+from drydock_provisioner.orchestrator.validations.bootaction_validity import BootactionDefined
+
+LOG = logging.getLogger(__name__)
+
+
+class TestBootactionsValidity(object):
+    def test_valid_bootaction(self, deckhand_ingester, drydock_state, setup,
+                              input_files, mock_get_build_data):
+        input_file = input_files.join("validation.yaml")
+        design_ref = "file://%s" % str(input_file)
+
+        orch = Orchestrator(
+            state_manager=drydock_state, ingester=deckhand_ingester)
+
+        status, site_design = Orchestrator.get_effective_site(orch, design_ref)
+
+        assert status.status == hd_fields.ValidationResult.Success
+
+        LOG.debug("%s" % status.to_dict())
+
+        validator = BootactionDefined()
+        message_list = validator.execute(site_design, orchestrator=orch)
+        msg = message_list[0].to_dict()
+
+        assert msg.get('error') is False
+        assert msg.get('level') == hd_fields.MessageLevels.INFO
+        assert len(message_list) == 1
+
+    def test_absent_bootaction(self, deckhand_ingester, drydock_state, setup,
+                               input_files, mock_get_build_data):
+        input_file = input_files.join("absent_bootaction.yaml")
+        design_ref = "file://%s" % str(input_file)
+
+        orch = Orchestrator(
+            state_manager=drydock_state, ingester=deckhand_ingester)
+
+        status, site_design = Orchestrator.get_effective_site(orch, design_ref)
+
+        ba_msg = [
+            msg for msg in status.message_list
+            if ("DD4001" in msg.name
+                and msg.level == hd_fields.MessageLevels.WARN)
+        ]
+
+        assert len(ba_msg) > 0
+
+    def test_invalid_bootaction_pkg_list(self, deckhand_ingester,
+                                         drydock_state, setup, input_files,
+                                         mock_get_build_data):
+        input_file = input_files.join("invalid_bootaction_pkg.yaml")
+        design_ref = "file://%s" % str(input_file)
+
+        orch = Orchestrator(
+            state_manager=drydock_state, ingester=deckhand_ingester)
+
+        status, site_design = Orchestrator.get_effective_site(orch, design_ref)
+
+        ba_doc = objects.DocumentReference(
+            doc_type=hd_fields.DocumentType.Deckhand,
+            doc_name="invalid_pkg_list",
+            doc_schema="drydock/BootAction/v1")
+
+        assert status.status == hd_fields.ValidationResult.Failure
+
+        ba_msg = [msg for msg in status.message_list if ba_doc in msg.docs]
+
+        assert len(ba_msg) > 0
+
+        for msg in ba_msg:
+            LOG.debug(msg)
+            assert ":) is not a valid format" in msg.message
+            assert msg.error

tests/yaml_samples/absent_bootaction.yaml

@@ -0,0 +1,128 @@
+#Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+schema: 'drydock/HostProfile/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: defaults
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  hardware_profile: HPGen9v3
+  oob:
+    type: ipmi
+    network: oob
+    account: admin
+    credential: admin
+  storage:
+    physical_devices:
+      sda:
+        labels:
+          role: rootdisk
+        partitions:
+          - name: root
+            size: 20g
+            bootable: true
+            filesystem:
+              mountpoint: '/'
+              fstype: 'ext4'
+              mount_options: 'defaults'
+          - name: boot
+            size: 1g
+            bootable: false
+            filesystem:
+              mountpoint: '/boot'
+              fstype: 'ext4'
+              mount_options: 'defaults'
+      sdb:
+        volume_group: 'log_vg'
+    volume_groups:
+      log_vg:
+        logical_volumes:
+          - name: 'log_lv'
+            size: '500m'
+            filesystem:
+              mountpoint: '/var/log'
+              fstype: 'xfs'
+              mount_options: 'defaults'
+  platform:
+    image: 'xenial'
+    kernel: 'ga-16.04'
+    kernel_params:
+      quiet: true
+      console: ttyS2
+  metadata:
+    owner_data:
+      foo: bar
+---
+schema: 'drydock/BaremetalNode/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: controller01
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  host_profile: defaults
+  addressing:
+    - network: pxe
+      address: dhcp
+    - network: mgmt
+      address: 172.16.1.20
+    - network: public
+      address: 172.16.3.20
+    - network: oob
+      address: 172.16.100.20
+  metadata:
+    rack: rack1
+---
+schema: 'drydock/HardwareProfile/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: HPGen9v3
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  vendor: HP
+  generation: '8'
+  hw_version: '3'
+  bios_version: '2.2.3'
+  boot_mode: bios
+  bootstrap_protocol: pxe
+  pxe_interface: 0
+  device_aliases:
+    prim_nic01:
+      address: '0000:00:03.0'
+      dev_type: '82540EM Gigabit Ethernet Controller'
+      bus_type: 'pci'
+    prim_nic02:
+      address: '0000:00:04.0'
+      dev_type: '82540EM Gigabit Ethernet Controller'
+      bus_type: 'pci'
+    primary_boot:
+      address: '2:0.0.0'
+      dev_type: 'VBOX HARDDISK'
+      bus_type: 'scsi'
+  cpu_sets:
+    sriov: '2,4'
+  hugepages:
+    sriov:
+      size: '1G'
+      count: 300
+    dpdk:
+      size: '2M'
+      count: 530000
+...

tests/yaml_samples/deckhand_fullsite_no_nodes.yaml

@@ -346,80 +346,4 @@ data:
     dpdk:
       size: '2M'
       count: 530000
----
-schema: 'drydock/BootAction/v1'
-metadata:
-  schema: 'metadata/Document/v1'
-  name: hw_filtered
-  storagePolicy: 'cleartext'
-  labels:
-    application: 'drydock'
-data:
-  signaling: false
-  node_filter:
-    filter_set_type: 'union'
-    filter_set:
-      - filter_type: 'union'
-        node_names:
-          - 'compute01'
-  assets:
-    - path: /var/tmp/hello.sh
-      type: file
-      permissions: '555'
-      data: |-
-        IyEvYmluL2Jhc2gKCmVjaG8gJ0hlbGxvIFdvcmxkISAtZnJvbSB7eyBub2RlLmhvc3RuYW1lIH19
-        Jwo=
-      data_pipeline:
-        - base64_decode
-        - utf8_decode
-        - template
-    - path: /lib/systemd/system/hello.service
-      type: unit
-      permissions: '600'
-      data: |-
-        W1VuaXRdCkRlc2NyaXB0aW9uPUhlbGxvIFdvcmxkCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4
-        ZWNTdGFydD0vdmFyL3RtcC9oZWxsby5zaAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIu
-        dGFyZ2V0Cg==
-      data_pipeline:
-        - base64_decode
-        - utf8_decode
-...
----
-schema: 'drydock/BootAction/v1'
-metadata:
-  schema: 'metadata/Document/v1'
-  name: helloworld
-  storagePolicy: 'cleartext'
-  labels:
-    application: 'drydock'
-data:
-  assets:
-    - path: /var/tmp/hello.sh
-      type: file
-      permissions: '555'
-      data: |-
-        IyEvYmluL2Jhc2gKCmVjaG8gJ0hlbGxvIFdvcmxkISAtZnJvbSB7eyBub2RlLmhvc3RuYW1lIH19
-        Jwo=
-      data_pipeline:
-        - base64_decode
-        - utf8_decode
-        - template
-    - path: /lib/systemd/system/hello.service
-      type: unit
-      permissions: '600'
-      data: |-
-        W1VuaXRdCkRlc2NyaXB0aW9uPUhlbGxvIFdvcmxkCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4
-        ZWNTdGFydD0vdmFyL3RtcC9oZWxsby5zaAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIu
-        dGFyZ2V0Cg==
-      data_pipeline:
-        - base64_decode
-        - utf8_decode
-    - path: /var/tmp/designref.sh
-      type: file
-      permissions: '500'
-      data: e3sgYWN0aW9uLmRlc2lnbl9yZWYgfX0K
-      data_pipeline:
-        - base64_decode
-        - utf8_decode
-        - template
 ...

tests/yaml_samples/invalid_bootaction_pkg.yaml

@@ -0,0 +1,14 @@
+---
+schema: 'drydock/BootAction/v1'
+metadata:
+  schema: 'metadata/Document/v1'
+  name: invalid_pkg_list
+  storagePolicy: 'cleartext'
+  labels:
+    application: 'drydock'
+data:
+  assets:
+    - type: 'pkg_list'
+      data:
+        foo: ':)'
+...