Add join-scripts endpoint

* Adds initial join-scripts API
* Updates resiliency test to leverage API for joining

Change-Id: Ibe0d42b8f4f4a3e1f6f102dee85a22cb8f78f8ec

docs/source/api.rst

@@ -38,7 +38,7 @@ static.labels
     Used to set configuration options in the generated script
 
 Responses
-- 204 No Content: Scripts generated successfully
+- 200 OK: Script returned as response body
 - 400 Bad Request: One or more query parameters is missing or misspelled
 
 

promenade/builder.py

@@ -80,6 +80,16 @@ class Builder:
     def build_node(self, node_document, *, output_dir):
         node_name = node_document['metadata']['name']
         LOG.info('Building script for node %s', node_name)
+        script = self.build_node_script(node_name)
+
+        _write_script(output_dir, _join_name(node_name), script)
+
+        if self.validators:
+            validate_script = self._build_node_validate_script(node_name)
+            _write_script(output_dir, 'validate-%s.sh' % node_name,
+                          validate_script)
+
+    def build_node_script(self, node_name):
         sub_config = self.config.extract_node_config(node_name)
         file_spec_paths = [
             f['path'] for f in self.config.get_path('HostSystem:files', [])
@@ -88,20 +98,17 @@ class Builder:
         tarball = renderer.build_tarball_from_roles(
             config=sub_config, roles=['common', 'join'], file_specs=file_specs)
 
-        script = renderer.render_template(
+        return renderer.render_template(
             sub_config,
             template='scripts/join.sh',
             context={
                 'tarball': tarball
             })
 
-        _write_script(output_dir, _join_name(node_name), script)
-
-        if self.validators:
-            validate_script = renderer.render_template(
-                sub_config, template='scripts/validate-join.sh')
-            _write_script(output_dir, 'validate-%s.sh' % node_name,
-                          validate_script)
+    def _build_node_validate_script(self, node_name):
+        sub_config = self.config.extract_node_config(node_name)
+        return renderer.render_template(
+            sub_config, template='scripts/validate-join.sh')
 
 
 def _fetch_tar_content(*, url, path):

promenade/config.py

@@ -2,6 +2,7 @@ from . import exceptions, logging, validation
 import copy
 import jinja2
 import jsonpath_ng
+import requests
 import yaml
 
 __all__ = ['Configuration']
@@ -32,6 +33,16 @@ class Configuration:
 
         return cls(documents=documents, **kwargs)
 
+    @classmethod
+    def from_design_ref(cls, design_ref):
+        response = requests.get(design_ref)
+        response.raise_for_status()
+
+        documents = list(yaml.safe_load_all(response.text))
+        validation.check_schemas(documents)
+
+        return cls(documents=documents)
+
     def __getitem__(self, path):
         value = self.get_path(path)
         if value:
@@ -125,6 +136,10 @@ class Configuration:
                 return data
         return default
 
+    def append(self, item):
+        validation.check_schema(item)
+        self.documents.append(item)
+
 
 def _matches_filter(document, *, schema, labels):
     matches = True

promenade/control/api.py

@@ -16,6 +16,7 @@ import falcon
 
 from promenade.control.base import BaseResource, PromenadeRequest
 from promenade.control.health_api import HealthResource
+from promenade.control.join_scripts import JoinScriptsResource
 from promenade.control.middleware import (AuthMiddleware, ContextMiddleware,
                                           LoggingMiddleware)
 from promenade import exceptions as exc
@@ -37,6 +38,7 @@ def start_api():
     v1_0_routes = [
         # API for managing region data
         ('/health', HealthResource()),
+        ('/join-scripts', JoinScriptsResource()),
     ]
 
     # Set up the 1.0 routes

promenade/control/join_scripts.py

@@ -0,0 +1,95 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from promenade.control.base import BaseResource
+from promenade.builder import Builder
+from promenade.config import Configuration
+from promenade import logging
+import falcon
+import kubernetes
+import random
+
+LOG = logging.getLogger(__name__)
+
+
+class JoinScriptsResource(BaseResource):
+    """
+    Lists the versions supported by this API
+    """
+
+    def on_get(self, req, resp):
+        design_ref = req.get_param('design_ref', required=True)
+        ip = req.get_param('ip', required=True)
+        hostname = req.get_param('hostname', required=True)
+
+        dynamic_labels = _get_param_list(req, 'labels.dynamic')
+        static_labels = _get_param_list(req, 'labels.static')
+
+        join_ip = _get_join_ip()
+
+        config = Configuration.from_design_ref(design_ref)
+        node_document = {
+            'schema': 'promenade/KubernetesNode/v1',
+            'metadata': {
+                'name': hostname,
+                'schema': 'metadata/Document/v1',
+                'layeringDefinition': {
+                    'abstract': False,
+                    'layer': 'site'
+                },
+            },
+            'data': {
+                'hostname': hostname,
+                'ip': ip,
+                'join_ip': join_ip,
+                'labels': {
+                    'dynamic': dynamic_labels,
+                    'static': static_labels,
+                },
+            },
+        }
+        config.append(node_document)
+
+        builder = Builder(config)
+        script = builder.build_node_script(hostname)
+
+        resp.body = script
+        resp.content_type = 'text/x-shellscript'
+        resp.status = falcon.HTTP_200
+
+
+def _get_join_ip():
+    # TODO(mark-burnett): Handle errors
+    kubernetes.config.load_incluster_config()
+    client = kubernetes.client.CoreV1Api()
+    response = client.list_node(label_selector='kubernetes-apiserver=enabled')
+
+    # Ignore bandit false positive: B311:blacklist
+    # The choice of which master to join to is a load-balancing concern, not a
+    # security concern.
+    return random.choice(list(map(_extract_ip, response.items)))  # nosec
+
+
+def _extract_ip(item):
+    for address in item.status.addresses:
+        if address.type == 'InternalIP':
+            return address.address
+
+
+def _get_param_list(req, name):
+    values = req.get_param_as_list(name)
+    if values:
+        return values
+    else:
+        return []

promenade/renderer.py

@@ -33,6 +33,8 @@ def insert_charts_into_bundler(bundler):
     for root, _dirnames, filenames in os.walk(
             '/opt/promenade/charts', followlinks=True):
         for source_filename in filenames:
+            if _source_file_is_excluded(source_filename):
+                continue
             source_path = os.path.join(root, source_filename)
             destination_path = os.path.join('etc/genesis/armada/assets/charts',
                                             os.path.relpath(
@@ -139,6 +141,10 @@ def _default_no_proxy(network_config):
     return ','.join(include)
 
 
+def _source_file_is_excluded(source_filename):
+    return source_filename.endswith('.tgz')
+
+
 def _yaml_safe_dump_all(documents):
     f = io.StringIO()
     yaml.safe_dump_all(documents, f)

requirements-direct.txt

@@ -4,7 +4,8 @@ jinja2==2.9.6
 jsonpath-ng==1.4.3
 jsonschema==2.6.0
 keystonemiddleware==4.17.0
-oslo.context==2.14.0
+kubernetes==3.0.0
+oslo.context==2.19.2
 pastedeploy==1.5.2
 pbr==3.0.1
 pyyaml==3.12

requirements-frozen.txt

@@ -1,24 +1,28 @@
 Babel==2.5.1
+cachetools==2.0.1
 certifi==2017.7.27.1
 chardet==3.0.4
 click==6.7
 debtcollector==1.18.0
 decorator==4.1.2
 falcon==1.2.0
+google-auth==1.2.0
 idna==2.6
+ipaddress==1.0.18
 iso8601==0.1.12
 Jinja2==2.9.6
 jsonpath-ng==1.4.3
 jsonschema==2.6.0
 keystoneauth1==3.2.0
 keystonemiddleware==4.17.0
+kubernetes==3.0.0
 MarkupSafe==1.0
 monotonic==1.4
 msgpack-python==0.4.8
 netaddr==0.7.19
 netifaces==0.10.6
 oslo.config==5.0.0
-oslo.context==2.14.0
+oslo.context==2.19.2
 oslo.i18n==3.18.0
 oslo.log==3.32.0
 oslo.serialization==2.21.2
@@ -27,6 +31,8 @@ PasteDeploy==1.5.2
 pbr==3.0.1
 ply==3.10
 positional==1.2.1
+pyasn1==0.3.7
+pyasn1-modules==0.1.5
 pycadf==2.6.0
 pyinotify==0.9.6
 pyparsing==2.2.0
@@ -37,9 +43,11 @@ pytz==2017.3
 PyYAML==3.12
 requests==2.18.4
 rfc3986==1.1.0
+rsa==3.4.2
 six==1.11.0
 stevedore==1.27.1
 urllib3==1.22
 uWSGI==2.0.15
 WebOb==1.7.3
+websocket-client==0.40.0
 wrapt==1.10.11

tools/g2/lib/all.sh

@@ -8,6 +8,7 @@ source "$LIB_DIR"/const.sh
 source "$LIB_DIR"/etcd.sh
 source "$LIB_DIR"/kube.sh
 source "$LIB_DIR"/log.sh
+source "$LIB_DIR"/nginx.sh
 source "$LIB_DIR"/promenade.sh
 source "$LIB_DIR"/registry.sh
 source "$LIB_DIR"/ssh.sh

tools/g2/lib/nginx.sh

@@ -0,0 +1,18 @@
+nginx_down() {
+    REGISTRY_ID=$(docker ps -qa -f name=promenade-nginx)
+    if [ "x${REGISTRY_ID}" != "x" ]; then
+        log Removing nginx server
+        docker rm -fv "${REGISTRY_ID}" &>> "${LOG_FILE}"
+    fi
+}
+
+nginx_up() {
+    log Starting nginx server to serve configuration files
+    mkdir -p "${TEMP_DIR}/nginx"
+    docker run -d \
+        -p 7777:80 \
+        --restart=always \
+        --name promenade-nginx \
+        -v "${TEMP_DIR}/nginx:/usr/share/nginx/html:ro" \
+            nginx:stable &>> "${LOG_FILE}"
+}

tools/g2/manifests/conformance.json

@@ -29,18 +29,42 @@
     },
     {
       "name": "Join Masters",
-      "script": "join-masters.sh",
+      "script": "join-nodes.sh",
       "arguments": [
-        "n1",
-        "n2",
-        "n3"
+        "-v", "n0",
+        "-n", "n1",
+        "-n", "n2",
+        "-n", "n3",
+        "-l", "calico-etcd=enabled",
+        "-l", "kubernetes-apiserver=enabled",
+        "-l", "kubernetes-controller-manager=enabled",
+        "-l", "kubernetes-etcd=enabled",
+        "-l", "kubernetes-scheduler=enabled",
+        "-l", "ucp-control-plane=enabled",
+        "-e", "kubernetes n0 genesis n1 n2 n3",
+        "-e", "calico n0 n0 n1 n2 n3"
       ]
     },
     {
-      "name": "Reprovision Genesis",
-      "script": "reprovision-genesis.sh",
+      "name": "Teardown Genesis",
+      "script": "teardown-nodes.sh",
       "arguments": [
-        "n1 n2 n3"
+        "-v", "n1",
+        "-n", "n0",
+        "-r",
+        "-e", "kubernetes n1 n1 n2 n3",
+        "-e", "calico n1 n1 n2 n3"
+      ]
+    },
+    {
+      "name": "Join n0 as Worker",
+      "script": "join-nodes.sh",
+      "arguments": [
+        "-v", "n1",
+        "-n", "n0",
+        "-l", "ucp-control-plane=enabled",
+        "-e", "kubernetes n1 n1 n2 n3",
+        "-e", "calico n1 n1 n2 n3"
       ]
     },
     {

tools/g2/manifests/resiliency.json

@@ -29,18 +29,42 @@
     },
     {
       "name": "Join Masters",
-      "script": "join-masters.sh",
+      "script": "join-nodes.sh",
       "arguments": [
-        "n1",
-        "n2",
-        "n3"
+        "-v", "n0",
+        "-n", "n1",
+        "-n", "n2",
+        "-n", "n3",
+        "-l", "calico-etcd=enabled",
+        "-l", "kubernetes-apiserver=enabled",
+        "-l", "kubernetes-controller-manager=enabled",
+        "-l", "kubernetes-etcd=enabled",
+        "-l", "kubernetes-scheduler=enabled",
+        "-l", "ucp-control-plane=enabled",
+        "-e", "kubernetes n0 genesis n1 n2 n3",
+        "-e", "calico n0 n0 n1 n2 n3"
       ]
     },
     {
-      "name": "Reprovision Genesis",
-      "script": "reprovision-genesis.sh",
+      "name": "Teardown Genesis",
+      "script": "teardown-nodes.sh",
       "arguments": [
-        "n1 n2 n3"
+        "-v", "n1",
+        "-n", "n0",
+        "-r",
+        "-e", "kubernetes n1 n1 n2 n3",
+        "-e", "calico n1 n1 n2 n3"
+      ]
+    },
+    {
+      "name": "Join n0 as Worker",
+      "script": "join-nodes.sh",
+      "arguments": [
+        "-v", "n1",
+        "-n", "n0",
+        "-l", "ucp-control-plane=enabled",
+        "-e", "kubernetes n1 n1 n2 n3",
+        "-e", "calico n1 n1 n2 n3"
       ]
     },
     {

tools/g2/manifests/two.json

@@ -1,50 +0,0 @@
-{
-  "configuration": [
-    "examples/basic"
-  ],
-  "stages": [
-    {
-      "name": "Gate Setup",
-      "script": "gate-setup.sh"
-    },
-    {
-      "name": "Build Image",
-      "script": "build-image.sh"
-    },
-    {
-      "name": "Generate Certificates",
-      "script": "generate-certificates.sh"
-    },
-    {
-      "name": "Build Scripts",
-      "script": "build-scripts.sh"
-    },
-    {
-      "name": "Create VMs",
-      "script": "create-vms.sh"
-    },
-    {
-      "name": "Genesis",
-      "script": "genesis.sh"
-    },
-    {
-      "name": "Join Masters",
-      "script": "join-masters.sh",
-      "arguments": [
-        "n1"
-      ]
-    },
-    {
-      "name": "Hard Reboot Cluster",
-      "script": "hard-reboot-cluster.sh"
-    }
-  ],
-  "vm": {
-    "memory": 2048,
-    "names": [
-      "n0",
-      "n1"
-    ],
-    "vcpus": 2
-  }
-}

tools/g2/manifests/ucp.json

@@ -1,57 +0,0 @@
-{
-  "configuration": [
-    "examples/complete"
-  ],
-  "stages": [
-    {
-      "name": "Gate Setup",
-      "script": "gate-setup.sh"
-    },
-    {
-      "name": "Build Image",
-      "script": "build-image.sh"
-    },
-    {
-      "name": "Generate Certificates",
-      "script": "generate-certificates.sh"
-    },
-    {
-      "name": "Build Scripts",
-      "script": "build-scripts.sh"
-    },
-    {
-      "name": "Create VMs",
-      "script": "create-vms.sh"
-    },
-    {
-      "name": "Genesis",
-      "script": "genesis.sh"
-    },
-    {
-      "name": "Join Masters",
-      "script": "join-masters.sh",
-      "arguments": [
-        "n1"
-      ]
-    },
-    {
-      "name": "Reprovision Genesis",
-      "script": "reprovision-genesis.sh",
-      "arguments": [
-        "n1"
-      ]
-    },
-    {
-      "name": "Hard Reboot Cluster",
-      "script": "hard-reboot-cluster.sh"
-    }
-  ],
-  "vm": {
-    "memory": 8096,
-    "names": [
-      "n0",
-      "n1"
-    ],
-    "vcpus": 4
-  }
-}

tools/g2/stages/build-scripts.sh

@@ -18,3 +18,5 @@ docker run --rm -t \
                 --validators \
                 -o scripts \
                 config/*.yaml
+
+cat "${TEMP_DIR}"/config/*.yaml > "${TEMP_DIR}/nginx/promenade.yaml"

tools/g2/stages/gate-setup.sh

@@ -8,6 +8,10 @@ source "${GATE_UTILS}"
 registry_up
 registry_populate
 
+# NginX for serving config files in the absence of Deckhand
+nginx_down
+nginx_up
+
 # SSH setup
 ssh_setup_declare
 

tools/g2/stages/join-masters.sh

@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-if [ $# -le 0 ]; then
-    echo "Must specify at least one vm to join"
-    exit 1
-fi
-
-source "${GATE_UTILS}"
-
-JOIN_TARGETS="${*}"
-
-for NAME in ${JOIN_TARGETS}; do
-    rsync_cmd "${TEMP_DIR}"/scripts/*"${NAME}"* "${NAME}:/root/promenade/"
-
-    ssh_cmd "${NAME}" "/root/promenade/join-${NAME}.sh"
-    ssh_cmd "${NAME}" "/root/promenade/validate-${NAME}.sh"
-
-    # NOTE(mark-burnett): Ensure disk cache is flushed after join.
-    ssh_cmd "${NAME}" sync
-done
-
-validate_cluster n0
-
-validate_etcd_membership kubernetes n0 genesis "${*}"
-validate_etcd_membership calico n0 n0 "${*}"

tools/g2/stages/join-nodes.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+set -eu
+
+source "${GATE_UTILS}"
+
+declare -a ETCD_CLUSTERS
+declare -a LABELS
+declare -a NODES
+
+while getopts "e:l:n:v:" opt; do
+    case "${opt}" in
+        e)
+            ETCD_CLUSTERS+=("${OPTARG}")
+            ;;
+        l)
+            LABELS+=("${OPTARG}")
+            ;;
+        n)
+            NODES+=("${OPTARG}")
+            ;;
+        v)
+            VIA=${OPTARG}
+            ;;
+        *)
+            echo "Unknown option"
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [ $# -gt 0 ]; then
+    echo "Unknown arguments specified: ${*}"
+    exit 1
+fi
+
+SCRIPT_DIR="${TEMP_DIR}/curled-scripts"
+
+echo Etcd Clusters: "${ETCD_CLUSTERS[@]}"
+echo Labels: "${LABELS[@]}"
+echo Nodes: "${NODES[@]}"
+
+render_curl_url() {
+    NAME=${1}
+    shift
+    LABELS=(${@})
+
+    LABEL_PARAMS=
+    for label in "${LABELS[@]}"; do
+        LABEL_PARAMS+="&labels.dynamic=${label}"
+    done
+
+    BASE_URL="http://promenade-api.ucp.svc.cluster.local/api/v1.0/join-scripts"
+    DESIGN_REF="design_ref=http://192.168.77.1:7777/promenade.yaml"
+    HOST_PARAMS="hostname=${NAME}&ip=$(config_vm_ip "${NAME}")"
+
+    echo "'${BASE_URL}?${DESIGN_REF}&${HOST_PARAMS}${LABEL_PARAMS}'"
+}
+
+mkdir -p "${SCRIPT_DIR}"
+
+for NAME in "${NODES[@]}"; do
+    log Building join script for node "${NAME}"
+
+    ssh_cmd "${VIA}" curl "$(render_curl_url "${NAME}" "${LABELS[@]}")" > "${SCRIPT_DIR}/join-${NAME}.sh"
+    chmod 755 "${SCRIPT_DIR}/join-${NAME}.sh"
+
+    log Joining node "${NAME}"
+    rsync_cmd "${SCRIPT_DIR}/join-${NAME}.sh" "${NAME}:/root/promenade/"
+    ssh_cmd "${NAME}" "/root/promenade/join-${NAME}.sh"
+done
+
+for etcd_validation_string in "${ETCD_CLUSTERS[@]}"; do
+    IFS=' ' read -a etcd_validation_args <<<"${etcd_validation_string}"
+    validate_etcd_membership "${etcd_validation_args[@]}"
+done

tools/g2/stages/teardown-nodes.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+set -eu
+
+source "${GATE_UTILS}"
+
+declare -a ETCD_CLUSTERS
+declare -a NODES
+
+RECREATE=0
+
+while getopts "e:n:rv:" opt; do
+    case "${opt}" in
+        e)
+            ETCD_CLUSTERS+=("${OPTARG}")
+            ;;
+        n)
+            NODES+=("${OPTARG}")
+            ;;
+        r)
+            RECREATE=1
+            ;;
+        v)
+            VIA=${OPTARG}
+            ;;
+        *)
+            echo "Unknown option"
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [ $# -gt 0 ]; then
+    echo "Unknown arguments specified: ${*}"
+    exit 1
+fi
+
+for NAME in "${NODES[@]}"; do
+    log Tearing down node "${NAME}"
+    promenade_teardown_node "${NAME}" "${VIA}"
+    vm_clean "${NAME}"
+    if [[ ${RECREATE} == "1" ]]; then
+        vm_create "${NAME}"
+    fi
+done
+
+for etcd_validation_string in "${ETCD_CLUSTERS[@]}"; do
+    IFS=' ' read -a etcd_validation_args <<<"${etcd_validation_string}"
+    validate_etcd_membership "${etcd_validation_args[@]}"
+done

tools/stop_gate.sh

@@ -14,3 +14,4 @@ source "${GATE_UTILS}"
 vm_clean_all
 net_clean
 registry_down
+nginx_down