diff --git a/.gitignore b/.gitignore index a3ae1ca18..cffa99bad 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ devenv/* .coverage coverage.xml .testrepository +imagebuild/coreos/build +imagebuild/coreos/dist +imagebuild/coreos/oem/authorized_keys +imagebuild/coreos/UPLOAD diff --git a/Dockerfile b/Dockerfile index 82836887a..7fa950b14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ RUN apt-get update && apt-get -y install \ # Install requirements separately, because pip understands a git+https url while setuptools doesn't RUN pip install -r /tmp/teeth-agent/requirements.txt -# This will succeed because all the dependencies (including pesky teeth_rest) were installed previously +# This will succeed because all the dependencies were installed previously RUN pip install /tmp/teeth-agent CMD [ "/usr/local/bin/teeth-agent" ] diff --git a/imagebuild/README.md b/imagebuild/README.md new file mode 100644 index 000000000..d073c9aad --- /dev/null +++ b/imagebuild/README.md @@ -0,0 +1,4 @@ +teeth-agent images +================== + +coreos - Builds a CoreOS Ramdisk and Kernel suitable for running teeth-agent diff --git a/imagebuild/coreos/Makefile b/imagebuild/coreos/Makefile new file mode 100644 index 000000000..26a1cc2b8 --- /dev/null +++ b/imagebuild/coreos/Makefile @@ -0,0 +1,17 @@ +default: docker coreos + +docker: + ./docker_build.bash + +coreos: + mkdir UPLOAD + ./coreos-oem-inject.py oem UPLOAD + +clean: + rm -rf teeth-agent + rm -f oem/container.tar.gz + rm -f UPLOAD/coreos_production_pxe_image-oem.cpio.gz + rm -f UPLOAD/coreos_production_pxe.vmlinuz + +docker_clean: + ./docker_clean.bash diff --git a/imagebuild/coreos/README.md b/imagebuild/coreos/README.md new file mode 100644 index 000000000..4798fe810 --- /dev/null +++ b/imagebuild/coreos/README.md @@ -0,0 +1,33 @@ +# teeth-agent CoreOS Image builder. + +Builds a CoreOS image suitable for running the teeth-agent on a server. + +# Requirements + +Must be run from a linux machine with a working docker installation and python-pip + +Run the following locally or from a virtualenv to install the python requirements +``` +pip install -r requirements.txt +``` + +# Instructions + +To create a docker repository and embed it into a CoreOS pxe image: +``` +make +``` + +To just create the docker repository in oem/container.tar.gz: +``` +make docker +``` + +To embed the oem/ directory into a CoreOS pxe image: + +Note: In order to have the ability to ssh into the created image, you need to +pass ssh keys in via the kernel command line for CoreOS, or create +oem/authorized_keys with the keys you need added before building the image. +``` +make coreos +``` diff --git a/imagebuild/coreos/coreos-oem-inject.py b/imagebuild/coreos/coreos-oem-inject.py new file mode 100755 index 000000000..e471b9a95 --- /dev/null +++ b/imagebuild/coreos/coreos-oem-inject.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python + +import os +import sys +import time +import requests +import tempfile +import shutil +from plumbum import local, cmd + +COREOS_VERSION="dev-channel" + +COREOS_ARCH="amd64-generic" +COREOS_BASE_URL="http://storage.core-os.net/coreos/{}/{}".format(COREOS_ARCH, COREOS_VERSION) +COREOS_PXE_DIGESTS="coreos_production_pxe_image.cpio.gz.DIGESTS.asc" +COREOS_PXE_KERNEL="coreos_production_pxe.vmlinuz" +COREOS_PXE_IMAGE="coreos_production_pxe_image.cpio.gz" +COREOS_PXE_IMAGE_URL = "{}/{}".format(COREOS_BASE_URL, COREOS_PXE_IMAGE) +COREOS_PXE_KERNEL_URL = "{}/{}".format(COREOS_BASE_URL, COREOS_PXE_KERNEL) +COREOS_PXE_DIGESTS_URL = "{}/{}".format(COREOS_BASE_URL, COREOS_PXE_DIGESTS) + + + +def get_etag(cache_name): + etag_file = "{}.etag".format(cache_name) + if not os.path.exists(etag_file): + return None + with open(etag_file, 'rb') as fp: + etag = fp.read() + etag.strip() + return etag + +def save_etag(cache_name, etag): + etag_file = "{}.etag".format(cache_name) + with open(etag_file, 'w+b') as fp: + fp.write(etag) + +def cache_file(cache_name, remote_url): + print("{} <- {}".format(cache_name, remote_url)) + etag = get_etag(cache_name) + headers = {} + if etag: + headers['If-None-Match'] = etag + + start = time.time() + r = requests.get(remote_url, headers=headers) + + if r.status_code == 304: + print("[etag-match]") + return + + if r.status_code != 200: + raise RuntimeError('Failed to download {}, got HTTP {} Status Code.'.format(remote_url, r.status_code)) + + with open(cache_name, 'w+b') as fp: + fp.write(r.content) + + print("{} bytes in {} seconds".format(len(r.content), time.time() - start)) + save_etag(cache_name, r.headers['etag']) + +def inject_oem(archive, oem_dir, output_file): + d = tempfile.mkdtemp(prefix="oem-inject") + try: + with local.cwd(d): + dest_oem_dir = os.path.join(d, 'usr', 'share', 'oem') + uz = cmd.gunzip["-c", archive] + extract = cmd.cpio["-iv"] + chain = uz | extract + print chain + chain() + + shutil.copytree(oem_dir, dest_oem_dir) + + find = cmd.find['.', '-depth', '-print'] + cpio = cmd.cpio['-o', '-H', 'newc'] + gz = cmd.gzip + chain = find | cmd.sort | cpio | gz > output_file + print chain + chain() + finally: + shutil.rmtree(d) + return output_file + +def validate_digests(digests, target, hash_type='sha1'): + with local.cwd(os.path.dirname(digests)): + gethashes = cmd.grep['-i', '-A1', '^# {} HASH$'.format(hash_type), digests] + forthis = cmd.grep[os.path.basename(target)] + viasum = local[hash_type + "sum"]['-c', '/dev/stdin'] + chain = gethashes | forthis | viasum + print chain + chain() + +def main(): + if len(sys.argv) != 3: + print("usage: {} [oem-directory-to-inject] [output-directory]".format(os.path.basename(__file__))) + return + + oem_dir = os.path.abspath(os.path.expanduser(sys.argv[1])) + output_dir = os.path.abspath(os.path.expanduser(sys.argv[2])) + + if not os.path.exists(oem_dir): + print("Error: {} doesn't exist.".format(oem_dir)) + return + + if not os.path.exists(os.path.join(oem_dir, 'run.sh')): + print("Error: {} is missing oem.sh".format(oem_dir)) + return + + here = os.path.abspath(os.path.dirname(__file__)) + + top_cache_dir = os.path.join(os.path.dirname(here), ".image_cache") + cache_dir = os.path.join(top_cache_dir, COREOS_ARCH, COREOS_VERSION) + + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + + orig_cpio = os.path.join(cache_dir, COREOS_PXE_IMAGE) + digests = os.path.join(cache_dir, COREOS_PXE_DIGESTS) + kernel = os.path.join(cache_dir, COREOS_PXE_KERNEL) + + cache_file(digests, COREOS_PXE_DIGESTS_URL) + gpg_verify_file(digests) + cache_file(kernel, COREOS_PXE_KERNEL_URL) + validate_digests(digests, kernel) + cache_file(orig_cpio, COREOS_PXE_IMAGE_URL) + validate_digests(digests, orig_cpio) + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + output_kernel = os.path.join(output_dir, os.path.basename(kernel)) + output_cpio = os.path.join(output_dir, os.path.basename(orig_cpio).replace('.cpio.gz', '-oem.cpio.gz')) + inject_oem(orig_cpio, oem_dir, output_cpio) + shutil.copy(kernel, output_kernel) + +def gpg_verify_file(ascfile): + d = tempfile.mkdtemp(prefix="oem-gpg-validate") + try: + tmpring = os.path.join(d, 'tmp.gpg') + key = os.path.join(d, 'coreos.key') + with open(key, 'w+b') as fp: + fp.write(gpg_key()) + + i = cmd.gpg['--batch', + '--no-default-keyring', + '--keyring', + tmpring, + '--import', + key] + print(i) + i() + + r = cmd.gpg['--batch', + '--no-default-keyring', + '--keyring', + tmpring, + '--verify', + ascfile] + print(r) + r() + + finally: + shutil.rmtree(d) + +def gpg_key(): + GPG_LONG_ID="50E0885593D2DCB4" + GPG_KEY="""-----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v2.0.20 (GNU/Linux) + + mQINBFIqVhQBEADjC7oxg5N9Xqmqqrac70EHITgjEXZfGm7Q50fuQlqDoeNWY+sN + szpw//dWz8lxvPAqUlTSeR+dl7nwdpG2yJSBY6pXnXFF9sdHoFAUI0uy1Pp6VU9b + /9uMzZo+BBaIfojwHCa91JcX3FwLly5sPmNAjgiTeYoFmeb7vmV9ZMjoda1B8k4e + 8E0oVPgdDqCguBEP80NuosAONTib3fZ8ERmRw4HIwc9xjFDzyPpvyc25liyPKr57 + UDoDbO/DwhrrKGZP11JZHUn4mIAO7pniZYj/IC47aXEEuZNn95zACGMYqfn8A9+K + mHIHwr4ifS+k8UmQ2ly+HX+NfKJLTIUBcQY+7w6C5CHrVBImVHzHTYLvKWGH3pmB + zn8cCTgwW7mJ8bzQezt1MozCB1CYKv/SelvxisIQqyxqYB9q41g9x3hkePDRlh1s + 5ycvN0axEpSgxg10bLJdkhE+CfYkuANAyjQzAksFRa1ZlMQ5I+VVpXEECTVpLyLt + QQH87vtZS5xFaHUQnArXtZFu1WC0gZvMkNkJofv3GowNfanZb8iNtNFE8r1+GjL7 + a9NhaD8She0z2xQ4eZm8+Mtpz9ap/F7RLa9YgnJth5bDwLlAe30lg+7WIZHilR09 + UBHapoYlLB3B6RF51wWVneIlnTpMIJeP9vOGFBUqZ+W1j3O3uoLij1FUuwARAQAB + tDZDb3JlT1MgQnVpbGRib3QgKE9mZmljYWwgQnVpbGRzKSA8YnVpbGRib3RAY29y + ZW9zLmNvbT6JAjkEEwECACMFAlIqVhQCGwMHCwkIBwMCAQYVCAIJCgsEFgIDAQIe + AQIXgAAKCRBQ4IhVk9LctFkGD/46/I3S392oQQs81pUOMbPulCitA7/ehYPuVlgy + mv6+SEZOtafEJuI9uiTzlAVremZfalyL20RBtU10ANJfejp14rOpMadlRqz0DCvc + Wuuhhn9FEQE59Yk3LQ7DBLLbeJwUvEAtEEXq8xVXWh4OWgDiP5/3oALkJ4Lb3sFx + KwMy2JjkImr1XgMY7M2UVIomiSFD7v0H5Xjxaow/R6twttESyoO7TSI6eVyVgkWk + GjOSVK5MZOZlux7hW+uSbyUGPoYrfF6TKM9+UvBqxWzz9GBG44AjcViuOn9eH/kF + NoOAwzLcL0wjKs9lN1G4mhYALgzQx/2ZH5XO0IbfAx5Z0ZOgXk25gJajLTiqtOkM + E6u691Dx4c87kST2g7Cp3JMCC+cqG37xilbV4u03PD0izNBt/FLaTeddNpPJyttz + gYqeoSv2xCYC8AM9N73Yp1nT1G1rnCpe5Jct8Mwq7j8rQWIBArt3lt6mYFNjuNpg + om+rZstK8Ut1c8vOhSwz7Qza+3YaaNjLwaxe52RZ5svt6sCfIVO2sKHf3iO3aLzZ + 5KrCLZ/8tJtVxlhxRh0TqJVqFvOneP7TxkZs9DkU5uq5lHc9FWObPfbW5lhrU36K + Pf5pn0XomaWqge+GCBCgF369ibWbUAyGPqYj5wr/jwmG6nedMiqcOwpeBljpDF1i + d9zMN4kCHAQQAQIABgUCUipXUQAKCRDAr7X91+bcxwvZD/0T4mVRyAp8+EhCta6f + Qnoiqc49oHhnKsoN7wDg45NRlQP84rH1knn4/nSpUzrB29bhY8OgAiXXMHVcS+Uk + hUsF0sHNlnunbY0GEuIziqnrjEisb1cdIGyfsWUPc/4+inzu31J1n3iQyxdOOkrA + ddd0iQxPtyEjwevAfptGUeAGvtFXP374XsEo2fbd+xHMdV1YkMImLGx0guOK8tgp + +ht7cyHkfsyymrCV/WGaTdGMwtoJOxNZyaS6l0ccneW4UhORda2wwD0mOHHk2EHG + dJuEN4SRSoXQ0zjXvFr/u3k7Qww11xU0V4c6ZPl0Rd/ziqbiDImlyODCx6KUlmJb + k4l77XhHezWD0l3ZwodCV0xSgkOKLkudtgHPOBgHnJSL0vy7Ts6UzM/QLX5GR7uj + do7P/v0FrhXB+bMKvB/fMVHsKQNqPepigfrJ4+dZki7qtpx0iXFOfazYUB4CeMHC + 0gGIiBjQxKorzzcc5DVaVaGmmkYoBpxZeUsAD3YNFr6AVm3AGGZO4JahEOsul2FF + V6B0BiSwhg1SnZzBjkCcTCPURFm82aYsFuwWwqwizObZZNDC/DcFuuAuuEaarhO9 + BGzShpdbM3Phb4tjKKEJ9Sps6FBC2Cf/1pmPyOWZToMXex5ZKB0XHGCI0DFlB4Tn + in95D/b2+nYGUehmneuAmgde87kCDQRSKlZGARAAuMYYnu48l3AvE8ZpTN6uXSt2 + RrXnOr9oEah6hw1fn9KYKVJi0ZGJHzQOeAHHO/3BKYPFZNoUoNOU6VR/KAn7gon1 + wkUwk9Tn0AXVIQ7wMFJNLvcinoTkLBT5tqcAz5MvAoI9sivAM0Rm2BgeujdHjRS+ + UQKq/EZtpnodeQKE8+pwe3zdf6A9FZY2pnBs0PxKJ0NZ1rZeAW9w+2WdbyrkWxUv + jYWMSzTUkWK6533PVi7RcdRmWrDMNVR/X1PfqqAIzQkQ8oGcXtRpYjFL30Z/LhKe + c9Awfm57rkZk2EMduIB/Y5VYqnOsmKgUghXjOo6JOcanQZ4sHAyQrB2Yd6UgdAfz + qa7AWNIAljSGy6/CfJAoVIgl1revG7GCsRD5Dr/+BLyauwZ/YtTH9mGDtg6hy/So + zzDAM8+79Y8VMBUtj64GQBgg2+0MVZYNsZCN209X+EGpGUmAGEFQLGLHwFoNlwwL + 1Uj+/5NTAhp2MQA/XRDTVx1nm8MZZXUOu6NTCUXtUmgTQuQEsKCosQzBuT/G+8Ia + R5jBVZ38/NJgLw+YcRPNVo2S2XSh7liw+Sl1sdjEW1nWQHotDAzd2MFG++KVbxwb + cXbDgJOB0+N0c362WQ7bzxpJZoaYGhNOVjVjNY8YkcOiDl0DqkCk45obz4hG2T08 + x0OoXN7Oby0FclbUkVsAEQEAAYkERAQYAQIADwUCUipWRgIbAgUJAeEzgAIpCRBQ + 4IhVk9LctMFdIAQZAQIABgUCUipWRgAKCRClQeyydOfjYdY6D/4+PmhaiyasTHqh + iui2DwDVdhwxdikQEl+KQQHtk7aqgbUAxgU1D4rbLxzXyhTbmql7D30nl+oZg0Be + yl67Xo6X/wHsP44651aTbwxVT9nzhOp6OEW5z/qxJaX1B9EBsYtjGO87N854xC6a + QEaGZPbNauRpcYEadkppSumBo5ujmRWc4S+H1VjQW4vGSCm9m4X7a7L7/063HJza + SYaHybbu/udWW8ymzuUf/UARH4141bGnZOtIa9vIGtFl2oWJ/ViyJew9vwdMqiI6 + Y86ISQcGV/lL/iThNJBn+pots0CqdsoLvEZQGF3ZozWJVCKnnn/kC8NNyd7Wst9C + +p7ZzN3BTz+74Te5Vde3prQPFG4ClSzwJZ/U15boIMBPtNd7pRYum2padTK9oHp1 + l5dI/cELluj5JXT58hs5RAn4xD5XRNb4ahtnc/wdqtle0Kr5O0qNGQ0+U6ALdy/f + IVpSXihfsiy45+nPgGpfnRVmjQvIWQelI25+cvqxX1dr827ksUj4h6af/Bm9JvPG + KKRhORXPe+OQM6y/ubJOpYPEq9fZxdClekjA9IXhojNA8C6QKy2Kan873XDE0H4K + Y2OMTqQ1/n1A6g3qWCWph/sPdEMCsfnybDPcdPZp3psTQ8uX/vGLz0AAORapVCbp + iFHbF3TduuvnKaBWXKjrr5tNY/njrU4zEADTzhgbtGW75HSGgN3wtsiieMdfbH/P + f7wcC2FlbaQmevXjWI5tyx2m3ejG9gqnjRSyN5DWPq0m5AfKCY+4Glfjf01l7wR2 + 5oOvwL9lTtyrFE68t3pylUtIdzDz3EG0LalVYpEDyTIygzrriRsdXC+Na1KXdr5E + GC0BZeG4QNS6XAsNS0/4SgT9ceA5DkgBCln58HRXabc25Tyfm2RiLQ70apWdEuoQ + TBoiWoMDeDmGLlquA5J2rBZh2XNThmpKU7PJ+2g3NQQubDeUjGEa6hvDwZ3vni6V + vVqsviCYJLcMHoHgJGtTTUoRO5Q6terCpRADMhQ014HYugZVBRdbbVGPo3YetrzU + /BuhvvROvb5dhWVi7zBUw2hUgQ0g0OpJB2TaJizXA+jIQ/x2HiO4QSUihp4JZJrL + 5G4P8dv7c7/BOqdj19VXV974RAnqDNSpuAsnmObVDO3Oy0eKj1J1eSIp5ZOA9Q3d + bHinx13rh5nMVbn3FxIemTYEbUFUbqa0eB3GRFoDz4iBGR4NqwIboP317S27NLDY + J8L6KmXTyNh8/Cm2l7wKlkwi3ItBGoAT+j3cOG988+3slgM9vXMaQRRQv9O1aTs1 + ZAai+Jq7AGjGh4ZkuG0cDZ2DuBy22XsUNboxQeHbQTsAPzQfvi+fQByUi6TzxiW0 + BeiJ6tEeDHDzdA== + =4Qn0 + -----END PGP PUBLIC KEY BLOCK----- + """ + return GPG_KEY + +if __name__ == "__main__": + main() diff --git a/imagebuild/coreos/docker_build.bash b/imagebuild/coreos/docker_build.bash new file mode 100755 index 000000000..d8b6715dc --- /dev/null +++ b/imagebuild/coreos/docker_build.bash @@ -0,0 +1,26 @@ +#!/bin/bash +# +# docker_build.bash - Prepares and outputs a tarball'd docker repository +# suitable for injection into a coreos pxe image +# + +set -e + +OUTPUT_FILE="oem/container.tar.gz" + +# If there's already a container.tar.gz, don't overwrite it -- instead, bail +if [[ -e "${OUTPUT_FILE}" ]]; then + echo "${OUTPUT_FILE} already exists. Will not overwrite. Exiting." + exit 1 +fi + +# Build the docker image +cd ../../teeth-agent +docker build -t oemdocker . +cd - + +# Export the oemdocker repository to a tarball so it can be embedded in CoreOS +# TODO: Investigate running a container and using "export" to flatten the +# image to shrink the CoreOS fs size. This will also require run.sh to +# use docker import instead of docker load as well. +docker save oemdocker | gzip > ${OUTPUT_FILE} diff --git a/imagebuild/coreos/docker_clean.bash b/imagebuild/coreos/docker_clean.bash new file mode 100755 index 000000000..091c8998a --- /dev/null +++ b/imagebuild/coreos/docker_clean.bash @@ -0,0 +1,16 @@ +#!/bin/bash +# +# Cleans up docker images and containers + +containers=$(docker ps -a -q) +images=$(docker images -q) + +# All the docker commands followed by || true because occassionally docker +# will fail to remove an image or container, & I want make to keep going anyway +if [[ ! -z "$containers" ]]; then + docker rm $containers || true +fi + +if [[ ! -z "$images" ]]; then + docker rmi $images || true +fi diff --git a/imagebuild/coreos/oem/run.sh b/imagebuild/coreos/oem/run.sh new file mode 100755 index 000000000..ff09be45d --- /dev/null +++ b/imagebuild/coreos/oem/run.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -e + +# CoreOS by default only has an OEM partition of 2GB. This isn't large enough +# for some images. Remount it with a larger size. Note: When CoreOS changes to +# r/w /, instead of remounting here, we'll use rootflags= to set the size. +mount -o remount,size=20G /media/state + +cd /usr/share/oem/ + +mkdir -pm 0700 /home/core/.ssh + +# TODO: Use proper https://github.com/coreos/init/blob/master/bin/update-ssh-keys script +if [[ -e authorized_keys ]]; then + cat authorized_keys >> /home/core/.ssh/authorized_keys +fi + +chown -R core:core /home/core/.ssh/ + +# We have to wait until docker is started to proceed +# In a perfect world I'd use inotifywait, but that doesn't exist on coreos +while [ ! -e /var/run/docker.sock ]; do + sleep 1; +done + +# TODO: Use docker import (and export the image) to shrink image size +docker load < container.tar.gz + +systemctl enable --runtime /usr/share/oem/system/* +systemctl start teeth-agent.service diff --git a/imagebuild/coreos/oem/system/teeth-agent.service b/imagebuild/coreos/oem/system/teeth-agent.service new file mode 100644 index 000000000..a25c2ba7d --- /dev/null +++ b/imagebuild/coreos/oem/system/teeth-agent.service @@ -0,0 +1,6 @@ +[Service] +ExecStart=/usr/bin/docker run -p 9999:9999 -privileged=true -v=/sys:/mnt/sys oemdocker /usr/local/bin/teeth-agent --ipaddr="`ip a | grep '10\.' | sed -e 's/inet \(10\.[0-9\.]\+\).*/\1/'`" +Restart=always + +[Install] +WantedBy=oem.target diff --git a/requirements.txt b/requirements.txt index 65ae9972f..b14a0bd96 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,9 @@ Werkzeug==0.9.4 requests==2.0.0 -cherrypy==3.2.4 stevedore==0.14 --e git+https://github.com/racker/teeth-rest.git@e876c0fddd5ce2f5223ab16936f711b0d57e19c4#egg=teeth_rest -structlog -ordereddict \ No newline at end of file +ordereddict>=1.1 +wsgiref>=0.1.2 +pecan>=0.4.5 +WSME>=0.6 +six>=1.5.2 +structlog==0.4.1 diff --git a/teeth_agent/agent.py b/teeth_agent/agent.py index b14863941..1b8883c2d 100644 --- a/teeth_agent/agent.py +++ b/teeth_agent/agent.py @@ -18,15 +18,14 @@ import random import threading import time -from cherrypy import wsgiserver import pkg_resources from stevedore import driver import structlog -from teeth_rest import encoding -from teeth_rest import errors as rest_errors +from wsgiref import simple_server -from teeth_agent import api +from teeth_agent.api import app from teeth_agent import base +from teeth_agent import encoding from teeth_agent import errors from teeth_agent import hardware from teeth_agent import overlord_agent_api @@ -39,7 +38,7 @@ class TeethAgentStatus(encoding.Serializable): self.started_at = started_at self.version = version - def serialize(self, view): + def serialize(self): """Turn the status into a dict.""" return utils.get_ordereddict([ ('mode', self.mode), @@ -107,12 +106,13 @@ class TeethAgentHeartbeater(threading.Thread): class TeethAgent(object): - def __init__(self, api_url, listen_address): + def __init__(self, api_url, listen_address, ipaddr): self.api_url = api_url self.listen_address = listen_address + self.ipaddr = ipaddr self.mode_implementation = None self.version = pkg_resources.get_distribution('teeth-agent').version - self.api = api.TeethAgentAPIServer(self) + self.api = app.VersionSelectorApplication(self) self.command_results = utils.get_ordereddict() self.heartbeater = TeethAgentHeartbeater(self) self.hardware = hardware.get_manager() @@ -180,14 +180,17 @@ class TeethAgent(object): try: result = self.mode_implementation.execute(command_part, **kwargs) - except rest_errors.InvalidContentError as e: + except errors.InvalidContentError as e: # Any command may raise a InvalidContentError which will be # returned to the caller directly. raise e except Exception as e: # Other errors are considered command execution errors, and are # recorded as an - result = base.SyncCommandResult(command_name, kwargs, False, e) + result = base.SyncCommandResult(command_name, + kwargs, + False, + unicode(e)) self.command_results[result.id] = result return result @@ -196,13 +199,16 @@ class TeethAgent(object): """Run the Teeth Agent.""" self.started_at = time.time() self.heartbeater.start() - server = wsgiserver.CherryPyWSGIServer(self.listen_address, self.api) + wsgi = simple_server.make_server( + self.listen_address[0], + self.listen_address[1], + self.api, + server_class=simple_server.WSGIServer) try: - server.start() + wsgi.serve_forever() except BaseException as e: self.log.error('shutting down', exception=e) - server.stop() self.heartbeater.stop() @@ -217,5 +223,5 @@ def _load_mode_implementation(mode_name): return mgr.driver -def build_agent(api_url, listen_host, listen_port): - return TeethAgent(api_url, (listen_host, listen_port)) +def build_agent(api_url, listen_host, listen_port, ipaddr): + return TeethAgent(api_url, (listen_host, listen_port), ipaddr) diff --git a/teeth_agent/api.py b/teeth_agent/api.py deleted file mode 100644 index 32be905c5..000000000 --- a/teeth_agent/api.py +++ /dev/null @@ -1,99 +0,0 @@ -""" -Copyright 2013 Rackspace, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -from teeth_rest import component -from teeth_rest import errors -from teeth_rest import responses - - -class AgentCommand(object): - def __init__(self, name, params): - self.name = name - self.params = params - - @classmethod - def deserialize(cls, obj): - for field in ['name', 'params']: - if field not in obj: - msg = 'Missing command \'{0}\' field.'.format(field) - raise errors.InvalidContentError(msg) - - if type(obj['params']) != dict: - raise errors.InvalidContentError( - 'Command params must be a dictionary.') - - return cls(obj['name'], obj['params']) - - -class TeethAgentAPI(component.APIComponent): - """The primary Teeth Agent API.""" - - def __init__(self, agent): - super(TeethAgentAPI, self).__init__() - self.agent = agent - - def add_routes(self): - """Called during initialization. Override to map relative routes to - methods. - """ - self.route('GET', '/status', self.get_agent_status) - self.route('GET', '/commands', self.list_command_results) - self.route('POST', '/commands', self.execute_command) - self.route('GET', - '/commands/', - self.get_command_result) - - def get_agent_status(self, request): - """Get the status of the agent.""" - return responses.ItemResponse(self.agent.get_status()) - - def list_command_results(self, request): - # TODO(russellhaering): pagination - command_results = self.agent.list_command_results() - return responses.PaginatedResponse(request, - command_results, - self.list_command_results, - None, - None) - - def execute_command(self, request): - """Execute a command on the agent.""" - command = AgentCommand.deserialize(self.parse_content(request)) - result = self.agent.execute_command(command.name, **command.params) - - wait = request.args.get('wait') - if wait and wait.lower() == 'true': - result.join() - - return responses.ItemResponse(result) - - def get_command_result(self, request, result_id): - """Retrieve the result of a command.""" - result = self.agent.get_command_result(result_id) - - wait = request.args.get('wait') - if wait and wait.lower() == 'true': - result.join() - - return responses.ItemResponse(result) - - -class TeethAgentAPIServer(component.APIServer): - """Server for the teeth agent API.""" - - def __init__(self, agent): - super(TeethAgentAPIServer, self).__init__() - self.add_component('/v1.0', TeethAgentAPI(agent)) diff --git a/teeth_agent/api/__init__.py b/teeth_agent/api/__init__.py new file mode 100644 index 000000000..2a30de06d --- /dev/null +++ b/teeth_agent/api/__init__.py @@ -0,0 +1,15 @@ +""" +Copyright 2014 Rackspace, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/teeth_agent/api/app.py b/teeth_agent/api/app.py new file mode 100644 index 000000000..b9560d4f4 --- /dev/null +++ b/teeth_agent/api/app.py @@ -0,0 +1,63 @@ +""" +Copyright 2014 Rackspace, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import pecan +from pecan import hooks + +from teeth_agent.api import config + + +class AgentHook(hooks.PecanHook): + def __init__(self, agent, *args, **kwargs): + super(AgentHook, self).__init__(*args, **kwargs) + self.agent = agent + + def before(self, state): + state.request.agent = self.agent + + +def get_pecan_config(): + # Set up the pecan configuration + filename = config.__file__.replace('.pyc', '.py') + return pecan.configuration.conf_from_file(filename) + + +def setup_app(agent, pecan_config=None, extra_hooks=None): + app_hooks = [AgentHook(agent)] + + if not pecan_config: + pecan_config = get_pecan_config() + + pecan.configuration.set_config(dict(pecan_config), overwrite=True) + + app = pecan.make_app( + pecan_config.app.root, + static_root=pecan_config.app.static_root, + debug=pecan_config.app.debug, + force_canonical=getattr(pecan_config.app, 'force_canonical', True), + hooks=app_hooks, + ) + + return app + + +class VersionSelectorApplication(object): + def __init__(self, agent): + pc = get_pecan_config() + self.v1 = setup_app(agent, pecan_config=pc) + + def __call__(self, environ, start_response): + return self.v1(environ, start_response) diff --git a/teeth_agent/api/app.wsgi b/teeth_agent/api/app.wsgi new file mode 100644 index 000000000..e69de29bb diff --git a/teeth_agent/api/config.py b/teeth_agent/api/config.py new file mode 100644 index 000000000..8814a1c1e --- /dev/null +++ b/teeth_agent/api/config.py @@ -0,0 +1,39 @@ +""" +Copyright 2014 Rackspace, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# Server Specific Configurations +# See https://pecan.readthedocs.org/en/latest/configuration.html#server-configuration # noqa +server = { + 'port': '9999', + 'host': '0.0.0.0' +} + +# Pecan Application Configurations +# See https://pecan.readthedocs.org/en/latest/configuration.html#application-configuration # noqa +app = { + 'root': 'teeth_agent.api.controllers.root.RootController', + 'modules': ['teeth_agent.api'], + 'static_root': '%(confdir)s/public', + 'debug': False, + 'enable_acl': True, + 'acl_public_routes': ['/', '/v1'], +} + +# WSME Configurations +# See https://wsme.readthedocs.org/en/latest/integrate.html#configuration +wsme = { + 'debug': False, +} diff --git a/teeth_agent/api/controllers/__init__.py b/teeth_agent/api/controllers/__init__.py new file mode 100644 index 000000000..2a30de06d --- /dev/null +++ b/teeth_agent/api/controllers/__init__.py @@ -0,0 +1,15 @@ +""" +Copyright 2014 Rackspace, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/teeth_agent/api/controllers/root.py b/teeth_agent/api/controllers/root.py new file mode 100644 index 000000000..4552e731b --- /dev/null +++ b/teeth_agent/api/controllers/root.py @@ -0,0 +1,96 @@ +# Copyright 2014 Rackspace, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pecan +from pecan import rest + +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from teeth_agent.api.controllers import v1 +from teeth_agent.api.controllers.v1 import base +from teeth_agent.api.controllers.v1 import link + + +class Version(base.APIBase): + """An API version representation.""" + + id = wtypes.text + "The ID of the version, also acts as the release number" + + links = [link.Link] + "A Link that point to a specific version of the API" + + @classmethod + def convert(self, id): + version = Version() + version.id = id + version.links = [link.Link.make_link('self', pecan.request.host_url, + id, '', bookmark=True)] + return version + + +class Root(base.APIBase): + + name = wtypes.text + "The name of the API" + + description = wtypes.text + "Some information about this API" + + versions = [Version] + "Links to all the versions available in this API" + + default_version = Version + "A link to the default version of the API" + + @classmethod + def convert(self): + root = Root() + root.name = 'OpenStack Ironic Python Agent API' + root.description = ('Ironic Python Agent is a provisioning agent for ' + 'OpenStack Ironic') + root.versions = [Version.convert('v1')] + root.default_version = Version.convert('v1') + return root + + +class RootController(rest.RestController): + + _versions = ['v1'] + "All supported API versions" + + _default_version = 'v1' + "The default API version" + + v1 = v1.Controller() + + @wsme_pecan.wsexpose(Root) + def get(self): + # NOTE: The reason why convert() it's being called for every + # request is because we need to get the host url from + # the request object to make the links. + return Root.convert() + + @pecan.expose() + def _route(self, args): + """Overrides the default routing behavior. + + It redirects the request to the default version of the ironic API + if the version number is not specified in the url. + """ + + if args[0] and args[0] not in self._versions: + args = [self._default_version] + args + return super(RootController, self)._route(args) diff --git a/teeth_agent/api/controllers/v1/__init__.py b/teeth_agent/api/controllers/v1/__init__.py new file mode 100644 index 000000000..3fbd9cd25 --- /dev/null +++ b/teeth_agent/api/controllers/v1/__init__.py @@ -0,0 +1,118 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Version 1 of the Ironic Python Agent API +""" + +import pecan +from pecan import rest + +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from teeth_agent.api.controllers.v1 import base +from teeth_agent.api.controllers.v1 import command +from teeth_agent.api.controllers.v1 import link +from teeth_agent.api.controllers.v1 import status + + +class MediaType(base.APIBase): + """A media type representation.""" + + base = wtypes.text + type = wtypes.text + + def __init__(self, base, type): + self.base = base + self.type = type + + +class V1(base.APIBase): + """The representation of the version 1 of the API.""" + + id = wtypes.text + "The ID of the version, also acts as the release number" + + media_types = [MediaType] + "An array of supported media types for this version" + + links = [link.Link] + "Links that point to a specific URL for this version and documentation" + + commands = [link.Link] + "Links to the command resource" + + status = [link.Link] + "Links to the status resource" + + @classmethod + def convert(self): + v1 = V1() + v1.id = "v1" + v1.links = [ + link.Link.make_link('self', + pecan.request.host_url, + 'v1', + '', + bookmark=True), + link.Link.make_link('describedby', + 'https://github.com', + 'rackerlabs', + 'teeth-agent', + bookmark=True, + type='text/html') + ] + v1.command = [ + link.Link.make_link('self', + pecan.request.host_url, + 'commands', + ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'commands', + '', + bookmark=True) + ] + v1.status = [ + link.Link.make_link('self', + pecan.request.host_url, + 'status', + ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'status', + '', + bookmark=True) + ] + v1.media_types = [MediaType('application/json', + ('application/vnd.openstack.' + 'ironic-python-agent.v1+json'))] + return v1 + + +class Controller(rest.RestController): + """Version 1 API controller root.""" + + commands = command.CommandController() + status = status.StatusController() + + @wsme_pecan.wsexpose(V1) + def get(self): + # NOTE: The reason why convert() it's being called for every + # request is because we need to get the host url from + # the request object to make the links. + return V1.convert() + +__all__ = (Controller) diff --git a/teeth_agent/api/controllers/v1/base.py b/teeth_agent/api/controllers/v1/base.py new file mode 100644 index 000000000..20af89648 --- /dev/null +++ b/teeth_agent/api/controllers/v1/base.py @@ -0,0 +1,73 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import six +from wsme import types as wtypes + + +class ExceptionType(wtypes.UserType): + basetype = wtypes.DictType + name = 'exception' + + def validate(self, value): + if not isinstance(value, BaseException): + raise ValueError('Value is not an exception') + return value + + def tobasetype(self, value): + """Turn a RESTError into a dict.""" + return { + 'type': value.__class__.__name__, + 'code': value.status_code, + 'message': value.message, + 'details': value.details, + } + + frombasetype = tobasetype + + +exception_type = ExceptionType() + + +class MultiType(wtypes.UserType): + """A complex type that represents one or more types. + + Used for validating that a value is an instance of one of the types. + + :param *types: Variable-length list of types. + + """ + def __init__(self, *types): + self.types = types + + def __str__(self): + return ' | '.join(map(str, self.types)) + + def validate(self, value): + for t in self.types: + if t is wtypes.text and isinstance(value, wtypes.bytes): + value = value.decode() + if isinstance(value, t): + return value + else: + raise ValueError( + "Wrong type. Expected '{type}', got '{value}'".format( + type=self.types, value=type(value))) + + +json_type = MultiType(list, dict, six.integer_types, wtypes.text) + + +class APIBase(wtypes.Base): + pass diff --git a/teeth_agent/api/controllers/v1/command.py b/teeth_agent/api/controllers/v1/command.py new file mode 100644 index 000000000..811000007 --- /dev/null +++ b/teeth_agent/api/controllers/v1/command.py @@ -0,0 +1,89 @@ +# Copyright 2014 Rackspace, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pecan +from pecan import rest +from wsme import types +from wsmeext import pecan as wsme_pecan + +from teeth_agent.api.controllers.v1 import base + + +class CommandResult(base.APIBase): + id = types.text + command_name = types.text + command_params = types.DictType(types.text, base.json_type) + command_status = types.text + command_error = base.exception_type + command_result = types.DictType(types.text, base.json_type) + + @classmethod + def from_result(cls, result): + instance = cls() + for field in ('id', 'command_name', 'command_params', 'command_status', + 'command_error', 'command_result'): + setattr(instance, field, getattr(result, field)) + return instance + + +class CommandResultList(base.APIBase): + commands = [CommandResult] + + @classmethod + def from_results(cls, results): + instance = cls() + instance.commands = [CommandResult.from_result(result) + for result in results] + return instance + + +class Command(base.APIBase): + """A command representation.""" + name = types.wsattr(types.text, mandatory=True) + params = types.wsattr(base.MultiType(dict), mandatory=True) + + +class CommandController(rest.RestController): + """Controller for issuing commands and polling for command status.""" + + @wsme_pecan.wsexpose(CommandResultList) + def get_all(self): + agent = pecan.request.agent + results = agent.list_command_results() + return CommandResultList.from_results(results) + + @wsme_pecan.wsexpose(CommandResult, types.text, types.text) + def get_one(self, result_id, wait=False): + agent = pecan.request.agent + result = agent.get_command_result(result_id) + + if wait and wait.lower() == 'true': + result.join() + + return CommandResult.from_result(result) + + @wsme_pecan.wsexpose(CommandResult, body=Command) + def post(self, wait=False, command=None): + # the POST body is always the last arg, + # so command must be a kwarg here + if command is None: + command = Command() + agent = pecan.request.agent + result = agent.execute_command(command.name, **command.params) + + if wait and wait.lower() == 'true': + result.join() + + return result diff --git a/teeth_agent/api/controllers/v1/link.py b/teeth_agent/api/controllers/v1/link.py new file mode 100644 index 000000000..987eb386d --- /dev/null +++ b/teeth_agent/api/controllers/v1/link.py @@ -0,0 +1,43 @@ +# Copyright 2014 Rackspace, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from wsme import types as wtypes + +from teeth_agent.api.controllers.v1 import base + + +class Link(base.APIBase): + """A link representation.""" + + href = wtypes.text + "The url of a link." + + rel = wtypes.text + "The name of a link." + + type = wtypes.text + "Indicates the type of document/link." + + @classmethod + def make_link(cls, rel_name, url, resource, resource_args, + bookmark=False, type=wtypes.Unset): + template = '%s/%s' if bookmark else '%s/v1/%s' + # FIXME(lucasagomes): I'm getting a 404 when doing a GET on + # a nested resource that the URL ends with a '/'. + # https://groups.google.com/forum/#!topic/pecan-dev/QfSeviLg5qs + template += '%s' if resource_args.startswith('?') else '/%s' + + return Link(href=(template) % (url, resource, resource_args), + rel=rel_name, type=type) diff --git a/teeth_agent/api/controllers/v1/status.py b/teeth_agent/api/controllers/v1/status.py new file mode 100644 index 000000000..c631ee59b --- /dev/null +++ b/teeth_agent/api/controllers/v1/status.py @@ -0,0 +1,44 @@ +# Copyright 2014 Rackspace, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pecan +from pecan import rest +from wsme import types +from wsmeext import pecan as wsme_pecan + +from teeth_agent.api.controllers.v1 import base + + +class AgentStatus(base.APIBase): + mode = types.text + started_at = base.MultiType(float) + version = types.text + + @classmethod + def from_agent_status(cls, status): + instance = cls() + for field in ('mode', 'started_at', 'version'): + setattr(instance, field, getattr(status, field)) + return instance + + +class StatusController(rest.RestController): + """Controller for getting agent status.""" + + @wsme_pecan.wsexpose(AgentStatus) + def get_all(self): + agent = pecan.request.agent + status = agent.get_status() + return AgentStatus.from_agent_status(status) diff --git a/teeth_agent/base.py b/teeth_agent/base.py index 03b611e03..c6f30a331 100644 --- a/teeth_agent/base.py +++ b/teeth_agent/base.py @@ -18,37 +18,35 @@ import threading import uuid import structlog -from teeth_rest import encoding -from teeth_rest import errors as rest_errors +from teeth_agent import encoding from teeth_agent import errors -from teeth_agent import utils class AgentCommandStatus(object): - RUNNING = 'RUNNING' - SUCCEEDED = 'SUCCEEDED' - FAILED = 'FAILED' + RUNNING = u'RUNNING' + SUCCEEDED = u'SUCCEEDED' + FAILED = u'FAILED' class BaseCommandResult(encoding.Serializable): def __init__(self, command_name, command_params): - self.id = str(uuid.uuid4()) + self.id = unicode(uuid.uuid4()) self.command_name = command_name self.command_params = command_params self.command_status = AgentCommandStatus.RUNNING self.command_error = None self.command_result = None - def serialize(self, view): - return utils.get_ordereddict([ - ('id', self.id), - ('command_name', self.command_name), - ('command_params', self.command_params), - ('command_status', self.command_status), - ('command_error', self.command_error), - ('command_result', self.command_result), - ]) + def serialize(self): + return dict(( + (u'id', self.id), + (u'command_name', self.command_name), + (u'command_params', self.command_params), + (u'command_status', self.command_status), + (u'command_error', self.command_error), + (u'command_result', self.command_result), + )) def is_done(self): return self.command_status != AgentCommandStatus.RUNNING @@ -83,9 +81,9 @@ class AsyncCommandResult(BaseCommandResult): self.execution_thread = threading.Thread(target=self.run, name=thread_name) - def serialize(self, view): + def serialize(self): with self.command_state_lock: - return super(AsyncCommandResult, self).serialize(view) + return super(AsyncCommandResult, self).serialize() def start(self): self.execution_thread.start() @@ -108,7 +106,7 @@ class AsyncCommandResult(BaseCommandResult): self.command_status = AgentCommandStatus.SUCCEEDED except Exception as e: - if not isinstance(e, rest_errors.RESTError): + if not isinstance(e, errors.RESTError): e = errors.CommandExecutionError(str(e)) with self.command_state_lock: diff --git a/teeth_agent/cmd/agent.py b/teeth_agent/cmd/agent.py index 6e0ca6988..32fbc7722 100644 --- a/teeth_agent/cmd/agent.py +++ b/teeth_agent/cmd/agent.py @@ -39,8 +39,13 @@ def run(): type=int, help='The port to listen on') + parser.add_argument('--ipaddr', + required=True, + help='The external IP address to advertise to ironic') + args = parser.parse_args() log.configure() agent.build_agent(args.api_url, args.listen_host, - args.listen_port).run() + args.listen_port, + args.ipaddr).run() diff --git a/teeth_agent/encoding.py b/teeth_agent/encoding.py new file mode 100644 index 000000000..a6386579b --- /dev/null +++ b/teeth_agent/encoding.py @@ -0,0 +1,53 @@ +""" +Copyright 2013 Rackspace, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import json +import uuid + + +class Serializable(object): + """Base class for things that can be serialized.""" + def serialize(self): + """Turn this object into a dict.""" + raise NotImplementedError() + + +class RESTJSONEncoder(json.JSONEncoder): + """A slightly customized JSON encoder.""" + def encode(self, o): + """Turn an object into JSON. + + Appends a newline to responses when configured to pretty-print, + in order to make use of curl less painful from most shells. + """ + delimiter = '' + + # if indent is None, newlines are still inserted, so we should too. + if self.indent is not None: + delimiter = '\n' + + return super(RESTJSONEncoder, self).encode(o) + delimiter + + def default(self, o): + """Turn an object into a serializable object. In particular, by + calling :meth:`.Serializable.serialize`. + """ + if isinstance(o, Serializable): + return o.serialize() + elif isinstance(o, uuid.UUID): + return str(o) + else: + return json.JSONEncoder.default(self, o) diff --git a/teeth_agent/errors.py b/teeth_agent/errors.py index 3d0f870b5..367508eea 100644 --- a/teeth_agent/errors.py +++ b/teeth_agent/errors.py @@ -14,10 +14,49 @@ See the License for the specific language governing permissions and limitations under the License. """ -from teeth_rest import errors +from teeth_agent import encoding +from teeth_agent import utils -class CommandExecutionError(errors.RESTError): +class RESTError(Exception, encoding.Serializable): + """Base class for errors generated in teeth.""" + message = 'An error occurred' + details = 'An unexpected error occurred. Please try back later.' + status_code = 500 + + def serialize(self): + """Turn a RESTError into a dict.""" + return utils.get_ordereddict([ + ('type', self.__class__.__name__), + ('code', self.status_code), + ('message', self.message), + ('details', self.details), + ]) + + +class InvalidContentError(RESTError): + """Error which occurs when a user supplies invalid content, either + because that content cannot be parsed according to the advertised + `Content-Type`, or due to a content validation error. + """ + message = 'Invalid request body' + status_code = 400 + + def __init__(self, details): + self.details = details + + +class NotFound(RESTError): + """Error which occurs when a user supplies invalid content, either + because that content cannot be parsed according to the advertised + `Content-Type`, or due to a content validation error. + """ + message = 'Not found' + status_code = 404 + details = 'The requested URL was not found.' + + +class CommandExecutionError(RESTError): """Error raised when a command fails to execute.""" message = 'Command execution failed' @@ -27,7 +66,7 @@ class CommandExecutionError(errors.RESTError): self.details = details -class InvalidCommandError(errors.InvalidContentError): +class InvalidCommandError(InvalidContentError): """Error which is raised when an unknown command is issued.""" messsage = 'Invalid command' @@ -36,7 +75,7 @@ class InvalidCommandError(errors.InvalidContentError): super(InvalidCommandError, self).__init__(details) -class InvalidCommandParamsError(errors.InvalidContentError): +class InvalidCommandParamsError(InvalidContentError): """Error which is raised when command parameters are invalid.""" message = 'Invalid command parameters' @@ -45,14 +84,14 @@ class InvalidCommandParamsError(errors.InvalidContentError): super(InvalidCommandParamsError, self).__init__(details) -class RequestedObjectNotFoundError(errors.NotFound): +class RequestedObjectNotFoundError(NotFound): def __init__(self, type_descr, obj_id): details = '{0} with id {1} not found.'.format(type_descr, obj_id) super(RequestedObjectNotFoundError, self).__init__(details) self.details = details -class OverlordAPIError(errors.RESTError): +class OverlordAPIError(RESTError): """Error raised when a call to the agent API fails.""" message = 'Error in call to teeth-agent-api.' @@ -71,7 +110,7 @@ class HeartbeatError(OverlordAPIError): super(HeartbeatError, self).__init__(details) -class ImageDownloadError(errors.RESTError): +class ImageDownloadError(RESTError): """Error raised when an image cannot be downloaded.""" message = 'Error downloading image.' @@ -81,7 +120,7 @@ class ImageDownloadError(errors.RESTError): self.details = 'Could not download image with id {0}.'.format(image_id) -class ImageChecksumError(errors.RESTError): +class ImageChecksumError(RESTError): """Error raised when an image fails to verify against its checksum.""" message = 'Error verifying image checksum.' @@ -92,7 +131,7 @@ class ImageChecksumError(errors.RESTError): self.details = self.details.format(image_id) -class ImageWriteError(errors.RESTError): +class ImageWriteError(RESTError): """Error raised when an image cannot be written to a device.""" message = 'Error writing image to device.' @@ -103,7 +142,7 @@ class ImageWriteError(errors.RESTError): self.details = self.details.format(device, exit_code) -class ConfigDriveWriteError(errors.RESTError): +class ConfigDriveWriteError(RESTError): """Error raised when a configdrive directory cannot be written to a device. """ @@ -118,7 +157,7 @@ class ConfigDriveWriteError(errors.RESTError): self.details = details -class SystemRebootError(errors.RESTError): +class SystemRebootError(RESTError): """Error raised when a system cannot reboot.""" message = 'Error rebooting system.' diff --git a/teeth_agent/hardware.py b/teeth_agent/hardware.py index 8753fd121..524fdafc2 100644 --- a/teeth_agent/hardware.py +++ b/teeth_agent/hardware.py @@ -21,10 +21,9 @@ import subprocess import stevedore import structlog +from teeth_agent import encoding from teeth_agent import utils -from teeth_rest import encoding - _global_manager = None @@ -50,7 +49,7 @@ class HardwareInfo(encoding.Serializable): self.type = type self.id = id - def serialize(self, view): + def serialize(self): return utils.get_ordereddict([ ('type', self.type), ('id', self.id), diff --git a/teeth_agent/log.py b/teeth_agent/log.py index a67ea3a64..13f218204 100644 --- a/teeth_agent/log.py +++ b/teeth_agent/log.py @@ -38,7 +38,7 @@ def _format_event(logger, method, event): have enough keys to format. """ if 'event' not in event: - # nothing to format, e.g. _log_request in teeth_rest/component + # nothing to format return event # Get a list of fields that need to be filled. formatter = string.Formatter() diff --git a/teeth_agent/overlord_agent_api.py b/teeth_agent/overlord_agent_api.py index 1ef250a32..e124769c3 100644 --- a/teeth_agent/overlord_agent_api.py +++ b/teeth_agent/overlord_agent_api.py @@ -17,8 +17,8 @@ limitations under the License. import json import requests -from teeth_rest import encoding +from teeth_agent import encoding from teeth_agent import errors @@ -28,8 +28,7 @@ class APIClient(object): def __init__(self, api_url): self.api_url = api_url.rstrip('/') self.session = requests.Session() - self.encoder = encoding.RESTJSONEncoder( - encoding.SerializationViews.PUBLIC) + self.encoder = encoding.RESTJSONEncoder() def _request(self, method, path, data=None): request_url = '{api_url}{path}'.format(api_url=self.api_url, path=path) diff --git a/teeth_agent/tests/agent.py b/teeth_agent/tests/agent.py index f1d4ed20c..f32c0a53f 100644 --- a/teeth_agent/tests/agent.py +++ b/teeth_agent/tests/agent.py @@ -20,11 +20,12 @@ import unittest import mock import pkg_resources +from wsgiref import simple_server -from teeth_rest import encoding from teeth_agent import agent from teeth_agent import base +from teeth_agent import encoding from teeth_agent import errors from teeth_agent import hardware @@ -117,11 +118,10 @@ class TestHeartbeater(unittest.TestCase): class TestBaseAgent(unittest.TestCase): def setUp(self): - self.encoder = encoding.RESTJSONEncoder( - encoding.SerializationViews.PUBLIC, - indent=4) + self.encoder = encoding.RESTJSONEncoder(indent=4) self.agent = agent.TeethAgent('https://fake_api.example.org:8081/', - ('localhost', 9999)) + ('localhost', 9999), + '192.168.1.1') def assertEqualEncoded(self, a, b): # Evidently JSONEncoder.default() can't handle None (??) so we have to @@ -136,7 +136,7 @@ class TestBaseAgent(unittest.TestCase): self.agent.started_at = started_at status = self.agent.get_status() - self.assertIsInstance(status, agent.TeethAgentStatus) + self.assertTrue(isinstance(status, agent.TeethAgentStatus)) self.assertEqual(status.started_at, started_at) self.assertEqual(status.version, pkg_resources.get_distribution('teeth-agent').version) @@ -156,7 +156,7 @@ class TestBaseAgent(unittest.TestCase): 'do_something', foo='bar') - @mock.patch('cherrypy.wsgiserver.CherryPyWSGIServer', autospec=True) + @mock.patch('wsgiref.simple_server.make_server', autospec=True) def test_run(self, wsgi_server_cls): wsgi_server = wsgi_server_cls.return_value wsgi_server.start.side_effect = KeyboardInterrupt() @@ -165,9 +165,12 @@ class TestBaseAgent(unittest.TestCase): self.agent.run() listen_addr = ('localhost', 9999) - wsgi_server_cls.assert_called_once_with(listen_addr, self.agent.api) - wsgi_server.start.assert_called_once_with() - wsgi_server.stop.assert_called_once_with() + wsgi_server_cls.assert_called_once_with( + listen_addr[0], + listen_addr[1], + self.agent.api, + server_class=simple_server.WSGIServer) + wsgi_server.serve_forever.assert_called_once() self.agent.heartbeater.start.assert_called_once_with() diff --git a/teeth_agent/tests/api.py b/teeth_agent/tests/api.py index fc0686640..bbcf47e29 100644 --- a/teeth_agent/tests/api.py +++ b/teeth_agent/tests/api.py @@ -22,10 +22,9 @@ import unittest from werkzeug import test from werkzeug import wrappers -from teeth_rest import encoding from teeth_agent import agent -from teeth_agent import api +from teeth_agent.api import app from teeth_agent import base @@ -44,13 +43,27 @@ class TestTeethAPI(unittest.TestCase): client = test.Client(api, wrappers.BaseResponse) return client.open(self._get_env_builder(method, path, data, query)) + def test_root(self): + mock_agent = mock.MagicMock() + api_server = app.setup_app(mock_agent) + + response = self._make_request(api_server, 'GET', '/') + self.assertEqual(response.status, '200 OK') + + def test_v1_root(self): + mock_agent = mock.MagicMock() + api_server = app.setup_app(mock_agent) + + response = self._make_request(api_server, 'GET', '/v1') + self.assertEqual(response.status, '200 OK') + def test_get_agent_status(self): status = agent.TeethAgentStatus('TEST_MODE', time.time(), 'v72ac9') mock_agent = mock.MagicMock() mock_agent.get_status.return_value = status - api_server = api.TeethAgentAPIServer(mock_agent) + api_server = app.setup_app(mock_agent) - response = self._make_request(api_server, 'GET', '/v1.0/status') + response = self._make_request(api_server, 'GET', '/v1/status') mock_agent.get_status.assert_called_once_with() self.assertEqual(response.status_code, 200) @@ -72,11 +85,11 @@ class TestTeethAPI(unittest.TestCase): mock_agent = mock.MagicMock() mock_agent.execute_command.return_value = result - api_server = api.TeethAgentAPIServer(mock_agent) + api_server = app.setup_app(mock_agent) response = self._make_request(api_server, 'POST', - '/v1.0/commands', + '/v1/commands/', data=command) self.assertEqual(mock_agent.execute_command.call_count, 1) @@ -85,55 +98,59 @@ class TestTeethAPI(unittest.TestCase): self.assertEqual(kwargs, {'key': 'value'}) self.assertEqual(response.status_code, 200) data = json.loads(response.data) - expected_result = result.serialize(encoding.SerializationViews.PUBLIC) + expected_result = result.serialize() self.assertEqual(data, expected_result) def test_execute_agent_command_validation(self): mock_agent = mock.MagicMock() - api_server = api.TeethAgentAPIServer(mock_agent) + api_server = app.setup_app(mock_agent) invalid_command = {} response = self._make_request(api_server, 'POST', - '/v1.0/commands', + '/v1/commands', data=invalid_command) self.assertEqual(response.status_code, 400) data = json.loads(response.data) - self.assertEqual(data['details'], 'Missing command \'name\' field.') + msg = 'Invalid input for field/attribute name.' + self.assertTrue(msg in data['faultstring']) + msg = 'Mandatory field missing' + self.assertTrue(msg in data['faultstring']) def test_execute_agent_command_params_validation(self): mock_agent = mock.MagicMock() - api_server = api.TeethAgentAPIServer(mock_agent) + api_server = app.setup_app(mock_agent) invalid_command = {'name': 'do_things', 'params': []} response = self._make_request(api_server, 'POST', - '/v1.0/commands', + '/v1/commands', data=invalid_command) self.assertEqual(response.status_code, 400) data = json.loads(response.data) - self.assertEqual(data['details'], - 'Command params must be a dictionary.') + # this message is actually much longer, but I'm ok with this + msg = 'Invalid input for field/attribute params.' + self.assertTrue(msg in data['faultstring']) def test_list_command_results(self): - cmd_result = base.SyncCommandResult('do_things', - {'key': 'value'}, + self.maxDiff = 10000 + cmd_result = base.SyncCommandResult(u'do_things', + {u'key': u'value'}, True, - {'test': 'result'}) + {u'test': u'result'}) mock_agent = mock.create_autospec(agent.TeethAgent) mock_agent.list_command_results.return_value = [ cmd_result, ] - api_server = api.TeethAgentAPIServer(mock_agent) - response = self._make_request(api_server, 'GET', '/v1.0/commands') + api_server = app.setup_app(mock_agent) + response = self._make_request(api_server, 'GET', '/v1/commands') self.assertEqual(response.status_code, 200) self.assertEqual(json.loads(response.data), { - 'items': [ - cmd_result.serialize(encoding.SerializationViews.PUBLIC), + u'commands': [ + cmd_result.serialize(), ], - 'links': [], }) def test_get_command_result(self): @@ -142,16 +159,15 @@ class TestTeethAPI(unittest.TestCase): True, {'test': 'result'}) - serialized_cmd_result = cmd_result.serialize( - encoding.SerializationViews.PUBLIC) + serialized_cmd_result = cmd_result.serialize() mock_agent = mock.create_autospec(agent.TeethAgent) mock_agent.get_command_result.return_value = cmd_result - api_server = api.TeethAgentAPIServer(mock_agent) + api_server = app.setup_app(mock_agent) response = self._make_request(api_server, 'GET', - '/v1.0/commands/abc123') + '/v1/commands/abc123') self.assertEqual(response.status_code, 200) data = json.loads(response.data) self.assertEqual(data, serialized_cmd_result)