From 141938ec3fe1b3557caf71e89617930a336c9fcc Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Tue, 4 Mar 2014 14:47:48 -0800 Subject: [PATCH 01/22] img(coreos): Initial commit of CoreOS image build - Mostly imported from github.com/racker/teeth-agent-image --- .gitignore | 4 + imagebuild/README.md | 4 + imagebuild/coreos/Makefile | 17 ++ imagebuild/coreos/README.md | 33 +++ imagebuild/coreos/coreos-oem-inject.py | 246 ++++++++++++++++++ imagebuild/coreos/docker_build.bash | 26 ++ imagebuild/coreos/docker_clean.bash | 16 ++ imagebuild/coreos/oem/run.sh | 31 +++ .../coreos/oem/system/teeth-agent.service | 6 + 9 files changed, 383 insertions(+) create mode 100644 imagebuild/README.md create mode 100644 imagebuild/coreos/Makefile create mode 100644 imagebuild/coreos/README.md create mode 100755 imagebuild/coreos/coreos-oem-inject.py create mode 100755 imagebuild/coreos/docker_build.bash create mode 100755 imagebuild/coreos/docker_clean.bash create mode 100755 imagebuild/coreos/oem/run.sh create mode 100644 imagebuild/coreos/oem/system/teeth-agent.service diff --git a/.gitignore b/.gitignore index 84e345ad8..b5cc23268 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,7 @@ ChangeLog devenv/* .coverage coverage.xml +imagebuild/coreos/build +imagebuild/coreos/dist +imagebuild/coreos/oem/authorized_keys +imagebuild/coreos/UPLOAD diff --git a/imagebuild/README.md b/imagebuild/README.md new file mode 100644 index 000000000..d073c9aad --- /dev/null +++ b/imagebuild/README.md @@ -0,0 +1,4 @@ +teeth-agent images +================== + +coreos - Builds a CoreOS Ramdisk and Kernel suitable for running teeth-agent diff --git a/imagebuild/coreos/Makefile b/imagebuild/coreos/Makefile new file mode 100644 index 000000000..26a1cc2b8 --- /dev/null +++ b/imagebuild/coreos/Makefile @@ -0,0 +1,17 @@ +default: docker coreos + +docker: + ./docker_build.bash + +coreos: + mkdir UPLOAD + ./coreos-oem-inject.py oem UPLOAD + +clean: + rm -rf teeth-agent + rm -f oem/container.tar.gz + rm -f UPLOAD/coreos_production_pxe_image-oem.cpio.gz + rm -f UPLOAD/coreos_production_pxe.vmlinuz + +docker_clean: + ./docker_clean.bash diff --git a/imagebuild/coreos/README.md b/imagebuild/coreos/README.md new file mode 100644 index 000000000..4798fe810 --- /dev/null +++ b/imagebuild/coreos/README.md @@ -0,0 +1,33 @@ +# teeth-agent CoreOS Image builder. + +Builds a CoreOS image suitable for running the teeth-agent on a server. + +# Requirements + +Must be run from a linux machine with a working docker installation and python-pip + +Run the following locally or from a virtualenv to install the python requirements +``` +pip install -r requirements.txt +``` + +# Instructions + +To create a docker repository and embed it into a CoreOS pxe image: +``` +make +``` + +To just create the docker repository in oem/container.tar.gz: +``` +make docker +``` + +To embed the oem/ directory into a CoreOS pxe image: + +Note: In order to have the ability to ssh into the created image, you need to +pass ssh keys in via the kernel command line for CoreOS, or create +oem/authorized_keys with the keys you need added before building the image. +``` +make coreos +``` diff --git a/imagebuild/coreos/coreos-oem-inject.py b/imagebuild/coreos/coreos-oem-inject.py new file mode 100755 index 000000000..d19aa879b --- /dev/null +++ b/imagebuild/coreos/coreos-oem-inject.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python + +import os +import sys +import time +import requests +import tempfile +import shutil +from plumbum import local, cmd + +COREOS_VERSION="197.0.0" + +COREOS_ARCH="amd64-generic" +COREOS_BASE_URL="http://storage.core-os.net/coreos/{}/{}".format(COREOS_ARCH, COREOS_VERSION) +COREOS_PXE_DIGESTS="coreos_production_pxe_image.cpio.gz.DIGESTS.asc" +COREOS_PXE_KERNEL="coreos_production_pxe.vmlinuz" +COREOS_PXE_IMAGE="coreos_production_pxe_image.cpio.gz" +COREOS_PXE_IMAGE_URL = "{}/{}".format(COREOS_BASE_URL, COREOS_PXE_IMAGE) +COREOS_PXE_KERNEL_URL = "{}/{}".format(COREOS_BASE_URL, COREOS_PXE_KERNEL) +COREOS_PXE_DIGESTS_URL = "{}/{}".format(COREOS_BASE_URL, COREOS_PXE_DIGESTS) + + + +def get_etag(cache_name): + etag_file = "{}.etag".format(cache_name) + if not os.path.exists(etag_file): + return None + with open(etag_file, 'rb') as fp: + etag = fp.read() + etag.strip() + return etag + +def save_etag(cache_name, etag): + etag_file = "{}.etag".format(cache_name) + with open(etag_file, 'w+b') as fp: + fp.write(etag) + +def cache_file(cache_name, remote_url): + print("{} <- {}".format(cache_name, remote_url)) + etag = get_etag(cache_name) + headers = {} + if etag: + headers['If-None-Match'] = etag + + start = time.time() + r = requests.get(remote_url, headers=headers) + + if r.status_code == 304: + print("[etag-match]") + return + + if r.status_code != 200: + raise RuntimeError('Failed to download {}, got HTTP {} Status Code.'.format(remote_url, r.status_code)) + + with open(cache_name, 'w+b') as fp: + fp.write(r.content) + + print("{} bytes in {} seconds".format(len(r.content), time.time() - start)) + save_etag(cache_name, r.headers['etag']) + +def inject_oem(archive, oem_dir, output_file): + d = tempfile.mkdtemp(prefix="oem-inject") + try: + with local.cwd(d): + dest_oem_dir = os.path.join(d, 'usr', 'share', 'oem') + uz = cmd.gunzip["-c", archive] + extract = cmd.cpio["-iv"] + chain = uz | extract + print chain + chain() + + shutil.copytree(oem_dir, dest_oem_dir) + + find = cmd.find['.', '-depth', '-print'] + cpio = cmd.cpio['-o', '-H', 'newc'] + gz = cmd.gzip + chain = find | cmd.sort | cpio | gz > output_file + print chain + chain() + finally: + shutil.rmtree(d) + return output_file + +def validate_digests(digests, target, hash_type='sha1'): + with local.cwd(os.path.dirname(digests)): + gethashes = cmd.grep['-i', '-A1', '^# {} HASH$'.format(hash_type), digests] + forthis = cmd.grep[os.path.basename(target)] + viasum = local[hash_type + "sum"]['-c', '/dev/stdin'] + chain = gethashes | forthis | viasum + print chain + chain() + +def main(): + if len(sys.argv) != 3: + print("usage: {} [oem-directory-to-inject] [output-directory]".format(os.path.basename(__file__))) + return + + oem_dir = os.path.abspath(os.path.expanduser(sys.argv[1])) + output_dir = os.path.abspath(os.path.expanduser(sys.argv[2])) + + if not os.path.exists(oem_dir): + print("Error: {} doesn't exist.".format(oem_dir)) + return + + if not os.path.exists(os.path.join(oem_dir, 'run.sh')): + print("Error: {} is missing oem.sh".format(oem_dir)) + return + + here = os.path.abspath(os.path.dirname(__file__)) + + top_cache_dir = os.path.join(os.path.dirname(here), ".image_cache") + cache_dir = os.path.join(top_cache_dir, COREOS_ARCH, COREOS_VERSION) + + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + + orig_cpio = os.path.join(cache_dir, COREOS_PXE_IMAGE) + digests = os.path.join(cache_dir, COREOS_PXE_DIGESTS) + kernel = os.path.join(cache_dir, COREOS_PXE_KERNEL) + + cache_file(digests, COREOS_PXE_DIGESTS_URL) + gpg_verify_file(digests) + cache_file(kernel, COREOS_PXE_KERNEL_URL) + validate_digests(digests, kernel) + cache_file(orig_cpio, COREOS_PXE_IMAGE_URL) + validate_digests(digests, orig_cpio) + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + output_kernel = os.path.join(output_dir, os.path.basename(kernel)) + output_cpio = os.path.join(output_dir, os.path.basename(orig_cpio).replace('.cpio.gz', '-oem.cpio.gz')) + inject_oem(orig_cpio, oem_dir, output_cpio) + shutil.copy(kernel, output_kernel) + +def gpg_verify_file(ascfile): + d = tempfile.mkdtemp(prefix="oem-gpg-validate") + try: + tmpring = os.path.join(d, 'tmp.gpg') + key = os.path.join(d, 'coreos.key') + with open(key, 'w+b') as fp: + fp.write(gpg_key()) + + i = cmd.gpg['--batch', + '--no-default-keyring', + '--keyring', + tmpring, + '--import', + key] + print(i) + i() + + r = cmd.gpg['--batch', + '--no-default-keyring', + '--keyring', + tmpring, + '--verify', + ascfile] + print(r) + r() + + finally: + shutil.rmtree(d) + +def gpg_key(): + GPG_LONG_ID="50E0885593D2DCB4" + GPG_KEY="""-----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v2.0.20 (GNU/Linux) + + mQINBFIqVhQBEADjC7oxg5N9Xqmqqrac70EHITgjEXZfGm7Q50fuQlqDoeNWY+sN + szpw//dWz8lxvPAqUlTSeR+dl7nwdpG2yJSBY6pXnXFF9sdHoFAUI0uy1Pp6VU9b + /9uMzZo+BBaIfojwHCa91JcX3FwLly5sPmNAjgiTeYoFmeb7vmV9ZMjoda1B8k4e + 8E0oVPgdDqCguBEP80NuosAONTib3fZ8ERmRw4HIwc9xjFDzyPpvyc25liyPKr57 + UDoDbO/DwhrrKGZP11JZHUn4mIAO7pniZYj/IC47aXEEuZNn95zACGMYqfn8A9+K + mHIHwr4ifS+k8UmQ2ly+HX+NfKJLTIUBcQY+7w6C5CHrVBImVHzHTYLvKWGH3pmB + zn8cCTgwW7mJ8bzQezt1MozCB1CYKv/SelvxisIQqyxqYB9q41g9x3hkePDRlh1s + 5ycvN0axEpSgxg10bLJdkhE+CfYkuANAyjQzAksFRa1ZlMQ5I+VVpXEECTVpLyLt + QQH87vtZS5xFaHUQnArXtZFu1WC0gZvMkNkJofv3GowNfanZb8iNtNFE8r1+GjL7 + a9NhaD8She0z2xQ4eZm8+Mtpz9ap/F7RLa9YgnJth5bDwLlAe30lg+7WIZHilR09 + UBHapoYlLB3B6RF51wWVneIlnTpMIJeP9vOGFBUqZ+W1j3O3uoLij1FUuwARAQAB + tDZDb3JlT1MgQnVpbGRib3QgKE9mZmljYWwgQnVpbGRzKSA8YnVpbGRib3RAY29y + ZW9zLmNvbT6JAjkEEwECACMFAlIqVhQCGwMHCwkIBwMCAQYVCAIJCgsEFgIDAQIe + AQIXgAAKCRBQ4IhVk9LctFkGD/46/I3S392oQQs81pUOMbPulCitA7/ehYPuVlgy + mv6+SEZOtafEJuI9uiTzlAVremZfalyL20RBtU10ANJfejp14rOpMadlRqz0DCvc + Wuuhhn9FEQE59Yk3LQ7DBLLbeJwUvEAtEEXq8xVXWh4OWgDiP5/3oALkJ4Lb3sFx + KwMy2JjkImr1XgMY7M2UVIomiSFD7v0H5Xjxaow/R6twttESyoO7TSI6eVyVgkWk + GjOSVK5MZOZlux7hW+uSbyUGPoYrfF6TKM9+UvBqxWzz9GBG44AjcViuOn9eH/kF + NoOAwzLcL0wjKs9lN1G4mhYALgzQx/2ZH5XO0IbfAx5Z0ZOgXk25gJajLTiqtOkM + E6u691Dx4c87kST2g7Cp3JMCC+cqG37xilbV4u03PD0izNBt/FLaTeddNpPJyttz + gYqeoSv2xCYC8AM9N73Yp1nT1G1rnCpe5Jct8Mwq7j8rQWIBArt3lt6mYFNjuNpg + om+rZstK8Ut1c8vOhSwz7Qza+3YaaNjLwaxe52RZ5svt6sCfIVO2sKHf3iO3aLzZ + 5KrCLZ/8tJtVxlhxRh0TqJVqFvOneP7TxkZs9DkU5uq5lHc9FWObPfbW5lhrU36K + Pf5pn0XomaWqge+GCBCgF369ibWbUAyGPqYj5wr/jwmG6nedMiqcOwpeBljpDF1i + d9zMN4kCHAQQAQIABgUCUipXUQAKCRDAr7X91+bcxwvZD/0T4mVRyAp8+EhCta6f + Qnoiqc49oHhnKsoN7wDg45NRlQP84rH1knn4/nSpUzrB29bhY8OgAiXXMHVcS+Uk + hUsF0sHNlnunbY0GEuIziqnrjEisb1cdIGyfsWUPc/4+inzu31J1n3iQyxdOOkrA + ddd0iQxPtyEjwevAfptGUeAGvtFXP374XsEo2fbd+xHMdV1YkMImLGx0guOK8tgp + +ht7cyHkfsyymrCV/WGaTdGMwtoJOxNZyaS6l0ccneW4UhORda2wwD0mOHHk2EHG + dJuEN4SRSoXQ0zjXvFr/u3k7Qww11xU0V4c6ZPl0Rd/ziqbiDImlyODCx6KUlmJb + k4l77XhHezWD0l3ZwodCV0xSgkOKLkudtgHPOBgHnJSL0vy7Ts6UzM/QLX5GR7uj + do7P/v0FrhXB+bMKvB/fMVHsKQNqPepigfrJ4+dZki7qtpx0iXFOfazYUB4CeMHC + 0gGIiBjQxKorzzcc5DVaVaGmmkYoBpxZeUsAD3YNFr6AVm3AGGZO4JahEOsul2FF + V6B0BiSwhg1SnZzBjkCcTCPURFm82aYsFuwWwqwizObZZNDC/DcFuuAuuEaarhO9 + BGzShpdbM3Phb4tjKKEJ9Sps6FBC2Cf/1pmPyOWZToMXex5ZKB0XHGCI0DFlB4Tn + in95D/b2+nYGUehmneuAmgde87kCDQRSKlZGARAAuMYYnu48l3AvE8ZpTN6uXSt2 + RrXnOr9oEah6hw1fn9KYKVJi0ZGJHzQOeAHHO/3BKYPFZNoUoNOU6VR/KAn7gon1 + wkUwk9Tn0AXVIQ7wMFJNLvcinoTkLBT5tqcAz5MvAoI9sivAM0Rm2BgeujdHjRS+ + UQKq/EZtpnodeQKE8+pwe3zdf6A9FZY2pnBs0PxKJ0NZ1rZeAW9w+2WdbyrkWxUv + jYWMSzTUkWK6533PVi7RcdRmWrDMNVR/X1PfqqAIzQkQ8oGcXtRpYjFL30Z/LhKe + c9Awfm57rkZk2EMduIB/Y5VYqnOsmKgUghXjOo6JOcanQZ4sHAyQrB2Yd6UgdAfz + qa7AWNIAljSGy6/CfJAoVIgl1revG7GCsRD5Dr/+BLyauwZ/YtTH9mGDtg6hy/So + zzDAM8+79Y8VMBUtj64GQBgg2+0MVZYNsZCN209X+EGpGUmAGEFQLGLHwFoNlwwL + 1Uj+/5NTAhp2MQA/XRDTVx1nm8MZZXUOu6NTCUXtUmgTQuQEsKCosQzBuT/G+8Ia + R5jBVZ38/NJgLw+YcRPNVo2S2XSh7liw+Sl1sdjEW1nWQHotDAzd2MFG++KVbxwb + cXbDgJOB0+N0c362WQ7bzxpJZoaYGhNOVjVjNY8YkcOiDl0DqkCk45obz4hG2T08 + x0OoXN7Oby0FclbUkVsAEQEAAYkERAQYAQIADwUCUipWRgIbAgUJAeEzgAIpCRBQ + 4IhVk9LctMFdIAQZAQIABgUCUipWRgAKCRClQeyydOfjYdY6D/4+PmhaiyasTHqh + iui2DwDVdhwxdikQEl+KQQHtk7aqgbUAxgU1D4rbLxzXyhTbmql7D30nl+oZg0Be + yl67Xo6X/wHsP44651aTbwxVT9nzhOp6OEW5z/qxJaX1B9EBsYtjGO87N854xC6a + QEaGZPbNauRpcYEadkppSumBo5ujmRWc4S+H1VjQW4vGSCm9m4X7a7L7/063HJza + SYaHybbu/udWW8ymzuUf/UARH4141bGnZOtIa9vIGtFl2oWJ/ViyJew9vwdMqiI6 + Y86ISQcGV/lL/iThNJBn+pots0CqdsoLvEZQGF3ZozWJVCKnnn/kC8NNyd7Wst9C + +p7ZzN3BTz+74Te5Vde3prQPFG4ClSzwJZ/U15boIMBPtNd7pRYum2padTK9oHp1 + l5dI/cELluj5JXT58hs5RAn4xD5XRNb4ahtnc/wdqtle0Kr5O0qNGQ0+U6ALdy/f + IVpSXihfsiy45+nPgGpfnRVmjQvIWQelI25+cvqxX1dr827ksUj4h6af/Bm9JvPG + KKRhORXPe+OQM6y/ubJOpYPEq9fZxdClekjA9IXhojNA8C6QKy2Kan873XDE0H4K + Y2OMTqQ1/n1A6g3qWCWph/sPdEMCsfnybDPcdPZp3psTQ8uX/vGLz0AAORapVCbp + iFHbF3TduuvnKaBWXKjrr5tNY/njrU4zEADTzhgbtGW75HSGgN3wtsiieMdfbH/P + f7wcC2FlbaQmevXjWI5tyx2m3ejG9gqnjRSyN5DWPq0m5AfKCY+4Glfjf01l7wR2 + 5oOvwL9lTtyrFE68t3pylUtIdzDz3EG0LalVYpEDyTIygzrriRsdXC+Na1KXdr5E + GC0BZeG4QNS6XAsNS0/4SgT9ceA5DkgBCln58HRXabc25Tyfm2RiLQ70apWdEuoQ + TBoiWoMDeDmGLlquA5J2rBZh2XNThmpKU7PJ+2g3NQQubDeUjGEa6hvDwZ3vni6V + vVqsviCYJLcMHoHgJGtTTUoRO5Q6terCpRADMhQ014HYugZVBRdbbVGPo3YetrzU + /BuhvvROvb5dhWVi7zBUw2hUgQ0g0OpJB2TaJizXA+jIQ/x2HiO4QSUihp4JZJrL + 5G4P8dv7c7/BOqdj19VXV974RAnqDNSpuAsnmObVDO3Oy0eKj1J1eSIp5ZOA9Q3d + bHinx13rh5nMVbn3FxIemTYEbUFUbqa0eB3GRFoDz4iBGR4NqwIboP317S27NLDY + J8L6KmXTyNh8/Cm2l7wKlkwi3ItBGoAT+j3cOG988+3slgM9vXMaQRRQv9O1aTs1 + ZAai+Jq7AGjGh4ZkuG0cDZ2DuBy22XsUNboxQeHbQTsAPzQfvi+fQByUi6TzxiW0 + BeiJ6tEeDHDzdA== + =4Qn0 + -----END PGP PUBLIC KEY BLOCK----- + """ + return GPG_KEY + +if __name__ == "__main__": + main() diff --git a/imagebuild/coreos/docker_build.bash b/imagebuild/coreos/docker_build.bash new file mode 100755 index 000000000..d8b6715dc --- /dev/null +++ b/imagebuild/coreos/docker_build.bash @@ -0,0 +1,26 @@ +#!/bin/bash +# +# docker_build.bash - Prepares and outputs a tarball'd docker repository +# suitable for injection into a coreos pxe image +# + +set -e + +OUTPUT_FILE="oem/container.tar.gz" + +# If there's already a container.tar.gz, don't overwrite it -- instead, bail +if [[ -e "${OUTPUT_FILE}" ]]; then + echo "${OUTPUT_FILE} already exists. Will not overwrite. Exiting." + exit 1 +fi + +# Build the docker image +cd ../../teeth-agent +docker build -t oemdocker . +cd - + +# Export the oemdocker repository to a tarball so it can be embedded in CoreOS +# TODO: Investigate running a container and using "export" to flatten the +# image to shrink the CoreOS fs size. This will also require run.sh to +# use docker import instead of docker load as well. +docker save oemdocker | gzip > ${OUTPUT_FILE} diff --git a/imagebuild/coreos/docker_clean.bash b/imagebuild/coreos/docker_clean.bash new file mode 100755 index 000000000..091c8998a --- /dev/null +++ b/imagebuild/coreos/docker_clean.bash @@ -0,0 +1,16 @@ +#!/bin/bash +# +# Cleans up docker images and containers + +containers=$(docker ps -a -q) +images=$(docker images -q) + +# All the docker commands followed by || true because occassionally docker +# will fail to remove an image or container, & I want make to keep going anyway +if [[ ! -z "$containers" ]]; then + docker rm $containers || true +fi + +if [[ ! -z "$images" ]]; then + docker rmi $images || true +fi diff --git a/imagebuild/coreos/oem/run.sh b/imagebuild/coreos/oem/run.sh new file mode 100755 index 000000000..ff09be45d --- /dev/null +++ b/imagebuild/coreos/oem/run.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -e + +# CoreOS by default only has an OEM partition of 2GB. This isn't large enough +# for some images. Remount it with a larger size. Note: When CoreOS changes to +# r/w /, instead of remounting here, we'll use rootflags= to set the size. +mount -o remount,size=20G /media/state + +cd /usr/share/oem/ + +mkdir -pm 0700 /home/core/.ssh + +# TODO: Use proper https://github.com/coreos/init/blob/master/bin/update-ssh-keys script +if [[ -e authorized_keys ]]; then + cat authorized_keys >> /home/core/.ssh/authorized_keys +fi + +chown -R core:core /home/core/.ssh/ + +# We have to wait until docker is started to proceed +# In a perfect world I'd use inotifywait, but that doesn't exist on coreos +while [ ! -e /var/run/docker.sock ]; do + sleep 1; +done + +# TODO: Use docker import (and export the image) to shrink image size +docker load < container.tar.gz + +systemctl enable --runtime /usr/share/oem/system/* +systemctl start teeth-agent.service diff --git a/imagebuild/coreos/oem/system/teeth-agent.service b/imagebuild/coreos/oem/system/teeth-agent.service new file mode 100644 index 000000000..dae67d05d --- /dev/null +++ b/imagebuild/coreos/oem/system/teeth-agent.service @@ -0,0 +1,6 @@ +[Service] +ExecStart=/usr/bin/docker run -p 9999:9999 -privileged=true -v=/sys:/mnt/sys oemdocker /usr/local/bin/teeth-agent --api-url http://10.127.75.252:8081 +Restart=always + +[Install] +WantedBy=oem.target From 18228e9093fc3f73e2216ace989989a22f343fd2 Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Fri, 7 Mar 2014 15:36:22 -0800 Subject: [PATCH 02/22] get things kind of working with pecan --- requirements.txt | 5 +- teeth_agent/agent.py | 15 ++- teeth_agent/api/__init__.py | 15 +++ teeth_agent/api/app.py | 66 ++++++++++ teeth_agent/api/app.wsgi | 0 teeth_agent/api/config.py | 39 ++++++ teeth_agent/api/controllers/__init__.py | 15 +++ teeth_agent/api/controllers/root.py | 100 +++++++++++++++ teeth_agent/api/controllers/v1/__init__.py | 135 +++++++++++++++++++++ teeth_agent/api/controllers/v1/base.py | 47 +++++++ teeth_agent/api/controllers/v1/link.py | 43 +++++++ teeth_agent/tests/agent.py | 12 +- 12 files changed, 481 insertions(+), 11 deletions(-) create mode 100644 teeth_agent/api/__init__.py create mode 100644 teeth_agent/api/app.py create mode 100644 teeth_agent/api/app.wsgi create mode 100644 teeth_agent/api/config.py create mode 100644 teeth_agent/api/controllers/__init__.py create mode 100644 teeth_agent/api/controllers/root.py create mode 100644 teeth_agent/api/controllers/v1/__init__.py create mode 100644 teeth_agent/api/controllers/v1/base.py create mode 100644 teeth_agent/api/controllers/v1/link.py diff --git a/requirements.txt b/requirements.txt index c38e4e3e9..148f0bcf8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,8 @@ Werkzeug==0.9.4 requests==2.0.0 -cherrypy==3.2.4 stevedore==0.14 -e git+https://github.com/racker/teeth-rest.git@e876c0fddd5ce2f5223ab16936f711b0d57e19c4#egg=teeth_rest +wsgiref>=0.1.2 +pecan>=0.4.5 +oslo.config>=1.2.0 +WSME>=0.6 diff --git a/teeth_agent/agent.py b/teeth_agent/agent.py index 3632bd0cb..3db736b61 100644 --- a/teeth_agent/agent.py +++ b/teeth_agent/agent.py @@ -19,14 +19,14 @@ import random import threading import time -from cherrypy import wsgiserver import pkg_resources from stevedore import driver import structlog from teeth_rest import encoding from teeth_rest import errors as rest_errors +from wsgiref import simple_server -from teeth_agent import api +from teeth_agent.api import app from teeth_agent import base from teeth_agent import errors from teeth_agent import hardware @@ -112,7 +112,7 @@ class TeethAgent(object): self.listen_address = listen_address self.mode_implementation = None self.version = pkg_resources.get_distribution('teeth-agent').version - self.api = api.TeethAgentAPIServer(self) + self.api = app.VersionSelectorApplication() self.command_results = collections.OrderedDict() self.heartbeater = TeethAgentHeartbeater(self) self.hardware = hardware.get_manager() @@ -196,13 +196,16 @@ class TeethAgent(object): """Run the Teeth Agent.""" self.started_at = time.time() self.heartbeater.start() - server = wsgiserver.CherryPyWSGIServer(self.listen_address, self.api) + wsgi = simple_server.make_server( + self.listen_address[0], + self.listen_address[1], + self.api, + server_class=simple_server.WSGIServer) try: - server.start() + wsgi.serve_forever() except BaseException as e: self.log.error('shutting down', exception=e) - server.stop() self.heartbeater.stop() diff --git a/teeth_agent/api/__init__.py b/teeth_agent/api/__init__.py new file mode 100644 index 000000000..2a30de06d --- /dev/null +++ b/teeth_agent/api/__init__.py @@ -0,0 +1,15 @@ +""" +Copyright 2014 Rackspace, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/teeth_agent/api/app.py b/teeth_agent/api/app.py new file mode 100644 index 000000000..3c25a6850 --- /dev/null +++ b/teeth_agent/api/app.py @@ -0,0 +1,66 @@ +""" +Copyright 2014 Rackspace, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from oslo.config import cfg +import pecan + +from teeth_agent.api import config + +CONF = cfg.CONF + + +def get_pecan_config(): + # Set up the pecan configuration + filename = config.__file__.replace('.pyc', '.py') + return pecan.configuration.conf_from_file(filename) + + +def setup_app(pecan_config=None, extra_hooks=None): + #policy.init() + + #app_hooks = [hooks.ConfigHook(), + #hooks.DBHook(), + #hooks.ContextHook(pecan_config.app.acl_public_routes), + #hooks.RPCHook(), + #hooks.NoExceptionTracebackHook()] + #if extra_hooks: + #app_hooks.extend(extra_hooks) + + if not pecan_config: + pecan_config = get_pecan_config() + + pecan.configuration.set_config(dict(pecan_config), overwrite=True) + + app = pecan.make_app( + pecan_config.app.root, + static_root=pecan_config.app.static_root, + debug=True, + #debug=CONF.debug, + force_canonical=getattr(pecan_config.app, 'force_canonical', True), + #hooks=app_hooks, + #wrap_app=middleware.ParsableErrorMiddleware, + ) + + return app + + +class VersionSelectorApplication(object): + def __init__(self): + pc = get_pecan_config() + self.v1 = setup_app(pecan_config=pc) + + def __call__(self, environ, start_response): + return self.v1(environ, start_response) diff --git a/teeth_agent/api/app.wsgi b/teeth_agent/api/app.wsgi new file mode 100644 index 000000000..e69de29bb diff --git a/teeth_agent/api/config.py b/teeth_agent/api/config.py new file mode 100644 index 000000000..8814a1c1e --- /dev/null +++ b/teeth_agent/api/config.py @@ -0,0 +1,39 @@ +""" +Copyright 2014 Rackspace, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# Server Specific Configurations +# See https://pecan.readthedocs.org/en/latest/configuration.html#server-configuration # noqa +server = { + 'port': '9999', + 'host': '0.0.0.0' +} + +# Pecan Application Configurations +# See https://pecan.readthedocs.org/en/latest/configuration.html#application-configuration # noqa +app = { + 'root': 'teeth_agent.api.controllers.root.RootController', + 'modules': ['teeth_agent.api'], + 'static_root': '%(confdir)s/public', + 'debug': False, + 'enable_acl': True, + 'acl_public_routes': ['/', '/v1'], +} + +# WSME Configurations +# See https://wsme.readthedocs.org/en/latest/integrate.html#configuration +wsme = { + 'debug': False, +} diff --git a/teeth_agent/api/controllers/__init__.py b/teeth_agent/api/controllers/__init__.py new file mode 100644 index 000000000..2a30de06d --- /dev/null +++ b/teeth_agent/api/controllers/__init__.py @@ -0,0 +1,15 @@ +""" +Copyright 2014 Rackspace, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/teeth_agent/api/controllers/root.py b/teeth_agent/api/controllers/root.py new file mode 100644 index 000000000..10776bfd8 --- /dev/null +++ b/teeth_agent/api/controllers/root.py @@ -0,0 +1,100 @@ +# -*- encoding: utf-8 -*- +# +# Copyright © 2012 New Dream Network, LLC (DreamHost) +# +# Author: Doug Hellmann +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pecan +from pecan import rest + +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from teeth_agent.api.controllers import v1 +from teeth_agent.api.controllers.v1 import base +from teeth_agent.api.controllers.v1 import link + + +class Version(base.APIBase): + """An API version representation.""" + + id = wtypes.text + "The ID of the version, also acts as the release number" + + links = [link.Link] + "A Link that point to a specific version of the API" + + @classmethod + def convert(self, id): + version = Version() + version.id = id + version.links = [link.Link.make_link('self', pecan.request.host_url, + id, '', bookmark=True)] + return version + + +class Root(base.APIBase): + + name = wtypes.text + "The name of the API" + + description = wtypes.text + "Some information about this API" + + versions = [Version] + "Links to all the versions available in this API" + + default_version = Version + "A link to the default version of the API" + + @classmethod + def convert(self): + root = Root() + root.name = "OpenStack Ironic API" + root.description = ("Ironic is an OpenStack project which aims to " + "provision baremetal machines.") + root.versions = [Version.convert('v1')] + root.default_version = Version.convert('v1') + return root + + +class RootController(rest.RestController): + + _versions = ['v1'] + "All supported API versions" + + _default_version = 'v1' + "The default API version" + + v1 = v1.Controller() + + @wsme_pecan.wsexpose(Root) + def get(self): + # NOTE: The reason why convert() it's being called for every + # request is because we need to get the host url from + # the request object to make the links. + return Root.convert() + + @pecan.expose() + def _route(self, args): + """Overrides the default routing behavior. + + It redirects the request to the default version of the ironic API + if the version number is not specified in the url. + """ + + if args[0] and args[0] not in self._versions: + args = [self._default_version] + args + return super(RootController, self)._route(args) diff --git a/teeth_agent/api/controllers/v1/__init__.py b/teeth_agent/api/controllers/v1/__init__.py new file mode 100644 index 000000000..a02b02e1b --- /dev/null +++ b/teeth_agent/api/controllers/v1/__init__.py @@ -0,0 +1,135 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Version 1 of the Ironic API + +NOTE: IN PROGRESS AND NOT FULLY IMPLEMENTED. + +Should maintain feature parity with Nova Baremetal Extension. + +Specification can be found at ironic/doc/api/v1.rst +""" + +import pecan +from pecan import rest + +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from teeth_agent.api.controllers.v1 import base +#from ironic.api.controllers.v1 import chassis +#from ironic.api.controllers.v1 import driver +from teeth_agent.api.controllers.v1 import link +#from ironic.api.controllers.v1 import node +#from ironic.api.controllers.v1 import port + + +class MediaType(base.APIBase): + """A media type representation.""" + + base = wtypes.text + type = wtypes.text + + def __init__(self, base, type): + self.base = base + self.type = type + + +class V1(base.APIBase): + """The representation of the version 1 of the API.""" + + id = wtypes.text + "The ID of the version, also acts as the release number" + + media_types = [MediaType] + "An array of supported media types for this version" + + links = [link.Link] + "Links that point to a specific URL for this version and documentation" + + #chassis = [link.Link] + #"Links to the chassis resource" + + #nodes = [link.Link] + #"Links to the nodes resource" + + #ports = [link.Link] + #"Links to the ports resource" + + #drivers = [link.Link] + #"Links to the drivers resource" + + @classmethod + def convert(self): + v1 = V1() + v1.id = "v1" + v1.links = [ + link.Link.make_link('self', pecan.request.host_url, + 'v1', '', bookmark=True), + link.Link.make_link('describedby', + 'http://docs.openstack.org', + 'developer/ironic/dev', + 'api-spec-v1.html', + bookmark=True, type='text/html') + ] + v1.media_types = [MediaType('application/json', + 'application/vnd.openstack.ironic.v1+json')] + #v1.chassis = [link.Link.make_link('self', pecan.request.host_url, + #'chassis', ''), + #link.Link.make_link('bookmark', + #pecan.request.host_url, + #'chassis', '', + #bookmark=True) + #] + #v1.nodes = [link.Link.make_link('self', pecan.request.host_url, + #'nodes', ''), + #link.Link.make_link('bookmark', + #pecan.request.host_url, + #'nodes', '', + #bookmark=True) + #] + #v1.ports = [link.Link.make_link('self', pecan.request.host_url, + #'ports', ''), + #link.Link.make_link('bookmark', + #pecan.request.host_url, + #'ports', '', + #bookmark=True) + #] + #v1.drivers = [link.Link.make_link('self', pecan.request.host_url, + #'drivers', ''), + #link.Link.make_link('bookmark', + #pecan.request.host_url, + #'drivers', '', + #bookmark=True) + #] + return v1 + + +class Controller(rest.RestController): + """Version 1 API controller root.""" + + #nodes = node.NodesController() + #ports = port.PortsController() + #chassis = chassis.ChassisController() + #drivers = driver.DriversController() + + @wsme_pecan.wsexpose(V1) + def get(self): + # NOTE: The reason why convert() it's being called for every + # request is because we need to get the host url from + # the request object to make the links. + return V1.convert() + +__all__ = (Controller) diff --git a/teeth_agent/api/controllers/v1/base.py b/teeth_agent/api/controllers/v1/base.py new file mode 100644 index 000000000..8a10ffbfc --- /dev/null +++ b/teeth_agent/api/controllers/v1/base.py @@ -0,0 +1,47 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime + +import wsme +from wsme import types as wtypes + + +class APIBase(wtypes.Base): + + created_at = wsme.wsattr(datetime.datetime, readonly=True) + "The time in UTC at which the object is created" + + updated_at = wsme.wsattr(datetime.datetime, readonly=True) + "The time in UTC at which the object is updated" + + def as_dict(self): + """Render this object as a dict of its fields.""" + return dict((k, getattr(self, k)) + for k in self.fields + if hasattr(self, k) and + getattr(self, k) != wsme.Unset) + + def unset_fields_except(self, except_list=None): + """Unset fields so they don't appear in the message body. + + :param except_list: A list of fields that won't be touched. + + """ + if except_list is None: + except_list = [] + + for k in self.as_dict(): + if k not in except_list: + setattr(self, k, wsme.Unset) diff --git a/teeth_agent/api/controllers/v1/link.py b/teeth_agent/api/controllers/v1/link.py new file mode 100644 index 000000000..d4e85000b --- /dev/null +++ b/teeth_agent/api/controllers/v1/link.py @@ -0,0 +1,43 @@ +# Copyright 2013 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from wsme import types as wtypes + +from teeth_agent.api.controllers.v1 import base + + +class Link(base.APIBase): + """A link representation.""" + + href = wtypes.text + "The url of a link." + + rel = wtypes.text + "The name of a link." + + type = wtypes.text + "Indicates the type of document/link." + + @classmethod + def make_link(cls, rel_name, url, resource, resource_args, + bookmark=False, type=wtypes.Unset): + template = '%s/%s' if bookmark else '%s/v1/%s' + # FIXME(lucasagomes): I'm getting a 404 when doing a GET on + # a nested resource that the URL ends with a '/'. + # https://groups.google.com/forum/#!topic/pecan-dev/QfSeviLg5qs + template += '%s' if resource_args.startswith('?') else '/%s' + + return Link(href=(template) % (url, resource, resource_args), + rel=rel_name, type=type) diff --git a/teeth_agent/tests/agent.py b/teeth_agent/tests/agent.py index f1d4ed20c..8fac1d865 100644 --- a/teeth_agent/tests/agent.py +++ b/teeth_agent/tests/agent.py @@ -20,6 +20,7 @@ import unittest import mock import pkg_resources +from wsgiref import simple_server from teeth_rest import encoding @@ -156,7 +157,7 @@ class TestBaseAgent(unittest.TestCase): 'do_something', foo='bar') - @mock.patch('cherrypy.wsgiserver.CherryPyWSGIServer', autospec=True) + @mock.patch('wsgiref.simple_server.make_server', autospec=True) def test_run(self, wsgi_server_cls): wsgi_server = wsgi_server_cls.return_value wsgi_server.start.side_effect = KeyboardInterrupt() @@ -165,9 +166,12 @@ class TestBaseAgent(unittest.TestCase): self.agent.run() listen_addr = ('localhost', 9999) - wsgi_server_cls.assert_called_once_with(listen_addr, self.agent.api) - wsgi_server.start.assert_called_once_with() - wsgi_server.stop.assert_called_once_with() + wsgi_server_cls.assert_called_once_with( + listen_addr[0], + listen_addr[1], + self.agent.api, + server_class=simple_server.WSGIServer) + wsgi_server.serve_forever.assert_called_once_with() self.agent.heartbeater.start.assert_called_once_with() From ecf8732ce529e875ed51bb426aa9b9a2cae81a3c Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Mon, 10 Mar 2014 13:45:50 -0700 Subject: [PATCH 03/22] remove api-url from unit file - Patch incoming to the agent for it to discover API url (and other params) via kernel command line --- imagebuild/coreos/oem/system/teeth-agent.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagebuild/coreos/oem/system/teeth-agent.service b/imagebuild/coreos/oem/system/teeth-agent.service index dae67d05d..60d853934 100644 --- a/imagebuild/coreos/oem/system/teeth-agent.service +++ b/imagebuild/coreos/oem/system/teeth-agent.service @@ -1,5 +1,5 @@ [Service] -ExecStart=/usr/bin/docker run -p 9999:9999 -privileged=true -v=/sys:/mnt/sys oemdocker /usr/local/bin/teeth-agent --api-url http://10.127.75.252:8081 +ExecStart=/usr/bin/docker run -p 9999:9999 -privileged=true -v=/sys:/mnt/sys oemdocker /usr/local/bin/teeth-agent Restart=always [Install] From c50c0805af6c3fa945a0e227327806ceaef3107d Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Mon, 10 Mar 2014 13:52:57 -0700 Subject: [PATCH 04/22] fix(coreos-oem-inject): Default to dev-channel - We'll switch this to a 'stable' channel once one exists for coreos --- imagebuild/coreos/coreos-oem-inject.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagebuild/coreos/coreos-oem-inject.py b/imagebuild/coreos/coreos-oem-inject.py index d19aa879b..e471b9a95 100755 --- a/imagebuild/coreos/coreos-oem-inject.py +++ b/imagebuild/coreos/coreos-oem-inject.py @@ -8,7 +8,7 @@ import tempfile import shutil from plumbum import local, cmd -COREOS_VERSION="197.0.0" +COREOS_VERSION="dev-channel" COREOS_ARCH="amd64-generic" COREOS_BASE_URL="http://storage.core-os.net/coreos/{}/{}".format(COREOS_ARCH, COREOS_VERSION) From 1c2389b99fdd8ab37464cd447abab662b313cdc7 Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Fri, 7 Mar 2014 13:30:45 -0800 Subject: [PATCH 05/22] add(gerrit support): Required to move to openstack Config taken from: http://ci.openstack.org/stackforge.html --- .gitreview | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitreview diff --git a/.gitreview b/.gitreview new file mode 100644 index 000000000..490ba065b --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=review.openstack.org +port=29418 +project=openstack/ironic-python-agent.git From 4f8bce3726b11321fce79b3249bc2f325a4038fb Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Tue, 11 Mar 2014 14:13:13 -0700 Subject: [PATCH 06/22] move everything into pecan --- requirements.txt | 1 + teeth_agent/agent.py | 7 +- teeth_agent/api/app.py | 20 ++++- teeth_agent/api/controllers/v1/__init__.py | 66 +++------------- teeth_agent/api/controllers/v1/base.py | 52 +++++++------ teeth_agent/api/controllers/v1/command.py | 88 ++++++++++++++++++++++ teeth_agent/base.py | 2 +- 7 files changed, 153 insertions(+), 83 deletions(-) create mode 100644 teeth_agent/api/controllers/v1/command.py diff --git a/requirements.txt b/requirements.txt index 148f0bcf8..19e997103 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ wsgiref>=0.1.2 pecan>=0.4.5 oslo.config>=1.2.0 WSME>=0.6 +six>=1.5.2 diff --git a/teeth_agent/agent.py b/teeth_agent/agent.py index 3db736b61..9083c989e 100644 --- a/teeth_agent/agent.py +++ b/teeth_agent/agent.py @@ -112,7 +112,7 @@ class TeethAgent(object): self.listen_address = listen_address self.mode_implementation = None self.version = pkg_resources.get_distribution('teeth-agent').version - self.api = app.VersionSelectorApplication() + self.api = app.VersionSelectorApplication(self) self.command_results = collections.OrderedDict() self.heartbeater = TeethAgentHeartbeater(self) self.hardware = hardware.get_manager() @@ -187,7 +187,10 @@ class TeethAgent(object): except Exception as e: # Other errors are considered command execution errors, and are # recorded as an - result = base.SyncCommandResult(command_name, kwargs, False, e) + result = base.SyncCommandResult(command_name, + kwargs, + False, + unicode(e)) self.command_results[result.id] = result return result diff --git a/teeth_agent/api/app.py b/teeth_agent/api/app.py index 3c25a6850..a850b6187 100644 --- a/teeth_agent/api/app.py +++ b/teeth_agent/api/app.py @@ -16,19 +16,29 @@ limitations under the License. from oslo.config import cfg import pecan +from pecan import hooks from teeth_agent.api import config CONF = cfg.CONF +class AgentHook(hooks.PecanHook): + def __init__(self, agent, *args, **kwargs): + super(AgentHook, self).__init__(*args, **kwargs) + self.agent = agent + + def before(self, state): + state.request.agent = self.agent + + def get_pecan_config(): # Set up the pecan configuration filename = config.__file__.replace('.pyc', '.py') return pecan.configuration.conf_from_file(filename) -def setup_app(pecan_config=None, extra_hooks=None): +def setup_app(agent, pecan_config=None, extra_hooks=None): #policy.init() #app_hooks = [hooks.ConfigHook(), @@ -39,6 +49,8 @@ def setup_app(pecan_config=None, extra_hooks=None): #if extra_hooks: #app_hooks.extend(extra_hooks) + app_hooks = [AgentHook(agent)] + if not pecan_config: pecan_config = get_pecan_config() @@ -50,7 +62,7 @@ def setup_app(pecan_config=None, extra_hooks=None): debug=True, #debug=CONF.debug, force_canonical=getattr(pecan_config.app, 'force_canonical', True), - #hooks=app_hooks, + hooks=app_hooks, #wrap_app=middleware.ParsableErrorMiddleware, ) @@ -58,9 +70,9 @@ def setup_app(pecan_config=None, extra_hooks=None): class VersionSelectorApplication(object): - def __init__(self): + def __init__(self, agent): pc = get_pecan_config() - self.v1 = setup_app(pecan_config=pc) + self.v1 = setup_app(agent, pecan_config=pc) def __call__(self, environ, start_response): return self.v1(environ, start_response) diff --git a/teeth_agent/api/controllers/v1/__init__.py b/teeth_agent/api/controllers/v1/__init__.py index a02b02e1b..8a120c118 100644 --- a/teeth_agent/api/controllers/v1/__init__.py +++ b/teeth_agent/api/controllers/v1/__init__.py @@ -13,13 +13,7 @@ # under the License. """ -Version 1 of the Ironic API - -NOTE: IN PROGRESS AND NOT FULLY IMPLEMENTED. - -Should maintain feature parity with Nova Baremetal Extension. - -Specification can be found at ironic/doc/api/v1.rst +Version 1 of the Ironic Python Agent API """ import pecan @@ -29,11 +23,8 @@ from wsme import types as wtypes import wsmeext.pecan as wsme_pecan from teeth_agent.api.controllers.v1 import base -#from ironic.api.controllers.v1 import chassis -#from ironic.api.controllers.v1 import driver +from teeth_agent.api.controllers.v1 import command from teeth_agent.api.controllers.v1 import link -#from ironic.api.controllers.v1 import node -#from ironic.api.controllers.v1 import port class MediaType(base.APIBase): @@ -59,17 +50,8 @@ class V1(base.APIBase): links = [link.Link] "Links that point to a specific URL for this version and documentation" - #chassis = [link.Link] - #"Links to the chassis resource" - - #nodes = [link.Link] - #"Links to the nodes resource" - - #ports = [link.Link] - #"Links to the ports resource" - - #drivers = [link.Link] - #"Links to the drivers resource" + commands = [link.Link] + "Links to the command resource" @classmethod def convert(self): @@ -84,46 +66,22 @@ class V1(base.APIBase): 'api-spec-v1.html', bookmark=True, type='text/html') ] + v1.command = [link.Link.make_link('self', pecan.request.host_url, + 'commands', ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'commands', '', + bookmark=True) + ] v1.media_types = [MediaType('application/json', 'application/vnd.openstack.ironic.v1+json')] - #v1.chassis = [link.Link.make_link('self', pecan.request.host_url, - #'chassis', ''), - #link.Link.make_link('bookmark', - #pecan.request.host_url, - #'chassis', '', - #bookmark=True) - #] - #v1.nodes = [link.Link.make_link('self', pecan.request.host_url, - #'nodes', ''), - #link.Link.make_link('bookmark', - #pecan.request.host_url, - #'nodes', '', - #bookmark=True) - #] - #v1.ports = [link.Link.make_link('self', pecan.request.host_url, - #'ports', ''), - #link.Link.make_link('bookmark', - #pecan.request.host_url, - #'ports', '', - #bookmark=True) - #] - #v1.drivers = [link.Link.make_link('self', pecan.request.host_url, - #'drivers', ''), - #link.Link.make_link('bookmark', - #pecan.request.host_url, - #'drivers', '', - #bookmark=True) - #] return v1 class Controller(rest.RestController): """Version 1 API controller root.""" - #nodes = node.NodesController() - #ports = port.PortsController() - #chassis = chassis.ChassisController() - #drivers = driver.DriversController() + commands = command.CommandController() @wsme_pecan.wsexpose(V1) def get(self): diff --git a/teeth_agent/api/controllers/v1/base.py b/teeth_agent/api/controllers/v1/base.py index 8a10ffbfc..15f3aec82 100644 --- a/teeth_agent/api/controllers/v1/base.py +++ b/teeth_agent/api/controllers/v1/base.py @@ -12,36 +12,44 @@ # License for the specific language governing permissions and limitations # under the License. -import datetime - +import six import wsme from wsme import types as wtypes +class MultiType(wtypes.UserType): + """A complex type that represents one or more types. + + Used for validating that a value is an instance of one of the types. + + :param *types: Variable-length list of types. + + """ + def __init__(self, *types): + self.types = types + + def __str__(self): + return ' | '.join(map(str, self.types)) + + def validate(self, value): + for t in self.types: + if t is wtypes.text and isinstance(value, wtypes.bytes): + value = value.decode() + if isinstance(value, t): + return value + else: + raise ValueError( + "Wrong type. Expected '{type}', got '{value}'".format( + type=self.types, value=type(value))) + + +json_type = MultiType(list, dict, six.integer_types, wtypes.text) + + class APIBase(wtypes.Base): - - created_at = wsme.wsattr(datetime.datetime, readonly=True) - "The time in UTC at which the object is created" - - updated_at = wsme.wsattr(datetime.datetime, readonly=True) - "The time in UTC at which the object is updated" - def as_dict(self): """Render this object as a dict of its fields.""" return dict((k, getattr(self, k)) for k in self.fields if hasattr(self, k) and getattr(self, k) != wsme.Unset) - - def unset_fields_except(self, except_list=None): - """Unset fields so they don't appear in the message body. - - :param except_list: A list of fields that won't be touched. - - """ - if except_list is None: - except_list = [] - - for k in self.as_dict(): - if k not in except_list: - setattr(self, k, wsme.Unset) diff --git a/teeth_agent/api/controllers/v1/command.py b/teeth_agent/api/controllers/v1/command.py new file mode 100644 index 000000000..89aa77406 --- /dev/null +++ b/teeth_agent/api/controllers/v1/command.py @@ -0,0 +1,88 @@ +# Copyright 2013 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pecan +from pecan import rest +from wsme import types +from wsmeext import pecan as wsme_pecan + +from teeth_agent.api.controllers.v1 import base + + +class CommandResult(base.APIBase): + id = types.text + command_name = types.text + command_params = base.json_type + command_status = types.text + command_error = types.text + command_result = types.text + + @classmethod + def from_result(cls, result): + instance = cls() + for field in ('id', 'command_name', 'command_params', 'command_status', + 'command_error', 'command_result'): + setattr(instance, field, getattr(result, field)) + return instance + + +class CommandResultList(base.APIBase): + commands = [CommandResult] + + @classmethod + def from_results(cls, results): + instance = cls() + instance.commands = [CommandResult.from_result(result) + for result in results] + return instance + + +class Command(base.APIBase): + """A command representation.""" + name = types.text + params = base.json_type + + @classmethod + def deserialize(cls, obj): + instance = cls() + instance.name = obj['name'] + instance.params = obj['params'] + return instance + + +class CommandController(rest.RestController): + """Controller for issuing commands and polling for command status.""" + + @wsme_pecan.wsexpose(CommandResultList) + def get_all(self): + agent = pecan.request.agent + results = agent.list_command_results() + return CommandResultList.from_results(results) + + @wsme_pecan.wsexpose(CommandResult, types.text, types.text) + def get_one(self, result_id, wait=False): + agent = pecan.request.agent + result = agent.get_command_result(result_id) + + #if wait and wait.lower() == 'true': + #result.join() + + return CommandResult.from_result(result) + + @wsme_pecan.wsexpose(CommandResult, body=Command) + def post(self, command): + agent = pecan.request.agent + result = agent.execute_command(command.name, **command.params) + return result diff --git a/teeth_agent/base.py b/teeth_agent/base.py index 3170678a6..f666a232f 100644 --- a/teeth_agent/base.py +++ b/teeth_agent/base.py @@ -112,7 +112,7 @@ class AsyncCommandResult(BaseCommandResult): e = errors.CommandExecutionError(str(e)) with self.command_state_lock: - self.command_error = e + self.command_error = '{}: {}'.format(e.message, e.details) self.command_status = AgentCommandStatus.FAILED From 5403d794ff5312d7cade4eb9a3faee15f92c2139 Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Tue, 11 Mar 2014 14:41:50 -0700 Subject: [PATCH 07/22] some cleanup --- teeth_agent/api.py | 99 ---------------------- teeth_agent/api/controllers/v1/__init__.py | 28 +++--- teeth_agent/api/controllers/v1/base.py | 8 +- 3 files changed, 19 insertions(+), 116 deletions(-) delete mode 100644 teeth_agent/api.py diff --git a/teeth_agent/api.py b/teeth_agent/api.py deleted file mode 100644 index 49a74aaef..000000000 --- a/teeth_agent/api.py +++ /dev/null @@ -1,99 +0,0 @@ -""" -Copyright 2013 Rackspace, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -from teeth_rest import component -from teeth_rest import errors -from teeth_rest import responses - - -class AgentCommand(object): - def __init__(self, name, params): - self.name = name - self.params = params - - @classmethod - def deserialize(cls, obj): - for field in ['name', 'params']: - if field not in obj: - msg = 'Missing command \'{}\' field.'.format(field) - raise errors.InvalidContentError(msg) - - if type(obj['params']) != dict: - raise errors.InvalidContentError( - 'Command params must be a dictionary.') - - return cls(obj['name'], obj['params']) - - -class TeethAgentAPI(component.APIComponent): - """The primary Teeth Agent API.""" - - def __init__(self, agent): - super(TeethAgentAPI, self).__init__() - self.agent = agent - - def add_routes(self): - """Called during initialization. Override to map relative routes to - methods. - """ - self.route('GET', '/status', self.get_agent_status) - self.route('GET', '/commands', self.list_command_results) - self.route('POST', '/commands', self.execute_command) - self.route('GET', - '/commands/', - self.get_command_result) - - def get_agent_status(self, request): - """Get the status of the agent.""" - return responses.ItemResponse(self.agent.get_status()) - - def list_command_results(self, request): - # TODO(russellhaering): pagination - command_results = self.agent.list_command_results() - return responses.PaginatedResponse(request, - command_results, - self.list_command_results, - None, - None) - - def execute_command(self, request): - """Execute a command on the agent.""" - command = AgentCommand.deserialize(self.parse_content(request)) - result = self.agent.execute_command(command.name, **command.params) - - wait = request.args.get('wait') - if wait and wait.lower() == 'true': - result.join() - - return responses.ItemResponse(result) - - def get_command_result(self, request, result_id): - """Retrieve the result of a command.""" - result = self.agent.get_command_result(result_id) - - wait = request.args.get('wait') - if wait and wait.lower() == 'true': - result.join() - - return responses.ItemResponse(result) - - -class TeethAgentAPIServer(component.APIServer): - """Server for the teeth agent API.""" - - def __init__(self, agent): - super(TeethAgentAPIServer, self).__init__() - self.add_component('/v1.0', TeethAgentAPI(agent)) diff --git a/teeth_agent/api/controllers/v1/__init__.py b/teeth_agent/api/controllers/v1/__init__.py index 8a120c118..a682eb448 100644 --- a/teeth_agent/api/controllers/v1/__init__.py +++ b/teeth_agent/api/controllers/v1/__init__.py @@ -58,21 +58,29 @@ class V1(base.APIBase): v1 = V1() v1.id = "v1" v1.links = [ - link.Link.make_link('self', pecan.request.host_url, - 'v1', '', bookmark=True), + link.Link.make_link('self', + pecan.request.host_url, + 'v1', + '', + bookmark=True), link.Link.make_link('describedby', 'http://docs.openstack.org', 'developer/ironic/dev', 'api-spec-v1.html', - bookmark=True, type='text/html') + bookmark=True, + type='text/html') + ] + v1.command = [ + link.Link.make_link('self', + pecan.request.host_url, + 'commands', + ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'commands', + '', + bookmark=True) ] - v1.command = [link.Link.make_link('self', pecan.request.host_url, - 'commands', ''), - link.Link.make_link('bookmark', - pecan.request.host_url, - 'commands', '', - bookmark=True) - ] v1.media_types = [MediaType('application/json', 'application/vnd.openstack.ironic.v1+json')] return v1 diff --git a/teeth_agent/api/controllers/v1/base.py b/teeth_agent/api/controllers/v1/base.py index 15f3aec82..465751f7c 100644 --- a/teeth_agent/api/controllers/v1/base.py +++ b/teeth_agent/api/controllers/v1/base.py @@ -13,7 +13,6 @@ # under the License. import six -import wsme from wsme import types as wtypes @@ -47,9 +46,4 @@ json_type = MultiType(list, dict, six.integer_types, wtypes.text) class APIBase(wtypes.Base): - def as_dict(self): - """Render this object as a dict of its fields.""" - return dict((k, getattr(self, k)) - for k in self.fields - if hasattr(self, k) and - getattr(self, k) != wsme.Unset) + pass From 4f4a8302327824ab4e4ea021ef57feb3d4f7a349 Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Tue, 11 Mar 2014 17:26:13 -0700 Subject: [PATCH 08/22] fix exception handling --- teeth_agent/api/controllers/v1/base.py | 24 +++++++++++++++++++++++ teeth_agent/api/controllers/v1/command.py | 2 +- teeth_agent/base.py | 2 +- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/teeth_agent/api/controllers/v1/base.py b/teeth_agent/api/controllers/v1/base.py index 465751f7c..20af89648 100644 --- a/teeth_agent/api/controllers/v1/base.py +++ b/teeth_agent/api/controllers/v1/base.py @@ -16,6 +16,30 @@ import six from wsme import types as wtypes +class ExceptionType(wtypes.UserType): + basetype = wtypes.DictType + name = 'exception' + + def validate(self, value): + if not isinstance(value, BaseException): + raise ValueError('Value is not an exception') + return value + + def tobasetype(self, value): + """Turn a RESTError into a dict.""" + return { + 'type': value.__class__.__name__, + 'code': value.status_code, + 'message': value.message, + 'details': value.details, + } + + frombasetype = tobasetype + + +exception_type = ExceptionType() + + class MultiType(wtypes.UserType): """A complex type that represents one or more types. diff --git a/teeth_agent/api/controllers/v1/command.py b/teeth_agent/api/controllers/v1/command.py index 89aa77406..5785f85cb 100644 --- a/teeth_agent/api/controllers/v1/command.py +++ b/teeth_agent/api/controllers/v1/command.py @@ -26,7 +26,7 @@ class CommandResult(base.APIBase): command_name = types.text command_params = base.json_type command_status = types.text - command_error = types.text + command_error = base.exception_type command_result = types.text @classmethod diff --git a/teeth_agent/base.py b/teeth_agent/base.py index f666a232f..3170678a6 100644 --- a/teeth_agent/base.py +++ b/teeth_agent/base.py @@ -112,7 +112,7 @@ class AsyncCommandResult(BaseCommandResult): e = errors.CommandExecutionError(str(e)) with self.command_state_lock: - self.command_error = '{}: {}'.format(e.message, e.details) + self.command_error = e self.command_status = AgentCommandStatus.FAILED From 470ca175e23abca44da9f032e5a2dd404db11adb Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Wed, 12 Mar 2014 09:50:32 -0700 Subject: [PATCH 09/22] get api tests passing --- teeth_agent/api/controllers/v1/command.py | 15 ++----- teeth_agent/base.py | 26 ++++++------ teeth_agent/tests/api.py | 50 ++++++++++++----------- 3 files changed, 44 insertions(+), 47 deletions(-) diff --git a/teeth_agent/api/controllers/v1/command.py b/teeth_agent/api/controllers/v1/command.py index 5785f85cb..be98355e1 100644 --- a/teeth_agent/api/controllers/v1/command.py +++ b/teeth_agent/api/controllers/v1/command.py @@ -24,10 +24,10 @@ from teeth_agent.api.controllers.v1 import base class CommandResult(base.APIBase): id = types.text command_name = types.text - command_params = base.json_type + command_params = types.DictType(types.text, base.json_type) command_status = types.text command_error = base.exception_type - command_result = types.text + command_result = types.DictType(types.text, base.json_type) @classmethod def from_result(cls, result): @@ -51,15 +51,8 @@ class CommandResultList(base.APIBase): class Command(base.APIBase): """A command representation.""" - name = types.text - params = base.json_type - - @classmethod - def deserialize(cls, obj): - instance = cls() - instance.name = obj['name'] - instance.params = obj['params'] - return instance + name = types.wsattr(types.text, mandatory=True) + params = types.wsattr(base.MultiType(dict), mandatory=True) class CommandController(rest.RestController): diff --git a/teeth_agent/base.py b/teeth_agent/base.py index 3170678a6..4ed81e7aa 100644 --- a/teeth_agent/base.py +++ b/teeth_agent/base.py @@ -26,29 +26,29 @@ from teeth_agent import errors class AgentCommandStatus(object): - RUNNING = 'RUNNING' - SUCCEEDED = 'SUCCEEDED' - FAILED = 'FAILED' + RUNNING = u'RUNNING' + SUCCEEDED = u'SUCCEEDED' + FAILED = u'FAILED' class BaseCommandResult(encoding.Serializable): def __init__(self, command_name, command_params): - self.id = str(uuid.uuid4()) + self.id = unicode(uuid.uuid4()) self.command_name = command_name self.command_params = command_params self.command_status = AgentCommandStatus.RUNNING self.command_error = None self.command_result = None - def serialize(self, view): - return collections.OrderedDict([ - ('id', self.id), - ('command_name', self.command_name), - ('command_params', self.command_params), - ('command_status', self.command_status), - ('command_error', self.command_error), - ('command_result', self.command_result), - ]) + def serialize(self): + return dict(( + (u'id', self.id), + (u'command_name', self.command_name), + (u'command_params', self.command_params), + (u'command_status', self.command_status), + (u'command_error', self.command_error), + (u'command_result', self.command_result), + )) def is_done(self): return self.command_status != AgentCommandStatus.RUNNING diff --git a/teeth_agent/tests/api.py b/teeth_agent/tests/api.py index fc0686640..50ff05ba8 100644 --- a/teeth_agent/tests/api.py +++ b/teeth_agent/tests/api.py @@ -25,7 +25,7 @@ from werkzeug import wrappers from teeth_rest import encoding from teeth_agent import agent -from teeth_agent import api +from teeth_agent.api import app from teeth_agent import base @@ -48,9 +48,9 @@ class TestTeethAPI(unittest.TestCase): status = agent.TeethAgentStatus('TEST_MODE', time.time(), 'v72ac9') mock_agent = mock.MagicMock() mock_agent.get_status.return_value = status - api_server = api.TeethAgentAPIServer(mock_agent) + api_server = app.setup_app(mock_agent) - response = self._make_request(api_server, 'GET', '/v1.0/status') + response = self._make_request(api_server, 'GET', '/v1/status') mock_agent.get_status.assert_called_once_with() self.assertEqual(response.status_code, 200) @@ -72,11 +72,11 @@ class TestTeethAPI(unittest.TestCase): mock_agent = mock.MagicMock() mock_agent.execute_command.return_value = result - api_server = api.TeethAgentAPIServer(mock_agent) + api_server = app.setup_app(mock_agent) response = self._make_request(api_server, 'POST', - '/v1.0/commands', + '/v1/commands/', data=command) self.assertEqual(mock_agent.execute_command.call_count, 1) @@ -90,50 +90,54 @@ class TestTeethAPI(unittest.TestCase): def test_execute_agent_command_validation(self): mock_agent = mock.MagicMock() - api_server = api.TeethAgentAPIServer(mock_agent) + api_server = app.setup_app(mock_agent) - invalid_command = {} + invalid_command = {'invalid': 'stuff'} response = self._make_request(api_server, 'POST', - '/v1.0/commands', + '/v1/commands', data=invalid_command) self.assertEqual(response.status_code, 400) data = json.loads(response.data) - self.assertEqual(data['details'], 'Missing command \'name\' field.') + msg = 'Invalid input for field/attribute name.' + self.assertTrue(msg in data['faultstring']) + msg = 'Mandatory field missing' + self.assertTrue(msg in data['faultstring']) def test_execute_agent_command_params_validation(self): mock_agent = mock.MagicMock() - api_server = api.TeethAgentAPIServer(mock_agent) + api_server = app.setup_app(mock_agent) invalid_command = {'name': 'do_things', 'params': []} response = self._make_request(api_server, 'POST', - '/v1.0/commands', + '/v1/commands', data=invalid_command) self.assertEqual(response.status_code, 400) data = json.loads(response.data) - self.assertEqual(data['details'], - 'Command params must be a dictionary.') + # this message is actually much longer, but I'm ok with this + msg = 'Invalid input for field/attribute params.' + self.assertTrue(msg in data['faultstring']) def test_list_command_results(self): - cmd_result = base.SyncCommandResult('do_things', - {'key': 'value'}, + self.maxDiff = 10000 + cmd_result = base.SyncCommandResult(u'do_things', + {u'key': u'value'}, True, - {'test': 'result'}) + {u'test': u'result'}) mock_agent = mock.create_autospec(agent.TeethAgent) mock_agent.list_command_results.return_value = [ cmd_result, ] - api_server = api.TeethAgentAPIServer(mock_agent) - response = self._make_request(api_server, 'GET', '/v1.0/commands') + api_server = app.setup_app(mock_agent) + response = self._make_request(api_server, 'GET', '/v1/commands') self.assertEqual(response.status_code, 200) self.assertEqual(json.loads(response.data), { - 'items': [ - cmd_result.serialize(encoding.SerializationViews.PUBLIC), + u'commands': [ + cmd_result.serialize(None), ], - 'links': [], }) def test_get_command_result(self): @@ -148,10 +152,10 @@ class TestTeethAPI(unittest.TestCase): mock_agent = mock.create_autospec(agent.TeethAgent) mock_agent.get_command_result.return_value = cmd_result - api_server = api.TeethAgentAPIServer(mock_agent) + api_server = app.setup_app(mock_agent) response = self._make_request(api_server, 'GET', - '/v1.0/commands/abc123') + '/v1/commands/abc123') self.assertEqual(response.status_code, 200) data = json.loads(response.data) self.assertEqual(data, serialized_cmd_result) From c60fde8aa75f083beb001824527fb857e98d5825 Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Wed, 12 Mar 2014 10:55:04 -0700 Subject: [PATCH 10/22] add /status and fix tests --- teeth_agent/api/controllers/v1/__init__.py | 16 ++++++++ teeth_agent/api/controllers/v1/status.py | 44 ++++++++++++++++++++++ teeth_agent/base.py | 3 +- 3 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 teeth_agent/api/controllers/v1/status.py diff --git a/teeth_agent/api/controllers/v1/__init__.py b/teeth_agent/api/controllers/v1/__init__.py index a682eb448..70635657c 100644 --- a/teeth_agent/api/controllers/v1/__init__.py +++ b/teeth_agent/api/controllers/v1/__init__.py @@ -25,6 +25,7 @@ import wsmeext.pecan as wsme_pecan from teeth_agent.api.controllers.v1 import base from teeth_agent.api.controllers.v1 import command from teeth_agent.api.controllers.v1 import link +from teeth_agent.api.controllers.v1 import status class MediaType(base.APIBase): @@ -53,6 +54,9 @@ class V1(base.APIBase): commands = [link.Link] "Links to the command resource" + status = [link.Link] + "Links to the status resource" + @classmethod def convert(self): v1 = V1() @@ -81,6 +85,17 @@ class V1(base.APIBase): '', bookmark=True) ] + v1.status = [ + link.Link.make_link('self', + pecan.request.host_url, + 'status', + ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'status', + '', + bookmark=True) + ] v1.media_types = [MediaType('application/json', 'application/vnd.openstack.ironic.v1+json')] return v1 @@ -90,6 +105,7 @@ class Controller(rest.RestController): """Version 1 API controller root.""" commands = command.CommandController() + status = status.StatusController() @wsme_pecan.wsexpose(V1) def get(self): diff --git a/teeth_agent/api/controllers/v1/status.py b/teeth_agent/api/controllers/v1/status.py new file mode 100644 index 000000000..c631ee59b --- /dev/null +++ b/teeth_agent/api/controllers/v1/status.py @@ -0,0 +1,44 @@ +# Copyright 2014 Rackspace, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pecan +from pecan import rest +from wsme import types +from wsmeext import pecan as wsme_pecan + +from teeth_agent.api.controllers.v1 import base + + +class AgentStatus(base.APIBase): + mode = types.text + started_at = base.MultiType(float) + version = types.text + + @classmethod + def from_agent_status(cls, status): + instance = cls() + for field in ('mode', 'started_at', 'version'): + setattr(instance, field, getattr(status, field)) + return instance + + +class StatusController(rest.RestController): + """Controller for getting agent status.""" + + @wsme_pecan.wsexpose(AgentStatus) + def get_all(self): + agent = pecan.request.agent + status = agent.get_status() + return AgentStatus.from_agent_status(status) diff --git a/teeth_agent/base.py b/teeth_agent/base.py index 4ed81e7aa..3c55ea026 100644 --- a/teeth_agent/base.py +++ b/teeth_agent/base.py @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. """ -import collections import threading import uuid @@ -40,7 +39,7 @@ class BaseCommandResult(encoding.Serializable): self.command_error = None self.command_result = None - def serialize(self): + def serialize(self, view): return dict(( (u'id', self.id), (u'command_name', self.command_name), From 415e3a1c98416a96748868895430534d8adac78a Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Wed, 12 Mar 2014 10:55:18 -0700 Subject: [PATCH 11/22] fix some copyrights --- teeth_agent/api/controllers/v1/command.py | 2 +- teeth_agent/api/controllers/v1/link.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/teeth_agent/api/controllers/v1/command.py b/teeth_agent/api/controllers/v1/command.py index be98355e1..c66ec3d05 100644 --- a/teeth_agent/api/controllers/v1/command.py +++ b/teeth_agent/api/controllers/v1/command.py @@ -1,4 +1,4 @@ -# Copyright 2013 Red Hat, Inc. +# Copyright 2014 Rackspace, Inc. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/teeth_agent/api/controllers/v1/link.py b/teeth_agent/api/controllers/v1/link.py index d4e85000b..987eb386d 100644 --- a/teeth_agent/api/controllers/v1/link.py +++ b/teeth_agent/api/controllers/v1/link.py @@ -1,4 +1,4 @@ -# Copyright 2013 Red Hat, Inc. +# Copyright 2014 Rackspace, Inc. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may From 73bef57ed35853b704b056ba0c0b6cb3f2c35302 Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Wed, 12 Mar 2014 11:09:58 -0700 Subject: [PATCH 12/22] remove dead code --- teeth_agent/api/app.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/teeth_agent/api/app.py b/teeth_agent/api/app.py index a850b6187..1486578dc 100644 --- a/teeth_agent/api/app.py +++ b/teeth_agent/api/app.py @@ -39,16 +39,6 @@ def get_pecan_config(): def setup_app(agent, pecan_config=None, extra_hooks=None): - #policy.init() - - #app_hooks = [hooks.ConfigHook(), - #hooks.DBHook(), - #hooks.ContextHook(pecan_config.app.acl_public_routes), - #hooks.RPCHook(), - #hooks.NoExceptionTracebackHook()] - #if extra_hooks: - #app_hooks.extend(extra_hooks) - app_hooks = [AgentHook(agent)] if not pecan_config: @@ -59,11 +49,11 @@ def setup_app(agent, pecan_config=None, extra_hooks=None): app = pecan.make_app( pecan_config.app.root, static_root=pecan_config.app.static_root, + # TODO(jimrollenhagen) move this to an oslo config debug=True, #debug=CONF.debug, force_canonical=getattr(pecan_config.app, 'force_canonical', True), hooks=app_hooks, - #wrap_app=middleware.ParsableErrorMiddleware, ) return app From 2fc74707cb86bb389f29ae3f30abc643471a1bb9 Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Thu, 13 Mar 2014 13:35:31 -0700 Subject: [PATCH 13/22] remove oslo dependency --- requirements.txt | 1 - teeth_agent/api/app.py | 7 +------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 19e997103..54d08099f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,5 @@ stevedore==0.14 -e git+https://github.com/racker/teeth-rest.git@e876c0fddd5ce2f5223ab16936f711b0d57e19c4#egg=teeth_rest wsgiref>=0.1.2 pecan>=0.4.5 -oslo.config>=1.2.0 WSME>=0.6 six>=1.5.2 diff --git a/teeth_agent/api/app.py b/teeth_agent/api/app.py index 1486578dc..b9560d4f4 100644 --- a/teeth_agent/api/app.py +++ b/teeth_agent/api/app.py @@ -14,14 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. """ -from oslo.config import cfg import pecan from pecan import hooks from teeth_agent.api import config -CONF = cfg.CONF - class AgentHook(hooks.PecanHook): def __init__(self, agent, *args, **kwargs): @@ -49,9 +46,7 @@ def setup_app(agent, pecan_config=None, extra_hooks=None): app = pecan.make_app( pecan_config.app.root, static_root=pecan_config.app.static_root, - # TODO(jimrollenhagen) move this to an oslo config - debug=True, - #debug=CONF.debug, + debug=pecan_config.app.debug, force_canonical=getattr(pecan_config.app, 'force_canonical', True), hooks=app_hooks, ) From 649ed621c251a3f546cf6550f96fa6fbb18ca0ee Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Thu, 13 Mar 2014 13:42:16 -0700 Subject: [PATCH 14/22] fix up some text --- teeth_agent/api/controllers/root.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/teeth_agent/api/controllers/root.py b/teeth_agent/api/controllers/root.py index 10776bfd8..4552e731b 100644 --- a/teeth_agent/api/controllers/root.py +++ b/teeth_agent/api/controllers/root.py @@ -1,8 +1,4 @@ -# -*- encoding: utf-8 -*- -# -# Copyright © 2012 New Dream Network, LLC (DreamHost) -# -# Author: Doug Hellmann +# Copyright 2014 Rackspace, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -62,9 +58,9 @@ class Root(base.APIBase): @classmethod def convert(self): root = Root() - root.name = "OpenStack Ironic API" - root.description = ("Ironic is an OpenStack project which aims to " - "provision baremetal machines.") + root.name = 'OpenStack Ironic Python Agent API' + root.description = ('Ironic Python Agent is a provisioning agent for ' + 'OpenStack Ironic') root.versions = [Version.convert('v1')] root.default_version = Version.convert('v1') return root From 5a596157da80ca3c1d890fe3e16f9c9abf492a27 Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Fri, 14 Mar 2014 15:52:17 -0700 Subject: [PATCH 15/22] add horrible method of getting the host IP --- imagebuild/coreos/oem/system/teeth-agent.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagebuild/coreos/oem/system/teeth-agent.service b/imagebuild/coreos/oem/system/teeth-agent.service index 60d853934..a25c2ba7d 100644 --- a/imagebuild/coreos/oem/system/teeth-agent.service +++ b/imagebuild/coreos/oem/system/teeth-agent.service @@ -1,5 +1,5 @@ [Service] -ExecStart=/usr/bin/docker run -p 9999:9999 -privileged=true -v=/sys:/mnt/sys oemdocker /usr/local/bin/teeth-agent +ExecStart=/usr/bin/docker run -p 9999:9999 -privileged=true -v=/sys:/mnt/sys oemdocker /usr/local/bin/teeth-agent --ipaddr="`ip a | grep '10\.' | sed -e 's/inet \(10\.[0-9\.]\+\).*/\1/'`" Restart=always [Install] From c1c6425e9381a686b9568f4a089750c101393db5 Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Fri, 14 Mar 2014 16:19:20 -0700 Subject: [PATCH 16/22] parse the ipaddr argument --- teeth_agent/agent.py | 7 ++++--- teeth_agent/cmd/agent.py | 7 ++++++- teeth_agent/tests/agent.py | 3 ++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/teeth_agent/agent.py b/teeth_agent/agent.py index 3632bd0cb..b8efe5f56 100644 --- a/teeth_agent/agent.py +++ b/teeth_agent/agent.py @@ -107,9 +107,10 @@ class TeethAgentHeartbeater(threading.Thread): class TeethAgent(object): - def __init__(self, api_url, listen_address): + def __init__(self, api_url, listen_address, ipaddr): self.api_url = api_url self.listen_address = listen_address + self.ipaddr = ipaddr self.mode_implementation = None self.version = pkg_resources.get_distribution('teeth-agent').version self.api = api.TeethAgentAPIServer(self) @@ -217,5 +218,5 @@ def _load_mode_implementation(mode_name): return mgr.driver -def build_agent(api_url, listen_host, listen_port): - return TeethAgent(api_url, (listen_host, listen_port)) +def build_agent(api_url, listen_host, listen_port, ipaddr): + return TeethAgent(api_url, (listen_host, listen_port), ipaddr) diff --git a/teeth_agent/cmd/agent.py b/teeth_agent/cmd/agent.py index bdfa8d41d..92f4ef386 100644 --- a/teeth_agent/cmd/agent.py +++ b/teeth_agent/cmd/agent.py @@ -39,8 +39,13 @@ def run(): type=int, help='The port to listen on') + parser.add_argument('--ipaddr', + required=True, + help='The external IP address to advertise to ironic') + args = parser.parse_args() logging.configure() agent.build_agent(args.api_url, args.listen_host, - args.listen_port).run() + args.listen_port, + args.ipaddr).run() diff --git a/teeth_agent/tests/agent.py b/teeth_agent/tests/agent.py index f1d4ed20c..5e46f6bd6 100644 --- a/teeth_agent/tests/agent.py +++ b/teeth_agent/tests/agent.py @@ -121,7 +121,8 @@ class TestBaseAgent(unittest.TestCase): encoding.SerializationViews.PUBLIC, indent=4) self.agent = agent.TeethAgent('https://fake_api.example.org:8081/', - ('localhost', 9999)) + ('localhost', 9999), + '192.168.1.1') def assertEqualEncoded(self, a, b): # Evidently JSONEncoder.default() can't handle None (??) so we have to From d3749befc3877db156e08f25236afcc8549f9b1b Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Fri, 14 Mar 2014 16:48:44 -0700 Subject: [PATCH 17/22] simplify tests a bit --- teeth_agent/tests/agent.py | 2 +- teeth_agent/tests/api.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/teeth_agent/tests/agent.py b/teeth_agent/tests/agent.py index 8fac1d865..ce5995396 100644 --- a/teeth_agent/tests/agent.py +++ b/teeth_agent/tests/agent.py @@ -171,7 +171,7 @@ class TestBaseAgent(unittest.TestCase): listen_addr[1], self.agent.api, server_class=simple_server.WSGIServer) - wsgi_server.serve_forever.assert_called_once_with() + wsgi_server.serve_forever.assert_called_once() self.agent.heartbeater.start.assert_called_once_with() diff --git a/teeth_agent/tests/api.py b/teeth_agent/tests/api.py index 50ff05ba8..ba964e134 100644 --- a/teeth_agent/tests/api.py +++ b/teeth_agent/tests/api.py @@ -92,7 +92,8 @@ class TestTeethAPI(unittest.TestCase): mock_agent = mock.MagicMock() api_server = app.setup_app(mock_agent) - invalid_command = {'invalid': 'stuff'} + #invalid_command = {'invalid': 'stuff'} + invalid_command = {} response = self._make_request(api_server, 'POST', '/v1/commands', From 809246e68d6cc8ba1c21b9d8b0494a6758538d88 Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Fri, 14 Mar 2014 16:49:16 -0700 Subject: [PATCH 18/22] add wait param --- teeth_agent/api/controllers/v1/command.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/teeth_agent/api/controllers/v1/command.py b/teeth_agent/api/controllers/v1/command.py index c66ec3d05..811000007 100644 --- a/teeth_agent/api/controllers/v1/command.py +++ b/teeth_agent/api/controllers/v1/command.py @@ -69,13 +69,21 @@ class CommandController(rest.RestController): agent = pecan.request.agent result = agent.get_command_result(result_id) - #if wait and wait.lower() == 'true': - #result.join() + if wait and wait.lower() == 'true': + result.join() return CommandResult.from_result(result) @wsme_pecan.wsexpose(CommandResult, body=Command) - def post(self, command): + def post(self, wait=False, command=None): + # the POST body is always the last arg, + # so command must be a kwarg here + if command is None: + command = Command() agent = pecan.request.agent result = agent.execute_command(command.name, **command.params) + + if wait and wait.lower() == 'true': + result.join() + return result From a9705715f98ebca5cb5801a4f4466641998c9089 Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Fri, 14 Mar 2014 16:49:30 -0700 Subject: [PATCH 19/22] fix up API docs a bit --- teeth_agent/api/controllers/v1/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/teeth_agent/api/controllers/v1/__init__.py b/teeth_agent/api/controllers/v1/__init__.py index 70635657c..998e6e446 100644 --- a/teeth_agent/api/controllers/v1/__init__.py +++ b/teeth_agent/api/controllers/v1/__init__.py @@ -68,9 +68,9 @@ class V1(base.APIBase): '', bookmark=True), link.Link.make_link('describedby', - 'http://docs.openstack.org', - 'developer/ironic/dev', - 'api-spec-v1.html', + 'https://github.com', + 'rackerlabs', + 'teeth-agent', bookmark=True, type='text/html') ] @@ -96,8 +96,7 @@ class V1(base.APIBase): '', bookmark=True) ] - v1.media_types = [MediaType('application/json', - 'application/vnd.openstack.ironic.v1+json')] + v1.media_types = [MediaType('application/json')] return v1 From 1cc511ca11551b66db2b6918d4ae45329c38e363 Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Fri, 14 Mar 2014 16:59:02 -0700 Subject: [PATCH 20/22] remove commented code --- teeth_agent/tests/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/teeth_agent/tests/api.py b/teeth_agent/tests/api.py index ba964e134..14348bb9c 100644 --- a/teeth_agent/tests/api.py +++ b/teeth_agent/tests/api.py @@ -92,7 +92,6 @@ class TestTeethAPI(unittest.TestCase): mock_agent = mock.MagicMock() api_server = app.setup_app(mock_agent) - #invalid_command = {'invalid': 'stuff'} invalid_command = {} response = self._make_request(api_server, 'POST', From 078b67dfa597f652181bf8d97909ef2482f3a183 Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Mon, 17 Mar 2014 10:13:17 -0700 Subject: [PATCH 21/22] Fix MediaType() call and add tests for / and /v1 --- teeth_agent/api/controllers/v1/__init__.py | 4 +++- teeth_agent/tests/api.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/teeth_agent/api/controllers/v1/__init__.py b/teeth_agent/api/controllers/v1/__init__.py index 998e6e446..3fbd9cd25 100644 --- a/teeth_agent/api/controllers/v1/__init__.py +++ b/teeth_agent/api/controllers/v1/__init__.py @@ -96,7 +96,9 @@ class V1(base.APIBase): '', bookmark=True) ] - v1.media_types = [MediaType('application/json')] + v1.media_types = [MediaType('application/json', + ('application/vnd.openstack.' + 'ironic-python-agent.v1+json'))] return v1 diff --git a/teeth_agent/tests/api.py b/teeth_agent/tests/api.py index 14348bb9c..655bbae1c 100644 --- a/teeth_agent/tests/api.py +++ b/teeth_agent/tests/api.py @@ -44,6 +44,20 @@ class TestTeethAPI(unittest.TestCase): client = test.Client(api, wrappers.BaseResponse) return client.open(self._get_env_builder(method, path, data, query)) + def test_root(self): + mock_agent = mock.MagicMock() + api_server = app.setup_app(mock_agent) + + response = self._make_request(api_server, 'GET', '/') + self.assertEqual(response.status, '200 OK') + + def test_v1_root(self): + mock_agent = mock.MagicMock() + api_server = app.setup_app(mock_agent) + + response = self._make_request(api_server, 'GET', '/v1') + self.assertEqual(response.status, '200 OK') + def test_get_agent_status(self): status = agent.TeethAgentStatus('TEST_MODE', time.time(), 'v72ac9') mock_agent = mock.MagicMock() From 2c77d82204d4708f5993cb7f1181aea52037ad0c Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Mon, 17 Mar 2014 10:58:39 -0700 Subject: [PATCH 22/22] Remove dependency on teeth-rest. This commit: - Removes all references to teeth-rest. - Brings in encoding.py and errors.py from teeth-rest. - Removes the "view" thing from the encoding module. - Adds structlog as a dep. This was missing and overlooked because teeth-rest was installing it in the environment. --- Dockerfile | 2 +- requirements.txt | 2 +- teeth_agent/agent.py | 7 ++-- teeth_agent/base.py | 11 +++--- teeth_agent/encoding.py | 53 ++++++++++++++++++++++++++ teeth_agent/errors.py | 62 +++++++++++++++++++++++++------ teeth_agent/hardware.py | 4 +- teeth_agent/logging.py | 2 +- teeth_agent/overlord_agent_api.py | 5 +-- teeth_agent/tests/agent.py | 6 +-- teeth_agent/tests/api.py | 8 ++-- 11 files changed, 124 insertions(+), 38 deletions(-) create mode 100644 teeth_agent/encoding.py diff --git a/Dockerfile b/Dockerfile index 82836887a..7fa950b14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ RUN apt-get update && apt-get -y install \ # Install requirements separately, because pip understands a git+https url while setuptools doesn't RUN pip install -r /tmp/teeth-agent/requirements.txt -# This will succeed because all the dependencies (including pesky teeth_rest) were installed previously +# This will succeed because all the dependencies were installed previously RUN pip install /tmp/teeth-agent CMD [ "/usr/local/bin/teeth-agent" ] diff --git a/requirements.txt b/requirements.txt index 54d08099f..7df51dbf8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ Werkzeug==0.9.4 requests==2.0.0 stevedore==0.14 --e git+https://github.com/racker/teeth-rest.git@e876c0fddd5ce2f5223ab16936f711b0d57e19c4#egg=teeth_rest wsgiref>=0.1.2 pecan>=0.4.5 WSME>=0.6 six>=1.5.2 +structlog==0.4.1 diff --git a/teeth_agent/agent.py b/teeth_agent/agent.py index 62292fdde..fbc263951 100644 --- a/teeth_agent/agent.py +++ b/teeth_agent/agent.py @@ -22,12 +22,11 @@ import time import pkg_resources from stevedore import driver import structlog -from teeth_rest import encoding -from teeth_rest import errors as rest_errors from wsgiref import simple_server from teeth_agent.api import app from teeth_agent import base +from teeth_agent import encoding from teeth_agent import errors from teeth_agent import hardware from teeth_agent import overlord_agent_api @@ -39,7 +38,7 @@ class TeethAgentStatus(encoding.Serializable): self.started_at = started_at self.version = version - def serialize(self, view): + def serialize(self): """Turn the status into a dict.""" return collections.OrderedDict([ ('mode', self.mode), @@ -181,7 +180,7 @@ class TeethAgent(object): try: result = self.mode_implementation.execute(command_part, **kwargs) - except rest_errors.InvalidContentError as e: + except errors.InvalidContentError as e: # Any command may raise a InvalidContentError which will be # returned to the caller directly. raise e diff --git a/teeth_agent/base.py b/teeth_agent/base.py index 3c55ea026..8147c43d9 100644 --- a/teeth_agent/base.py +++ b/teeth_agent/base.py @@ -18,9 +18,8 @@ import threading import uuid import structlog -from teeth_rest import encoding -from teeth_rest import errors as rest_errors +from teeth_agent import encoding from teeth_agent import errors @@ -39,7 +38,7 @@ class BaseCommandResult(encoding.Serializable): self.command_error = None self.command_result = None - def serialize(self, view): + def serialize(self): return dict(( (u'id', self.id), (u'command_name', self.command_name), @@ -82,9 +81,9 @@ class AsyncCommandResult(BaseCommandResult): self.execution_thread = threading.Thread(target=self.run, name=thread_name) - def serialize(self, view): + def serialize(self): with self.command_state_lock: - return super(AsyncCommandResult, self).serialize(view) + return super(AsyncCommandResult, self).serialize() def start(self): self.execution_thread.start() @@ -107,7 +106,7 @@ class AsyncCommandResult(BaseCommandResult): self.command_status = AgentCommandStatus.SUCCEEDED except Exception as e: - if not isinstance(e, rest_errors.RESTError): + if not isinstance(e, errors.RESTError): e = errors.CommandExecutionError(str(e)) with self.command_state_lock: diff --git a/teeth_agent/encoding.py b/teeth_agent/encoding.py new file mode 100644 index 000000000..a6386579b --- /dev/null +++ b/teeth_agent/encoding.py @@ -0,0 +1,53 @@ +""" +Copyright 2013 Rackspace, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import json +import uuid + + +class Serializable(object): + """Base class for things that can be serialized.""" + def serialize(self): + """Turn this object into a dict.""" + raise NotImplementedError() + + +class RESTJSONEncoder(json.JSONEncoder): + """A slightly customized JSON encoder.""" + def encode(self, o): + """Turn an object into JSON. + + Appends a newline to responses when configured to pretty-print, + in order to make use of curl less painful from most shells. + """ + delimiter = '' + + # if indent is None, newlines are still inserted, so we should too. + if self.indent is not None: + delimiter = '\n' + + return super(RESTJSONEncoder, self).encode(o) + delimiter + + def default(self, o): + """Turn an object into a serializable object. In particular, by + calling :meth:`.Serializable.serialize`. + """ + if isinstance(o, Serializable): + return o.serialize() + elif isinstance(o, uuid.UUID): + return str(o) + else: + return json.JSONEncoder.default(self, o) diff --git a/teeth_agent/errors.py b/teeth_agent/errors.py index 0e1abf1dd..2b32a9033 100644 --- a/teeth_agent/errors.py +++ b/teeth_agent/errors.py @@ -14,10 +14,50 @@ See the License for the specific language governing permissions and limitations under the License. """ -from teeth_rest import errors +import collections + +from teeth_agent import encoding -class CommandExecutionError(errors.RESTError): +class RESTError(Exception, encoding.Serializable): + """Base class for errors generated in teeth.""" + message = 'An error occurred' + details = 'An unexpected error occurred. Please try back later.' + status_code = 500 + + def serialize(self): + """Turn a RESTError into a dict.""" + return collections.OrderedDict([ + ('type', self.__class__.__name__), + ('code', self.status_code), + ('message', self.message), + ('details', self.details), + ]) + + +class InvalidContentError(RESTError): + """Error which occurs when a user supplies invalid content, either + because that content cannot be parsed according to the advertised + `Content-Type`, or due to a content validation error. + """ + message = 'Invalid request body' + status_code = 400 + + def __init__(self, details): + self.details = details + + +class NotFound(RESTError): + """Error which occurs when a user supplies invalid content, either + because that content cannot be parsed according to the advertised + `Content-Type`, or due to a content validation error. + """ + message = 'Not found' + status_code = 404 + details = 'The requested URL was not found.' + + +class CommandExecutionError(RESTError): """Error raised when a command fails to execute.""" message = 'Command execution failed' @@ -27,7 +67,7 @@ class CommandExecutionError(errors.RESTError): self.details = details -class InvalidCommandError(errors.InvalidContentError): +class InvalidCommandError(InvalidContentError): """Error which is raised when an unknown command is issued.""" messsage = 'Invalid command' @@ -36,7 +76,7 @@ class InvalidCommandError(errors.InvalidContentError): super(InvalidCommandError, self).__init__(details) -class InvalidCommandParamsError(errors.InvalidContentError): +class InvalidCommandParamsError(InvalidContentError): """Error which is raised when command parameters are invalid.""" message = 'Invalid command parameters' @@ -45,14 +85,14 @@ class InvalidCommandParamsError(errors.InvalidContentError): super(InvalidCommandParamsError, self).__init__(details) -class RequestedObjectNotFoundError(errors.NotFound): +class RequestedObjectNotFoundError(NotFound): def __init__(self, type_descr, obj_id): details = '{} with id {} not found.'.format(type_descr, obj_id) super(RequestedObjectNotFoundError, self).__init__(details) self.details = details -class OverlordAPIError(errors.RESTError): +class OverlordAPIError(RESTError): """Error raised when a call to the agent API fails.""" message = 'Error in call to teeth-agent-api.' @@ -71,7 +111,7 @@ class HeartbeatError(OverlordAPIError): super(HeartbeatError, self).__init__(details) -class ImageDownloadError(errors.RESTError): +class ImageDownloadError(RESTError): """Error raised when an image cannot be downloaded.""" message = 'Error downloading image.' @@ -81,7 +121,7 @@ class ImageDownloadError(errors.RESTError): self.details = 'Could not download image with id {}.'.format(image_id) -class ImageChecksumError(errors.RESTError): +class ImageChecksumError(RESTError): """Error raised when an image fails to verify against its checksum.""" message = 'Error verifying image checksum.' @@ -92,7 +132,7 @@ class ImageChecksumError(errors.RESTError): self.details = self.details.format(image_id) -class ImageWriteError(errors.RESTError): +class ImageWriteError(RESTError): """Error raised when an image cannot be written to a device.""" message = 'Error writing image to device.' @@ -103,7 +143,7 @@ class ImageWriteError(errors.RESTError): self.details = self.details.format(device, exit_code) -class ConfigDriveWriteError(errors.RESTError): +class ConfigDriveWriteError(RESTError): """Error raised when a configdrive directory cannot be written to a device. """ @@ -117,7 +157,7 @@ class ConfigDriveWriteError(errors.RESTError): self.details = details -class SystemRebootError(errors.RESTError): +class SystemRebootError(RESTError): """Error raised when a system cannot reboot.""" message = 'Error rebooting system.' diff --git a/teeth_agent/hardware.py b/teeth_agent/hardware.py index bc79db391..a8e4e0fd9 100644 --- a/teeth_agent/hardware.py +++ b/teeth_agent/hardware.py @@ -22,7 +22,7 @@ import subprocess import stevedore import structlog -from teeth_rest import encoding +from teeth_agent import encoding _global_manager = None @@ -49,7 +49,7 @@ class HardwareInfo(encoding.Serializable): self.type = type self.id = id - def serialize(self, view): + def serialize(self): return collections.OrderedDict([ ('type', self.type), ('id', self.id), diff --git a/teeth_agent/logging.py b/teeth_agent/logging.py index 1429f9932..5c66bc900 100644 --- a/teeth_agent/logging.py +++ b/teeth_agent/logging.py @@ -38,7 +38,7 @@ def _format_event(logger, method, event): have enough keys to format. """ if 'event' not in event: - # nothing to format, e.g. _log_request in teeth_rest/component + # nothing to format return event # Get a list of fields that need to be filled. formatter = string.Formatter() diff --git a/teeth_agent/overlord_agent_api.py b/teeth_agent/overlord_agent_api.py index 7fa41ade5..969d375cb 100644 --- a/teeth_agent/overlord_agent_api.py +++ b/teeth_agent/overlord_agent_api.py @@ -17,8 +17,8 @@ limitations under the License. import json import requests -from teeth_rest import encoding +from teeth_agent import encoding from teeth_agent import errors @@ -28,8 +28,7 @@ class APIClient(object): def __init__(self, api_url): self.api_url = api_url.rstrip('/') self.session = requests.Session() - self.encoder = encoding.RESTJSONEncoder( - encoding.SerializationViews.PUBLIC) + self.encoder = encoding.RESTJSONEncoder() def _request(self, method, path, data=None): request_url = '{api_url}{path}'.format(api_url=self.api_url, path=path) diff --git a/teeth_agent/tests/agent.py b/teeth_agent/tests/agent.py index 33b011267..83313606f 100644 --- a/teeth_agent/tests/agent.py +++ b/teeth_agent/tests/agent.py @@ -22,10 +22,10 @@ import mock import pkg_resources from wsgiref import simple_server -from teeth_rest import encoding from teeth_agent import agent from teeth_agent import base +from teeth_agent import encoding from teeth_agent import errors from teeth_agent import hardware @@ -118,9 +118,7 @@ class TestHeartbeater(unittest.TestCase): class TestBaseAgent(unittest.TestCase): def setUp(self): - self.encoder = encoding.RESTJSONEncoder( - encoding.SerializationViews.PUBLIC, - indent=4) + self.encoder = encoding.RESTJSONEncoder(indent=4) self.agent = agent.TeethAgent('https://fake_api.example.org:8081/', ('localhost', 9999), '192.168.1.1') diff --git a/teeth_agent/tests/api.py b/teeth_agent/tests/api.py index 655bbae1c..bbcf47e29 100644 --- a/teeth_agent/tests/api.py +++ b/teeth_agent/tests/api.py @@ -22,7 +22,6 @@ import unittest from werkzeug import test from werkzeug import wrappers -from teeth_rest import encoding from teeth_agent import agent from teeth_agent.api import app @@ -99,7 +98,7 @@ class TestTeethAPI(unittest.TestCase): self.assertEqual(kwargs, {'key': 'value'}) self.assertEqual(response.status_code, 200) data = json.loads(response.data) - expected_result = result.serialize(encoding.SerializationViews.PUBLIC) + expected_result = result.serialize() self.assertEqual(data, expected_result) def test_execute_agent_command_validation(self): @@ -150,7 +149,7 @@ class TestTeethAPI(unittest.TestCase): self.assertEqual(response.status_code, 200) self.assertEqual(json.loads(response.data), { u'commands': [ - cmd_result.serialize(None), + cmd_result.serialize(), ], }) @@ -160,8 +159,7 @@ class TestTeethAPI(unittest.TestCase): True, {'test': 'result'}) - serialized_cmd_result = cmd_result.serialize( - encoding.SerializationViews.PUBLIC) + serialized_cmd_result = cmd_result.serialize() mock_agent = mock.create_autospec(agent.TeethAgent) mock_agent.get_command_result.return_value = cmd_result