From 95eee6bc11e97105cb0356d7475bee699d404bda Mon Sep 17 00:00:00 2001
From: German Eichberger <German.eichberger@rackspace.com>
Date: Wed, 21 Mar 2018 15:20:55 -0700
Subject: [PATCH] Downloads amphora images from artifact storage

This will download a test amphora image per default form the
Open Stack artifact storage to speed up tests. Operators can
configure their won artifact storage to simplify deploys.

Change-Id: I408d4128d35aab889dbe89fd9497d83a7830129b
---
 defaults/main.yml                             |  17 ++-
 doc/source/configure-octavia.rst              |  14 +++
 .../image_download-754d13e7df9b7891.yaml      |  16 +++
 tasks/main.yml                                |   4 +
 tasks/octavia_amp_image.yml                   | 105 ++++++++++++++++++
 tasks/octavia_post_install.yml                |  48 --------
 tests/os_octavia-overrides.yml                |   1 +
 tests/test-configure-octavia.yml              |  30 -----
 tests/test-octavia.yml                        |  11 --
 9 files changed, 153 insertions(+), 93 deletions(-)
 create mode 100644 releasenotes/notes/image_download-754d13e7df9b7891.yaml
 create mode 100644 tasks/octavia_amp_image.yml

diff --git a/defaults/main.yml b/defaults/main.yml
index 82fad4b1..5f3f982a 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -238,12 +238,21 @@ octavia_glance_image_tag: octavia-amphora-image
 octavia_amp_image_owner_id:
 # add here the glance image id if tagging is not used (not recommended for prod)
 octavia_amp_image_id:
-# add here the file name of the image if it should be uplaoded automatically
-# Note: This does not support image tags so don't use for prod
+# download the image from an artefact server
+# Note: The default is the Octavia test image so don't use that in prod
+octavia_download_artefact: True
+# The host to download images to if enabled
+# Options are ['deployment-host', 'target-host']
+octavia_image_downloader: "deployment-host"
+# The URL to downlaod from
+octavia_artefact_url: http://tarballs.openstack.org/octavia/test-images/test-only-amphora-x64-haproxy-ubuntu-xenial.qcow2
+# the directory to store the downloaded file to
+octavia_amp_image_path: "~/"
+# add here the file name of the image if it should be uploaded automatically
 octavia_amp_image_file_name:
 # enable uploading image to glance automatically
-# Note: This does not support image tags so don't use for prod
-octavia_amp_image_upload_enabled: False
+octavia_amp_image_upload_enabled: "{{ octavia_download_artefact }}"
+
 # Name of the Octavia security group
 octavia_security_group_name: octavia_sec_grp
 # Restrict access to only authorized hosts
diff --git a/doc/source/configure-octavia.rst b/doc/source/configure-octavia.rst
index 248f2f23..9aac629d 100644
--- a/doc/source/configure-octavia.rst
+++ b/doc/source/configure-octavia.rst
@@ -76,6 +76,16 @@ ips which overlap with ips assigned to hosts or containers (see the
 Building Octavia images
 ~~~~~~~~~~~~~~~~~~~~~~~
 
+.. note::
+    The default behavior is to download a test image from the OpenStack artifact
+    storage the Octavia team provides daily. Because this image doesn't apply
+    operating system security patches in a timely manner it is unsuited
+    for production use.
+
+    Some Operating System vendors might provide official amphora builds or an
+    organization might maintain their own artifact storage - for those cases the
+    automatic download can be leveraged, too.
+
 Images using the ``diskimage-builder`` must be built outside of a container.
 For this process, use one of the physical hosts within the environment.
 
@@ -115,6 +125,10 @@ For this process, use one of the physical hosts within the environment.
       glance image-create --name amphora-x64-haproxy --visibility private --disk-format qcow2 \
          --container-format bare --tags octavia-amphora-image </var/lib/octavia/amphora-x64-haproxy.qcow2
 
+   .. note::
+        Alternatively you can specify the new image in the appropriate settings and rerun the
+        ansible with an appropriate tag.
+
 You can find more information abpout the diskimage script and the process at
 https://github.com/openstack/octavia/tree/master/diskimage-create
 
diff --git a/releasenotes/notes/image_download-754d13e7df9b7891.yaml b/releasenotes/notes/image_download-754d13e7df9b7891.yaml
new file mode 100644
index 00000000..c034e504
--- /dev/null
+++ b/releasenotes/notes/image_download-754d13e7df9b7891.yaml
@@ -0,0 +1,16 @@
+---
+features:
+  - |
+    This consolidates the amphora image tasks in a common file and adds a way
+    to download an amphora image from an artefact storage over http(s). With
+    the Octavia team providing test images the tests were modified to not
+    build images any longer but download them.
+security:
+  - |
+    It is commonly considered bad practice to downlaod random images from the
+    Internet expecially the test images the Octavia team provides which could
+    potentially include unpatched operating system packages - so for any
+    production deploy adjust the download url to an artifact storage your
+    organization controls. The system also does not authenticate the image
+    (e.g. with an md5) so should only be used on networks your organization
+    controls.
diff --git a/tasks/main.yml b/tasks/main.yml
index 2b73c0fc..1beb30c3 100644
--- a/tasks/main.yml
+++ b/tasks/main.yml
@@ -87,6 +87,10 @@
     - octavia-install
     - octavia-config
 
+- include: octavia_amp_image.yml
+  tags:
+    - octavia-config
+
 
 - name: Flush handlers
   meta: flush_handlers
diff --git a/tasks/octavia_amp_image.yml b/tasks/octavia_amp_image.yml
new file mode 100644
index 00000000..3cd451ad
--- /dev/null
+++ b/tasks/octavia_amp_image.yml
@@ -0,0 +1,105 @@
+---
+  # Copyright 2018, Rackspace US, Inc.
+  #
+  # Licensed under the Apache License, Version 2.0 (the "License");
+  # you may not use this file except in compliance with the License.
+  # You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+
+- name: Download image from artefact server
+  get_url:
+    url: "{{ octavia_artefact_url }}"
+    dest: "{{ octavia_amp_image_path }}"
+  retries: 10
+  delay: 10
+  register: octavia_download_result
+  when:
+    - octavia_download_artefact|bool
+  delegate_to: "{{ (octavia_image_downloader == 'deployment-host') | ternary('localhost', inventory_hostname) }}"
+
+- name: Set the filename fact
+  set_fact:
+    octavia_amp_image_file_name: "{{ octavia_download_result.dest }}"
+  when:
+    - octavia_image_downloader == "deployment-host"
+    - octavia_download_artefact|bool
+
+- name: Copy download images from deployment-host to target-host(s)
+  copy:
+    src: "{{ octavia_amp_image_file_name }}"
+    dest: "~/{{ octavia_amp_image_file_name|basename }}"
+  when:
+    - octavia_amp_image_upload_enabled
+    - octavia_image_downloader == "deployment-host"
+  register: octavia_amp_image_copy_result
+  until: octavia_amp_image_copy_result | success
+  retries: 6
+  delay: 5
+
+- name: Set if we need to upload an image
+  set_fact:
+    octavia_amp_image_needs_upload: "{{ (octavia_image_downloader != 'deployment-host')|ternary(octavia_download_result|changed, octavia_amp_image_copy_result|changed) }}"
+    octavia_dst_amp_image_path: "{{ ((octavia_image_downloader == 'deployment-host') and (octavia_download_artefact|bool))|ternary(octavia_amp_image_copy_result.dest, octavia_download_result.dest) }}"
+  when:
+    - octavia_amp_image_upload_enabled
+
+- name: Get curremt image id
+  os_image_facts:
+    auth:
+      auth_url: "{{ keystone_service_adminurl }}"
+      username: "{{ octavia_service_user_name }}"
+      password: "{{ octavia_service_password }}"
+      project_name: "{{ octavia_service_project_name }}"
+      user_domain_name: "{{ octavia_service_user_domain_id }}"
+      project_domain_name: "{{ octavia_service_project_domain_id }}"
+    endpoint_type: "{{ octavia_ansible_endpoint_type }}"
+    region_name: "{{ octavia_service_region }}"
+    validate_certs: "{{ keystone_service_adminuri_insecure }}"
+    auth_type: "{{ octavia_keystone_auth_plugin }}"
+    image: amphora-x64-haproxy
+  when:
+     - octavia_amp_image_needs_upload|default(False)
+
+# use shell since os_image doesn't support tags
+- name: Upload new image to glance
+  shell: |
+    . {{ ansible_env.HOME }}/openrc
+    openstack image create --file  {{ octavia_dst_amp_image_path }} --disk-format qcow2 \
+    --tag {{ octavia_glance_image_tag }}  --private --project service amphora-x64-haproxy
+  when:
+    - octavia_amp_image_needs_upload|default(False)
+  run_once: True
+  tags:
+    - skip_ansible_lint
+
+- name: Delete old image from glance
+  os_image:
+    auth:
+      auth_url: "{{ keystone_service_adminurl }}"
+      username: "{{ octavia_service_user_name }}"
+      password: "{{ octavia_service_password }}"
+      project_name: "{{ octavia_service_project_name }}"
+      user_domain_name: "{{ octavia_service_user_domain_id }}"
+      project_domain_name: "{{ octavia_service_project_domain_id }}"
+    endpoint_type: "{{ octavia_ansible_endpoint_type }}"
+    region_name: "{{ octavia_service_region }}"
+    validate_certs: "{{ keystone_service_adminuri_insecure }}"
+    auth_type: "{{ octavia_keystone_auth_plugin }}"
+    id: "{{ openstack.id }}"
+    state: absent
+  when:
+   - openstack is defined # result from os_image_facts
+
+
+
+
+
+
+
diff --git a/tasks/octavia_post_install.yml b/tasks/octavia_post_install.yml
index 3e92c6af..a8190601 100644
--- a/tasks/octavia_post_install.yml
+++ b/tasks/octavia_post_install.yml
@@ -56,54 +56,6 @@
     - src: "{{ octavia_ca_private_key }}"
       dest: "/etc/octavia/certs/ca_key.pem"
 
-- name: Copy image
-  copy:
-    src: "{{ octavia_amp_image_file_name}}"
-    dest: "{{ octavia_amp_image_file_name}}"
-  when: octavia_amp_image_upload_enabled
-
-- name: Upload image to glance
-  os_image:
-    auth:
-      auth_url: "{{ keystone_service_adminurl }}"
-      username: "{{ octavia_service_user_name }}"
-      password: "{{ octavia_service_password }}"
-      project_name: "{{ octavia_service_project_name }}"
-      user_domain_name: "{{ octavia_service_user_domain_id }}"
-      project_domain_name: "{{ octavia_service_project_domain_id }}"
-    endpoint_type: "{{ octavia_ansible_endpoint_type }}"
-    region_name: "{{ octavia_service_region }}"
-    validate_certs: "{{ keystone_service_adminuri_insecure }}"
-    auth_type: "{{ octavia_keystone_auth_plugin }}"
-    name: amphora-x64-haproxy
-    container_format: bare
-    disk_format: qcow2
-    state: present
-    filename: "{{ octavia_amp_image_file_name}}"
-    is_public:
-  when: octavia_amp_image_upload_enabled
-
-- name: Get image uuid
-  os_image_facts:
-    auth:
-      auth_url: "{{ keystone_service_adminurl }}"
-      username: "{{ octavia_service_user_name }}"
-      password: "{{ octavia_service_password }}"
-      project_name: "{{ octavia_service_project_name }}"
-      user_domain_name: "{{ octavia_service_user_domain_id }}"
-      project_domain_name: "{{ octavia_service_project_domain_id }}"
-    endpoint_type: "{{ octavia_ansible_endpoint_type }}"
-    region_name: "{{ octavia_service_region }}"
-    validate_certs: "{{ keystone_service_adminuri_insecure }}"
-    auth_type: "{{ octavia_keystone_auth_plugin }}"
-    image: amphora-x64-haproxy
-  when: octavia_amp_image_upload_enabled
-
-- name: Set image UUID fact
-  set_fact:
-     octavia_amp_image_id: "{{ openstack_image.id }}"
-  when: octavia_amp_image_upload_enabled
-
 - name: Copy user provided HAProxy templates
   copy:
     src: "{{ item.src }}"
diff --git a/tests/os_octavia-overrides.yml b/tests/os_octavia-overrides.yml
index b7ed8d6f..5bbe2ad4 100644
--- a/tests/os_octavia-overrides.yml
+++ b/tests/os_octavia-overrides.yml
@@ -15,6 +15,7 @@
 ## Octavia rewuires currently nested virtualization (VT-X) which only one jenkins
 ## gate cloud provides reliabley. To cut down on gate errors this will
 ## disbale the tests which spin up an amphora
+## As of 4/6/18 zuul runs out of memory testing with this enabled
 test_octavia_amphora: False
 
 # Test Octavia standalone
diff --git a/tests/test-configure-octavia.yml b/tests/test-configure-octavia.yml
index 95f8ccb1..7509735f 100644
--- a/tests/test-configure-octavia.yml
+++ b/tests/test-configure-octavia.yml
@@ -28,24 +28,7 @@
       retries: 5
       delay: 2
       with_items:
-        - qemu
-        - uuid-runtime
-        - curl
-        - kpartx
         - git
-    - name: Install pip requirements
-      pip:
-        name: "{{ item }}"
-        state: "{{ octavia_pip_package_state }}"
-      register: install_packages
-      until: install_packages|success
-      retries: 5
-      delay: 2
-      with_items:
-        - argparse
-        - "Babel>=1.3"
-        - dib-utils
-        - PyYAML
     - name: Clone Octavia
       git:
         repo: "https://git.openstack.org/openstack/octavia"
@@ -53,19 +36,6 @@
         version: "{{ octavia_git_install_branch }}"
       tags:
         - skip_ansible_lint
-    - name: Clone Diskimage-Builder
-      git:
-        repo: "https://git.openstack.org/openstack/diskimage-builder"
-        dest: "{{ octavia_system_home_folder }}/diskimage-builder"
-        version: "2.9.0"
-    - name: Create amphora image
-      shell: "./diskimage-create.sh -o {{ octavia_system_home_folder }}/amphora-x64-haproxy.qcow2"
-      args:
-        chdir: "{{ octavia_system_home_folder }}/octavia/diskimage-create"
-        creates: "{{ octavia_system_home_folder }}/amphora-x64-haproxy.qcow2"
-      when: test_octavia_amphora | bool
-      tags:
-        - skip_ansible_lint
     - name: Change permission
       file:
         path: "{{ octavia_system_home_folder }}/octavia/bin/create_certificates.sh"
diff --git a/tests/test-octavia.yml b/tests/test-octavia.yml
index 8ae51f53..58efcd46 100644
--- a/tests/test-octavia.yml
+++ b/tests/test-octavia.yml
@@ -77,17 +77,6 @@
         network_name: "public"
         name: "public-subnet"
         cidr: "10.1.3.0/24"
-    - name: Upload image to glance
-      shell: >-
-          glance image-create --name amphora-x64-haproxy --visibility private --disk-format qcow2 \
-            --container-format bare --tags octavia-amphora-image <{{ octavia_system_home_folder }}/amphora-x64-haproxy.qcow2 \
-            && touch {{ octavia_system_home_folder }}/image
-      args:
-        creates: "{{ octavia_system_home_folder }}/image"
-      environment: "{{ env }}"
-      when: test_octavia_amphora | bool
-      tags:
-        - skip_ansible_lint
     - name: Create ssh-key
       shell: >
           cat /dev/zero | ssh-keygen -q -N ""