Synchronize upload-image-s3 to upload-image-swift

This updates the s3 image upload role with the corresponding recent
changes to the swift image upload role.

Change-Id: I2aa663333c81b4834c5193216fff1fe10dc0301e

roles/upload-image-s3/README.rst

@@ -4,6 +4,10 @@ This uploads a filesystem image (for example, one built by diskimage
 builder) to an S3 bucket.  The role returns an artifact to Zuul
 suitable for use by the zuul-launcher.
 
+If a `raw` or `vhd` image is provided and the `zstd` command is
+available, it will be compressed in the way that zuul-launcher
+expects.
+
 **Role Variables**
 
 .. zuul:rolevar:: upload_image_s3_endpoint
@@ -41,7 +45,7 @@ suitable for use by the zuul-launcher.
    automatic expiration of objects in this bucket.
 
 .. zuul:rolevar:: upload_image_s3_image_name
-   :default: `{{ build_diskimage_image_name }}`
+   :default: `{{ build_diskimage_image_name | default(zuul.image_build_name) }}`
 
    The Zuul image name for use by zuul-launcher (e.g., `debian-bookworm`).
 

roles/upload-image-s3/defaults/main.yaml

@@ -1,4 +1,4 @@
-upload_image_s3_image_name: '{{ build_diskimage_image_name }}'
+upload_image_s3_image_name: '{{ build_diskimage_image_name | default(zuul.image_build_name) }}'
 upload_image_s3_delete_after: 0
 upload_image_s3_filename: '{{ build_diskimage_image_root }}/{{ build_diskimage_image_name }}.{{ upload_image_s3_extension }}'
 upload_image_s3_name: '{{ zuul.build }}-{{ build_diskimage_image_name }}.{{ upload_image_s3_extension }}'

roles/upload-image-s3/tasks/main.yaml

@@ -1,4 +1,4 @@
-# Run the checksums in the background while we're uploading
+# Run the checksums in the background while we're working
 - name: Get sha256 hash
   stat:
     path: '{{ upload_image_s3_filename }}'
@@ -15,6 +15,28 @@
   poll: 0
   register: md5_task
 
+- name: Check if zstd is installed
+  shell: "command -v zstd || exit 1"
+  register: zstd_installed
+  failed_when: false
+
+- name: Compress image
+  when:
+    - "zstd_installed.rc == 0"
+    - "upload_image_s3_format in ['raw', 'vhd']"
+  command: zstd '{{ upload_image_s3_filename }}'
+
+- name: Set extension
+  set_fact:
+    zj_upload_image_s3_extension: ''
+
+- name: Set extension
+  when:
+    - "zstd_installed.rc == 0"
+    - "upload_image_s3_format in ['raw', 'vhd']"
+  set_fact:
+    zj_upload_image_s3_extension: '.zst'
+
 - name: Upload image to S3
   no_log: true
   upload_image_s3:
@@ -22,11 +44,15 @@
     bucket: "{{ upload_image_s3_bucket }}"
     aws_access_key: "{{ upload_image_s3_aws_access_key }}"
     aws_secret_key: "{{ upload_image_s3_aws_secret_key }}"
-    filename: '{{ upload_image_s3_filename }}'
-    name: '{{ upload_image_s3_name }}'
+    filename: '{{ upload_image_s3_filename }}{{ zj_upload_image_s3_extension }}'
+    name: '{{ upload_image_s3_name }}{{ zj_upload_image_s3_extension }}'
     delete_after: '{{ upload_image_s3_delete_after }}'
   register: upload_results
 
+- name: Delete uncompressed image
+  when: "upload_image_s3_format in ['raw', 'vhd']"
+  command: rm '{{ upload_image_s3_filename }}'
+
 - name: Wait for sha256
   async_status:
     jid: "{{ sha256_task.ansible_job_id }}"

test-playbooks/upload-image-s3.yaml

@@ -13,6 +13,13 @@
     - ensure-docker
     - ensure-pip
   tasks:
+    - name: Install zstd
+      become: true
+      package:
+        state: present
+        name:
+          - zstd
+
     - name: Install boto
       pip:
         extra_args: "--break-system-packages"
@@ -58,6 +65,14 @@
       wait_for:
         timeout: 5
 
+    - name: Add content to tempfile
+      copy:
+        content: "{{ test_content }}"
+        dest: "{{ test_filename }}"
+
+    - name: Remove temp file so we can run the role again
+      command: rm -f /tmp/testfile.zst
+
     # This should delete the first object then replace it.
     - name: Upload file to s3
       include_role:
@@ -86,4 +101,4 @@
       command: "{{ ansible_user_dir }}/mc find local/zuul"
 
     - name: Check for testfile in minio bucket
-      command: "{{ ansible_user_dir }}/mc find local/zuul/{{ test_objectname }}"
+      command: "{{ ansible_user_dir }}/mc find local/zuul/{{ test_objectname }}.zst"