CI: Test kayobe_control_host_become

This change adds support for the kayobe_control_host_become variable to
CI jobs. This variable will cause any uses of become on localhost to
fail. This allows us to test that the kayobe_control_host_become
configuration variable has successfully removed all uses of become.

I've enabled this on a subset of jobs to get good test coverage.

Depends-On: https://review.opendev.org/c/openstack/kolla-ansible/+/949758
Change-Id: I6da98e055714f75112cf08ff8aa53dd84de2e425

dev/functions

@@ -909,6 +909,14 @@ function to_bool {
     fi
 }
 
+function is_absolute_path {
+    path="$1"
+    case "$path" in
+    /*) true ;;
+    *) false ;;
+    esac
+}
+
 function configure_iptables {
     # NOTE(wszumski): adapted from the ironic devstack plugin, see:
     # https://github.com/openstack/ironic/blob/36e87dc5b472d79470b783fbba9ce396e3cbb96e/devstack/lib/ironic#L2132

dev/kayobe-control-host-become-sudo-checker

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+IFS='' read -r -d '' ERR_MSG <<"EOF"
+This task tried to use become, but kayobe_control_host_become is set to false. Please change become: true, to become: "{{ kayobe_control_host_become | bool }}", e.g:
+
+- name: Run a command
+  command: echo hi
+  become: true
+
+Should be:
+
+- name: Run a command
+  command: echo hi
+  become: "{{ kayobe_control_host_become | bool }}"
+
+Hint: You may need to write any files to a user controlled directory.
+ErrorCode: CONTROL_HOST_BECOME_VIOLATION
+EOF
+
+
+>&2 echo "$ERR_MSG"
+exit 1

dev/kolla-control-host-become-sudo-checker

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+IFS='' read -r -d '' ERR_MSG <<"EOF"
+This task tried to use become, but kolla_ansible_control_host_become is set to false.
+
+The task will need to be changed in kolla-ansible to support running as an
+unprivileged user.
+
+Hint: You may need to write any files to a user controlled directory.
+ErrorCode: CONTROL_HOST_BECOME_VIOLATION
+EOF
+
+
+>&2 echo "$ERR_MSG"
+exit 1

dev/playbook-run.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+# Script to run a custom playbook
+
+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${PARENT}/functions"
+
+function main {
+    local playbook_path
+    playbook=$1
+    args=("${@:2}")
+    shift $#
+    config_init
+    environment_setup
+    # Use eval so we can do something like: playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/test.yml'
+    # NOTE: KAYOBE_CONFIG_PATH gets defined by kayobe_init
+    playbook_path="$(eval echo $playbook)"
+    if ! is_absolute_path "$playbook_path"; then
+        # Default to a path relative to repository root
+        playbook_path="$KAYOBE_CONFIG_ROOT/$playbook_path"
+    fi
+    if [ ! -f "$playbook_path" ]; then
+        die $LINENO "Playbook path does not exist: $playbook_path"
+    fi
+    run_kayobe playbook run "$playbook_path" "${args[@]}"
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    if [ "$#" -lt 1 ]; then
+        die $LINENO "Error: You must provide a playbook to run." \
+            "Usage: playbook-run.sh <playbook>"
+    fi
+    main "${@:1}"
+fi

playbooks/kayobe-base/overrides.yml.j2

@@ -0,0 +1,3 @@
+---
+# Test support for not escalating privileges
+kayobe_control_host_become: "{{ kayobe_control_host_become }}"

playbooks/kayobe-base/post.yml

@@ -1,4 +1,32 @@
 ---
+- hosts: primary
+  environment:
+    KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"
+  tasks:
+    # Checks become validator was correctly configured in pre.yml.
+    - block:
+        - name: Create a playbook to test become validator was configured
+          copy:
+            content: |
+              ---
+              - hosts: localhost
+                tasks:
+                  - name: Testing become fails
+                    command: "true"
+                    become: true
+                    register: result
+                    failed_when: '"CONTROL_HOST_BECOME_VIOLATION" not in result.module_stderr'
+            dest: /tmp/test-control-host-become.yml
+
+        - name: Check that that kayobe become validator was correctly configured
+          shell:
+            cmd: "{{ kayobe_src_dir }}/dev/playbook-run.sh /tmp/test-control-host-become.yml &> {{ logs_dir }}/ansible/kayobe-test-control-host-become"
+            executable: /bin/bash
+          failed_when: false
+          register: become_check_result
+
+      when: not kayobe_control_host_become | bool
+
 - hosts: all
   roles:
     - role: kayobe-diagnostics
@@ -7,3 +35,11 @@
       kayobe_diagnostics_config_dir: "{{ kayobe_config_src_dir }}"
       kayobe_diagnostics_previous_config_dir: "{{ previous_kayobe_config_src_dir }}"
       kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+
+- hosts: primary
+  tasks:
+    # Fail after logs have been posted
+    - name: Fail run if "Check that that kayobe become validator was correctly configured" failed
+      assert:
+        that: become_check_result.rc == 0
+      when: become_check_result is not skipped

playbooks/kayobe-base/pre.yml

@@ -88,6 +88,49 @@
         kayobe_galaxy_requirements_src_dir: "{{ kolla_ansible_src_dir }}"
         kayobe_galaxy_requirements_dest_path: "/tmp/kolla-ansible-requirements.yml"
 
+    # NOTE(wszumski): I explored running as an unprivileged user, but it looked like
+    # a world of pain, so I've gone for this simpler approach (for now).
+    - block:
+        - name: Ensure inventory host_vars directories exist
+          file:
+            state: directory
+            path: "{{ item }}"
+          loop:
+            - "{{ kayobe_config_src_dir }}/etc/kayobe/inventory/host_vars/"
+            - "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/inventory/host_vars/"
+
+        - name: Configure the become checker for localhost (kayobe)
+          # NOTE(wszumski): This will cause all uses of become to fail when running
+          # kayobe playbooks against localhost. This should not happen since we
+          # have disabled escalation of privileges with kayobe_control_host_become.
+          # If you hit this error you will need to change your task to respect
+          # that variable.
+          copy:
+            content: |
+              ansible_become_exe: "{{ kayobe_src_dir }}/dev/kayobe-control-host-become-sudo-checker"
+            dest: "{{ kayobe_config_src_dir }}/etc/kayobe/inventory/host_vars/localhost"
+
+        - name: Configure the become checker for localhost (kolla)
+          # NOTE(wszumski): This will cause all uses of become to fail when running
+          # kolla playbooks against localhost. This should not happen since we
+          # have disabled escalation of privileges with kayobe_control_host_become.
+          # If you hit this error you will need to change kolla-ansible to not use
+          # become unless necessary.
+          copy:
+            content: |
+              ansible_become_exe: "{{ kayobe_src_dir }}/dev/kolla-control-host-become-sudo-checker"
+            dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/inventory/host_vars/localhost"
+
+      when: not kayobe_control_host_become | bool
+
+    # NOTE(wszumski): Use the name zz-10-overrides.yml to ensure this takes
+    # precedence over the standard config files, but can control order with the
+    # priority (number after zz).
+    - name: Ensure kayobe-config override config file exists
+      template:
+        src: overrides.yml.j2
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-10-overrides.yml"
+
     - block:
         - name: Ensure previous kayobe directory exists
           file:

playbooks/kayobe-infra-vm-base/pre.yml

@@ -32,12 +32,13 @@
         value: 1
       become: true
 
-    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
-    # precedence over the standard config files.
+    # NOTE(mgoddard): Use the name zz-20-overrides.yml to ensure this takes
+    # precedence over the standard config files, but can control order with the
+    # priority (number after zz).
     - name: Ensure kayobe-config override config file exists
       template:
         src: overrides.yml.j2
-        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-20-overrides.yml"
 
     - name: Ensure infra-vms group variables exist
       template:

playbooks/kayobe-overcloud-base/pre.yml

@@ -13,12 +13,13 @@
         bridge_prefix: 24
         bridge_port_interface: dummy1
 
-    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
-    # precedence over the standard config files.
+    # NOTE(mgoddard): Use the name zz-20-overrides.yml to ensure this takes
+    # precedence over the standard config files, but can control order with the
+    # priority (number after zz).
     - name: Ensure kayobe-config override config file exists
       template:
         src: overrides.yml.j2
-        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-20-overrides.yml"
 
     - name: Ensure kolla-ansible globals.yml override config file exists
       template:

playbooks/kayobe-overcloud-host-configure-base/pre.yml

@@ -28,13 +28,13 @@
         virtualenv: "{{ testinfra_venv }}"
         virtualenv_command: "{{ cmd }}"
 
-    # NOTE(mgoddard): Use the name zzz-overrides.yml to ensure this takes
-    # precedence over the standard config files and zz-overrides.yml from
+    # NOTE(mgoddard): Use the name zz-30-overrides.yml to ensure this takes
+    # precedence over the standard config files and zz-20-overrides.yml from
     # kayobe-overcloud-base.
     - name: Ensure kayobe-config override config file exists
       template:
         src: overrides.yml.j2
-        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zzz-overrides.yml"
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-30-overrides.yml"
 
     # NOTE(mgoddard): Create two loopback devices backed by files. These will
     # be added to a software RAID volume, then added to an LVM volume group.

playbooks/kayobe-overcloud-upgrade-base/pre.yml

@@ -11,12 +11,13 @@
         bridge_prefix: 24
         bridge_port_interface: dummy1
 
-    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
-    # precedence over the standard config files.
+    # NOTE(mgoddard): Use the name zz-20-overrides.yml to ensure this takes
+    # precedence over the standard config files, but can control order with the
+    # priority (number after zz).
     - name: Ensure kayobe-config override config file exists
       template:
         src: overrides.yml.j2
-        dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+        dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/zz-20-overrides.yml"
       vars:
         is_previous_release: true
 

playbooks/kayobe-overcloud-upgrade-base/run.yml

@@ -47,12 +47,13 @@
         - etc/kolla/public-openrc.sh
         - etc/kolla/clouds.yaml
 
-    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
-    # precedence over the standard config files.
+    # NOTE(mgoddard): Use the name zz-20-overrides.yml to ensure this takes
+    # precedence over the standard config files, but can control order with the
+    # priority (number after zz).
     - name: Ensure kayobe-config override config file exists
       template:
         src: overrides.yml.j2
-        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-20-overrides.yml"
 
     - name: Ensure kolla-ansible globals.yml override config file exists
       template:

playbooks/kayobe-seed-base/pre.yml

@@ -13,12 +13,13 @@
         bridge_prefix: 24
         bridge_port_interface: dummy1
 
-    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
-    # precedence over the standard config files.
+    # NOTE(mgoddard): Use the name zz-20-overrides.yml to ensure this takes
+    # precedence over the standard config files, but can control order with the
+    # priority (number after zz).
     - name: Ensure kayobe-config override config file exists
       template:
         src: overrides.yml.j2
-        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-20-overrides.yml"
 
     - name: Ensure kolla-ansible globals.yml override config file exists
       template:

playbooks/kayobe-seed-upgrade-base/pre.yml

@@ -11,12 +11,13 @@
         bridge_prefix: 24
         bridge_port_interface: dummy1
 
-    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
-    # precedence over the standard config files.
+    # NOTE(mgoddard): Use the name zz-20-overrides.yml to ensure this takes
+    # precedence over the standard config files, but can control order with the
+    # priority (number after zz).
     - name: Ensure kayobe-config override config file exists
       template:
         src: overrides.yml.j2
-        dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+        dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/zz-20-overrides.yml"
       vars:
         is_previous_release: true
 

playbooks/kayobe-seed-upgrade-base/run.yml

@@ -38,12 +38,13 @@
         - etc/kayobe/kolla/passwords.yml
         - etc/kayobe/kolla/config/bifrost/dib.yml
 
-    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
-    # precedence over the standard config files.
+    # NOTE(mgoddard): Use the name zz-20-overrides.yml to ensure this takes
+    # precedence over the standard config files, but can control order with the
+    # priority (number after zz).
     - name: Ensure kayobe-config override config file exists
       template:
         src: overrides.yml.j2
-        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-20-overrides.yml"
 
     - name: Ensure kolla-ansible globals.yml override config file exists
       template:

playbooks/kayobe-seed-vm-base/pre.yml

@@ -32,12 +32,13 @@
         value: 1
       become: true
 
-    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
-    # precedence over the standard config files.
+    # NOTE(mgoddard): Use the name zz-20-overrides.yml to ensure this takes
+    # precedence over the standard config files, but can control order with the
+    # priority (number after zz).
     - name: Ensure kayobe-config override config file exists
       template:
         src: overrides.yml.j2
-        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-20-overrides.yml"
 
     - name: Ensure seed group variables exist
       template:

roles/kayobe-ci-prep/tasks/main.yml

@@ -1,7 +1,17 @@
 ---
+- block:
     - name: Install dbus for debian system
       apt:
         name: dbus
+
+    - name: Install packages needed for unprivileged mode
+      apt:
+        name: "{{ item }}"
+      loop:
+        - libssl-dev
+        - python3-pip
+        - vim
+      when: not kayobe_control_host_become | bool
   when:
     - ansible_facts.os_family == 'Debian'
   become: true
@@ -17,5 +27,14 @@
 
     - name: Enable the EPEL repository
       command: dnf config-manager --disable epel
+
+    - name: Install packages needed for unprivileged mode
+      package:
+        name: "{{ item }}"
+        state: present
+      loop:
+        - openssl-devel
+      when: not kayobe_control_host_become | bool
+
   when: ansible_facts.os_family == 'RedHat'
   become: true

zuul.d/jobs.yaml

@@ -96,6 +96,7 @@
       ansible_collection_kolla_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/ansible-collection-kolla'].src_dir }}"
       kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kayobe'].src_dir }}"
       kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kayobe-config-dev'].src_dir }}"
+      kayobe_control_host_become: true
       kolla_ansible_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla-ansible'].src_dir }}"
       previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}"
       previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
@@ -129,6 +130,8 @@
 - job:
     name: kayobe-overcloud-rocky9
     parent: kayobe-overcloud-base
+    vars:
+      kayobe_control_host_become: false
     nodeset: kayobe-rocky9
 
 - job:
@@ -149,6 +152,7 @@
     nodeset: kayobe-ubuntu-noble
     vars:
       container_engine: podman
+      kayobe_control_host_become: false
 
 - job:
     name: kayobe-overcloud-tls-base
@@ -269,6 +273,7 @@
     vars:
       overcloud_container_image_regex: "^base"
       seed_container_image_regex: "^base"
+      kayobe_control_host_become: false
 
 - job:
     name: kayobe-seed-images-rocky9-podman
@@ -283,6 +288,7 @@
     nodeset: kayobe-ubuntu-noble
     vars:
       seed_container_image_regex: "^base"
+      kayobe_control_host_become: false
 
 - job:
     name: kayobe-seed-images-ubuntu-noble-podman
@@ -377,6 +383,8 @@
     name: kayobe-seed-vm-ubuntu-noble
     parent: kayobe-seed-vm-base
     nodeset: kayobe-ubuntu-noble
+    vars:
+      kayobe_control_host_become: false
 
 - job:
     name: kayobe-seed-vm-efi-base
@@ -397,6 +405,8 @@
     name: kayobe-seed-vm-rocky9-efi
     parent: kayobe-seed-vm-efi-base
     nodeset: kayobe-rocky9
+    vars:
+      kayobe_control_host_become: false
 
 - job:
     name: kayobe-seed-vm-ubuntu-noble-efi
@@ -425,6 +435,8 @@
     name: kayobe-infra-vm-rocky9
     parent: kayobe-infra-vm-base
     nodeset: kayobe-rocky9
+    vars:
+      kayobe_control_host_become: false
 
 - job:
     name: kayobe-infra-vm-ubuntu-noble