diff --git a/tests/run.yml b/tests/run.yml
index d28ccb2d7c..b08bf1e972 100644
--- a/tests/run.yml
+++ b/tests/run.yml
@@ -26,6 +26,7 @@
         upper_constraints_file: "{{ ansible_env.HOME }}/src/opendev.org/openstack/requirements/upper-constraints.txt"
         docker_image_tag_suffix: "{{ '-aarch64' if ansible_architecture == 'aarch64' else '' }}"
         kolla_ansible_venv_path: "{{ ansible_env.HOME }}/kolla-ansible-venv"
+        kolla_runtime_venv_path: "{{ ansible_env.HOME }}/kolla-runtime-venv"
         kolla_internal_fqdn: "kolla.example.com"
 
     - name: Install dig for Designate testing
@@ -52,6 +53,30 @@
       when:
         - scenario == "lets-encrypt"
 
+    # NOTE(kevko): Rocky Linux has Python 3.9 as the default, but we want to use Python 3.12 instead.
+    - name: Install Python3.12 and dependencies on RHEL derivatives
+      dnf:
+        name:
+          - python3.12
+          - python3.12-devel
+          - python3.12-pip
+        state: latest
+      when: ansible_facts.os_family == 'RedHat'
+      become: true
+
+    # NOTE(kevko): While in Rocky, virtualenv is included as part of the packages above, in Debuntu, virtualenv is in a separate package.
+    - name: Install python3-virtualenv
+      package:
+        name: python3-virtualenv
+        state: present
+        update_cache: "{{ True if ansible_facts.os_family == 'Debian' else omit }}"
+      become: true
+      when: ansible_facts.os_family != 'RedHat'
+
+    - name: Create Kolla runtime venv
+      command:
+        cmd: "{{ 'python3.12' if ansible_facts.os_family == 'RedHat' else 'python3' }} -m venv {{ kolla_runtime_venv_path }}"
+        creates: "{{ kolla_runtime_venv_path }}"
 
 - hosts: primary
   any_errors_fatal: true
@@ -236,37 +261,11 @@
         state: directory
       become: true
 
-    - name: Install Python3.12 and dependencies needed for dbus-python on RHEL derivatives
-      dnf:
-        name:
-          - python3.12
-          - python3.12-devel
-          - python3.12-pip
-          - gcc
-          - dbus-devel
-          - glib2-devel
-          - dbus-x11
-        state: latest
-      when: ansible_facts.os_family == 'RedHat'
-      become: true
-
     - name: Create Kolla Ansible venv
       command:
         cmd: "{{ 'python3.12' if ansible_facts.os_family == 'RedHat' else 'python3' }} -m venv {{ kolla_ansible_venv_path }}"
         creates: "{{ kolla_ansible_venv_path }}"
 
-    - name: Ensure the latest tested pip
-      pip:
-        name: "pip==23.*"
-        state: latest
-        virtualenv: "{{ kolla_ansible_venv_path }}"
-
-    - name: Ensure the latest tested setuptools
-      pip:
-        name: "setuptools==67.2.0"
-        state: latest
-        virtualenv: "{{ kolla_ansible_venv_path }}"
-
     - name: Install kolla-ansible and dependencies
       pip:
         extra_args: "-c {{ upper_constraints_file }}"
diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2
index 9a8b3b709e..08b882b45b 100644
--- a/tests/templates/globals-default.j2
+++ b/tests/templates/globals-default.j2
@@ -1,5 +1,6 @@
 ---
-ansible_python_interpreter: /usr/bin/python3
+virtualenv: "{{ kolla_runtime_venv_path }}"
+ansible_python_interpreter: "{{ kolla_runtime_venv_path }}/bin/python"
 
 kolla_base_distro: "{{ base_distro }}"
 network_interface: "{{ api_interface_name }}"
diff --git a/tests/upgrade.sh b/tests/upgrade.sh
index e75699ca99..a53a440e5b 100755
--- a/tests/upgrade.sh
+++ b/tests/upgrade.sh
@@ -13,6 +13,11 @@ function upgrade {
     source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
 
     kolla-ansible certificates -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/certificates
+    # Previous versions had older docker, requests requirements for example
+    # Therefore we need to run bootstrap again to ensure libraries are in
+    # proper versions (ansible-collection-kolla is different for new version, potentionally
+    # also dependencies).
+    kolla-ansible bootstrap-servers -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade-bootstrap
     # Skip rabbitmq-ha-precheck before the queues are migrated.
     kolla-ansible prechecks -i ${RAW_INVENTORY} --skip-tags rabbitmq-ha-precheck -vvv &> /tmp/logs/ansible/upgrade-prechecks-pre-rabbitmq