diff --git a/ansible/group_vars/all/apt b/ansible/group_vars/all/apt index fad722dcd..ded0cdf0a 100644 --- a/ansible/group_vars/all/apt +++ b/ansible/group_vars/all/apt @@ -10,3 +10,30 @@ apt_proxy_http: # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}. apt_proxy_https: "{{ apt_proxy_http }}" + +# List of apt keys. Each item is a dict containing the following keys: +# * url: URL of key +# * filename: Name of a file in which to store the downloaded key. The +# extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise. +# Default is an empty list. +apt_keys: [] + +# A list of Apt repositories. Each item is a dict with the following keys: +# * types: whitespace-separated list of repository types, e.g. deb or deb-src +# (optional, default is 'deb') +# * url: URL of the repository +# * suites: whitespace-separated list of suites, e.g. focal (optional, default +# is ansible_facts.distribution_release) +# * components: whitespace-separated list of components, e.g. main (optional, +# default is 'main') +# * signed_by: whitespace-separated list of names of GPG keyring files in +# apt_keys_path (optional, default is unset) +# * architecture: whitespace-separated list of architectures that will be used +# (optional, default is unset) +# Default is an empty list. +apt_repositories: [] + +# Whether to disable repositories in /etc/apt/sources.list. This may be used +# when replacing the distribution repositories via apt_repositories. +# Default is false. +apt_disable_sources_list: false diff --git a/ansible/roles/apt/defaults/main.yml b/ansible/roles/apt/defaults/main.yml index fad722dcd..43ce85b36 100644 --- a/ansible/roles/apt/defaults/main.yml +++ b/ansible/roles/apt/defaults/main.yml @@ -10,3 +10,33 @@ apt_proxy_http: # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}. apt_proxy_https: "{{ apt_proxy_http }}" + +# Directory containing GPG keyrings for apt repos. +apt_keys_path: "/usr/local/share/keyrings" + +# List of apt keys. Each item is a dict containing the following keys: +# * url: URL of key +# * filename: Name of a file in which to store the downloaded key. The +# extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise. +# Default is an empty list. +apt_keys: [] + +# A list of Apt repositories. Each item is a dict with the following keys: +# * types: whitespace-separated list of repository types, e.g. deb or deb-src +# (optional, default is 'deb') +# * url: URL of the repository +# * suites: whitespace-separated list of suites, e.g. focal (optional, default +# is ansible_facts.distribution_release) +# * components: whitespace-separated list of components, e.g. main (optional, +# default is 'main') +# * signed_by: whitespace-separated list of names of GPG keyring files in +# apt_keys_path (optional, default is unset) +# * architecture: whitespace-separated list of architectures that will be used +# (optional, default is unset) +# Default is an empty list. +apt_repositories: [] + +# Whether to disable repositories in /etc/apt/sources.list. This may be used +# when replacing the distribution repositories via apt_repositories. +# Default is false. +apt_disable_sources_list: false diff --git a/ansible/roles/apt/handlers/main.yml b/ansible/roles/apt/handlers/main.yml new file mode 100644 index 000000000..2d39add43 --- /dev/null +++ b/ansible/roles/apt/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Update apt cache + package: + update_cache: true + become: true diff --git a/ansible/roles/apt/tasks/keys.yml b/ansible/roles/apt/tasks/keys.yml new file mode 100644 index 000000000..4c1cda1e0 --- /dev/null +++ b/ansible/roles/apt/tasks/keys.yml @@ -0,0 +1,19 @@ +--- +- name: Ensure keys directory exists + file: + path: "{{ apt_keys_path }}" + owner: root + group: root + mode: 0755 + state: directory + become: true + +- name: Ensure keys exist + get_url: + url: "{{ item.url }}" + dest: "{{ apt_keys_path ~ '/' ~ item.filename | basename }}" + owner: root + group: root + mode: 0644 + loop: "{{ apt_keys }}" + become: true diff --git a/ansible/roles/apt/tasks/main.yml b/ansible/roles/apt/tasks/main.yml index 16205b6be..4bbd0b665 100644 --- a/ansible/roles/apt/tasks/main.yml +++ b/ansible/roles/apt/tasks/main.yml @@ -1,17 +1,6 @@ --- -- name: Configure apt proxy - template: - src: "01proxy.j2" - dest: /etc/apt/apt.conf.d/01proxy - owner: root - group: root - mode: 0664 - become: true - when: apt_proxy_http | default('', true) | length > 0 or apt_proxy_https | default('', true) | length > 0 +- import_tasks: proxy.yml -- name: Remove old apt proxy config - file: - path: /etc/apt/apt.conf.d/01proxy - state: absent - become: true - when: apt_proxy_http | default('', true) | length == 0 and apt_proxy_https | default('', true) | length == 0 +- import_tasks: keys.yml + +- import_tasks: repos.yml diff --git a/ansible/roles/apt/tasks/proxy.yml b/ansible/roles/apt/tasks/proxy.yml new file mode 100644 index 000000000..16205b6be --- /dev/null +++ b/ansible/roles/apt/tasks/proxy.yml @@ -0,0 +1,17 @@ +--- +- name: Configure apt proxy + template: + src: "01proxy.j2" + dest: /etc/apt/apt.conf.d/01proxy + owner: root + group: root + mode: 0664 + become: true + when: apt_proxy_http | default('', true) | length > 0 or apt_proxy_https | default('', true) | length > 0 + +- name: Remove old apt proxy config + file: + path: /etc/apt/apt.conf.d/01proxy + state: absent + become: true + when: apt_proxy_http | default('', true) | length == 0 and apt_proxy_https | default('', true) | length == 0 diff --git a/ansible/roles/apt/tasks/repos.yml b/ansible/roles/apt/tasks/repos.yml new file mode 100644 index 000000000..ef401bb5e --- /dev/null +++ b/ansible/roles/apt/tasks/repos.yml @@ -0,0 +1,22 @@ +--- +# NOTE(mgoddard): Use the modern deb822 repository format rather than the old +# format used by the apt_repository module. +- name: Configure apt repositories + template: + src: "kayobe.sources.j2" + dest: "/etc/apt/sources.list.d/kayobe.sources" + owner: root + group: root + mode: 0644 + become: true + notify: + - Update apt cache + +- name: Disable repositories in /etc/apt/sources.list + replace: + backup: true + path: /etc/apt/sources.list + regexp: '^(deb.*)' + replace: '# \1' + when: apt_disable_sources_list | bool + become: true diff --git a/ansible/roles/apt/templates/kayobe.sources.j2 b/ansible/roles/apt/templates/kayobe.sources.j2 new file mode 100644 index 000000000..91f6bf6b4 --- /dev/null +++ b/ansible/roles/apt/templates/kayobe.sources.j2 @@ -0,0 +1,15 @@ +# {{ ansible_managed }} + +{% for repo in apt_repositories %} +Types: {{ repo.types | default('deb') }} +URIs: {{ repo.url }} +Suites: {{ repo.suites | default(ansible_facts.distribution_release) }} +Components: {{ repo.components | default('main') }} +{% if repo.signed_by is defined %} +Signed-by: {{ apt_keys_path }}/{{ repo.signed_by }} +{% endif %} +{% if repo.architecture is defined %} +Architecture: {{ repo.architecture }} +{% endif %} + +{% endfor %} diff --git a/doc/source/configuration/reference/hosts.rst b/doc/source/configuration/reference/hosts.rst index 68ee55c1f..ed50bfd79 100644 --- a/doc/source/configuration/reference/hosts.rst +++ b/doc/source/configuration/reference/hosts.rst @@ -316,8 +316,7 @@ oversight or testing. Apt === -On Ubuntu, Apt is used to manage packages and package repositories. Currently -Kayobe does not provide support for configuring custom Apt repositories. +On Ubuntu, Apt is used to manage packages and package repositories. Apt cache --------- @@ -325,10 +324,100 @@ Apt cache The Apt cache timeout may be configured via ``apt_cache_valid_time`` (in seconds) in ``etc/kayobe/apt.yml``, and defaults to 3600. +Apt proxy +--------- + Apt can be configured to use a proxy via ``apt_proxy_http`` and ``apt_proxy_https`` in ``etc/kayobe/apt.yml``. These should be set to the full URL of the relevant proxy (e.g. ``http://squid.example.com:3128``). +Apt repositories +---------------- + +Kayobe supports configuration of custom Apt repositories via the +``apt_repositories`` variable in ``etc/kayobe/apt.yml`` since the Yoga release. +The format is a list, with each item mapping to a dict/map with the following +items: + +* ``types``: whitespace-separated list of repository types, e.g. ``deb`` or + ``deb-src`` (optional, default is ``deb``) +* ``url``: URL of the repository +* ``suites``: whitespace-separated list of suites, e.g. ``focal`` (optional, + default is ``ansible_facts.distribution_release``) +* ``components``: whitespace-separated list of components, e.g. ``main`` + (optional, default is ``main``) +* ``signed_by``: whitespace-separated list of names of GPG keyring files in + ``apt_keys_path`` (optional, default is unset) +* ``architecture``: whitespace-separated list of architectures that will be used + (optional, default is unset) + +The default of ``apt_repositories`` is an empty list. + +For example, the following configuration defines a single Apt repository: + +.. code-block:: yaml + :caption: ``apt.yml`` + + apt_repositories: + - types: deb + url: https://example.com/repo + suites: focal + components: all + +In the following example, the Ubuntu Focal 20.04 repositories are consumed from +a local package mirror. The ``apt_disable_sources_list`` variable is set to +``true``, which disables all repositories in ``/etc/apt/sources.list``, +including the default Ubuntu ones. + +.. code-block:: yaml + :caption: ``apt.yml`` + + apt_repositories: + - url: http://mirror.example.com/ubuntu/ + suites: focal focal-updates + components: main restricted universe multiverse + - url: http://mirror.example.com/ubuntu/ + suites: focal-security + components: main restricted universe multiverse + + apt_disable_sources_list: true + +Apt keys +-------- + +Some repositories may be signed by a key that is not one of Apt's trusted keys. +Kayobe avoids the use of the deprecated ``apt-key`` utility, and instead allows +keys to be downloaded to a directory. This enables repositories to use the +``SignedBy`` option to state that they are signed by a specific key. This +approach is more secure than using globally trusted keys. + +Keys to be downloaded are defined by the ``apt_keys`` variable. The format is a +list, with each item mapping to a dict/map with the following items: + +* ``url``: URL of key +* ``filename``: Name of a file in which to store the downloaded key in + ``apt_keys_path``. The extension should be ``.asc`` for ASCII-armoured keys, + or ``.gpg`` otherwise. + +The default value of ``apt_keys`` is an empty list. + +In the following example, a key is downloaded, and a repository is configured +that is signed by the key. + +.. code-block:: yaml + :caption: ``apt.yml`` + + apt_keys: + - url: https://example.com/GPG-key + filename: example-key.asc + + apt_repositories: + - types: deb + url: https://example.com/repo + suites: focal + components: all + signed_by: example-key.asc + SELinux ======= *tags:* diff --git a/etc/kayobe/apt.yml b/etc/kayobe/apt.yml index 5f278e322..c4314a0aa 100644 --- a/etc/kayobe/apt.yml +++ b/etc/kayobe/apt.yml @@ -11,6 +11,33 @@ # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}. #apt_proxy_https: +# List of apt keys. Each item is a dict containing the following keys: +# * url: URL of key +# * filename: Name of a file in which to store the downloaded key. The +# extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise. +# Default is an empty list. +#apt_keys: + +# A list of Apt repositories. Each item is a dict with the following keys: +# * types: whitespace-separated list of repository types, e.g. deb or deb-src +# (optional, default is 'deb') +# * url: URL of the repository +# * suites: whitespace-separated list of suites, e.g. focal (optional, default +# is ansible_facts.distribution_release) +# * components: whitespace-separated list of components, e.g. main (optional, +# default is 'main') +# * signed_by: whitespace-separated list of names of GPG keyring files in +# apt_keys_path (optional, default is unset) +# * architecture: whitespace-separated list of architectures that will be used +# (optional, default is unset) +# Default is an empty list. +#apt_repositories: + +# Whether to disable repositories in /etc/apt/sources.list. This may be used +# when replacing the distribution repositories via apt_repositories. +# Default is false. +#apt_disable_sources_list: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 index a6de4e74f..6591d5e68 100644 --- a/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 +++ b/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 @@ -114,6 +114,25 @@ docker_storage_driver: devicemapper # Set Honolulu time. timezone: Pacific/Honolulu +{% if ansible_os_family == "Debian" %} +apt_keys: + - url: https://packages.treasuredata.com/GPG-KEY-td-agent + filename: td-agent.asc +apt_repositories: + # Ubuntu focal repositories. + - url: "http://{{ zuul_site_mirror_fqdn }}/ubuntu/" + suites: focal focal-updates + components: main restricted universe multiverse + - url: "http://{{ zuul_site_mirror_fqdn }}/ubuntu/" + suites: focal-security + components: main restricted universe multiverse + # Treasuredata repository. + - url: http://packages.treasuredata.com/4/ubuntu/focal/ + components: contrib + signed_by: td-agent.asc +apt_disable_sources_list: true +{% endif %} + {% if ansible_os_family in ['RedHat', 'Rocky'] %} # Use a local DNF mirror. dnf_use_local_mirror: true diff --git a/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py b/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py index 8b335c11b..a15429be8 100644 --- a/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py +++ b/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py @@ -16,6 +16,11 @@ def _is_firewalld_supported(): return info in ['centos', 'rocky'] +def _is_apt(): + info = distro.linux_distribution() + return info[0].startswith('Ubuntu') + + def _is_dnf(): info = distro.id() return info in ['centos', 'rocky'] @@ -187,6 +192,13 @@ def test_ntp_clock_synchronized(host): assert "synchronized: yes" in status_output +@pytest.mark.skipif(not _is_apt(), reason="Apt only supported on Ubuntu") +def test_apt_custom_package_repository_is_available(host): + with host.sudo(): + host.check_output("apt -y install td-agent") + assert host.package("td-agent").is_installed + + @pytest.mark.parametrize('repo', ["appstream", "baseos", "extras", "epel", "epel-modular"]) @pytest.mark.skipif(not _is_dnf_mirror(), reason="DNF OpenDev mirror only for CentOS 8") diff --git a/releasenotes/notes/apt-repositories-850efef70ba34946.yaml b/releasenotes/notes/apt-repositories-850efef70ba34946.yaml new file mode 100644 index 000000000..1e82c8a1e --- /dev/null +++ b/releasenotes/notes/apt-repositories-850efef70ba34946.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds support for configuring Apt repositories on Ubuntu hosts. See `story + 2009655 <https://storyboard.openstack.org/#!/story/2009655>`__ for details.