Merge "PoC: manifest-driven image sync for test images"
This commit is contained in:
@@ -1,10 +1,70 @@
|
||||
// Example Docker registry configuration.
|
||||
//
|
||||
// This file is used as the default by ConfigurationManager.get_docker_config(),
|
||||
// unless overridden via the --docker_config_file CLI option or programmatically.
|
||||
//
|
||||
// Registry endpoints, credentials, and image manifest paths can be overridden at runtime
|
||||
// using --docker_config_file or set programmatically via ConfigurationManager.
|
||||
//
|
||||
// Registry Resolution Behavior:
|
||||
// - The "source_registry" field (if present) on an individual image in the manifest takes highest precedence.
|
||||
// This allows different images within the same manifest to pull from different source registries.
|
||||
// - Use "manifest_registry_map" to override the source registry per manifest.
|
||||
// - Use "default_source_registry" as a global fallback if neither of the above is set.
|
||||
// This is useful when all your images come from a single upstream source like DockerHub.
|
||||
// The resolution priority is:
|
||||
// 1. source_registry (per image)
|
||||
// 2. manifest_registry_map (per manifest)
|
||||
// 3. default_source_registry (global fallback)
|
||||
//
|
||||
// Notes:
|
||||
// - Each registry must define a unique "registry_name", which acts as a logical key.
|
||||
// This is referenced by:
|
||||
// * the "source_registry" field in image manifests
|
||||
// * the "manifest_registry_map" in this config file
|
||||
// * the "default_source_registry" fallback below
|
||||
//
|
||||
// - "image_manifest_files" may include one or more YAML files.
|
||||
// - Each image listed in a manifest is pulled from its resolved source registry
|
||||
// and pushed into the "local_registry" defined below.
|
||||
|
||||
{
|
||||
registries: {
|
||||
"local_registry" : {
|
||||
"registry_name": "local_registry",
|
||||
"registry_url": "registry.local:9001",
|
||||
"user_name": "test_user",
|
||||
"password": "test_password",
|
||||
}
|
||||
"default_source_registry": "dockerhub",
|
||||
|
||||
"image_manifest_files": [
|
||||
"resources/image_manifests/stx-test-images.yaml",
|
||||
"resources/image_manifests/stx-test-images-invalid.yaml",
|
||||
// "resources/image_manifests/harbor-test-images.yaml",
|
||||
// "resources/stx-networking-images.yaml",
|
||||
],
|
||||
|
||||
"manifest_registry_map": {
|
||||
"resources/image_manifests/stx-test-images.yaml": "dockerhub",
|
||||
// "resources/image_manifests/stx-test-images-invalid.yaml": "dockerhub",
|
||||
// "resources/image_manifests/harbor-test-images.yaml": "harbor",
|
||||
},
|
||||
|
||||
"registries": {
|
||||
"dockerhub": {
|
||||
"registry_name": "dockerhub",
|
||||
"registry_url": "docker.io",
|
||||
"user_name": "",
|
||||
"password": "",
|
||||
},
|
||||
|
||||
// Example entry for a private registry such as Harbor:
|
||||
// "harbor": {
|
||||
// "registry_name": "harbor",
|
||||
// "registry_url": "harbor.example.org:5000",
|
||||
// "user_name": "robot_user",
|
||||
// "password": "robot_token",
|
||||
// }
|
||||
|
||||
"local_registry": {
|
||||
"registry_name": "local_registry",
|
||||
"registry_url": "registry.local:9001",
|
||||
"user_name": "test_user",
|
||||
"password": "test_password",
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@@ -1,32 +1,128 @@
|
||||
from typing import List
|
||||
|
||||
import json5
|
||||
|
||||
from config.docker.objects.registry import Registry
|
||||
|
||||
|
||||
class DockerConfig:
|
||||
"""
|
||||
Class to hold configuration of the Cloud Platform's Docker Registries
|
||||
Holds configuration for Docker registries and image sync manifests.
|
||||
|
||||
This class parses the contents of a Docker config JSON5 file, exposing registry
|
||||
definitions and optional image manifest paths for use by automation keywords.
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
self.registry_list: [Registry] = []
|
||||
def __init__(self, config: str):
|
||||
"""
|
||||
Initializes the DockerConfig object by loading the specified config file.
|
||||
|
||||
Args:
|
||||
config (str): Path to the Docker configuration file (e.g., default.json5).
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the file is not found.
|
||||
ValueError: If the config is missing required fields.
|
||||
"""
|
||||
self.registry_list: List[Registry] = []
|
||||
|
||||
try:
|
||||
json_data = open(config)
|
||||
with open(config) as f:
|
||||
self._config_dict = json5.load(f)
|
||||
except FileNotFoundError:
|
||||
print(f"Could not find the docker config file: {config}")
|
||||
print(f"Could not find the Docker config file: {config}")
|
||||
raise
|
||||
|
||||
docker_dict = json5.load(json_data)
|
||||
for registry in docker_dict['registries']:
|
||||
registry_dict = docker_dict['registries'][registry]
|
||||
reg = Registry(registry_dict['registry_name'], registry_dict['registry_url'], registry_dict['user_name'], registry_dict['password'])
|
||||
for registry_key in self._config_dict.get("registries", {}):
|
||||
registry_dict = self._config_dict["registries"][registry_key]
|
||||
reg = Registry(registry_name=registry_dict["registry_name"], registry_url=registry_dict["registry_url"], user_name=registry_dict["user_name"], password=registry_dict["password"])
|
||||
self.registry_list.append(reg)
|
||||
|
||||
def get_registry(self, registry_name) -> Registry:
|
||||
"""
|
||||
Getter for the KUBECONFIG environment variable on the lab where we want to run.
|
||||
def get_registry(self, registry_name: str) -> Registry:
|
||||
"""
|
||||
Retrieves a registry object by logical name.
|
||||
|
||||
registries = list(filter(lambda registry: registry.get_registry_name() == registry_name, self.registry_list))
|
||||
Args:
|
||||
registry_name (str): Logical name (e.g., 'dockerhub', 'local_registry').
|
||||
|
||||
Returns:
|
||||
Registry: Matching registry object.
|
||||
|
||||
Raises:
|
||||
ValueError: If the registry name is not found.
|
||||
"""
|
||||
registries = list(filter(lambda r: r.get_registry_name() == registry_name, self.registry_list))
|
||||
if not registries:
|
||||
raise ValueError(f"No registry with the name {registry_name} was found")
|
||||
raise ValueError(f"No registry with the name '{registry_name}' was found")
|
||||
return registries[0]
|
||||
|
||||
def get_image_manifest_files(self) -> List[str]:
|
||||
"""
|
||||
Returns the list of image manifest file paths defined in the config.
|
||||
|
||||
Returns:
|
||||
List[str]: List of paths to manifest YAML files.
|
||||
|
||||
Raises:
|
||||
ValueError: If the value is neither a string nor a list.
|
||||
"""
|
||||
manifests = self._config_dict.get("image_manifest_files", [])
|
||||
if isinstance(manifests, str):
|
||||
return [manifests]
|
||||
if not isinstance(manifests, list):
|
||||
raise ValueError("image_manifest_files must be a string or list of strings")
|
||||
return manifests
|
||||
|
||||
def get_default_source_registry_name(self) -> str:
|
||||
"""
|
||||
Returns the default source registry name defined in config (if any).
|
||||
|
||||
Returns:
|
||||
str: Logical registry name (e.g., 'dockerhub'), or empty string.
|
||||
"""
|
||||
return self._config_dict.get("default_source_registry", "")
|
||||
|
||||
def get_registry_for_manifest(self, manifest_path: str) -> str:
|
||||
"""
|
||||
Returns the default registry name for a given manifest path, if defined.
|
||||
|
||||
Args:
|
||||
manifest_path (str): Full relative path to the manifest file.
|
||||
|
||||
Returns:
|
||||
str: Logical registry name (e.g., 'dockerhub'), or empty string.
|
||||
"""
|
||||
return self._config_dict.get("manifest_registry_map", {}).get(manifest_path, "")
|
||||
|
||||
def get_manifest_registry_map(self) -> dict:
|
||||
"""
|
||||
Returns the mapping of manifest file paths to registry names.
|
||||
|
||||
Returns:
|
||||
dict: Mapping of manifest file path -> logical registry name.
|
||||
"""
|
||||
return self._config_dict.get("manifest_registry_map", {})
|
||||
|
||||
def get_effective_source_registry_name(self, image: dict, manifest_filename: str) -> str:
|
||||
"""
|
||||
Resolves the source registry name for a given image using the following precedence:
|
||||
|
||||
1. The "source_registry" field in the image entry (if present).
|
||||
2. A per-manifest registry mapping defined in "manifest_registry_map" in the config.
|
||||
3. The global "default_source_registry" defined in the config.
|
||||
|
||||
Args:
|
||||
image (dict): An image entry from the manifest.
|
||||
manifest_filename (str): Filename of the manifest (e.g., 'stx-test-images.yaml').
|
||||
|
||||
Returns:
|
||||
str: The resolved logical registry name (e.g., 'dockerhub').
|
||||
"""
|
||||
if "source_registry" in image:
|
||||
return image["source_registry"]
|
||||
|
||||
manifest_map = self.get_manifest_registry_map()
|
||||
if manifest_filename in manifest_map:
|
||||
return manifest_map[manifest_filename]
|
||||
|
||||
return self.get_default_source_registry_name()
|
||||
|
122
keywords/docker/images/docker_sync_images_keywords.py
Normal file
122
keywords/docker/images/docker_sync_images_keywords.py
Normal file
@@ -0,0 +1,122 @@
|
||||
import yaml
|
||||
|
||||
from config.configuration_manager import ConfigurationManager
|
||||
from framework.exceptions.keyword_exception import KeywordException
|
||||
from framework.logging.automation_logger import get_logger
|
||||
from framework.ssh.ssh_connection import SSHConnection
|
||||
from keywords.base_keyword import BaseKeyword
|
||||
from keywords.docker.images.docker_images_keywords import DockerImagesKeywords
|
||||
from keywords.docker.images.docker_load_image_keywords import DockerLoadImageKeywords
|
||||
|
||||
|
||||
class DockerSyncImagesKeywords(BaseKeyword):
|
||||
"""
|
||||
Provides functionality for Docker image synchronization across registries.
|
||||
|
||||
Supports pulling from source, tagging, and pushing to the local registry
|
||||
based on manifest-driven configuration.
|
||||
"""
|
||||
|
||||
def __init__(self, ssh_connection: SSHConnection):
|
||||
"""
|
||||
Initialize DockerSyncImagesKeywords with an SSH connection.
|
||||
|
||||
Args:
|
||||
ssh_connection (SSHConnection): Active SSH connection to the system under test.
|
||||
"""
|
||||
self.ssh_connection = ssh_connection
|
||||
self.docker_images_keywords = DockerImagesKeywords(ssh_connection)
|
||||
self.docker_load_keywords = DockerLoadImageKeywords(ssh_connection)
|
||||
|
||||
def sync_images_from_manifest(self, manifest_path: str) -> None:
|
||||
"""
|
||||
Syncs Docker images listed in a YAML manifest from a source registry into the local registry.
|
||||
|
||||
For each image:
|
||||
- Pull from the resolved source registry.
|
||||
- Tag for the local registry (e.g., registry.local:9001).
|
||||
- Push to the local registry.
|
||||
|
||||
Registry credentials and mappings are resolved using ConfigurationManager.get_docker_config(),
|
||||
which loads config from `config/docker/files/default.json5` or a CLI override.
|
||||
|
||||
Registry resolution priority (from most to least specific):
|
||||
1. "source_registry" field on the individual image entry (in the manifest)
|
||||
2. "manifest_registry_map" entry matching the full manifest path (in config)
|
||||
3. "default_source_registry" defined globally in config
|
||||
|
||||
Expected manifest format:
|
||||
```yaml
|
||||
images:
|
||||
- name: "starlingx/test-image"
|
||||
tag: "tag-x"
|
||||
# Optional: source_registry: "dockerhub"
|
||||
```
|
||||
|
||||
Notes:
|
||||
- Registry URLs and credentials must be defined in config, not in the manifest.
|
||||
Any such values in the manifest are ignored.
|
||||
- Each image entry must include "name" and "tag".
|
||||
|
||||
Args:
|
||||
manifest_path (str): Full path to the YAML manifest file.
|
||||
|
||||
Raises:
|
||||
KeywordException: If one or more image sync operations fail.
|
||||
ValueError: If no registry can be resolved for an image.
|
||||
"""
|
||||
docker_config = ConfigurationManager.get_docker_config()
|
||||
local_registry = docker_config.get_registry("local_registry")
|
||||
default_registry_name = docker_config.get_default_source_registry_name()
|
||||
|
||||
with open(manifest_path, "r") as f:
|
||||
manifest = yaml.safe_load(f)
|
||||
|
||||
manifest_registry_name = docker_config.get_registry_for_manifest(manifest_path)
|
||||
|
||||
if "images" not in manifest:
|
||||
raise ValueError(f"Manifest at {manifest_path} is missing required 'images' key")
|
||||
|
||||
failures = []
|
||||
|
||||
for image in manifest["images"]:
|
||||
name = image["name"]
|
||||
tag = image["tag"]
|
||||
|
||||
# Resolve source registry in order of precedence:
|
||||
# 1) per-image override ("source_registry" in manifest)
|
||||
# 2) per-manifest default (manifest_registry_map in config)
|
||||
# 3) global fallback (default_source_registry in config)
|
||||
source_registry_name = image.get("source_registry") or manifest_registry_name or default_registry_name
|
||||
|
||||
if not source_registry_name:
|
||||
raise ValueError(f"Image '{name}:{tag}' has no 'source_registry' and no default_source_registry is set in config.")
|
||||
try:
|
||||
source_registry = docker_config.get_registry(source_registry_name)
|
||||
|
||||
source_image = f"{source_registry.get_registry_url()}/{name}:{tag}"
|
||||
target_image = f"{local_registry.get_registry_url()}/{name}:{tag}"
|
||||
|
||||
get_logger().log_info(f"Pulling {source_image}")
|
||||
self.docker_images_keywords.pull_image(source_image)
|
||||
|
||||
get_logger().log_info(f"Tagging {source_image} -> {target_image}")
|
||||
self.docker_load_keywords.tag_docker_image_for_registry(
|
||||
image_name=source_image,
|
||||
tag_name=f"{name}:{tag}",
|
||||
registry=local_registry,
|
||||
)
|
||||
|
||||
get_logger().log_info(f"Pushing {target_image}")
|
||||
self.docker_load_keywords.push_docker_image_to_registry(
|
||||
tag_name=f"{name}:{tag}",
|
||||
registry=local_registry,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to sync image {name}:{tag} from {source_registry_name}: {e}"
|
||||
get_logger().log_error(error_msg)
|
||||
failures.append(error_msg)
|
||||
|
||||
if failures:
|
||||
raise KeywordException(f"Image sync failed for manifest '{manifest_path}':\n - " + "\n - ".join(failures))
|
31
resources/image_manifests/harbor-test-images.yaml
Normal file
31
resources/image_manifests/harbor-test-images.yaml
Normal file
@@ -0,0 +1,31 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Image Manifest: harbor-test-images.yaml
|
||||
#
|
||||
# This manifest defines one or more Docker images to be synchronized from a
|
||||
# Harbor registry into the local StarlingX registry.
|
||||
#
|
||||
# Purpose:
|
||||
# - Demonstrates syncing images from a private registry (Harbor).
|
||||
# - Used to verify that authentication and sync logic function correctly.
|
||||
#
|
||||
# Notes:
|
||||
# - Each image entry must include both `name` and `tag`.
|
||||
# - Use logical registry names (not raw URLs) in the `source_registry` field.
|
||||
# - Must match a key under `registries` in `config/docker/files/default.json5`.
|
||||
# - Examples: `harbor`, `dockerhub`
|
||||
# - Image names must include full namespace (e.g., `examplecorp/my-image`).
|
||||
# - Credentials for accessing private registries (like Harbor) are not specified here.
|
||||
# - Authentication details (username, password, etc.) are configured in
|
||||
# `config/docker/files/default.json5` under the corresponding `registries` entry.
|
||||
# - This file defaults to `config/docker/files/default.json5`, but can be overridden using `--docker_config_file`.
|
||||
# - Registry resolution is handled dynamically via `ConfigurationManager`.
|
||||
# - Resolution priority (from most to least specific):
|
||||
# 1. `source_registry` field on the image entry (optional)
|
||||
# 2. `manifest_registry_map` in `config/docker/files/default.json5`
|
||||
# 3. `default_source_registry` in `config/docker/files/default.json5`
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
images:
|
||||
- name: "harbor_user/network-soak-test"
|
||||
tag: "latest" # Example only; avoid using 'latest' in production or automated test environments.
|
||||
source_registry: "harbor" # Logical registry name from docker config e.g. config/docker/files/default.json5.
|
37
resources/image_manifests/stx-test-images-invalid.yaml
Normal file
37
resources/image_manifests/stx-test-images-invalid.yaml
Normal file
@@ -0,0 +1,37 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Image Manifest: `stx-test-images-invalid.yaml`
|
||||
#
|
||||
# This manifest defines invalid or non-existent Docker images to simulate partial
|
||||
# sync failures for testing purposes.
|
||||
#
|
||||
# Purpose:
|
||||
# - Verifies sync logic when some images fail (e.g., bad tags or missing images).
|
||||
# - Used for testing failure paths in `test_docker_image_sync.py`.
|
||||
#
|
||||
# Notes:
|
||||
# - Each image entry must include both `name` and `tag`.
|
||||
# - Image names must include their full namespace (e.g., `starlingx/stx-keystone`).
|
||||
# - Registry URLs and credentials are not listed here. They are defined in:
|
||||
# `config/docker/files/default.json5`
|
||||
# - Registry resolution is handled dynamically via `ConfigurationManager`.
|
||||
# - Resolution priority (from most to least specific):
|
||||
# 1. `source_registry` field on the individual image entry (optional)
|
||||
# 2. `manifest_registry_map` entry in `config/docker/files/default.json5`
|
||||
# 3. `default_source_registry` in `config/docker/files/default.json5`
|
||||
# ------------------------------------------------------------------------------
|
||||
images:
|
||||
- name: "starlingx/stx-platformclients"
|
||||
tag: "stx.11.0-v1.0.1"
|
||||
# This image and tag are valid
|
||||
|
||||
- name: "starlingx/stx-platformclients"
|
||||
tag: "made-up-tag"
|
||||
# This tag is invalid
|
||||
|
||||
- name: "starlingx/stx-keystone"
|
||||
tag: "master-debian-stable-20250530T120001Z.0"
|
||||
# This image and tag are valid
|
||||
|
||||
- name: "starlingx/stx-non-existent"
|
||||
tag: "stx.11.0-v1.0.1"
|
||||
# This image does not exist
|
@@ -0,0 +1,31 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Image Manifest: stx-test-images-mixed-registries.yaml
|
||||
#
|
||||
# This manifest defines a mix of Docker images to be synchronized from multiple
|
||||
# source registries (e.g., DockerHub and Harbor) into the local registry.
|
||||
#
|
||||
# Purpose:
|
||||
# - Validates manifest-driven image sync logic across different registry backends.
|
||||
# - Demonstrates per-image control over the source registry via `source_registry`.
|
||||
#
|
||||
# Notes:
|
||||
# - Each image entry must include both `name` and `tag`.
|
||||
# - Use logical registry names (not raw URLs) in the `source_registry` field.
|
||||
# - The name must match a key under `registries` in the Docker config file.
|
||||
# - This file defaults to `config/docker/files/default.json5`, but can be overridden using `--docker_config_file`.
|
||||
# - Examples: `dockerhub`, `harbor`
|
||||
# - Image names must include full namespace (e.g., starlingx/stx-keystone).
|
||||
# - Registry resolution is handled dynamically via ConfigurationManager.
|
||||
# - Resolution priority (from most to least specific):
|
||||
# 1. `source_registry` field on the image entry (optional)
|
||||
# 2. `manifest_registry_map` in config/docker/files/default.json5
|
||||
# 3. `default_source_registry` in config/docker/files/default.json5
|
||||
# ------------------------------------------------------------------------------
|
||||
images:
|
||||
- name: "starlingx/stx-keystone"
|
||||
tag: "master-debian-stable-20250530T120001Z.0"
|
||||
source_registry: "dockerhub"
|
||||
|
||||
- name: "examplecorp/network-soak-test"
|
||||
tag: "1.4.2"
|
||||
source_registry: "harbor" # Logical registry name from docker config e.g. config/docker/files/default.json5.
|
24
resources/image_manifests/stx-test-images.yaml
Normal file
24
resources/image_manifests/stx-test-images.yaml
Normal file
@@ -0,0 +1,24 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# Image Manifest: `stx-test-images.yaml`
|
||||
#
|
||||
# This manifest defines Docker images to be synchronized from a source registry
|
||||
# (e.g., DockerHub) into the local StarlingX registry (e.g., `registry.local:9001`).
|
||||
#
|
||||
# Notes:
|
||||
# - Each image entry must include both `name` and `tag`.
|
||||
# - Image names must include their full namespace (e.g., `starlingx/stx-platformclients`).
|
||||
# - Registry URLs and credentials are not listed here. They are defined in:
|
||||
# `config/docker/files/default.json5`
|
||||
# - Registry resolution is handled dynamically via `ConfigurationManager`.
|
||||
# - Resolution priority (from most to least specific):
|
||||
# 1. `source_registry` field on the individual image entry (optional)
|
||||
# 2. `manifest_registry_map` entry in `config/docker/files/default.json5`
|
||||
# 3. `default_source_registry` in `config/docker/files/default.json5`
|
||||
# ------------------------------------------------------------------------------
|
||||
images:
|
||||
- name: "starlingx/stx-platformclients"
|
||||
tag: "stx.11.0-v1.0.1"
|
||||
# source_registry: "dockerhub" # Optional field to specify a custom source registry
|
||||
|
||||
- name: "starlingx/stx-keystone"
|
||||
tag: "master-debian-stable-20250530T120001Z.0"
|
233
testcases/cloud_platform/images/test_docker_image_sync.py
Normal file
233
testcases/cloud_platform/images/test_docker_image_sync.py
Normal file
@@ -0,0 +1,233 @@
|
||||
"""
|
||||
Docker Image Sync Tests Using Manifest-Based Configuration
|
||||
|
||||
This module implements foundational tests that verify Docker images listed
|
||||
in YAML manifest files can be pulled from remote registries (e.g., DockerHub),
|
||||
tagged, and pushed into the local StarlingX registry (registry.local:9001).
|
||||
|
||||
Tests validate both positive and negative scenarios using a config-driven
|
||||
approach that resolves registries dynamically via ConfigurationManager.
|
||||
|
||||
Key Behaviors:
|
||||
- Validates sync logic from manifest to local registry via SSH.
|
||||
- Verifies registry resolution order: source_registry, then manifest_registry_map,
|
||||
then default_source_registry.
|
||||
- Supports flexible test-driven control over which manifests are synced.
|
||||
- Logs missing images or partial sync failures for improved debugging.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from pytest import FixtureRequest, fail, raises
|
||||
|
||||
from config.configuration_manager import ConfigurationManager
|
||||
from framework.exceptions.keyword_exception import KeywordException
|
||||
from framework.logging.automation_logger import get_logger
|
||||
from keywords.cloud_platform.ssh.lab_connection_keywords import LabConnectionKeywords
|
||||
from keywords.docker.images.docker_images_keywords import DockerImagesKeywords
|
||||
from keywords.docker.images.docker_sync_images_keywords import DockerSyncImagesKeywords
|
||||
|
||||
|
||||
def run_manifest_sync_test(request: FixtureRequest, manifest_filename: str) -> None:
|
||||
"""
|
||||
Executes a manifest-based sync test, pulling Docker images from source registries
|
||||
and pushing them to the local registry. Verifies that all expected images appear
|
||||
in the local registry.
|
||||
|
||||
Args:
|
||||
request (FixtureRequest): pytest request object used to register cleanup finalizer.
|
||||
manifest_filename (str): Path to the manifest file in resources/.
|
||||
|
||||
Raises:
|
||||
AssertionError: If any expected image is missing from the local registry.
|
||||
"""
|
||||
ssh_connection = LabConnectionKeywords().get_active_controller_ssh()
|
||||
|
||||
docker_config = ConfigurationManager.get_docker_config()
|
||||
local_registry = docker_config.get_registry("local_registry")
|
||||
manifest_paths = docker_config.get_image_manifest_files()
|
||||
|
||||
manifest_path = next((p for p in manifest_paths if Path(p).name == manifest_filename), None)
|
||||
if not manifest_path:
|
||||
raise FileNotFoundError(f"Manifest {manifest_filename} not found in docker config.")
|
||||
|
||||
DockerSyncImagesKeywords(ssh_connection).sync_images_from_manifest(manifest_path=manifest_path)
|
||||
|
||||
with open(manifest_path, "r") as f:
|
||||
manifest = yaml.safe_load(f)
|
||||
|
||||
docker_image_keywords = DockerImagesKeywords(ssh_connection)
|
||||
|
||||
def cleanup():
|
||||
"""
|
||||
Cleans up Docker images listed in the manifest from the local system.
|
||||
|
||||
For each image, up to three tag formats are removed from the local Docker cache:
|
||||
1. source_registry/image:tag (skipped if source is docker.io; see note below)
|
||||
2. local_registry/image:tag (e.g., image pushed to registry.local)
|
||||
3. image:tag (default short form used by Docker)
|
||||
|
||||
Purpose:
|
||||
- Ensures complete removal regardless of how the image was tagged during pull/push operations.
|
||||
- Supports idempotent cleanup, avoiding reliance on a single canonical tag.
|
||||
- Handles cases where Docker implicitly normalizes or aliases tag references.
|
||||
|
||||
Notes:
|
||||
- Full `docker.io/...` references are skipped during cleanup, as Docker stores these as `image:tag`.
|
||||
"""
|
||||
get_logger().log_info(f"Cleaning up images listed in {manifest_filename}...")
|
||||
|
||||
ssh_connection = LabConnectionKeywords().get_active_controller_ssh()
|
||||
docker_image_keywords_cleanup = DockerImagesKeywords(ssh_connection)
|
||||
|
||||
for image in manifest.get("images", []):
|
||||
name = image["name"]
|
||||
tag = image["tag"]
|
||||
|
||||
source_registry_name = docker_config.get_effective_source_registry_name(image, manifest_filename)
|
||||
if not source_registry_name:
|
||||
get_logger().log_debug(f"Skipping cleanup for image {name}:{tag} (no source registry resolved)")
|
||||
continue
|
||||
|
||||
source_registry = docker_config.get_registry(source_registry_name)
|
||||
source_url = source_registry.get_registry_url()
|
||||
|
||||
# Always try to remove these two references
|
||||
refs = [
|
||||
f"{local_registry.get_registry_url()}/{name}:{tag}",
|
||||
f"{name}:{tag}",
|
||||
]
|
||||
|
||||
# Optionally add full source registry tag if not DockerHub
|
||||
if "docker.io" not in source_url:
|
||||
refs.insert(0, f"{source_url}/{name}:{tag}")
|
||||
else:
|
||||
get_logger().log_debug(f"Skipping full docker.io-prefixed tag for {source_url}/{name}:{tag}")
|
||||
|
||||
for ref in refs:
|
||||
docker_image_keywords_cleanup.remove_image(ref)
|
||||
|
||||
request.addfinalizer(cleanup)
|
||||
|
||||
images = docker_image_keywords.list_images()
|
||||
actual_repos = [img.get_repository() for img in images]
|
||||
validation_errors = []
|
||||
|
||||
for image in manifest["images"]:
|
||||
name = image["name"]
|
||||
tag = image["tag"]
|
||||
expected_ref = f"{local_registry.get_registry_url()}/{name}"
|
||||
|
||||
get_logger().log_info(f"Checking local registry for: {expected_ref}:{tag}")
|
||||
if expected_ref not in actual_repos:
|
||||
msg = f"[{manifest_filename}] Expected image not found: {expected_ref}"
|
||||
get_logger().log_warning(msg)
|
||||
validation_errors.append(msg)
|
||||
|
||||
if validation_errors:
|
||||
raise AssertionError("One or more expected images were not found:\n - " + "\n - ".join(validation_errors))
|
||||
|
||||
|
||||
def test_sync_docker_images_valid_manifest_stx_dockerhub(request):
|
||||
"""
|
||||
Validates that all images from a well-formed manifest can be pulled and synced into the local registry.
|
||||
"""
|
||||
run_manifest_sync_test(request, "stx-test-images.yaml")
|
||||
|
||||
|
||||
def test_sync_docker_images_invalid_manifest(request):
|
||||
"""
|
||||
Negative test: verifies that syncing an invalid manifest raises KeywordException.
|
||||
|
||||
This simulates real-world scenarios where image tags are missing or incorrectly referenced.
|
||||
|
||||
Very simple brittle string matching is used to verify the exception message.
|
||||
"""
|
||||
with raises(KeywordException, match="Image sync failed"):
|
||||
run_manifest_sync_test(request, "stx-test-images-invalid.yaml")
|
||||
|
||||
|
||||
def test_sync_all_manifests_from_config(request):
|
||||
"""
|
||||
Verifies that all manifest files listed in the Docker config can be successfully synced to local registry.
|
||||
|
||||
This test ensures that ConfigurationManager.get_docker_config().get_image_manifest_files()
|
||||
is functional and can drive the sync logic dynamically.
|
||||
|
||||
Note: Expected to currently fail if any manifest is invalid or any image fail sync fails.
|
||||
"""
|
||||
manifest_paths = ConfigurationManager.get_docker_config().get_image_manifest_files()
|
||||
|
||||
get_logger().log_info("Found image manifest paths in config: " + ", ".join(manifest_paths))
|
||||
|
||||
for manifest_path in manifest_paths:
|
||||
manifest_name = Path(manifest_path).name
|
||||
run_manifest_sync_test(request, manifest_name)
|
||||
|
||||
get_logger().log_info(f"All manifests synced successfully. Manifests: {', '.join(manifest_paths)}")
|
||||
|
||||
|
||||
def test_sync_explicit_manifests(request):
|
||||
"""
|
||||
Verifies that all manifest files listed in the test case can be successfully synced to local registry.
|
||||
|
||||
Note: Expected to currently fail if any manifest is invalid or any image sync fails.
|
||||
"""
|
||||
manifest_paths = [
|
||||
"stx-test-images.yaml",
|
||||
# "stx-test-images-invalid.yaml"
|
||||
# Uncomment the above line to include the invalid manifest in the test (and expect failure).
|
||||
]
|
||||
|
||||
get_logger().log_info("Found image manifest paths in config: " + ", ".join(manifest_paths))
|
||||
|
||||
for manifest_path in manifest_paths:
|
||||
manifest_name = Path(manifest_path).name
|
||||
run_manifest_sync_test(request, manifest_name)
|
||||
|
||||
get_logger().log_info(f"All manifests synced successfully. Manifests: {', '.join(manifest_paths)}")
|
||||
|
||||
|
||||
def test_invalid_manifest_logging(request):
|
||||
"""
|
||||
Negative test: verifies that syncing an invalid manifest raises KeywordException.
|
||||
|
||||
Logs only the image references that actually failed during sync.
|
||||
"""
|
||||
manifest_path = "stx-test-images-invalid.yaml"
|
||||
|
||||
try:
|
||||
run_manifest_sync_test(request, manifest_path)
|
||||
except KeywordException as e:
|
||||
# Parse individual failure lines from the exception message
|
||||
failure_lines = [line.strip(" -") for line in str(e).splitlines() if line.strip().startswith("-")]
|
||||
|
||||
# Extract just the image reference (everything between 'image ' and ' from ')
|
||||
failed_images = []
|
||||
for line in failure_lines:
|
||||
if "image " in line and " from " in line:
|
||||
parts = line.split("image ", 1)[-1].split(" from ")[0].strip()
|
||||
failed_images.append(parts)
|
||||
else:
|
||||
failed_images.append(line) # Fallback: use the whole line
|
||||
|
||||
formatted_images = "\n\t- " + "\n\t- ".join(failed_images)
|
||||
get_logger().log_info(f"Negative image sync test passed.\n" f"\tManifest file: {manifest_path}\n" f"\tFailed images:{formatted_images}")
|
||||
|
||||
else:
|
||||
fail("Expected KeywordException was not raised.")
|
||||
|
||||
|
||||
# def test_sync_docker_images_valid_manifest_harbor(request):
|
||||
# """
|
||||
# Validates that all images from a well-formed manifest can be pulled and synced into the local registry from a harbor regsitry.
|
||||
# """
|
||||
# run_manifest_sync_test(request, "harbor-test-images.yaml")
|
||||
|
||||
|
||||
# def test_sync_docker_images_mixed_registries(request):
|
||||
# """
|
||||
# Validates that images from a manifest with mixed registries (DockerHub and Harbor) can be pulled and synced into the local registry.
|
||||
# """
|
||||
# run_manifest_sync_test(request, "stx-test-images-mixed-registries.yaml")
|
Reference in New Issue
Block a user