Merge "PoC: manifest-driven image sync for test images"

This commit is contained in:
Zuul
2025-06-13 15:46:19 +00:00
committed by Gerrit Code Review
8 changed files with 656 additions and 22 deletions

View File

@@ -1,10 +1,70 @@
// Example Docker registry configuration.
//
// This file is used as the default by ConfigurationManager.get_docker_config(),
// unless overridden via the --docker_config_file CLI option or programmatically.
//
// Registry endpoints, credentials, and image manifest paths can be overridden at runtime
// using --docker_config_file or set programmatically via ConfigurationManager.
//
// Registry Resolution Behavior:
// - The "source_registry" field (if present) on an individual image in the manifest takes highest precedence.
// This allows different images within the same manifest to pull from different source registries.
// - Use "manifest_registry_map" to override the source registry per manifest.
// - Use "default_source_registry" as a global fallback if neither of the above is set.
// This is useful when all your images come from a single upstream source like DockerHub.
// The resolution priority is:
// 1. source_registry (per image)
// 2. manifest_registry_map (per manifest)
// 3. default_source_registry (global fallback)
//
// Notes:
// - Each registry must define a unique "registry_name", which acts as a logical key.
// This is referenced by:
// * the "source_registry" field in image manifests
// * the "manifest_registry_map" in this config file
// * the "default_source_registry" fallback below
//
// - "image_manifest_files" may include one or more YAML files.
// - Each image listed in a manifest is pulled from its resolved source registry
// and pushed into the "local_registry" defined below.
{
registries: {
"local_registry" : {
"registry_name": "local_registry",
"registry_url": "registry.local:9001",
"user_name": "test_user",
"password": "test_password",
}
"default_source_registry": "dockerhub",
"image_manifest_files": [
"resources/image_manifests/stx-test-images.yaml",
"resources/image_manifests/stx-test-images-invalid.yaml",
// "resources/image_manifests/harbor-test-images.yaml",
// "resources/stx-networking-images.yaml",
],
"manifest_registry_map": {
"resources/image_manifests/stx-test-images.yaml": "dockerhub",
// "resources/image_manifests/stx-test-images-invalid.yaml": "dockerhub",
// "resources/image_manifests/harbor-test-images.yaml": "harbor",
},
"registries": {
"dockerhub": {
"registry_name": "dockerhub",
"registry_url": "docker.io",
"user_name": "",
"password": "",
},
// Example entry for a private registry such as Harbor:
// "harbor": {
// "registry_name": "harbor",
// "registry_url": "harbor.example.org:5000",
// "user_name": "robot_user",
// "password": "robot_token",
// }
"local_registry": {
"registry_name": "local_registry",
"registry_url": "registry.local:9001",
"user_name": "test_user",
"password": "test_password",
},
}
}
}

View File

@@ -1,32 +1,128 @@
from typing import List
import json5
from config.docker.objects.registry import Registry
class DockerConfig:
"""
Class to hold configuration of the Cloud Platform's Docker Registries
Holds configuration for Docker registries and image sync manifests.
This class parses the contents of a Docker config JSON5 file, exposing registry
definitions and optional image manifest paths for use by automation keywords.
"""
def __init__(self, config):
self.registry_list: [Registry] = []
def __init__(self, config: str):
"""
Initializes the DockerConfig object by loading the specified config file.
Args:
config (str): Path to the Docker configuration file (e.g., default.json5).
Raises:
FileNotFoundError: If the file is not found.
ValueError: If the config is missing required fields.
"""
self.registry_list: List[Registry] = []
try:
json_data = open(config)
with open(config) as f:
self._config_dict = json5.load(f)
except FileNotFoundError:
print(f"Could not find the docker config file: {config}")
print(f"Could not find the Docker config file: {config}")
raise
docker_dict = json5.load(json_data)
for registry in docker_dict['registries']:
registry_dict = docker_dict['registries'][registry]
reg = Registry(registry_dict['registry_name'], registry_dict['registry_url'], registry_dict['user_name'], registry_dict['password'])
for registry_key in self._config_dict.get("registries", {}):
registry_dict = self._config_dict["registries"][registry_key]
reg = Registry(registry_name=registry_dict["registry_name"], registry_url=registry_dict["registry_url"], user_name=registry_dict["user_name"], password=registry_dict["password"])
self.registry_list.append(reg)
def get_registry(self, registry_name) -> Registry:
"""
Getter for the KUBECONFIG environment variable on the lab where we want to run.
def get_registry(self, registry_name: str) -> Registry:
"""
Retrieves a registry object by logical name.
registries = list(filter(lambda registry: registry.get_registry_name() == registry_name, self.registry_list))
Args:
registry_name (str): Logical name (e.g., 'dockerhub', 'local_registry').
Returns:
Registry: Matching registry object.
Raises:
ValueError: If the registry name is not found.
"""
registries = list(filter(lambda r: r.get_registry_name() == registry_name, self.registry_list))
if not registries:
raise ValueError(f"No registry with the name {registry_name} was found")
raise ValueError(f"No registry with the name '{registry_name}' was found")
return registries[0]
def get_image_manifest_files(self) -> List[str]:
"""
Returns the list of image manifest file paths defined in the config.
Returns:
List[str]: List of paths to manifest YAML files.
Raises:
ValueError: If the value is neither a string nor a list.
"""
manifests = self._config_dict.get("image_manifest_files", [])
if isinstance(manifests, str):
return [manifests]
if not isinstance(manifests, list):
raise ValueError("image_manifest_files must be a string or list of strings")
return manifests
def get_default_source_registry_name(self) -> str:
"""
Returns the default source registry name defined in config (if any).
Returns:
str: Logical registry name (e.g., 'dockerhub'), or empty string.
"""
return self._config_dict.get("default_source_registry", "")
def get_registry_for_manifest(self, manifest_path: str) -> str:
"""
Returns the default registry name for a given manifest path, if defined.
Args:
manifest_path (str): Full relative path to the manifest file.
Returns:
str: Logical registry name (e.g., 'dockerhub'), or empty string.
"""
return self._config_dict.get("manifest_registry_map", {}).get(manifest_path, "")
def get_manifest_registry_map(self) -> dict:
"""
Returns the mapping of manifest file paths to registry names.
Returns:
dict: Mapping of manifest file path -> logical registry name.
"""
return self._config_dict.get("manifest_registry_map", {})
def get_effective_source_registry_name(self, image: dict, manifest_filename: str) -> str:
"""
Resolves the source registry name for a given image using the following precedence:
1. The "source_registry" field in the image entry (if present).
2. A per-manifest registry mapping defined in "manifest_registry_map" in the config.
3. The global "default_source_registry" defined in the config.
Args:
image (dict): An image entry from the manifest.
manifest_filename (str): Filename of the manifest (e.g., 'stx-test-images.yaml').
Returns:
str: The resolved logical registry name (e.g., 'dockerhub').
"""
if "source_registry" in image:
return image["source_registry"]
manifest_map = self.get_manifest_registry_map()
if manifest_filename in manifest_map:
return manifest_map[manifest_filename]
return self.get_default_source_registry_name()

View File

@@ -0,0 +1,122 @@
import yaml
from config.configuration_manager import ConfigurationManager
from framework.exceptions.keyword_exception import KeywordException
from framework.logging.automation_logger import get_logger
from framework.ssh.ssh_connection import SSHConnection
from keywords.base_keyword import BaseKeyword
from keywords.docker.images.docker_images_keywords import DockerImagesKeywords
from keywords.docker.images.docker_load_image_keywords import DockerLoadImageKeywords
class DockerSyncImagesKeywords(BaseKeyword):
"""
Provides functionality for Docker image synchronization across registries.
Supports pulling from source, tagging, and pushing to the local registry
based on manifest-driven configuration.
"""
def __init__(self, ssh_connection: SSHConnection):
"""
Initialize DockerSyncImagesKeywords with an SSH connection.
Args:
ssh_connection (SSHConnection): Active SSH connection to the system under test.
"""
self.ssh_connection = ssh_connection
self.docker_images_keywords = DockerImagesKeywords(ssh_connection)
self.docker_load_keywords = DockerLoadImageKeywords(ssh_connection)
def sync_images_from_manifest(self, manifest_path: str) -> None:
"""
Syncs Docker images listed in a YAML manifest from a source registry into the local registry.
For each image:
- Pull from the resolved source registry.
- Tag for the local registry (e.g., registry.local:9001).
- Push to the local registry.
Registry credentials and mappings are resolved using ConfigurationManager.get_docker_config(),
which loads config from `config/docker/files/default.json5` or a CLI override.
Registry resolution priority (from most to least specific):
1. "source_registry" field on the individual image entry (in the manifest)
2. "manifest_registry_map" entry matching the full manifest path (in config)
3. "default_source_registry" defined globally in config
Expected manifest format:
```yaml
images:
- name: "starlingx/test-image"
tag: "tag-x"
# Optional: source_registry: "dockerhub"
```
Notes:
- Registry URLs and credentials must be defined in config, not in the manifest.
Any such values in the manifest are ignored.
- Each image entry must include "name" and "tag".
Args:
manifest_path (str): Full path to the YAML manifest file.
Raises:
KeywordException: If one or more image sync operations fail.
ValueError: If no registry can be resolved for an image.
"""
docker_config = ConfigurationManager.get_docker_config()
local_registry = docker_config.get_registry("local_registry")
default_registry_name = docker_config.get_default_source_registry_name()
with open(manifest_path, "r") as f:
manifest = yaml.safe_load(f)
manifest_registry_name = docker_config.get_registry_for_manifest(manifest_path)
if "images" not in manifest:
raise ValueError(f"Manifest at {manifest_path} is missing required 'images' key")
failures = []
for image in manifest["images"]:
name = image["name"]
tag = image["tag"]
# Resolve source registry in order of precedence:
# 1) per-image override ("source_registry" in manifest)
# 2) per-manifest default (manifest_registry_map in config)
# 3) global fallback (default_source_registry in config)
source_registry_name = image.get("source_registry") or manifest_registry_name or default_registry_name
if not source_registry_name:
raise ValueError(f"Image '{name}:{tag}' has no 'source_registry' and no default_source_registry is set in config.")
try:
source_registry = docker_config.get_registry(source_registry_name)
source_image = f"{source_registry.get_registry_url()}/{name}:{tag}"
target_image = f"{local_registry.get_registry_url()}/{name}:{tag}"
get_logger().log_info(f"Pulling {source_image}")
self.docker_images_keywords.pull_image(source_image)
get_logger().log_info(f"Tagging {source_image} -> {target_image}")
self.docker_load_keywords.tag_docker_image_for_registry(
image_name=source_image,
tag_name=f"{name}:{tag}",
registry=local_registry,
)
get_logger().log_info(f"Pushing {target_image}")
self.docker_load_keywords.push_docker_image_to_registry(
tag_name=f"{name}:{tag}",
registry=local_registry,
)
except Exception as e:
error_msg = f"Failed to sync image {name}:{tag} from {source_registry_name}: {e}"
get_logger().log_error(error_msg)
failures.append(error_msg)
if failures:
raise KeywordException(f"Image sync failed for manifest '{manifest_path}':\n - " + "\n - ".join(failures))

View File

@@ -0,0 +1,31 @@
# ------------------------------------------------------------------------------
# Image Manifest: harbor-test-images.yaml
#
# This manifest defines one or more Docker images to be synchronized from a
# Harbor registry into the local StarlingX registry.
#
# Purpose:
# - Demonstrates syncing images from a private registry (Harbor).
# - Used to verify that authentication and sync logic function correctly.
#
# Notes:
# - Each image entry must include both `name` and `tag`.
# - Use logical registry names (not raw URLs) in the `source_registry` field.
# - Must match a key under `registries` in `config/docker/files/default.json5`.
# - Examples: `harbor`, `dockerhub`
# - Image names must include full namespace (e.g., `examplecorp/my-image`).
# - Credentials for accessing private registries (like Harbor) are not specified here.
# - Authentication details (username, password, etc.) are configured in
# `config/docker/files/default.json5` under the corresponding `registries` entry.
# - This file defaults to `config/docker/files/default.json5`, but can be overridden using `--docker_config_file`.
# - Registry resolution is handled dynamically via `ConfigurationManager`.
# - Resolution priority (from most to least specific):
# 1. `source_registry` field on the image entry (optional)
# 2. `manifest_registry_map` in `config/docker/files/default.json5`
# 3. `default_source_registry` in `config/docker/files/default.json5`
# ------------------------------------------------------------------------------
images:
- name: "harbor_user/network-soak-test"
tag: "latest" # Example only; avoid using 'latest' in production or automated test environments.
source_registry: "harbor" # Logical registry name from docker config e.g. config/docker/files/default.json5.

View File

@@ -0,0 +1,37 @@
# ------------------------------------------------------------------------------
# Image Manifest: `stx-test-images-invalid.yaml`
#
# This manifest defines invalid or non-existent Docker images to simulate partial
# sync failures for testing purposes.
#
# Purpose:
# - Verifies sync logic when some images fail (e.g., bad tags or missing images).
# - Used for testing failure paths in `test_docker_image_sync.py`.
#
# Notes:
# - Each image entry must include both `name` and `tag`.
# - Image names must include their full namespace (e.g., `starlingx/stx-keystone`).
# - Registry URLs and credentials are not listed here. They are defined in:
# `config/docker/files/default.json5`
# - Registry resolution is handled dynamically via `ConfigurationManager`.
# - Resolution priority (from most to least specific):
# 1. `source_registry` field on the individual image entry (optional)
# 2. `manifest_registry_map` entry in `config/docker/files/default.json5`
# 3. `default_source_registry` in `config/docker/files/default.json5`
# ------------------------------------------------------------------------------
images:
- name: "starlingx/stx-platformclients"
tag: "stx.11.0-v1.0.1"
# This image and tag are valid
- name: "starlingx/stx-platformclients"
tag: "made-up-tag"
# This tag is invalid
- name: "starlingx/stx-keystone"
tag: "master-debian-stable-20250530T120001Z.0"
# This image and tag are valid
- name: "starlingx/stx-non-existent"
tag: "stx.11.0-v1.0.1"
# This image does not exist

View File

@@ -0,0 +1,31 @@
# ------------------------------------------------------------------------------
# Image Manifest: stx-test-images-mixed-registries.yaml
#
# This manifest defines a mix of Docker images to be synchronized from multiple
# source registries (e.g., DockerHub and Harbor) into the local registry.
#
# Purpose:
# - Validates manifest-driven image sync logic across different registry backends.
# - Demonstrates per-image control over the source registry via `source_registry`.
#
# Notes:
# - Each image entry must include both `name` and `tag`.
# - Use logical registry names (not raw URLs) in the `source_registry` field.
# - The name must match a key under `registries` in the Docker config file.
# - This file defaults to `config/docker/files/default.json5`, but can be overridden using `--docker_config_file`.
# - Examples: `dockerhub`, `harbor`
# - Image names must include full namespace (e.g., starlingx/stx-keystone).
# - Registry resolution is handled dynamically via ConfigurationManager.
# - Resolution priority (from most to least specific):
# 1. `source_registry` field on the image entry (optional)
# 2. `manifest_registry_map` in config/docker/files/default.json5
# 3. `default_source_registry` in config/docker/files/default.json5
# ------------------------------------------------------------------------------
images:
- name: "starlingx/stx-keystone"
tag: "master-debian-stable-20250530T120001Z.0"
source_registry: "dockerhub"
- name: "examplecorp/network-soak-test"
tag: "1.4.2"
source_registry: "harbor" # Logical registry name from docker config e.g. config/docker/files/default.json5.

View File

@@ -0,0 +1,24 @@
# ------------------------------------------------------------------------------
# Image Manifest: `stx-test-images.yaml`
#
# This manifest defines Docker images to be synchronized from a source registry
# (e.g., DockerHub) into the local StarlingX registry (e.g., `registry.local:9001`).
#
# Notes:
# - Each image entry must include both `name` and `tag`.
# - Image names must include their full namespace (e.g., `starlingx/stx-platformclients`).
# - Registry URLs and credentials are not listed here. They are defined in:
# `config/docker/files/default.json5`
# - Registry resolution is handled dynamically via `ConfigurationManager`.
# - Resolution priority (from most to least specific):
# 1. `source_registry` field on the individual image entry (optional)
# 2. `manifest_registry_map` entry in `config/docker/files/default.json5`
# 3. `default_source_registry` in `config/docker/files/default.json5`
# ------------------------------------------------------------------------------
images:
- name: "starlingx/stx-platformclients"
tag: "stx.11.0-v1.0.1"
# source_registry: "dockerhub" # Optional field to specify a custom source registry
- name: "starlingx/stx-keystone"
tag: "master-debian-stable-20250530T120001Z.0"

View File

@@ -0,0 +1,233 @@
"""
Docker Image Sync Tests Using Manifest-Based Configuration
This module implements foundational tests that verify Docker images listed
in YAML manifest files can be pulled from remote registries (e.g., DockerHub),
tagged, and pushed into the local StarlingX registry (registry.local:9001).
Tests validate both positive and negative scenarios using a config-driven
approach that resolves registries dynamically via ConfigurationManager.
Key Behaviors:
- Validates sync logic from manifest to local registry via SSH.
- Verifies registry resolution order: source_registry, then manifest_registry_map,
then default_source_registry.
- Supports flexible test-driven control over which manifests are synced.
- Logs missing images or partial sync failures for improved debugging.
"""
from pathlib import Path
import yaml
from pytest import FixtureRequest, fail, raises
from config.configuration_manager import ConfigurationManager
from framework.exceptions.keyword_exception import KeywordException
from framework.logging.automation_logger import get_logger
from keywords.cloud_platform.ssh.lab_connection_keywords import LabConnectionKeywords
from keywords.docker.images.docker_images_keywords import DockerImagesKeywords
from keywords.docker.images.docker_sync_images_keywords import DockerSyncImagesKeywords
def run_manifest_sync_test(request: FixtureRequest, manifest_filename: str) -> None:
"""
Executes a manifest-based sync test, pulling Docker images from source registries
and pushing them to the local registry. Verifies that all expected images appear
in the local registry.
Args:
request (FixtureRequest): pytest request object used to register cleanup finalizer.
manifest_filename (str): Path to the manifest file in resources/.
Raises:
AssertionError: If any expected image is missing from the local registry.
"""
ssh_connection = LabConnectionKeywords().get_active_controller_ssh()
docker_config = ConfigurationManager.get_docker_config()
local_registry = docker_config.get_registry("local_registry")
manifest_paths = docker_config.get_image_manifest_files()
manifest_path = next((p for p in manifest_paths if Path(p).name == manifest_filename), None)
if not manifest_path:
raise FileNotFoundError(f"Manifest {manifest_filename} not found in docker config.")
DockerSyncImagesKeywords(ssh_connection).sync_images_from_manifest(manifest_path=manifest_path)
with open(manifest_path, "r") as f:
manifest = yaml.safe_load(f)
docker_image_keywords = DockerImagesKeywords(ssh_connection)
def cleanup():
"""
Cleans up Docker images listed in the manifest from the local system.
For each image, up to three tag formats are removed from the local Docker cache:
1. source_registry/image:tag (skipped if source is docker.io; see note below)
2. local_registry/image:tag (e.g., image pushed to registry.local)
3. image:tag (default short form used by Docker)
Purpose:
- Ensures complete removal regardless of how the image was tagged during pull/push operations.
- Supports idempotent cleanup, avoiding reliance on a single canonical tag.
- Handles cases where Docker implicitly normalizes or aliases tag references.
Notes:
- Full `docker.io/...` references are skipped during cleanup, as Docker stores these as `image:tag`.
"""
get_logger().log_info(f"Cleaning up images listed in {manifest_filename}...")
ssh_connection = LabConnectionKeywords().get_active_controller_ssh()
docker_image_keywords_cleanup = DockerImagesKeywords(ssh_connection)
for image in manifest.get("images", []):
name = image["name"]
tag = image["tag"]
source_registry_name = docker_config.get_effective_source_registry_name(image, manifest_filename)
if not source_registry_name:
get_logger().log_debug(f"Skipping cleanup for image {name}:{tag} (no source registry resolved)")
continue
source_registry = docker_config.get_registry(source_registry_name)
source_url = source_registry.get_registry_url()
# Always try to remove these two references
refs = [
f"{local_registry.get_registry_url()}/{name}:{tag}",
f"{name}:{tag}",
]
# Optionally add full source registry tag if not DockerHub
if "docker.io" not in source_url:
refs.insert(0, f"{source_url}/{name}:{tag}")
else:
get_logger().log_debug(f"Skipping full docker.io-prefixed tag for {source_url}/{name}:{tag}")
for ref in refs:
docker_image_keywords_cleanup.remove_image(ref)
request.addfinalizer(cleanup)
images = docker_image_keywords.list_images()
actual_repos = [img.get_repository() for img in images]
validation_errors = []
for image in manifest["images"]:
name = image["name"]
tag = image["tag"]
expected_ref = f"{local_registry.get_registry_url()}/{name}"
get_logger().log_info(f"Checking local registry for: {expected_ref}:{tag}")
if expected_ref not in actual_repos:
msg = f"[{manifest_filename}] Expected image not found: {expected_ref}"
get_logger().log_warning(msg)
validation_errors.append(msg)
if validation_errors:
raise AssertionError("One or more expected images were not found:\n - " + "\n - ".join(validation_errors))
def test_sync_docker_images_valid_manifest_stx_dockerhub(request):
"""
Validates that all images from a well-formed manifest can be pulled and synced into the local registry.
"""
run_manifest_sync_test(request, "stx-test-images.yaml")
def test_sync_docker_images_invalid_manifest(request):
"""
Negative test: verifies that syncing an invalid manifest raises KeywordException.
This simulates real-world scenarios where image tags are missing or incorrectly referenced.
Very simple brittle string matching is used to verify the exception message.
"""
with raises(KeywordException, match="Image sync failed"):
run_manifest_sync_test(request, "stx-test-images-invalid.yaml")
def test_sync_all_manifests_from_config(request):
"""
Verifies that all manifest files listed in the Docker config can be successfully synced to local registry.
This test ensures that ConfigurationManager.get_docker_config().get_image_manifest_files()
is functional and can drive the sync logic dynamically.
Note: Expected to currently fail if any manifest is invalid or any image fail sync fails.
"""
manifest_paths = ConfigurationManager.get_docker_config().get_image_manifest_files()
get_logger().log_info("Found image manifest paths in config: " + ", ".join(manifest_paths))
for manifest_path in manifest_paths:
manifest_name = Path(manifest_path).name
run_manifest_sync_test(request, manifest_name)
get_logger().log_info(f"All manifests synced successfully. Manifests: {', '.join(manifest_paths)}")
def test_sync_explicit_manifests(request):
"""
Verifies that all manifest files listed in the test case can be successfully synced to local registry.
Note: Expected to currently fail if any manifest is invalid or any image sync fails.
"""
manifest_paths = [
"stx-test-images.yaml",
# "stx-test-images-invalid.yaml"
# Uncomment the above line to include the invalid manifest in the test (and expect failure).
]
get_logger().log_info("Found image manifest paths in config: " + ", ".join(manifest_paths))
for manifest_path in manifest_paths:
manifest_name = Path(manifest_path).name
run_manifest_sync_test(request, manifest_name)
get_logger().log_info(f"All manifests synced successfully. Manifests: {', '.join(manifest_paths)}")
def test_invalid_manifest_logging(request):
"""
Negative test: verifies that syncing an invalid manifest raises KeywordException.
Logs only the image references that actually failed during sync.
"""
manifest_path = "stx-test-images-invalid.yaml"
try:
run_manifest_sync_test(request, manifest_path)
except KeywordException as e:
# Parse individual failure lines from the exception message
failure_lines = [line.strip(" -") for line in str(e).splitlines() if line.strip().startswith("-")]
# Extract just the image reference (everything between 'image ' and ' from ')
failed_images = []
for line in failure_lines:
if "image " in line and " from " in line:
parts = line.split("image ", 1)[-1].split(" from ")[0].strip()
failed_images.append(parts)
else:
failed_images.append(line) # Fallback: use the whole line
formatted_images = "\n\t- " + "\n\t- ".join(failed_images)
get_logger().log_info(f"Negative image sync test passed.\n" f"\tManifest file: {manifest_path}\n" f"\tFailed images:{formatted_images}")
else:
fail("Expected KeywordException was not raised.")
# def test_sync_docker_images_valid_manifest_harbor(request):
# """
# Validates that all images from a well-formed manifest can be pulled and synced into the local registry from a harbor regsitry.
# """
# run_manifest_sync_test(request, "harbor-test-images.yaml")
# def test_sync_docker_images_mixed_registries(request):
# """
# Validates that images from a manifest with mixed registries (DockerHub and Harbor) can be pulled and synced into the local registry.
# """
# run_manifest_sync_test(request, "stx-test-images-mixed-registries.yaml")