From be52337e6f75203cf2098b15c8d3c79ddac80068 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Mon, 17 Aug 2026 22:17:15 +0000 Subject: [PATCH 1/2] integration: Adds BaseSourceMinionPoolProvider to test source provider TestExportProvider now implements BaseSourceMinionPoolProvider, mirroring TestImportProvider's container-backed minion pattern. Creates common.py for the test providers, containing common code between then, which include the minion pool related code. --- .../tests/integration/test_provider/common.py | 183 ++++++++++++++++++ .../tests/integration/test_provider/exp.py | 119 +++++++----- .../tests/integration/test_provider/imp.py | 143 +------------- coriolis/tests/integration/utils.py | 14 +- 4 files changed, 273 insertions(+), 186 deletions(-) create mode 100644 coriolis/tests/integration/test_provider/common.py diff --git a/coriolis/tests/integration/test_provider/common.py b/coriolis/tests/integration/test_provider/common.py new file mode 100644 index 00000000..18d600dd --- /dev/null +++ b/coriolis/tests/integration/test_provider/common.py @@ -0,0 +1,183 @@ +# Copyright 2026 Cloudbase Solutions Srl +# All Rights Reserved. + +""" +Shared functionality between the import and export test providers. +""" + +import os +import uuid + +import paramiko + +from coriolis import utils as coriolis_utils +from coriolis.tests.integration import utils as test_utils + + +class TestProviderMixin: + """Shared provider methods between TestImportProvider and TestExportProvider.""" + + def __init__(self, event_handler): + self._event_handler = event_handler + + # BaseProvider / BaseEndpointProvider + + def get_connection_info_schema(self): + return { + "type": "object", + "properties": { + "pkey_path": {"type": "string"}, + "role": {"type": "string"}, + }, + "required": ["pkey_path"], + } + + def validate_connection(self, ctxt, connection_info): + pkey_path = connection_info["pkey_path"] + if not os.path.exists(pkey_path): + raise ValueError("SSH private key not found: %s" % pkey_path) + + def _create_minion( + self, + name_prefix, + connection_info, + devices=None, + volumes=None, + device_cgroup_rules=None, + ): + """Create a data-minion container and return its SSH connection info.""" + pkey_path = connection_info["pkey_path"] + container_name = "%s-%s" % (name_prefix, uuid.uuid4().hex[:8]) + + container_id = test_utils.run_container( + test_utils.DATA_MINION_IMAGE, + container_name, + is_systemd=True, + ssh_key=f"{pkey_path}.pub", + devices=devices, + volumes=volumes, + device_cgroup_rules=device_cgroup_rules, + ) + + try: + container_ip = test_utils.get_container_ip(container_id) + test_utils.wait_for_ssh(container_ip, 22, "root", pkey_path) + + pkey = paramiko.RSAKey.from_private_key_file(pkey_path) + ssh_conn_info = { + "ip": container_ip, + "port": 22, + "username": "root", + "pkey": coriolis_utils.serialize_key(pkey), + } + + return { + "container_id": container_id, + "ssh_connection_info": ssh_conn_info, + } + except Exception: + test_utils.remove_container(container_id) + raise + + # BaseSourceMinionPoolProvider / BaseDestinationMinionPoolProvider + + def validate_minion_compatibility_for_transfer( + self, ctxt, connection_info, export_info, environment_options, minion_properties + ): + pass + + def validate_minion_pool_environment_options( + self, ctxt, connection_info, environment_options + ): + pass + + def set_up_pool_shared_resources( + self, ctxt, connection_info, environment_options, pool_identifier + ): + return {} + + def tear_down_pool_shared_resources( + self, ctxt, connection_info, environment_options, pool_shared_resources + ): + pass + + def delete_minion(self, ctxt, connection_info, minion_properties): + container_id = (minion_properties or {}).get("container_id") + if container_id: + test_utils.remove_container(container_id) + + def shutdown_minion(self, ctxt, connection_info, minion_properties): + container_id = (minion_properties or {}).get("container_id") + if container_id: + test_utils.stop_container(container_id) + + def start_minion(self, ctxt, connection_info, minion_properties): + container_id = (minion_properties or {}).get("container_id") + if container_id: + test_utils.start_container(container_id) + + def attach_volumes_to_minion( + self, + ctxt, + connection_info, + minion_properties, + minion_connection_info, + volumes_info, + ): + container_id = minion_properties["container_id"] + + for vol in volumes_info: + if "volume_dev" in vol: + # Destination side: the device was already resolved by + # deploy_replica_disks, or left empty for a shared disk owned by another + # instance of a clustered transfer, in which case there is nothing to + # attach here. + device_path = vol["volume_dev"] + if not device_path: + continue + else: + # Source side: derive it from the disk_id. + device_path = "/dev/%s" % vol["disk_id"] + + test_utils.hotplug_device_to_container(container_id, device_path) + vol["volume_dev"] = device_path + + return { + "minion_properties": minion_properties, + "volumes_info": volumes_info, + } + + def detach_volumes_from_minion( + self, + ctxt, + connection_info, + minion_properties, + minion_connection_info, + volumes_info, + ): + container_id = (minion_properties or {}).get("container_id") + if not container_id: + return + + for vol in volumes_info or []: + dev_path = vol.get("volume_dev") + if not dev_path: + continue + + test_utils.unplug_device_from_container(container_id, dev_path) + + return { + "minion_properties": minion_properties, + "volumes_info": volumes_info, + } + + def healthcheck_minion( + self, ctxt, connection_info, minion_properties, minion_connection_info + ): + ip = minion_connection_info.get("ip") + port = minion_connection_info.get("port", 22) + username = minion_connection_info.get("username", "root") + pkey = minion_connection_info.get("pkey") + + client = coriolis_utils.connect_ssh(ip, port, username, pkey=pkey) + client.close() diff --git a/coriolis/tests/integration/test_provider/exp.py b/coriolis/tests/integration/test_provider/exp.py index 84c46089..c99dba63 100644 --- a/coriolis/tests/integration/test_provider/exp.py +++ b/coriolis/tests/integration/test_provider/exp.py @@ -14,7 +14,6 @@ import unittest import uuid -import paramiko from oslo_config import cfg from oslo_log import log as logging @@ -27,10 +26,12 @@ BaseEndpointSourceOptionsProvider, BaseReplicaExportProvider, BaseReplicaExportValidationProvider, + BaseSourceMinionPoolProvider, BaseUpdateSourceReplicaProvider, ) from coriolis.tests.integration import provider_test_base from coriolis.tests.integration import utils as test_utils +from coriolis.tests.integration.test_provider import common CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -49,6 +50,7 @@ class TestExportProvider( + common.TestProviderMixin, BaseEndpointInstancesProvider, BaseEndpointInventoryExportProvider, BaseEndpointSourceOptionsProvider, @@ -56,6 +58,7 @@ class TestExportProvider( BaseReplicaExportProvider, BaseReplicaExportValidationProvider, provider_test_base.BaseTestExportProvider, + BaseSourceMinionPoolProvider, ): """Source-side provider backed by a local loop device. @@ -133,20 +136,13 @@ def _event_manager(self): def _make_replicator(self, conn_info, event_mgr, volumes_info, repl_state): """Build a Replicator that connects via SSH to *conn_info*. - *conn_info* must contain ``ip``, ``port``, ``username``, and - ``pkey_path`` keys. An optional ``use_tunnel`` key forces the - replicator client to connect through an SSH tunnel instead of - directly to the replicator's TCP port. + *conn_info* must contain ``ip``, ``port``, ``username``, and a ``pkey``, as + returned by ``TestProviderMixin._create_minion``'s ``ssh_connection_info``. + An optional ``use_tunnel`` key forces the replicator client to connect through + an SSH tunnel instead of directly to the replicator's TCP port. """ - pkey = paramiko.RSAKey.from_private_key_file(conn_info["pkey_path"]) - repl_conn_info = { - "ip": conn_info["ip"], - "port": conn_info.get("port", 22), - "username": conn_info.get("username", "root"), - "pkey": pkey, - } return replicator_module.Replicator( - repl_conn_info, + conn_info, event_mgr, volumes_info, repl_state, @@ -154,23 +150,6 @@ def _make_replicator(self, conn_info, event_mgr, volumes_info, repl_state): _allow_loop_devices=True, ) - # BaseProvider / BaseEndpointProvider - - def get_connection_info_schema(self): - return { - "type": "object", - "properties": { - "pkey_path": {"type": "string"}, - "role": {"type": "string"}, - }, - "required": ["pkey_path"], - } - - def validate_connection(self, ctxt, connection_info): - pkey_path = connection_info["pkey_path"] - if not os.path.exists(pkey_path): - raise ValueError("SSH private key not found: %s" % pkey_path) - # BaseExportInstanceProvider def get_source_environment_schema(self): @@ -313,28 +292,17 @@ def deploy_replica_source_resources( ): block_devices = source_environment.get("instance_block_devices", {}) block_device_paths = block_devices.get(export_info["instance_name"], []) - pkey_path = connection_info["pkey_path"] - - container_name = "coriolis-replicator-%s" % uuid.uuid4().hex[:8] - container_id = test_utils.run_container( - test_utils.DATA_MINION_IMAGE, - container_name, - is_systemd=True, - ssh_key=f"{pkey_path}.pub", + + info = self._create_minion( + "coriolis-replicator", + connection_info, devices=block_device_paths, ) + container_id = info["container_id"] + src_conn_info = info["ssh_connection_info"] + src_conn_info["use_tunnel"] = source_environment.get("use_tunnel", False) try: - container_ip = test_utils.get_container_ip(container_id) - test_utils.wait_for_ssh(container_ip, 22, "root", pkey_path) - - src_conn_info = { - "ip": container_ip, - "port": 22, - "username": "root", - "pkey_path": pkey_path, - "use_tunnel": source_environment.get("use_tunnel", False), - } replicator = self._make_replicator( src_conn_info, self._event_manager(), [], None ) @@ -375,13 +343,27 @@ def replicate_disks( ): repl_state = _extract_repl_state(volumes_info) if incremental else None + disk_mappings = source_resources.get("disk_mappings") + if disk_mappings is None: + # Minion pool case: "source_resources" only carries the pool minion's + # container_id. the block devices were never attached to the container, + # so hotplug them now. + container_id = source_resources["container_id"] + block_devices = source_environment.get("instance_block_devices", {}) + block_device_paths = block_devices.get(instance_name, []) + for path in block_device_paths: + test_utils.hotplug_device_to_container(container_id, path) + + disk_mappings = { + os.path.basename(path): path for path in block_device_paths + } + replicator = self._make_replicator( source_conn_info, self._event_manager(), volumes_info, repl_state ) replicator.init_replicator() replicator.wait_for_chunks() - disk_mappings = source_resources.get("disk_mappings", {}) source_volumes_info = [ { "disk_id": vol["disk_id"], @@ -429,6 +411,45 @@ def validate_replica_export_input( ): return {} + # BaseSourceMinionPoolProvider + + def get_minion_pool_environment_schema(self): + return self.get_source_environment_schema() + + def get_minion_pool_options( + self, ctxt, connection_info, env=None, option_names=None + ): + return self.get_source_environment_options( + ctxt, connection_info, env, option_names + ) + + def create_minion( + self, + ctxt, + connection_info, + environment_options, + pool_identifier, + pool_os_type, + pool_shared_resources, + new_minion_identifier, + ): + # Devices are hotplugged after container creation via mknod / nsenter. + # We must pre-authorize all block devices through the + # --device-cgroup-rule option, otherwise any device added will be + # inaccessible ("operation not permitted" error on open). + result = self._create_minion( + "coriolis-pool-minion", + connection_info, + device_cgroup_rules=["b *:* rwm"], + ) + + return { + "connection_info": result["ssh_connection_info"], + "minion_provider_properties": { + "container_id": result["container_id"], + }, + } + # Helpers def _get_block_device_size(device): diff --git a/coriolis/tests/integration/test_provider/imp.py b/coriolis/tests/integration/test_provider/imp.py index 84cbef8c..708865b6 100644 --- a/coriolis/tests/integration/test_provider/imp.py +++ b/coriolis/tests/integration/test_provider/imp.py @@ -11,13 +11,10 @@ import os import unittest -import uuid -import paramiko from oslo_log import log as logging from coriolis import constants -from coriolis import utils as coriolis_utils from coriolis.providers import backup_writers from coriolis.providers.base import ( BaseDestinationMinionPoolProvider, @@ -31,7 +28,7 @@ ) from coriolis.tests.integration import provider_test_base from coriolis.tests.integration import utils as test_utils -from coriolis.tests.integration.test_provider import osmorphing +from coriolis.tests.integration.test_provider import common, osmorphing LOG = logging.getLogger(__name__) @@ -47,6 +44,7 @@ class TestImportProvider( + common.TestProviderMixin, BaseEndpointProvider, BaseEndpointDestinationOptionsProvider, BaseEndpointNetworksProvider, @@ -76,9 +74,6 @@ class TestImportProvider( platform = "test-dest" - def __init__(self, event_handler): - self._event_handler = event_handler - @classmethod def supports_shared_disks(cls) -> bool: return True @@ -111,23 +106,6 @@ def check_prerequisites(self): % (test_utils.DATA_MINION_IMAGE, test_utils.DATA_MINION_IMAGE) ) - # BaseProvider / BaseEndpointProvider - - def get_connection_info_schema(self): - return { - "type": "object", - "properties": { - "pkey_path": {"type": "string"}, - "role": {"type": "string"}, - }, - "required": ["pkey_path"], - } - - def validate_connection(self, ctxt, connection_info): - pkey_path = connection_info["pkey_path"] - if not os.path.exists(pkey_path): - raise ValueError("SSH private key not found: %s" % pkey_path) - # BaseImportInstanceProvider def get_target_environment_schema(self): @@ -244,36 +222,17 @@ def _create_minion( setup_writer=True, writer_backend=backup_writers.BACKUP_WRITER_HTTP, ): - pkey_path = connection_info["pkey_path"] - container_name = "%s-%s" % (name_prefix, uuid.uuid4().hex[:8]) - - container_id = test_utils.run_container( - test_utils.DATA_MINION_IMAGE, - container_name, - is_systemd=True, - ssh_key=f"{pkey_path}.pub", + info = super()._create_minion( + name_prefix, + connection_info, devices=devices, volumes=volumes, device_cgroup_rules=device_cgroup_rules, ) try: - container_ip = test_utils.get_container_ip(container_id) - test_utils.wait_for_ssh(container_ip, 22, "root", pkey_path) - - pkey = paramiko.RSAKey.from_private_key_file(pkey_path) - ssh_conn_info = { - "ip": container_ip, - "port": 22, - "username": "root", - "pkey": coriolis_utils.serialize_key(pkey), - } - - info = { - "container_id": container_id, - "ssh_connection_info": ssh_conn_info, - } if setup_writer: + ssh_conn_info = info["ssh_connection_info"] if writer_backend == backup_writers.BACKUP_WRITER_SSH: info["backup_writer_connection_info"] = { "backend": backup_writers.BACKUP_WRITER_SSH, @@ -291,7 +250,7 @@ def _create_minion( return info except Exception: - test_utils.remove_container(container_id) + test_utils.remove_container(info["container_id"]) raise def delete_replica_target_resources( @@ -466,26 +425,6 @@ def get_minion_pool_options( ctxt, connection_info, env, option_names ) - def validate_minion_compatibility_for_transfer( - self, ctxt, connection_info, export_info, environment_options, minion_properties - ): - pass - - def validate_minion_pool_environment_options( - self, ctxt, connection_info, environment_options - ): - pass - - def set_up_pool_shared_resources( - self, ctxt, connection_info, environment_options, pool_identifier - ): - return {} - - def tear_down_pool_shared_resources( - self, ctxt, connection_info, environment_options, pool_shared_resources - ): - pass - def create_minion( self, ctxt, @@ -521,74 +460,6 @@ def create_minion( }, } - def delete_minion(self, ctxt, connection_info, minion_properties): - container_id = (minion_properties or {}).get("container_id") - if container_id: - test_utils.remove_container(container_id) - - def shutdown_minion(self, ctxt, connection_info, minion_properties): - container_id = (minion_properties or {}).get("container_id") - if container_id: - test_utils.stop_container(container_id) - - def start_minion(self, ctxt, connection_info, minion_properties): - container_id = (minion_properties or {}).get("container_id") - if container_id: - test_utils.start_container(container_id) - - def attach_volumes_to_minion( - self, - ctxt, - connection_info, - minion_properties, - minion_connection_info, - volumes_info, - ): - container_id = minion_properties["container_id"] - for vol in volumes_info: - device_path = vol["volume_dev"] - test_utils.hotplug_device_to_container(container_id, device_path) - - return { - "minion_properties": minion_properties, - "volumes_info": volumes_info, - } - - def detach_volumes_from_minion( - self, - ctxt, - connection_info, - minion_properties, - minion_connection_info, - volumes_info, - ): - container_id = (minion_properties or {}).get("container_id") - if not container_id: - return - - for vol in volumes_info or []: - dev_path = vol.get("volume_dev") - if not dev_path: - continue - - test_utils.unplug_device_from_container(container_id, dev_path) - - return { - "minion_properties": minion_properties, - "volumes_info": volumes_info, - } - - def healthcheck_minion( - self, ctxt, connection_info, minion_properties, minion_connection_info - ): - ip = minion_connection_info.get("ip") - port = minion_connection_info.get("port", 22) - username = minion_connection_info.get("username", "root") - pkey = minion_connection_info.get("pkey") - - client = coriolis_utils.connect_ssh(ip, port, username, pkey=pkey) - client.close() - def validate_osmorphing_minion_compatibility_for_transfer( self, ctxt, connection_info, export_info, environment_options, minion_properties ): diff --git a/coriolis/tests/integration/utils.py b/coriolis/tests/integration/utils.py index 5a724d04..cc4c9896 100644 --- a/coriolis/tests/integration/utils.py +++ b/coriolis/tests/integration/utils.py @@ -317,8 +317,20 @@ def _get_container_pid(container_id): def hotplug_device_to_container(container_id, device_path): - """Create a device node for *device_path* in *container_id*'s namespace.""" + """Create a device node for *device_path* in *container_id*'s namespace. + + Noop if the device node already exists in the container (e.g.: it was already + hotplugged by a previous call, such as an earlier incremental replication pass). + """ pid = _get_container_pid(container_id) + + exists = _run( + ["nsenter", "--target", str(pid), "--mount", "--", "test", "-e", device_path], + check=False, + ) + if exists.returncode == 0: + return + stat_result = os.stat(device_path) major = os.major(stat_result.st_rdev) minor = os.minor(stat_result.st_rdev) From 7ba61610fb03908f9531ac0ccd7735bc4dce911f Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Mon, 17 Aug 2026 22:17:15 +0000 Subject: [PATCH 2/2] integration: Adds tests for source minion pools New tests: source pool CRUD / allocate / deallocate, get_source_minion_pool_options, source-pool-backed transfer. --- coriolis/tests/integration/base.py | 166 ++++++++++++------ .../deployments/test_osmorphing.py | 2 +- coriolis/tests/integration/harness.py | 1 + coriolis/tests/integration/test_endpoints.py | 10 ++ .../integration/test_failure_recovery.py | 4 +- .../tests/integration/test_minion_pools.py | 49 +++++- .../integration/transfers/test_transfer.py | 15 +- 7 files changed, 183 insertions(+), 64 deletions(-) diff --git a/coriolis/tests/integration/base.py b/coriolis/tests/integration/base.py index 9ff8c2f8..edee3844 100644 --- a/coriolis/tests/integration/base.py +++ b/coriolis/tests/integration/base.py @@ -70,7 +70,8 @@ def setUpClass(cls): cls._imp_conn_info = cls._harness.imp_conn_info cls._imp_env_options = cls._harness.imp_env_options cls._storage_mappings = cls._harness.imp_storage_mappings - cls._pool_env = cls._harness.imp_minion_pool_environment + cls._imp_pool_env = cls._harness.imp_minion_pool_environment + cls._exp_pool_env = cls._harness.exp_minion_pool_environment cls._client = cls.get_client() @@ -168,13 +169,19 @@ def _create_pool( name="test-pool", skip_allocation=True, wait_for_allocation=False, + platform=constants.PROVIDER_PLATFORM_DESTINATION, ): + env_options = ( + cls._imp_pool_env + if platform == constants.PROVIDER_PLATFORM_DESTINATION + else cls._exp_pool_env + ) pool = cls._client.minion_pools.create( name=name, endpoint=endpoint_id, - platform=constants.PROVIDER_PLATFORM_DESTINATION, + platform=platform, os_type=constants.OS_TYPE_LINUX, - environment_options=cls._pool_env, + environment_options=env_options, minimum_minions=1, maximum_minions=1, minion_max_idle_time=3600, @@ -241,6 +248,39 @@ def _get_db_context(): is_admin=True, ) + def assertPoolAllocated(self, pool_id): + """Assert the pool is healthy and still in ALLOCATED status.""" + ctxt = self._get_db_context() + pool = db_api.get_minion_pool(ctxt, pool_id) + self.assertIsNotNone(pool, "Pool %s not found" % pool_id) + self.assertEqual( + constants.MINION_POOL_STATUS_ALLOCATED, + pool.status, + "Pool %s is not ALLOCATED (got %s)" % (pool_id, pool.status), + ) + + def assertMachinesAvailable(self, pool_id): + """Assert all machines in the pool are AVAILABLE and have been used.""" + ctxt = self._get_db_context() + pool = db_api.get_minion_pool(ctxt, pool_id, include_machines=True) + self.assertIsNotNone(pool, "Pool %s not found" % pool_id) + self.assertTrue( + pool.minion_machines, + "Pool %s has no minion machines" % pool_id, + ) + for machine in pool.minion_machines: + self.assertEqual( + constants.MINION_MACHINE_STATUS_AVAILABLE, + machine.allocation_status, + "Machine %s in pool %s is not AVAILABLE (got %s)" + % (machine.id, pool_id, machine.allocation_status), + ) + self.assertIsNotNone( + machine.last_used_at, + "Machine %s in pool %s has no last_used_at; " + "it may not have been used by the transfer" % (machine.id, pool_id), + ) + @staticmethod def _ignoreExc(func, ignored_exc=Exception): """Wrap the given function, ignoring exceptions.""" @@ -255,7 +295,8 @@ def f(*args, **kwargs): class ReplicaIntegrationTestBase(CoriolisIntegrationTestBase): - _CREATE_MINION_POOLS = False + _CREATE_DST_MINION_POOL = False + _CREATE_SRC_MINION_POOL = False _SRC_DEVICE_SIZE_MB = 16 # Extra source_environment entries merged into the default transfer's @@ -281,15 +322,27 @@ def setUpClass(cls): ) # Create minion pool if needed. - cls._pool_id = None - if cls._CREATE_MINION_POOLS: + cls._dst_pool_id = None + if cls._CREATE_DST_MINION_POOL: pool = cls._create_pool( cls._dst_endpoint.id, - "transfer-pool", + "dst-transfer-pool", skip_allocation=False, wait_for_allocation=True, ) - cls._pool_id = pool.id + cls._dst_pool_id = pool.id + + # Create source minion pool if needed. + cls._src_pool_id = None + if cls._CREATE_SRC_MINION_POOL: + pool = cls._create_pool( + cls._src_endpoint.id, + "src-transfer-pool", + skip_allocation=False, + wait_for_allocation=True, + platform=constants.PROVIDER_PLATFORM_SOURCE, + ) + cls._src_pool_id = pool.id def setUp(self): super().setUp() @@ -303,7 +356,8 @@ def setUp(self): self._src_endpoint.id, self._dst_endpoint.id, instances=[self._instance_name], - destination_minion_pool_id=self._pool_id, + destination_minion_pool_id=self._dst_pool_id, + origin_minion_pool_id=self._src_pool_id, source_environment={ **extra_source_env, **self._exp_env_options, @@ -591,7 +645,28 @@ def _slow_call(*args, **kwargs): self.addCleanup(patcher.stop) -class MinionPoolTestBase(CoriolisIntegrationTestBase): +class SourceMinionPoolTestBase(CoriolisIntegrationTestBase): + """Base class for source minion pool integration tests. + + Skips the entire test class when the export provider does not advertise + ``PROVIDER_TYPE_SOURCE_MINION_POOL`` support. + """ + + @classmethod + def setUpClass(cls): + h = harness._IntegrationHarness.get() + available = providers_factory.get_available_providers() + exp_types = available.get(h.exp_provider_platform, {}).get("types", []) + if constants.PROVIDER_TYPE_SOURCE_MINION_POOL not in exp_types: + raise unittest.SkipTest( + "Export provider '%s' does not support minion pools" + % h.exp_provider_platform + ) + + super().setUpClass() + + +class DestinationMinionPoolTestBase(CoriolisIntegrationTestBase): """Base class for minion pool integration tests. Skips the entire test class when the import provider does not advertise @@ -615,59 +690,50 @@ def setUpClass(cls): super().setUpClass() -class MinionPoolReplicaTestBase(MinionPoolTestBase, ReplicaIntegrationTestBase): - """Base class for replica integration tests using minion pools. +class MinionPoolReplicaTestBase( + DestinationMinionPoolTestBase, ReplicaIntegrationTestBase +): + """Base class for replica integration tests using destination minion pools. Extends the assertions to also verify that the minions in the pool have been used, and that the minions and the pool returns to an available state. """ - _CREATE_MINION_POOLS = True + _CREATE_DST_MINION_POOL = True def _execute_and_wait(self, transfer_id, timeout=600): super()._execute_and_wait(transfer_id, timeout=timeout) - self.assertPoolAllocated(self._pool_id) - self.assertMachinesAvailable(self._pool_id) + self.assertPoolAllocated(self._dst_pool_id) + self.assertMachinesAvailable(self._dst_pool_id) def assertExecutionCompleted(self, execution_id, timeout=600): super().assertExecutionCompleted(execution_id, timeout=timeout) - self.assertPoolAllocated(self._pool_id) - self.assertMachinesAvailable(self._pool_id) + self.assertPoolAllocated(self._dst_pool_id) + self.assertMachinesAvailable(self._dst_pool_id) def assertDeploymentCompleted(self, deployment_id, timeout=600): super().assertDeploymentCompleted(deployment_id, timeout=timeout) - self.assertPoolAllocated(self._pool_id) - self.assertMachinesAvailable(self._pool_id) + self.assertPoolAllocated(self._dst_pool_id) + self.assertMachinesAvailable(self._dst_pool_id) - def assertPoolAllocated(self, pool_id): - """Assert the pool is healthy and still in ALLOCATED status.""" - ctxt = self._get_db_context() - pool = db_api.get_minion_pool(ctxt, pool_id) - self.assertIsNotNone(pool, "Pool %s not found" % pool_id) - self.assertEqual( - constants.MINION_POOL_STATUS_ALLOCATED, - pool.status, - "Pool %s is not ALLOCATED (got %s)" % (pool_id, pool.status), - ) - def assertMachinesAvailable(self, pool_id): - """Assert all machines in the pool are AVAILABLE and have been used.""" - ctxt = self._get_db_context() - pool = db_api.get_minion_pool(ctxt, pool_id, include_machines=True) - self.assertIsNotNone(pool, "Pool %s not found" % pool_id) - self.assertTrue( - pool.minion_machines, - "Pool %s has no minion machines" % pool_id, - ) - for machine in pool.minion_machines: - self.assertEqual( - constants.MINION_MACHINE_STATUS_AVAILABLE, - machine.allocation_status, - "Machine %s in pool %s is not AVAILABLE (got %s)" - % (machine.id, pool_id, machine.allocation_status), - ) - self.assertIsNotNone( - machine.last_used_at, - "Machine %s in pool %s has no last_used_at; " - "it may not have been used by the transfer" % (machine.id, pool_id), - ) +class SourceMinionPoolReplicaTestBase( + SourceMinionPoolTestBase, ReplicaIntegrationTestBase +): + """Base class for replica integration tests using source minion pools. + + Extends the assertions to also verify that the minions in the pool have + been used, and that the minions and the pool returns to an available state. + """ + + _CREATE_SRC_MINION_POOL = True + + def _execute_and_wait(self, transfer_id, timeout=600): + super()._execute_and_wait(transfer_id, timeout=timeout) + self.assertPoolAllocated(self._src_pool_id) + self.assertMachinesAvailable(self._src_pool_id) + + def assertExecutionCompleted(self, execution_id, timeout=600): + super().assertExecutionCompleted(execution_id, timeout=timeout) + self.assertPoolAllocated(self._src_pool_id) + self.assertMachinesAvailable(self._src_pool_id) diff --git a/coriolis/tests/integration/deployments/test_osmorphing.py b/coriolis/tests/integration/deployments/test_osmorphing.py index 759f4b41..e533401a 100644 --- a/coriolis/tests/integration/deployments/test_osmorphing.py +++ b/coriolis/tests/integration/deployments/test_osmorphing.py @@ -194,7 +194,7 @@ def test_os_morphing_global_script_first_boot(self): class OsMorphingMinionPoolDeploymentTest( - integration_base.MinionPoolTestBase, OsMorphingDeploymentTestBase + integration_base.DestinationMinionPoolTestBase, OsMorphingDeploymentTestBase ): """OS morphing deployment using a minion pool for the OS morphing phase.""" diff --git a/coriolis/tests/integration/harness.py b/coriolis/tests/integration/harness.py index e39f28c3..09fbd05b 100644 --- a/coriolis/tests/integration/harness.py +++ b/coriolis/tests/integration/harness.py @@ -400,6 +400,7 @@ def __init__(self): self.exp_provider.initialize(self.exp_conn_info, providers_config["source"]) self.exp_provider.check_prerequisites() self.exp_env_options = providers_config["source"]["environment"] + self.exp_minion_pool_environment = {} # Init importer. imp_provider_cls = providers_config["destination"]["provider_cls"] diff --git a/coriolis/tests/integration/test_endpoints.py b/coriolis/tests/integration/test_endpoints.py index fac52574..5111e77b 100644 --- a/coriolis/tests/integration/test_endpoints.py +++ b/coriolis/tests/integration/test_endpoints.py @@ -10,6 +10,7 @@ - get_storage (list and default) - get_source_environment_options - get_target_environment_options +- get_source_minion_pool_options - get_destination_minion_pool_options - get_inventory_csv - endpoint_instances.list and endpoint_instances.get @@ -98,6 +99,15 @@ def test_list_destination_options(self): self.assertIsInstance(options, list) self.assertTrue(len(options) > 0, "Expected at least one destination option") + def test_list_source_minion_pool_options(self): + options = self._client.endpoint_source_minion_pool_options.list( + self._src_endpoint.id + ) + self.assertIsInstance(options, list) + self.assertTrue( + len(options) > 0, "Expected at least one source minion pool option" + ) + def test_list_destination_minion_pool_options(self): if not isinstance( self._imp_provider, provider_base.BaseDestinationMinionPoolProvider diff --git a/coriolis/tests/integration/test_failure_recovery.py b/coriolis/tests/integration/test_failure_recovery.py index 9328b457..1d7ee0f4 100644 --- a/coriolis/tests/integration/test_failure_recovery.py +++ b/coriolis/tests/integration/test_failure_recovery.py @@ -166,13 +166,13 @@ def test_transfer_minion_allocation_failure_cleans_up(self): mock_create.assert_called() # The pool itself stays usable. - self.assertPoolAllocated(self._pool_id) + self.assertPoolAllocated(self._dst_pool_id) # Its only machine failed both the healthcheck and the recreation # attempt. ending up as UNINITIALIZED. It then gets deleted, rather # than left dangling in a broken intermediate status. ctxt = self._get_db_context() - pool = db_api.get_minion_pool(ctxt, self._pool_id, include_machines=True) + pool = db_api.get_minion_pool(ctxt, self._dst_pool_id, include_machines=True) self.assertEqual( [], pool.minion_machines, diff --git a/coriolis/tests/integration/test_minion_pools.py b/coriolis/tests/integration/test_minion_pools.py index 901ec50b..77ad6b11 100644 --- a/coriolis/tests/integration/test_minion_pools.py +++ b/coriolis/tests/integration/test_minion_pools.py @@ -21,15 +21,8 @@ CONF = cfg.CONF -class MinionPoolLifecycleTest(base.MinionPoolTestBase): - def setUp(self): - super().setUp() - - self._endpoint = self._create_endpoint( - name="pool-dst", - endpoint_type=self._imp_platform, - connection_info=self._imp_conn_info, - ) +class MinionPoolLifecycleTestMixin: + _MINION_PLATFORM = None def _wait_for_machine_status(self, pool_id, status, timeout=120): """Poll the DB until the pool's single machine reaches *status*.""" @@ -54,6 +47,7 @@ def test_minion_pool_crud(self): pool = self._create_pool(self._endpoint.id) self.assertEqual("test-pool", pool.name) + self.assertEqual(self._MINION_PLATFORM, pool.platform) self.assertEqual(constants.MINION_POOL_STATUS_DEALLOCATED, pool.status) # List @@ -115,6 +109,22 @@ def test_allocate_deallocate(self): "Pool deallocation ended in unexpected status '%s'" % final.status, ) + +class MinionPoolLifecycleTests( + MinionPoolLifecycleTestMixin, base.DestinationMinionPoolTestBase +): + _MINION_PLATFORM = constants.PROVIDER_PLATFORM_DESTINATION + + def setUp(self): + super().setUp() + + self._endpoint = self._create_endpoint( + name="pool-dst", + endpoint_type=self._imp_platform, + connection_info=self._imp_conn_info, + ) + self._pool_env = self._imp_pool_env + def test_cron_triggered_refresh(self): """Cron-scheduled refresh. @@ -163,3 +173,24 @@ def test_cron_triggered_refresh(self): "Minion pool machine '%s' was not refreshed by the automatic " "cron job in time" % pool.id, ) + + +class SourceMinionPoolLifecycleTests( + MinionPoolLifecycleTestMixin, base.SourceMinionPoolTestBase +): + _MINION_PLATFORM = constants.PROVIDER_PLATFORM_SOURCE + + def setUp(self): + super().setUp() + + self._endpoint = self._create_endpoint( + name="pool-src", + endpoint_type=self._exp_platform, + connection_info=self._exp_conn_info, + ) + self._pool_env = self._exp_pool_env + + def _create_pool(self, endpoint_id, **kwargs): + return super()._create_pool( + endpoint_id, platform=constants.PROVIDER_PLATFORM_SOURCE, **kwargs + ) diff --git a/coriolis/tests/integration/transfers/test_transfer.py b/coriolis/tests/integration/transfers/test_transfer.py index c777f5da..1eb1dfa2 100644 --- a/coriolis/tests/integration/transfers/test_transfer.py +++ b/coriolis/tests/integration/transfers/test_transfer.py @@ -436,8 +436,8 @@ class MinionPoolTransferTest( def test_transfer(self): super().test_transfer() - self.assertPoolAllocated(self._pool_id) - self.assertMachinesAvailable(self._pool_id) + self.assertPoolAllocated(self._dst_pool_id) + self.assertMachinesAvailable(self._dst_pool_id) class ReplicaTransferViaSSHTunnelTest(base.ReplicaIntegrationTestBase): @@ -480,3 +480,14 @@ def _spy_get_ssh_tunnel(client_self): test_utils.devices_match(self._src_device, self._dst_device), "Devices do not match after transfer via SSH tunnel", ) + + +class SourceMinionPoolTransferTest( + base.SourceMinionPoolReplicaTestBase, _ReplicaTransferTestsMixin +): + """Transfer execution that uses a pre-allocated source minion pool.""" + + def test_transfer(self): + super().test_transfer() + self.assertPoolAllocated(self._src_pool_id) + self.assertMachinesAvailable(self._src_pool_id)