From ddf31b514d291d490963634f183d2b625bccba4b Mon Sep 17 00:00:00 2001 From: aemous Date: Mon, 11 May 2026 13:30:44 -0400 Subject: [PATCH 1/5] Make PrefixTrie public and implement LazyCommand class to defer loading plugins until commands are invoked. --- awscli/botocore/hooks.py | 4 +- awscli/lazy.py | 60 +++++++++++++ tests/functional/test_lazy.py | 87 +++++++++++++++++++ tests/unit/botocore/test_hooks.py | 54 +++++++++++- tests/unit/test_lazy.py | 139 ++++++++++++++++++++++++++++++ 5 files changed, 341 insertions(+), 3 deletions(-) create mode 100644 awscli/lazy.py create mode 100644 tests/functional/test_lazy.py create mode 100644 tests/unit/test_lazy.py diff --git a/awscli/botocore/hooks.py b/awscli/botocore/hooks.py index 90db67d6fe69..1530bbf158ee 100644 --- a/awscli/botocore/hooks.py +++ b/awscli/botocore/hooks.py @@ -196,7 +196,7 @@ def __init__(self): # read only access (we never modify self._handlers). # A cache of event name to handler list. self._lookup_cache = {} - self._handlers = _PrefixTrie() + self._handlers = PrefixTrie() # This is used to ensure that unique_id's are only # registered once. self._unique_id_handlers = {} @@ -398,7 +398,7 @@ def __copy__(self): return new_instance -class _PrefixTrie: +class PrefixTrie: """Specialized prefix trie that handles wildcards. The prefixes in this case are based on dot separated diff --git a/awscli/lazy.py b/awscli/lazy.py new file mode 100644 index 000000000000..4cad0964443d --- /dev/null +++ b/awscli/lazy.py @@ -0,0 +1,60 @@ +import importlib + +from awscli.commands import CLICommand + + +class LazyCommand(CLICommand): + """A command-table entry that defers importing its real implementation. + + Sits in the command table like any other CLICommand, but only imports + the actual module (and creates the real command object) when the command + is invoked or its help is accessed. + """ + + def __init__(self, name, session, module_path, class_name): + self._name = name + self._session = session + self._module_path = module_path + self._class_name = class_name + self._real = None + self._lineage = [self] + + def _resolve(self): + if self._real is None: + mod = importlib.import_module(self._module_path) + cls = getattr(mod, self._class_name) + self._real = cls(self._session) + self._real.lineage = self._lineage + return self._real + + def __call__(self, args, parsed_globals): + return self._resolve()(args, parsed_globals) + + def create_help_command(self): + return self._resolve().create_help_command() + + @property + def arg_table(self): + return self._resolve().arg_table + + @property + def subcommand_table(self): + return self._resolve().subcommand_table + + @property + def name(self): + return self._name + + @name.setter + def name(self, value): + self._name = value + + @property + def lineage(self): + return self._lineage + + @lineage.setter + def lineage(self, value): + self._lineage = value + if self._real is not None: + self._real.lineage = value diff --git a/tests/functional/test_lazy.py b/tests/functional/test_lazy.py new file mode 100644 index 000000000000..3ba067e13aad --- /dev/null +++ b/tests/functional/test_lazy.py @@ -0,0 +1,87 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import pytest + +from awscli.handlers_registry import MAIN_COMMAND_TABLE_OPS +from awscli.lazy import LazyCommand +from awscli.testutils import BaseAWSHelpOutputTest, mock + +# Derive test parameters from MAIN_COMMAND_TABLE_OPS. +_ADD_OPS = [op for op in MAIN_COMMAND_TABLE_OPS if op[0] == 'add'] +_RENAME_OPS = [op for op in MAIN_COMMAND_TABLE_OPS if op[0] == 'rename'] +_ADD_CMD_NAMES = [op[1] for op in _ADD_OPS] +_RENAME_NEW_NAMES = [op[2] for op in _RENAME_OPS] + + +class TestLazyCommandHelpRenders(BaseAWSHelpOutputTest): + def test_added_command_help_renders(self): + for cmd_name in _ADD_CMD_NAMES: + with self.subTest(cmd_name=cmd_name): + self.driver.main([cmd_name, 'help']) + self.assert_contains(cmd_name) + + def test_renamed_command_help_renders(self): + for new_name in _RENAME_NEW_NAMES: + with self.subTest(new_name=new_name): + self.driver.main([new_name, 'help']) + self.assert_contains(new_name) + + +class TestLazyCommandIsTransparent(BaseAWSHelpOutputTest): + def test_added_commands_appear_in_top_level_help(self): + self.driver.main(['help']) + for cmd_name in _ADD_CMD_NAMES: + self.assert_contains(cmd_name) + + def test_lazy_command_has_subcommands(self): + command_table = self.driver.subcommand_table + s3_cmd = command_table['s3'] + assert isinstance(s3_cmd, LazyCommand) + subcommands = s3_cmd.subcommand_table + assert 'ls' in subcommands + assert 'cp' in subcommands + + +class TestLazyCommandErrorPaths: + def test_invalid_module_path_raises_on_resolve(self): + session = mock.MagicMock() + cmd = LazyCommand( + 'bad-cmd', + session, + 'awscli.nonexistent.module', + 'FakeCommand', + ) + with pytest.raises(ModuleNotFoundError): + cmd([], mock.MagicMock()) + + def test_invalid_class_name_raises_on_resolve(self): + session = mock.MagicMock() + cmd = LazyCommand( + 'bad-cmd', + session, + 'awscli.customizations.dynamodb.ddb', + 'NonexistentClass', + ) + with pytest.raises(AttributeError): + cmd([], mock.MagicMock()) + + def test_invalid_module_path_raises_on_help(self): + session = mock.MagicMock() + cmd = LazyCommand( + 'bad-cmd', + session, + 'awscli.nonexistent.module', + 'FakeCommand', + ) + with pytest.raises(ModuleNotFoundError): + cmd.create_help_command() diff --git a/tests/unit/botocore/test_hooks.py b/tests/unit/botocore/test_hooks.py index 23d9b4985d6c..4defdd5511dc 100644 --- a/tests/unit/botocore/test_hooks.py +++ b/tests/unit/botocore/test_hooks.py @@ -14,7 +14,11 @@ import functools from functools import partial -from botocore.hooks import HierarchicalEmitter, first_non_none_response +from botocore.hooks import ( + HierarchicalEmitter, + PrefixTrie, + first_non_none_response, +) from tests import unittest @@ -584,5 +588,53 @@ def handler(a, b, **kwargs): ) +class TestPrefixTrie(unittest.TestCase): + def setUp(self): + self.trie = PrefixTrie() + + def test_append_and_prefix_search_exact_match(self): + self.trie.append_item('building-command-table.main', 'handler1') + results = self.trie.prefix_search('building-command-table.main') + self.assertIn('handler1', results) + + def test_prefix_search_matches_parent(self): + self.trie.append_item('building-command-table', 'handler1') + results = self.trie.prefix_search('building-command-table.main') + self.assertIn('handler1', results) + + def test_prefix_search_does_not_match_sibling(self): + self.trie.append_item('building-command-table.ecs', 'handler1') + results = self.trie.prefix_search('building-command-table.main') + self.assertNotIn('handler1', results) + + def test_prefix_search_does_not_match_child(self): + self.trie.append_item('building-command-table.main.sub', 'handler1') + results = self.trie.prefix_search('building-command-table.main') + self.assertNotIn('handler1', results) + + def test_wildcard_match(self): + self.trie.append_item('building-command-table.*', 'handler1') + results = self.trie.prefix_search('building-command-table.main') + self.assertIn('handler1', results) + + def test_multiple_items_at_same_key(self): + self.trie.append_item('building-command-table.main', 'handler1') + self.trie.append_item('building-command-table.main', 'handler2') + results = self.trie.prefix_search('building-command-table.main') + self.assertIn('handler1', results) + self.assertIn('handler2', results) + + def test_multiple_levels_all_returned(self): + self.trie.append_item('building-command-table', 'parent') + self.trie.append_item('building-command-table.main', 'exact') + results = self.trie.prefix_search('building-command-table.main') + self.assertIn('parent', results) + self.assertIn('exact', results) + + def test_empty_trie_returns_empty(self): + results = self.trie.prefix_search('building-command-table.main') + self.assertEqual(len(results), 0) + + if __name__ == '__main__': unittest.main() diff --git a/tests/unit/test_lazy.py b/tests/unit/test_lazy.py new file mode 100644 index 000000000000..63a1ff5ea44b --- /dev/null +++ b/tests/unit/test_lazy.py @@ -0,0 +1,139 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from unittest.mock import MagicMock, patch + +import pytest + +from awscli.lazy import LazyCommand + + +@pytest.fixture +def session(): + return MagicMock() + + +@pytest.fixture +def mock_command_class(): + cls = MagicMock() + instance = MagicMock() + cls.return_value = instance + return cls + + +@pytest.fixture +def mock_module(mock_command_class): + module = MagicMock() + module.MyCommand = mock_command_class + return module + + +class TestLazyCommandResolution: + def test_does_not_import_on_construction(self, session): + with patch('importlib.import_module') as imp: + LazyCommand('cmd', session, 'some.module', 'MyCommand') + imp.assert_not_called() + + def test_imports_module_on_call(self, session, mock_module): + cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand') + with patch('importlib.import_module', return_value=mock_module): + cmd(['arg1'], MagicMock()) + mock_module.MyCommand.assert_called_once_with(session) + + def test_imports_module_on_help(self, session, mock_module): + cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand') + with patch('importlib.import_module', return_value=mock_module): + cmd.create_help_command() + mock_module.MyCommand.assert_called_once_with(session) + + def test_imports_module_on_arg_table(self, session, mock_module): + cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand') + with patch('importlib.import_module', return_value=mock_module): + _ = cmd.arg_table + mock_module.MyCommand.assert_called_once_with(session) + + def test_imports_module_on_subcommand_table(self, session, mock_module): + cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand') + with patch('importlib.import_module', return_value=mock_module): + _ = cmd.subcommand_table + mock_module.MyCommand.assert_called_once_with(session) + + def test_resolves_only_once(self, session, mock_module): + cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand') + with patch('importlib.import_module', return_value=mock_module) as imp: + cmd(['arg1'], MagicMock()) + cmd(['arg2'], MagicMock()) + imp.assert_called_once_with('some.module') + + def test_delegates_call_to_real_command(self, session, mock_module): + cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand') + args = ['arg1'] + parsed_globals = MagicMock() + with patch('importlib.import_module', return_value=mock_module): + cmd(args, parsed_globals) + mock_module.MyCommand.return_value.assert_called_once_with( + args, parsed_globals + ) + + def test_delegates_help_to_real_command(self, session, mock_module): + cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand') + with patch('importlib.import_module', return_value=mock_module): + result = cmd.create_help_command() + assert ( + result == mock_module.MyCommand.return_value.create_help_command() + ) + + +class TestLazyCommandProperties: + def test_name_returns_initial_name(self, session): + cmd = LazyCommand('my-cmd', session, 'some.module', 'MyCommand') + assert cmd.name == 'my-cmd' + + def test_name_setter_updates_name(self, session): + cmd = LazyCommand('old-name', session, 'some.module', 'MyCommand') + cmd.name = 'new-name' + assert cmd.name == 'new-name' + + def test_lineage_defaults_to_self(self, session): + cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand') + assert cmd.lineage == [cmd] + + def test_lineage_setter_updates_lineage(self, session): + cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand') + new_lineage = [MagicMock(), cmd] + cmd.lineage = new_lineage + assert cmd.lineage == new_lineage + + def test_lineage_propagated_to_real_on_resolve(self, session, mock_module): + cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand') + new_lineage = [MagicMock(), cmd] + cmd.lineage = new_lineage + with patch('importlib.import_module', return_value=mock_module): + cmd.create_help_command() + assert mock_module.MyCommand.return_value.lineage == new_lineage + + def test_lineage_setter_propagates_to_already_resolved( + self, session, mock_module + ): + cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand') + with patch('importlib.import_module', return_value=mock_module): + cmd.create_help_command() + new_lineage = [MagicMock(), cmd] + cmd.lineage = new_lineage + assert mock_module.MyCommand.return_value.lineage == new_lineage + + def test_lineage_not_propagated_if_not_resolved(self, session): + cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand') + new_lineage = [MagicMock(), cmd] + # Should not raise even though underlying command is not resolved. + cmd.lineage = new_lineage + assert cmd.lineage == new_lineage From dab19fb02ba42ff07547dab9f07bb1f58f9389d7 Mon Sep 17 00:00:00 2001 From: aemous Date: Mon, 11 May 2026 13:56:34 -0400 Subject: [PATCH 2/5] Remove parts of functional/test_lazy.py that depend on LazyInitEmitter. --- tests/functional/test_lazy.py | 38 +---------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/tests/functional/test_lazy.py b/tests/functional/test_lazy.py index 3ba067e13aad..fc8d33edd359 100644 --- a/tests/functional/test_lazy.py +++ b/tests/functional/test_lazy.py @@ -12,44 +12,8 @@ # language governing permissions and limitations under the License. import pytest -from awscli.handlers_registry import MAIN_COMMAND_TABLE_OPS from awscli.lazy import LazyCommand -from awscli.testutils import BaseAWSHelpOutputTest, mock - -# Derive test parameters from MAIN_COMMAND_TABLE_OPS. -_ADD_OPS = [op for op in MAIN_COMMAND_TABLE_OPS if op[0] == 'add'] -_RENAME_OPS = [op for op in MAIN_COMMAND_TABLE_OPS if op[0] == 'rename'] -_ADD_CMD_NAMES = [op[1] for op in _ADD_OPS] -_RENAME_NEW_NAMES = [op[2] for op in _RENAME_OPS] - - -class TestLazyCommandHelpRenders(BaseAWSHelpOutputTest): - def test_added_command_help_renders(self): - for cmd_name in _ADD_CMD_NAMES: - with self.subTest(cmd_name=cmd_name): - self.driver.main([cmd_name, 'help']) - self.assert_contains(cmd_name) - - def test_renamed_command_help_renders(self): - for new_name in _RENAME_NEW_NAMES: - with self.subTest(new_name=new_name): - self.driver.main([new_name, 'help']) - self.assert_contains(new_name) - - -class TestLazyCommandIsTransparent(BaseAWSHelpOutputTest): - def test_added_commands_appear_in_top_level_help(self): - self.driver.main(['help']) - for cmd_name in _ADD_CMD_NAMES: - self.assert_contains(cmd_name) - - def test_lazy_command_has_subcommands(self): - command_table = self.driver.subcommand_table - s3_cmd = command_table['s3'] - assert isinstance(s3_cmd, LazyCommand) - subcommands = s3_cmd.subcommand_table - assert 'ls' in subcommands - assert 'cp' in subcommands +from awscli.testutils import mock class TestLazyCommandErrorPaths: From 494bd6a047a10694478ef663ba9a1ce720ebf156 Mon Sep 17 00:00:00 2001 From: aemous Date: Mon, 11 May 2026 14:04:28 -0400 Subject: [PATCH 3/5] Add license header to awscli/lazy.py. --- awscli/lazy.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/awscli/lazy.py b/awscli/lazy.py index 4cad0964443d..58eccd7df515 100644 --- a/awscli/lazy.py +++ b/awscli/lazy.py @@ -1,3 +1,15 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. import importlib from awscli.commands import CLICommand From 40f4027c699a18d34f28f3d6d0f7bb27090e4dc0 Mon Sep 17 00:00:00 2001 From: aemous Date: Wed, 13 May 2026 11:18:59 -0400 Subject: [PATCH 4/5] Switch new test suite to use PyTest instead of unittest.TestCase. --- tests/unit/botocore/test_hooks.py | 96 ++++++++++++++++--------------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/tests/unit/botocore/test_hooks.py b/tests/unit/botocore/test_hooks.py index 4defdd5511dc..cb7e1d738131 100644 --- a/tests/unit/botocore/test_hooks.py +++ b/tests/unit/botocore/test_hooks.py @@ -14,6 +14,8 @@ import functools from functools import partial +import pytest + from botocore.hooks import ( HierarchicalEmitter, PrefixTrie, @@ -588,52 +590,54 @@ def handler(a, b, **kwargs): ) -class TestPrefixTrie(unittest.TestCase): - def setUp(self): - self.trie = PrefixTrie() - - def test_append_and_prefix_search_exact_match(self): - self.trie.append_item('building-command-table.main', 'handler1') - results = self.trie.prefix_search('building-command-table.main') - self.assertIn('handler1', results) - - def test_prefix_search_matches_parent(self): - self.trie.append_item('building-command-table', 'handler1') - results = self.trie.prefix_search('building-command-table.main') - self.assertIn('handler1', results) - - def test_prefix_search_does_not_match_sibling(self): - self.trie.append_item('building-command-table.ecs', 'handler1') - results = self.trie.prefix_search('building-command-table.main') - self.assertNotIn('handler1', results) - - def test_prefix_search_does_not_match_child(self): - self.trie.append_item('building-command-table.main.sub', 'handler1') - results = self.trie.prefix_search('building-command-table.main') - self.assertNotIn('handler1', results) - - def test_wildcard_match(self): - self.trie.append_item('building-command-table.*', 'handler1') - results = self.trie.prefix_search('building-command-table.main') - self.assertIn('handler1', results) - - def test_multiple_items_at_same_key(self): - self.trie.append_item('building-command-table.main', 'handler1') - self.trie.append_item('building-command-table.main', 'handler2') - results = self.trie.prefix_search('building-command-table.main') - self.assertIn('handler1', results) - self.assertIn('handler2', results) - - def test_multiple_levels_all_returned(self): - self.trie.append_item('building-command-table', 'parent') - self.trie.append_item('building-command-table.main', 'exact') - results = self.trie.prefix_search('building-command-table.main') - self.assertIn('parent', results) - self.assertIn('exact', results) - - def test_empty_trie_returns_empty(self): - results = self.trie.prefix_search('building-command-table.main') - self.assertEqual(len(results), 0) +@pytest.fixture +def trie(): + return PrefixTrie() + + +class TestPrefixTrie: + def test_append_and_prefix_search_exact_match(self, trie): + trie.append_item('building-command-table.main', 'handler1') + results = trie.prefix_search('building-command-table.main') + assert 'handler1' in results + + def test_prefix_search_matches_parent(self, trie): + trie.append_item('building-command-table', 'handler1') + results = trie.prefix_search('building-command-table.main') + assert 'handler1' in results + + def test_prefix_search_does_not_match_sibling(self, trie): + trie.append_item('building-command-table.ecs', 'handler1') + results = trie.prefix_search('building-command-table.main') + assert 'handler1' not in results + + def test_prefix_search_does_not_match_child(self, trie): + trie.append_item('building-command-table.main.sub', 'handler1') + results = trie.prefix_search('building-command-table.main') + assert 'handler1' not in results + + def test_wildcard_match(self, trie): + trie.append_item('building-command-table.*', 'handler1') + results = trie.prefix_search('building-command-table.main') + assert 'handler1' in results + + def test_multiple_items_at_same_key(self, trie): + trie.append_item('building-command-table.main', 'handler1') + trie.append_item('building-command-table.main', 'handler2') + results = trie.prefix_search('building-command-table.main') + assert 'handler1' in results + assert 'handler2' in results + + def test_multiple_levels_all_returned(self, trie): + trie.append_item('building-command-table', 'parent') + trie.append_item('building-command-table.main', 'exact') + results = trie.prefix_search('building-command-table.main') + assert 'parent' in results + assert 'exact' in results + + def test_empty_trie_returns_empty(self, trie): + results = trie.prefix_search('building-command-table.main') + assert len(results) == 0 if __name__ == '__main__': From fdddde11812c7f6e9531a0b81c10b5d328d1b04f Mon Sep 17 00:00:00 2001 From: aemous Date: Wed, 13 May 2026 11:19:36 -0400 Subject: [PATCH 5/5] Formatting. --- tests/unit/botocore/test_hooks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/botocore/test_hooks.py b/tests/unit/botocore/test_hooks.py index cb7e1d738131..b55e98bd1e77 100644 --- a/tests/unit/botocore/test_hooks.py +++ b/tests/unit/botocore/test_hooks.py @@ -15,7 +15,6 @@ from functools import partial import pytest - from botocore.hooks import ( HierarchicalEmitter, PrefixTrie,