[v2] Implement LazyCommand for deferring modules until their commands are invoked, and make PrefixTrie public

awscli/botocore/hooks.py

@@ -196,7 +196,7 @@ def __init__(self):
         # read only access (we never modify self._handlers).
         # A cache of event name to handler list.
         self._lookup_cache = {}
-        self._handlers = _PrefixTrie()
+        self._handlers = PrefixTrie()
         # This is used to ensure that unique_id's are only
         # registered once.
         self._unique_id_handlers = {}
@@ -398,7 +398,7 @@ def __copy__(self):
         return new_instance
 
 
-class _PrefixTrie:
+class PrefixTrie:
     """Specialized prefix trie that handles wildcards.
 
     The prefixes in this case are based on dot separated

awscli/lazy.py

@@ -0,0 +1,72 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import importlib
+
+from awscli.commands import CLICommand
+
+
+class LazyCommand(CLICommand):
+    """A command-table entry that defers importing its real implementation.
+
+    Sits in the command table like any other CLICommand, but only imports
+    the actual module (and creates the real command object) when the command
+    is invoked or its help is accessed.
+    """
+
+    def __init__(self, name, session, module_path, class_name):
+        self._name = name
+        self._session = session
+        self._module_path = module_path
+        self._class_name = class_name
+        self._real = None
+        self._lineage = [self]
+
+    def _resolve(self):
+        if self._real is None:
+            mod = importlib.import_module(self._module_path)
+            cls = getattr(mod, self._class_name)
+            self._real = cls(self._session)
+            self._real.lineage = self._lineage
+        return self._real
+
+    def __call__(self, args, parsed_globals):
+        return self._resolve()(args, parsed_globals)
+
+    def create_help_command(self):
+        return self._resolve().create_help_command()
+
+    @property
+    def arg_table(self):
+        return self._resolve().arg_table
+
+    @property
+    def subcommand_table(self):
+        return self._resolve().subcommand_table
+
+    @property
+    def name(self):
+        return self._name
+
+    @name.setter
+    def name(self, value):
+        self._name = value
+
+    @property
+    def lineage(self):
+        return self._lineage
+
+    @lineage.setter
+    def lineage(self, value):
+        self._lineage = value
+        if self._real is not None:
+            self._real.lineage = value

tests/functional/test_lazy.py

@@ -0,0 +1,51 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import pytest
+
+from awscli.lazy import LazyCommand
+from awscli.testutils import mock
+
+
+class TestLazyCommandErrorPaths:
+    def test_invalid_module_path_raises_on_resolve(self):
+        session = mock.MagicMock()
+        cmd = LazyCommand(
+            'bad-cmd',
+            session,
+            'awscli.nonexistent.module',
+            'FakeCommand',
+        )
+        with pytest.raises(ModuleNotFoundError):
+            cmd([], mock.MagicMock())
+
+    def test_invalid_class_name_raises_on_resolve(self):
+        session = mock.MagicMock()
+        cmd = LazyCommand(
+            'bad-cmd',
+            session,
+            'awscli.customizations.dynamodb.ddb',
+            'NonexistentClass',
+        )
+        with pytest.raises(AttributeError):
+            cmd([], mock.MagicMock())
+
+    def test_invalid_module_path_raises_on_help(self):
+        session = mock.MagicMock()
+        cmd = LazyCommand(
+            'bad-cmd',
+            session,
+            'awscli.nonexistent.module',
+            'FakeCommand',
+        )
+        with pytest.raises(ModuleNotFoundError):
+            cmd.create_help_command()

tests/unit/botocore/test_hooks.py

@@ -14,7 +14,12 @@
 import functools
 from functools import partial
 
-from botocore.hooks import HierarchicalEmitter, first_non_none_response
+import pytest
+from botocore.hooks import (
+    HierarchicalEmitter,
+    PrefixTrie,
+    first_non_none_response,
+)
 
 from tests import unittest
 
@@ -584,5 +589,55 @@ def handler(a, b, **kwargs):
         )
 
 
+@pytest.fixture
+def trie():
+    return PrefixTrie()
+
+
+class TestPrefixTrie:
+    def test_append_and_prefix_search_exact_match(self, trie):
+        trie.append_item('building-command-table.main', 'handler1')
+        results = trie.prefix_search('building-command-table.main')
+        assert 'handler1' in results
+
+    def test_prefix_search_matches_parent(self, trie):
+        trie.append_item('building-command-table', 'handler1')
+        results = trie.prefix_search('building-command-table.main')
+        assert 'handler1' in results
+
+    def test_prefix_search_does_not_match_sibling(self, trie):
+        trie.append_item('building-command-table.ecs', 'handler1')
+        results = trie.prefix_search('building-command-table.main')
+        assert 'handler1' not in results
+
+    def test_prefix_search_does_not_match_child(self, trie):
+        trie.append_item('building-command-table.main.sub', 'handler1')
+        results = trie.prefix_search('building-command-table.main')
+        assert 'handler1' not in results
+
+    def test_wildcard_match(self, trie):
+        trie.append_item('building-command-table.*', 'handler1')
+        results = trie.prefix_search('building-command-table.main')
+        assert 'handler1' in results
+
+    def test_multiple_items_at_same_key(self, trie):
+        trie.append_item('building-command-table.main', 'handler1')
+        trie.append_item('building-command-table.main', 'handler2')
+        results = trie.prefix_search('building-command-table.main')
+        assert 'handler1' in results
+        assert 'handler2' in results
+
+    def test_multiple_levels_all_returned(self, trie):
+        trie.append_item('building-command-table', 'parent')
+        trie.append_item('building-command-table.main', 'exact')
+        results = trie.prefix_search('building-command-table.main')
+        assert 'parent' in results
+        assert 'exact' in results
+
+    def test_empty_trie_returns_empty(self, trie):
+        results = trie.prefix_search('building-command-table.main')
+        assert len(results) == 0
+
+
 if __name__ == '__main__':
     unittest.main()

tests/unit/test_lazy.py

@@ -0,0 +1,139 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from awscli.lazy import LazyCommand
+
+
+@pytest.fixture
+def session():
+    return MagicMock()
+
+
+@pytest.fixture
+def mock_command_class():
+    cls = MagicMock()
+    instance = MagicMock()
+    cls.return_value = instance
+    return cls
+
+
+@pytest.fixture
+def mock_module(mock_command_class):
+    module = MagicMock()
+    module.MyCommand = mock_command_class
+    return module
+
+
+class TestLazyCommandResolution:
+    def test_does_not_import_on_construction(self, session):
+        with patch('importlib.import_module') as imp:
+            LazyCommand('cmd', session, 'some.module', 'MyCommand')
+        imp.assert_not_called()
+
+    def test_imports_module_on_call(self, session, mock_module):
+        cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand')
+        with patch('importlib.import_module', return_value=mock_module):
+            cmd(['arg1'], MagicMock())
+        mock_module.MyCommand.assert_called_once_with(session)
+
+    def test_imports_module_on_help(self, session, mock_module):
+        cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand')
+        with patch('importlib.import_module', return_value=mock_module):
+            cmd.create_help_command()
+        mock_module.MyCommand.assert_called_once_with(session)
+
+    def test_imports_module_on_arg_table(self, session, mock_module):
+        cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand')
+        with patch('importlib.import_module', return_value=mock_module):
+            _ = cmd.arg_table
+        mock_module.MyCommand.assert_called_once_with(session)
+
+    def test_imports_module_on_subcommand_table(self, session, mock_module):
+        cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand')
+        with patch('importlib.import_module', return_value=mock_module):
+            _ = cmd.subcommand_table
+        mock_module.MyCommand.assert_called_once_with(session)
+
+    def test_resolves_only_once(self, session, mock_module):
+        cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand')
+        with patch('importlib.import_module', return_value=mock_module) as imp:
+            cmd(['arg1'], MagicMock())
+            cmd(['arg2'], MagicMock())
+        imp.assert_called_once_with('some.module')
+
+    def test_delegates_call_to_real_command(self, session, mock_module):
+        cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand')
+        args = ['arg1']
+        parsed_globals = MagicMock()
+        with patch('importlib.import_module', return_value=mock_module):
+            cmd(args, parsed_globals)
+        mock_module.MyCommand.return_value.assert_called_once_with(
+            args, parsed_globals
+        )
+
+    def test_delegates_help_to_real_command(self, session, mock_module):
+        cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand')
+        with patch('importlib.import_module', return_value=mock_module):
+            result = cmd.create_help_command()
+        assert (
+            result == mock_module.MyCommand.return_value.create_help_command()
+        )
+
+
+class TestLazyCommandProperties:
+    def test_name_returns_initial_name(self, session):
+        cmd = LazyCommand('my-cmd', session, 'some.module', 'MyCommand')
+        assert cmd.name == 'my-cmd'
+
+    def test_name_setter_updates_name(self, session):
+        cmd = LazyCommand('old-name', session, 'some.module', 'MyCommand')
+        cmd.name = 'new-name'
+        assert cmd.name == 'new-name'
+
+    def test_lineage_defaults_to_self(self, session):
+        cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand')
+        assert cmd.lineage == [cmd]
+
+    def test_lineage_setter_updates_lineage(self, session):
+        cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand')
+        new_lineage = [MagicMock(), cmd]
+        cmd.lineage = new_lineage
+        assert cmd.lineage == new_lineage
+
+    def test_lineage_propagated_to_real_on_resolve(self, session, mock_module):
+        cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand')
+        new_lineage = [MagicMock(), cmd]
+        cmd.lineage = new_lineage
+        with patch('importlib.import_module', return_value=mock_module):
+            cmd.create_help_command()
+        assert mock_module.MyCommand.return_value.lineage == new_lineage
+
+    def test_lineage_setter_propagates_to_already_resolved(
+        self, session, mock_module
+    ):
+        cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand')
+        with patch('importlib.import_module', return_value=mock_module):
+            cmd.create_help_command()
+        new_lineage = [MagicMock(), cmd]
+        cmd.lineage = new_lineage
+        assert mock_module.MyCommand.return_value.lineage == new_lineage
+
+    def test_lineage_not_propagated_if_not_resolved(self, session):
+        cmd = LazyCommand('cmd', session, 'some.module', 'MyCommand')
+        new_lineage = [MagicMock(), cmd]
+        # Should not raise even though underlying command is not resolved.
+        cmd.lineage = new_lineage
+        assert cmd.lineage == new_lineage