diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5469b3a..75bcba0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,7 +34,7 @@ jobs: run: "python -m tox" - name: "Report to coveralls" - # coverage is only created in the py39 environment + # coverage is only created in the py314 environment # --service=github is a workaround for bug # https://github.com/coveralls-clients/coveralls-python/issues/251 if: "matrix.python-version == '3.14'" diff --git a/.gitignore b/.gitignore index b250aa7..456e39d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ pip-wheel-metadata # Virtualenv /env/ -/src/ +/.venv/ # Unit test / coverage reports .cache @@ -20,4 +20,8 @@ htmlcov .tox # Sphinx documentation +/doc/_build/ /doc/build/ + +# Type checkers +.mypy_cache/ diff --git a/CHANGES.txt b/CHANGES.txt index 80ea5d5..661ce8c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,8 +1,12 @@ CHANGES ******* -0.3 (unreleased) ----------------- +1.0.0 (unreleased) +------------------ + +- Skip importing packages in scan() since they're already imported in walk_packages(). + +- Add type annotations. - Add support for Python 3.10 - 3.14. @@ -12,6 +16,10 @@ CHANGES - Use GitHub Actions for CI. +- Add tests for 100% coverage. + +- Update docs. + 0.2 (2020-01-29) ---------------- diff --git a/README.rst b/README.rst index f14e91b..955e4a6 100644 --- a/README.rst +++ b/README.rst @@ -20,4 +20,4 @@ import a package and its sub-modules and sub-packages. Documentation_. -.. _Documentation: http://importscan.readthedocs.org +.. _Documentation: https://importscan.readthedocs.org diff --git a/develop_requirements.txt b/develop_requirements.txt index e076727..68e0793 100644 --- a/develop_requirements.txt +++ b/develop_requirements.txt @@ -1,11 +1,8 @@ # development --e '.[test,coverage,lint]' +-e '.[test,coverage,lint,docs,mypy,pyright]' pre-commit tox >= 4 radon -# documentation -sphinx - # releaser zest.releaser[recommended] diff --git a/doc/_build/doctest/output.txt b/doc/_build/doctest/output.txt new file mode 100644 index 0000000..358007f --- /dev/null +++ b/doc/_build/doctest/output.txt @@ -0,0 +1,9 @@ +Results of doctest builder run on 2026-08-07 09:26:02 +===================================================== + +Doctest summary +=============== + 0 tests + 0 failures in tests + 0 failures in setup code + 0 failures in cleanup code diff --git a/doc/_build/doctrees/__intersphinx_cache__/bowerstatic_objects.inv b/doc/_build/doctrees/__intersphinx_cache__/bowerstatic_objects.inv new file mode 100644 index 0000000..bad868b Binary files /dev/null and b/doc/_build/doctrees/__intersphinx_cache__/bowerstatic_objects.inv differ diff --git a/doc/_build/doctrees/__intersphinx_cache__/reg_objects.inv b/doc/_build/doctrees/__intersphinx_cache__/reg_objects.inv new file mode 100644 index 0000000..1b57648 Binary files /dev/null and b/doc/_build/doctrees/__intersphinx_cache__/reg_objects.inv differ diff --git a/doc/_build/doctrees/api.doctree b/doc/_build/doctrees/api.doctree new file mode 100644 index 0000000..f7a4b53 Binary files /dev/null and b/doc/_build/doctrees/api.doctree differ diff --git a/doc/_build/doctrees/changes.doctree b/doc/_build/doctrees/changes.doctree new file mode 100644 index 0000000..3b016c9 Binary files /dev/null and b/doc/_build/doctrees/changes.doctree differ diff --git a/doc/_build/doctrees/developing.doctree b/doc/_build/doctrees/developing.doctree new file mode 100644 index 0000000..4992a69 Binary files /dev/null and b/doc/_build/doctrees/developing.doctree differ diff --git a/doc/_build/doctrees/environment.pickle b/doc/_build/doctrees/environment.pickle new file mode 100644 index 0000000..028b509 Binary files /dev/null and b/doc/_build/doctrees/environment.pickle differ diff --git a/doc/_build/doctrees/index.doctree b/doc/_build/doctrees/index.doctree new file mode 100644 index 0000000..9751228 Binary files /dev/null and b/doc/_build/doctrees/index.doctree differ diff --git a/doc/conf.py b/doc/conf.py index 8f723d2..701b690 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -10,7 +10,7 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import pkg_resources +from importlib import metadata # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -24,16 +24,20 @@ # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ["sphinx.ext.autodoc", "sphinx.ext.intersphinx"] +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.doctest", +] autoclass_content = "both" autodoc_member_order = "groupwise" intersphinx_mapping = { - "reg": ("http://reg.readthedocs.io/en/latest", None), - "webob": ("http://docs.webob.org/en/latest", None), - "bowerstatic": ("http://bowerstatic.readthedocs.io/en/latest", None), + "reg": ("https://reg.readthedocs.io/en/latest", None), + "webob": ("https://docs.pylonsproject.org/projects/webob/en/latest", None), + "bowerstatic": ("https://bowerstatic.readthedocs.io/en/latest", None), } # Add any paths that contain templates here, relative to this directory. @@ -60,7 +64,28 @@ # built documents. # # The short X.Y version. -version = pkg_resources.get_distribution("importscan").version +try: + version = metadata.version("importscan") +except metadata.PackageNotFoundError: + # Fallback for ReadTheDocs and other environments where the package isn't installed + # Try to get version from pyproject.toml + import os + import re + + try: + pyproject_path = os.path.join( + os.path.dirname(__file__), "..", "pyproject.toml" + ) + with open(pyproject_path) as f: + content = f.read() + # Simple regex to extract version + version_match = re.search(r'version\s*=\s*["\']([^"\']+)["\']', content) + if version_match: + version = version_match.group(1) + else: + version = "0.0.0" + except (FileNotFoundError, Exception): + version = "0.0.0" # The full version, including alpha/beta/rc tags. release = version @@ -69,7 +94,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: @@ -82,7 +107,7 @@ exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all -# documents. +# documents.n # default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. @@ -142,7 +167,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [] +html_static_path: list[str] = [] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied @@ -209,7 +234,7 @@ # -- Options for LaTeX output --------------------------------------------- -latex_elements = { +latex_elements: dict[str, str] = { # The paper size ('letterpaper' or 'a4paper'). # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). @@ -297,4 +322,4 @@ # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {"https://docs.python.org/": None} +# intersphinx_mapping = {"https://docs.python.org/": None} diff --git a/doc/developing.rst b/doc/developing.rst index b235452..cea7200 100644 --- a/doc/developing.rst +++ b/doc/developing.rst @@ -8,7 +8,7 @@ Install Importscan for development Clone Importscan from github:: - $ git clone git@github.com:faassen/importscan.git + $ git clone git@github.com:morepath/importscan.git If this doesn't work and you get an error 'Permission denied (publickey)', you need to upload your ssh public key to github_. @@ -21,15 +21,11 @@ Make sure you have virtualenv_ installed. Create a new virtualenv for Python 3 inside the importscan directory:: - $ virtualenv -p python3 env/py3 + $ python -m venv --upgrade-deps .venv Activate the virtualenv:: - $ source env/py3/bin/activate - -Make sure you have recent setuptools and pip installed:: - - $ pip install -U setuptools pip + $ source .venv/bin/activate Install the various dependencies and development tools from develop_requirements.txt:: @@ -42,9 +38,8 @@ For upgrading the requirements just run the command again. The following commands work only if you have the virtualenv activated. -.. _github: https://help.github.com/articles/generating-an-ssh-key - -.. _virtualenv: https://pypi.python.org/pypi/virtualenv +.. _github: https://docs.github.com/en/authentication/connecting-to-github-with-ssh +.. _virtualenv: https://pypi.org/project/virtualenv Install pre-commit hook for Black integration --------------------------------------------- @@ -54,7 +49,7 @@ install the `pre-commit hook`_ for Black integration before committing:: $ pre-commit install -.. _`pre-commit hook`: https://black.readthedocs.io/en/stable/version_control_integration.html +.. _`pre-commit hook`: https://black.readthedocs.io/en/stable/integrations/source_version_control.html Running the tests ----------------- @@ -71,7 +66,7 @@ You can then point your web browser to the ``htmlcov/index.html`` file in the project directory and click on modules to see detailed coverage information. -.. _`py.test`: http://pytest.org/latest/ +.. _`py.test`: https://pytest.org/latest/ Black ----- @@ -115,11 +110,11 @@ To also show cyclomatic complexity, use this command:: $ flake8 --max-complexity=10 importscan -.. _flake8: https://pypi.python.org/pypi/flake8 +.. _flake8: https://pypi.org/project/flake8 -.. _pyflakes: https://pypi.python.org/pypi/pyflakes +.. _pyflakes: https://pypi.org/project/pyflakes -.. _pep8: http://www.python.org/dev/peps/pep-0008/ +.. _pep8: https://peps.python.org/pep-0008 .. _`cyclomatic complexity`: https://en.wikipedia.org/wiki/Cyclomatic_complexity @@ -151,4 +146,4 @@ You can also specify a test environment to run e.g.:: $ tox -e lint $ tox -e coverage -.. _pyenv: https://github.com/yyuu/pyenv +.. _pyenv: https://github.com/pyenv/pyenv diff --git a/importscan/scan.py b/importscan/scan.py index f081af5..424607c 100644 --- a/importscan/scan.py +++ b/importscan/scan.py @@ -2,13 +2,14 @@ import sys from pkgutil import iter_modules -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast if TYPE_CHECKING: from collections.abc import Callable, Generator, Iterable from importlib.abc import Loader from types import ModuleType from typing_extensions import TypeIs + from importscan.types import IgnoreModule, ModuleInfo, StrOrBytesPath @@ -119,23 +120,26 @@ def handle_error(name, e): # into iter_modules for submodules/subpackages. There's also # the additional issue that not all finders will implement the # non-standard iter_modules method, but without it there's - # no way to list all of the modules. Also since walk_packages - # already imports all of the packages, why are we importing - # them again here? Shouldn't we only import modules here? - # Also why do we do only use `import_module` here, but not - # in `walk_packages`? Doesn't that mean that the additional - # check in `import_module` doesn't do anything for packages? - loader = importer.find_spec(modname).loader # type: ignore + # no way to list all of the modules. + # + # Note: Packages are already imported in walk_packages() to access + # their __path__ for recursion and have already been error-checked. + # Only modules need to be imported here. + if ispkg: + continue + + spec = importer.find_spec(modname, None) + loader = spec.loader if spec else None assert loader is not None try: import_module(modname, loader, handle_error) finally: if hasattr(loader, "file") and hasattr( - loader.file, # pyright: ignore[reportAttributeAccessIssue] + getattr(loader, "file"), "close", ): - loader.file.close() # pyright: ignore[reportAttributeAccessIssue] + getattr(loader, "file").close() def import_module( @@ -143,13 +147,17 @@ def import_module( loader: Loader, handle_error: Callable[[str, Exception], object] | None, ) -> None: - get_filename = getattr(loader, "get_filename", None) + get_filename: Callable[..., str] | None = getattr( + loader, "get_filename", None + ) if get_filename is None: get_filename = loader._get_filename # type: ignore[attr-defined] try: - fn = get_filename(modname) + fn: str = cast( + str, get_filename(modname) # pyright: ignore[reportOptionalCall] + ) except TypeError: - fn = get_filename() + fn = cast(str, get_filename()) # pyright: ignore[reportOptionalCall] # only scan non-orphaned source files and package directories if fn.endswith((".pyc", ".pyo", "$py.class")): return diff --git a/importscan/tests/fixtures/importerror_pkg/__init__.py b/importscan/tests/fixtures/importerror_pkg/__init__.py new file mode 100644 index 0000000..c51def1 --- /dev/null +++ b/importscan/tests/fixtures/importerror_pkg/__init__.py @@ -0,0 +1 @@ +# pkg diff --git a/importscan/tests/fixtures/importerror_pkg/sub/__init__.py b/importscan/tests/fixtures/importerror_pkg/sub/__init__.py new file mode 100644 index 0000000..abe851f --- /dev/null +++ b/importscan/tests/fixtures/importerror_pkg/sub/__init__.py @@ -0,0 +1 @@ +raise ImportError("cannot import sub") diff --git a/importscan/tests/fixtures/importerror_pkg_handle_error/__init__.py b/importscan/tests/fixtures/importerror_pkg_handle_error/__init__.py new file mode 100644 index 0000000..c51def1 --- /dev/null +++ b/importscan/tests/fixtures/importerror_pkg_handle_error/__init__.py @@ -0,0 +1 @@ +# pkg diff --git a/importscan/tests/fixtures/importerror_pkg_handle_error/sub/__init__.py b/importscan/tests/fixtures/importerror_pkg_handle_error/sub/__init__.py new file mode 100644 index 0000000..abe851f --- /dev/null +++ b/importscan/tests/fixtures/importerror_pkg_handle_error/sub/__init__.py @@ -0,0 +1 @@ +raise ImportError("cannot import sub") diff --git a/importscan/tests/test_importscan.py b/importscan/tests/test_importscan.py index 5db700a..61b55a4 100644 --- a/importscan/tests/test_importscan.py +++ b/importscan/tests/test_importscan.py @@ -5,10 +5,14 @@ import re import sys from typing import TYPE_CHECKING +from unittest.mock import MagicMock, patch import pytest from pytest import raises + from importscan import scan +from importscan.scan import import_module + from . import fixtures if TYPE_CHECKING: @@ -185,3 +189,85 @@ def test_module_in_zipped() -> None: scan(moduleinzipped) assert fixtures.calls == 1 + + +def test_scan_loader_file_close() -> None: + # __init__.py overwrites the 'scan' attribute with the function, so + # `import importscan.scan as x` resolves via getattr and yields the + # function; sys.modules gives the actual module object. + scan_mod = sys.modules["importscan.scan"] + + from .fixtures import package + + mock_file = MagicMock() + mock_loader = MagicMock() + mock_loader.file = mock_file + mock_spec = MagicMock() + mock_spec.loader = mock_loader + mock_importer = MagicMock() + mock_importer.find_spec.return_value = mock_spec + + # patch.object avoids ambiguity from importscan.__init__ re-exporting scan + with patch.object(scan_mod, "walk_packages") as mock_wp: + mock_wp.return_value = iter( + [(mock_importer, "importscan.tests.fixtures.package.module", False)] + ) + with patch.object(scan_mod, "import_module"): + scan(package) + + mock_file.close.assert_called_once() + + +class _LoaderUnderscoreGetFilename: + def _get_filename(self, modname: str | None = None) -> str: + return "some_module.py" + + +class _LoaderGetFilenameTypeError: + def get_filename(self, modname: str | None = None) -> str: + if modname is not None: + raise TypeError + return "some_module.py" + + +class _LoaderPycFilename: + def get_filename(self, modname: str | None = None) -> str: + return "some_module.pyc" + + +def test_import_module_underscore_get_filename() -> None: + loader = _LoaderUnderscoreGetFilename() + with patch("builtins.__import__"): + import_module("somemodule", loader, None) # type: ignore[arg-type] + + +def test_import_module_get_filename_typeerror() -> None: + loader = _LoaderGetFilenameTypeError() + with patch("builtins.__import__"): + import_module("somemodule", loader, None) # type: ignore[arg-type] + + +def test_import_module_pyc_skipped() -> None: + loader = _LoaderPycFilename() + with patch("builtins.__import__") as mock_import: + import_module("somemodule", loader, None) # type: ignore[arg-type] + mock_import.assert_not_called() + + +def test_walk_packages_importerror_subpackage() -> None: + from .fixtures import importerror_pkg + + with raises(ImportError): + scan(importerror_pkg) + + +def test_walk_packages_importerror_subpackage_handle_error() -> None: + from .fixtures import importerror_pkg_handle_error + + errors: list[str] = [] + + def handle_error(name: str, e: Exception) -> None: + errors.append(name) + + scan(importerror_pkg_handle_error, handle_error=handle_error) + assert len(errors) == 1 diff --git a/pyproject.toml b/pyproject.toml index acb45e6..da3a6a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ Changelog = "https://github.com/morepath/importscan/blob/master/CHANGES.txt" [project.optional-dependencies] test = ["pytest >= 8", "pytest-env"] coverage = ["pytest-cov"] -lint = ["black", "flake8", "flake8-pyproject"] +lint = ["black", "flake8", "flake8-pyproject", "isort"] docs = ["sphinx"] mypy = ["mypy", "pytest"] pyright = ["pyright", "pytest"] @@ -71,15 +71,20 @@ max-line-length = 88 python_version = "3.10" strict = true warn_unreachable = true +warn_unused_ignores = true [[tool.mypy.overrides]] module = "importscan.tests.fixtures.*" ignore_errors = true [tool.pyright] +typeCheckingMode = "strict" +venvPath = "." +venv = ".venv" exclude = [ - "**/tests/fixtures/**", + "**/tests/fixtures/**", "**/__pycache__", ] +reportUnnecessaryTypeIgnoreComment = "warning" [tool.tox] requires = ["tox>=4"] @@ -154,3 +159,29 @@ commands = [ ["pyright", "importscan", "--pythonversion", "3.13"], ["pyright", "importscan", "--pythonversion", "3.14"], ] + +[tool.black] +target-version = ['py310', 'py311', 'py312', 'py313', 'py314'] +include = '\.pyi?$' +exclude = ''' +( + /( + \.git + | \.tox + | env + | venv + | .venv + | __pycache__ + | build + | dist + | src + )/ +) +''' + +[tool.isort] +profile = 'black' +py_version = 310 +skip_gitignore = true +extra_standard_library = ["typing_extensions"] +known_first_party = ["dectate", "importscan", "reg"]