diff --git a/.github/scripts/verify_provenance.py b/.github/scripts/verify_provenance.py new file mode 100644 index 0000000..c768517 --- /dev/null +++ b/.github/scripts/verify_provenance.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Verify that every file of a PyPI release carries PEP 740 provenance. + +The script talks to the public PyPI Integrity API: + + https://pypi.org/integrity/{project}/{version}/{filename}/provenance + +For each distribution file of the given release it checks that at least one +attestation bundle is present and prints the trusted publisher identity the +attestations are bound to. This is what makes the "provenance" indicator on +the PyPI project page show up, so the release workflow runs it after every +upload. + +Exit codes: + 0 - every file is attested (or --allow-missing was given) + 1 - at least one file has no provenance + 2 - the release could not be queried at all + +Usage: + python .github/scripts/verify_provenance.py --project localsitemap --version 2.1.0 +""" + +import argparse +import json +import sys +import time +import urllib.error +import urllib.request + +PYPI_JSON = "https://pypi.org/pypi/{project}/{version}/json" +PYPI_INTEGRITY = "https://pypi.org/integrity/{project}/{version}/{filename}/provenance" +USER_AGENT = "localsitemap-release-check/1.0" + + +def _get_json(url, timeout=30): + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + + +def _release_files(project, version): + data = _get_json(PYPI_JSON.format(project=project, version=version)) + return [item["filename"] for item in data.get("urls", [])] + + +def _provenance(project, version, filename): + url = PYPI_INTEGRITY.format(project=project, version=version, filename=filename) + try: + return _get_json(url) + except urllib.error.HTTPError as error: + if error.code == 404: + return None + raise + + +def _describe(bundles): + descriptions = [] + for bundle in bundles: + publisher = bundle.get("publisher") or {} + parts = [ + str(publisher.get("kind", "unknown")), + str(publisher.get("repository", "")), + str(publisher.get("workflow", "")), + ] + environment = publisher.get("environment") + if environment: + parts.append("env=%s" % environment) + descriptions.append(" ".join(part for part in parts if part)) + return "; ".join(descriptions) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project", default="localsitemap") + parser.add_argument("--version", required=True) + parser.add_argument( + "--allow-missing", + action="store_true", + help="do not fail when a file has no provenance", + ) + parser.add_argument( + "--attempts", type=int, default=5, + help="how many times to retry while PyPI propagates a new release", + ) + parser.add_argument("--delay", type=float, default=6.0, help="seconds between retries") + args = parser.parse_args(argv) + + filenames = [] + for attempt in range(1, max(1, args.attempts) + 1): + try: + filenames = _release_files(args.project, args.version) + if filenames: + break + except (urllib.error.URLError, ValueError) as error: + print("could not query PyPI (%s), attempt %d" % (error, attempt)) + if attempt < args.attempts: + time.sleep(args.delay) + + if not filenames: + print( + "error: no files found for %s %s on PyPI" % (args.project, args.version), + file=sys.stderr, + ) + return 2 + + missing = [] + for filename in filenames: + data = None + for attempt in range(1, max(1, args.attempts) + 1): + try: + data = _provenance(args.project, args.version, filename) + if data: + break + except (urllib.error.URLError, ValueError) as error: + print(" %s: query failed (%s), attempt %d" % (filename, error, attempt)) + if attempt < args.attempts: + time.sleep(args.delay) + + bundles = (data or {}).get("attestation_bundles") or [] + if bundles: + print("OK %s -> %s" % (filename, _describe(bundles))) + else: + print("MISS %s has no provenance on PyPI" % filename) + missing.append(filename) + + if missing: + if args.allow_missing: + print("warning: %d file(s) without provenance" % len(missing)) + return 0 + print( + "error: %d file(s) were published without provenance: %s" + % (len(missing), ", ".join(missing)), + file=sys.stderr, + ) + return 1 + + print("All %d file(s) are attested with verifiable provenance." % len(filenames)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml index 4a9ec6b..12efaa0 100644 --- a/.github/workflows/build_and_publish.yml +++ b/.github/workflows/build_and_publish.yml @@ -1,34 +1,142 @@ name: Publish Python 🐍 distributions 📦 to PyPI +# Publishing uses PyPI Trusted Publishing (OIDC) - no API token is stored in +# this repository. The publisher configured on PyPI must match: +# owner/repository : Infinitode/LocalSiteMap +# workflow : build_and_publish.yml +# environment : (left blank - therefore no `environment:` key below) +# +# Every upload is signed with PEP 740 attestations, which is what powers the +# "provenance" indicator on the PyPI project page. + on: push: tags: - - '*' + - 'v[0-9]*' + - '[0-9]*' + workflow_dispatch: + +permissions: + contents: read jobs: - build-n-publish: - name: Build and publish Python 🐍 distributions 📦 to PyPI + build: + name: Build distribution 📦 + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install build tooling + run: >- + python -m pip install --upgrade + build + twine + setuptools + wheel + + - name: Run the test suite + run: python -m unittest -v test_localsitemap + + - name: Determine the release version + id: version + run: | + set -euo pipefail + if [[ "${GITHUB_REF}" == refs/tags/* ]]; then + VERSION="${GITHUB_REF_NAME#v}" + else + VERSION="$(python -c 'from localsitemap._version import __version__; print(__version__)')" + fi + python - "${VERSION}" <<'PY' + import pathlib + import re + import sys + + version = sys.argv[1].strip() + pattern = r"^[0-9]+(\.[0-9]+)*((a|b|rc)[0-9]+)?(\.post[0-9]+)?(\.dev[0-9]+)?$" + if not re.match(pattern, version): + sys.exit("Invalid release version: %r" % version) + + target = pathlib.Path("localsitemap/_version.py") + target.write_text( + '"""Single source of truth for the LocalSiteMap version.\n\n' + "The release workflow rewrites this file from the pushed git tag, so the\n" + 'version published to PyPI always matches the tag.\n"""\n\n' + '__version__ = "%s"\n' % version, + encoding="utf-8", + ) + print("Building version", version) + PY + echo "version=${VERSION}" >> "${GITHUB_OUTPUT}" + + - name: Build sdist and wheel + run: python -m build + + - name: Check the distribution metadata + run: python -m twine check --strict dist/* + + - name: Upload the distribution artifacts + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + if-no-files-found: error + + publish: + name: Publish distribution 📦 to PyPI + needs: build + # Only real tag pushes publish. A manual "Run workflow" therefore builds, + # tests and validates the distributions without touching PyPI, which also + # makes it safe to run before Trusted Publishing is configured. + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + # NOTE: intentionally no `environment:` key. The PyPI trusted publisher is + # configured with a blank environment name, and adding one here would make + # the OIDC token claim mismatch and the upload fail. + permissions: + id-token: write # required for Trusted Publishing and attestations + steps: + - name: Download the distribution artifacts + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + attestations: true + print-hash: true + + verify-provenance: + name: Verify published provenance 🛡️ + needs: + - build + - publish + if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@master - - name: Set up Python 3.12 - uses: actions/setup-python@v3 - with: - python-version: '3.12' - - name: Install pypa/setuptools - run: >- - python -m - pip install setuptools wheel - - name: Extract tag name - id: tag - run: echo ::set-output name=TAG_NAME::$(echo $GITHUB_REF | cut -d / -f 3) - - name: Update version in setup.py - run: >- - sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" setup.py - - name: Build a binary wheel - run: >- - python setup.py sdist bdist_wheel - - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Confirm every published file carries provenance + env: + VERSION: ${{ needs.build.outputs.version }} + run: >- + python .github/scripts/verify_provenance.py + --project localsitemap + --version "${VERSION}" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..fa6269b --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,92 @@ +name: Tests + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install ruff + run: python -m pip install --upgrade ruff + + - name: Run ruff + run: ruff check . + + test: + name: Test on Python ${{ matrix.python-version }} (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # Python 3.9 ships in the tool cache of the 22.04 image. + - os: ubuntu-22.04 + python-version: '3.9' + - os: ubuntu-latest + python-version: '3.10' + - os: ubuntu-latest + python-version: '3.11' + - os: ubuntu-latest + python-version: '3.12' + - os: ubuntu-latest + python-version: '3.13' + - os: ubuntu-latest + python-version: '3.14' + - os: windows-latest + python-version: '3.12' + steps: + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Run the test suite + run: python -m unittest -v test_localsitemap + + build: + name: Build the distributions + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install build tooling + run: python -m pip install --upgrade build twine + + - name: Build sdist and wheel + run: python -m build + + - name: Check the distribution metadata + run: python -m twine check --strict dist/* + + - name: Install the built wheel and smoke test it + run: | + python -m pip install --force-reinstall dist/*.whl + cd "${RUNNER_TEMP}" + python -m localsitemap --version + python -c "import localsitemap; print(localsitemap.__version__)" diff --git a/.gitignore b/.gitignore index 2f92536..47c03d3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,15 @@ -__pycache__ -*.pyc -dist/ +__pycache__/ +*.py[cod] +*.egg-info/ build/ +dist/ venv/ -.venv/ \ No newline at end of file +.venv/ +.tox/ +.nox/ +.pytest_cache/ +.mypy_cache/ +.coverage +htmlcov/ + +.ruff_cache/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bee6980..e77c1ca 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,13 +8,19 @@ Thank you for considering contributing to **LocalSiteMap**! Contributions, wheth Create a fork of this repository to your GitHub account and clone it locally. 2. **Set Up the Environment** - Ensure you have Python 3.6+ installed. You should also install compatible versions of the necessary dependencies for the project. + Ensure you have Python 3.9+ installed. LocalSiteMap has no runtime dependencies, so there is nothing else to install for development (a virtual environment is still recommended). 3. **Run Tests** - Before making changes, run tests to confirm that everything works. + Before making changes, run tests to confirm that everything works: + + ```bash + python -m unittest -v test_localsitemap + ``` + + The same suite runs on GitHub Actions for Python 3.9 – 3.14 on Linux, plus Windows. > [!TIP] -> We sometimes include testing scripts in our projects for easy testing. +> We sometimes include testing scripts in our projects for easy testing. `test.py` is a small manual demo you can point at any directory. ## How to Contribute @@ -33,6 +39,38 @@ Thank you for considering contributing to **LocalSiteMap**! Contributions, wheth - Submit a pull request with a detailed explanation of what you've changed or added. - Ensure your PR passes all automated tests and adheres to the contribution guidelines. +## Building Locally + +All project metadata lives in `pyproject.toml`, and the version is read from `localsitemap/_version.py`. + +```bash +python -m pip install --upgrade build twine +python -m build +python -m twine check --strict dist/* +``` + +## Releasing + +1. Update `__version__` in `localsitemap/_version.py` and the changelog in `README.md`. +2. Commit and push, then push a matching tag (`2.1.0` or `v2.1.0`): + + ```bash + git tag 2.1.0 + git push origin 2.1.0 + ``` + +3. The `Publish Python 🐍 distributions 📦 to PyPI` workflow builds the sdist and wheel, runs the tests, and publishes to PyPI using **Trusted Publishing** — no API token or repository secret is required. + + The publisher configured on PyPI must match this repository: + + | Field | Value | + | --- | --- | + | Owner / Repository | `Infinitode/LocalSiteMap` | + | Workflow | `build_and_publish.yml` | + | Environment | *(leave blank — the workflow does not set `environment:`)* | + +4. The workflow signs every uploaded file with a PEP 740 attestation and then runs `.github/scripts/verify_provenance.py`, which fails the release if any published file is missing provenance. That is what makes the provenance badge and the PyPI "provenance" indicator accurate. + ## Community Guidelines To maintain a positive and collaborative community, we ask that all contributors adhere to the following principles: @@ -44,6 +82,6 @@ To maintain a positive and collaborative community, we ask that all contributors Offer helpful and actionable suggestions during code reviews. 3. **Follow Licensing Requirements** - Ensure that any derivative works comply with the [license](https://github.com/infinitode/localsitemap/blob/main/LICENSE). + Ensure that any derivative works comply with the [license](https://github.com/Infinitode/LocalSiteMap/blob/main/LICENSE). -By contributing, you agree to abide by these guidelines and this project's [license](https://github.com/infinitode/localsitemap/blob/main/LICENSE). Thank you for helping make **LocalSiteMap** better! +By contributing, you agree to abide by these guidelines and this project's [license](https://github.com/Infinitode/LocalSiteMap/blob/main/LICENSE). Thank you for helping make **LocalSiteMap** better! diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..60bbc8c --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include CONTRIBUTING.md +include LICENSE +include README.md +include test.py +include test_localsitemap.py +recursive-include .github/scripts *.py diff --git a/README.md b/README.md index 33c7e1c..b61dd7e 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,44 @@ # LocalSiteMap -![Python Version](https://img.shields.io/badge/python-3.13-blue.svg) -[![Code Size](https://img.shields.io/github/languages/code-size/infinitode/localsitemap)](https://github.com/infinitode/localsitemap) -![Downloads](https://pepy.tech/badge/localsitemap) -![License Compliance](https://img.shields.io/badge/license-compliance-brightgreen.svg) -![PyPI Version](https://img.shields.io/pypi/v/localsitemap) -LocalSiteMap is an open-source Python package designed for generating sitemaps and sitemap indexes from local files. It crawls your root website directory and automatically builds standard-compliant sitemaps. +[![PyPI Version](https://img.shields.io/pypi/v/localsitemap)](https://pypi.org/project/localsitemap/) +[![PyPI - Python Versions](https://img.shields.io/pypi/pyversions/localsitemap)](https://pypi.org/project/localsitemap/) +[![PyPI provenance](https://img.shields.io/badge/PyPI%20provenance-attested-blueviolet?logo=pypi&logoColor=white)](https://pypi.org/project/localsitemap/#files) +[![Tests](https://github.com/Infinitode/LocalSiteMap/actions/workflows/tests.yml/badge.svg)](https://github.com/Infinitode/LocalSiteMap/actions/workflows/tests.yml) +[![Downloads](https://pepy.tech/badge/localsitemap)](https://pepy.tech/project/localsitemap) +[![Code Size](https://img.shields.io/github/languages/code-size/infinitode/localsitemap)](https://github.com/Infinitode/LocalSiteMap) +![License](https://img.shields.io/badge/license-MIT%20(modified)-brightgreen.svg) -### Changes in version 2.0.0 (Major Release): +LocalSiteMap is an open-source Python package designed for generating sitemaps and sitemap indexes from local files. It crawls your root website directory and automatically builds standard-compliant sitemaps. No third-party dependencies are required. + +> [!IMPORTANT] +> `LocalSiteMap` crawls any directory found in the root folder, be sure to add exclusions for important directories, or hidden/resource/file directories. + +## Changes in version 2.1.0 + +### Breaking change +- **Python 3.9 or newer is now required.** Python 3.6 – 3.8 are long past end-of-life and are no longer supported by modern packaging tooling. Version `2.0.0` on PyPI still serves those interpreters (pip selects it automatically), and the runtime code has no 3.9-only syntax. + +### Bug fixes +- **Exclusions no longer match parent folders**: exclusions are now resolved *inside* the site root, so a site checked out in (for example) a folder called `media` is not wiped out when `media` is excluded. +- **Priority map keys now behave as documented**: folder keys may be written with a trailing slash (`"blog/"`), Windows separators are accepted, and extension keys are matched case-insensitively (`".PHP"` matches `.php`). +- **URLs are percent-encoded**: file and folder names containing spaces, `#`, `&`, `?` or non-ASCII characters (e.g. `my page.html`, `café/`) now produce valid, crawlable URLs. +- **Extension matching is case-insensitive**: `PAGE.HTML` is picked up by the default configuration, and `page.html.bak` is not mistaken for `page.html`. +- **Inputs are validated**: a missing `root_path`, a `base_url` without a scheme, or a non-positive `max_urls` now raise a clear error instead of silently writing a broken sitemap. `max_urls` above the protocol limit of 50 000 is clamped with a warning. +- **Priorities are clamped** to the `0.00`–`1.00` range required by the sitemap protocol. +- **Deterministic, deduplicated output**: entries are sorted, duplicate `` values are dropped, and repeated runs produce byte-identical files. +- **Stale split sitemaps are removed**: shrinking a site no longer leaves orphaned `sitemap_N.xml` parts behind. +- **Output is written atomically** and missing output directories are created, so an interrupted run cannot leave a truncated sitemap behind. +- **Much faster XML writing**: the DOM round-trip (`minidom`) was replaced with a streaming writer, which also removes the stray blank lines from the output. + +### New features +- **Command line interface**: `localsitemap` / `python -m localsitemap` with exclusions, extensions, priorities and a JSON priority map. +- **Glob exclusions**: `["draft-*", "*.bak"]`. +- **New options**: `exclude_hidden`, `include_homepage`, `sort_entries`. +- **`generate_sitemap()` returns the written files**, which is handy for CI and build scripts. +- **Modern packaging**: metadata moved to `pyproject.toml` with the version read from `localsitemap/_version.py`. +- **Release pipeline**: published from GitHub Actions with PyPI Trusted Publishing (OIDC) and PEP 740 attestations — see [Release provenance](#release-provenance). + +### Changes in version 2.0.0 (Major Release) - **Precise Exclusions**: Rewrote exclusion matching so only exact folder or filename matches (not partial substring matches) are excluded. - **Traversal Optimization**: Subdirectories matching the exclusion list are pruned in-place, eliminating the overhead of traversing skipped directories. - **UTC Timestamps**: Fixed timezone bug to retrieve and format all file modification timestamps as true, accurate UTC standard dates. @@ -15,18 +46,15 @@ LocalSiteMap is an open-source Python package designed for generating sitemaps a - **Flexible Priority Mappings**: Custom mapping for directory/page priorities using a dictionary or callable mapping function. - **Sitemap Splitting & Sitemap Indexing**: Automatically splits large sitemaps into chunks and creates a standard `` parent if the URL limit (`max_urls`, defaults to 50,000) is exceeded. -### Changes in version 1.0.2: +### Changes in version 1.0.2 - Removed the unnecessary `install_requires` from the `setup.py` as the package utilizes none of the required packages. -### Changes in version 1.0.1: +### Changes in version 1.0.1 - Added `show_progress` boolean to print out mapping progress, pages mapped, and files found. -### Changes in version 1.0.0: +### Changes in version 1.0.0 - Added initial package code, with automatic directory crawling to generate the `sitemap.xml` file. -> [!IMPORTANT] -> `LocalSiteMap` crawls any directory found in the root folder, be sure to add exclusions for important directories, or hidden/resource/file directories. - ## Installation You can install LocalSiteMap using pip: @@ -37,14 +65,14 @@ pip install localsitemap ## Supported Python Versions -LocalSiteMap supports the following Python versions: +LocalSiteMap supports Python 3.9 and newer, and every release is tested on CPython 3.9 – 3.14 on Linux, plus Windows: -- Python 3.6 -- Python 3.7 -- Python 3.8 - Python 3.9 - Python 3.10 -- Python 3.11/Later (Preferred) +- Python 3.11 +- Python 3.12 +- Python 3.13/Later (Preferred) +- Python 3.14 ## Features @@ -73,8 +101,14 @@ base_url_of_your_website = "https://example.com" excluded = ["auth", "forms", "template.html", "media", ".git", ".vscode", "node_modules"] # Generate the sitemap -generate_sitemap(root_directory, base_url_of_your_website, "sitemap.xml", excluded, show_progress=True) -print("Sitemap generated in sitemap.xml") +written = generate_sitemap( + root_directory, + base_url_of_your_website, + "sitemap.xml", + excluded, + show_progress=True, +) +print("Sitemap generated:", ", ".join(written)) ``` ### Advanced Features (Extensions, Mappings & Splitting) @@ -82,33 +116,79 @@ print("Sitemap generated in sitemap.xml") ```python from localsitemap import generate_sitemap -# Custom priority dictionary or callable mapper +# Custom priority dictionary or callable mapper. +# Keys may be a relative path, a folder name (trailing slash optional), +# a file name, or an extension (case-insensitive). priority_mapping = { "about/index.html": 0.95, "blog/": 0.70, - ".php": 0.50 + ".php": 0.50, } generate_sitemap( root_path=r"path/to/website", base_url="https://example.com", output_file="sitemap.xml", - excluded_paths=["node_modules", ".git"], + excluded_paths=["node_modules", ".git", "draft-*", "*.bak"], allowed_extensions=[".html", ".htm", ".php"], default_priority=0.80, homepage_priority=1.00, priority_mapping=priority_mapping, - max_urls=1000 # Will automatically split and output a sitemap index if URLs exceed 1000 + max_urls=1000, # Automatically splits and writes a sitemap index above 1000 URLs ) ``` +### Exclusions + +Exclusions are always interpreted relative to `root_path`: + +| Pattern | Meaning | +| --- | --- | +| `"node_modules"` | any file or folder with that name, at any depth | +| `"blog/posts"` | that relative path and everything below it | +| `"/srv/site/private"` | an absolute path inside the site root | +| `"draft-*"`, `"*.bak"` | shell-style glob patterns | + +`excluded_paths=["auth"]` therefore excludes `auth/` but keeps `authority/`. + +### Command line + +```bash +# After pip install localsitemap (or with `python -m localsitemap`) +localsitemap path/to/website https://example.com \ + -o public/sitemap.xml \ + --exclude auth --exclude "draft-*" \ + --extensions .html,.htm,.php \ + --max-urls 1000 \ + --progress +``` + +Run `localsitemap --help` for every option, including `--priority-map priorities.json` and `--exclude-hidden`. + +### Release provenance + +Releases are built and published by GitHub Actions through [PyPI Trusted Publishing](https://docs.pypi.org/trusted-publishers/) (OIDC) — there is no long-lived PyPI API token in this repository. Every uploaded file is signed with a [PEP 740](https://peps.python.org/pep-0740/) digital attestation, which is what makes PyPI display the *provenance* indicator for the release. You can verify it yourself with the bundled script: + +```bash +python .github/scripts/verify_provenance.py --project localsitemap --version 2.1.0 +``` + +or straight from the public Integrity API: + +```bash +curl -s "https://pypi.org/integrity/localsitemap/2.1.0/localsitemap-2.1.0-py3-none-any.whl/provenance" \ + | python -m json.tool +``` + +A non-`404` response containing an `attestation_bundles` entry proves the file was built by this repository's workflow. + ## Contributing -Contributions are welcome! If you encounter any issues, have suggestions, or want to contribute to LocalSiteMap, please open an issue or submit a pull request on [GitHub](https://github.com/infinitode/localsitemap). +Contributions are welcome! If you encounter any issues, have suggestions, or want to contribute to LocalSiteMap, please open an issue or submit a pull request on [GitHub](https://github.com/Infinitode/LocalSiteMap). See [CONTRIBUTING.md](CONTRIBUTING.md) for the test and release workflow. ## License -LocalSiteMap is released under the terms of the **MIT License (Modified)**. Please see the [LICENSE](https://github.com/infinitode/localsitemap/blob/main/LICENSE) file for the full text. +LocalSiteMap is released under the terms of the **MIT License (Modified)**. Please see the [LICENSE](https://github.com/Infinitode/LocalSiteMap/blob/main/LICENSE) file for the full text. **Modified License Clause** diff --git a/localsitemap/__init__.py b/localsitemap/__init__.py index 5456ff4..7918e98 100644 --- a/localsitemap/__init__.py +++ b/localsitemap/__init__.py @@ -1,125 +1,522 @@ -import os -import datetime -from xml.etree.ElementTree import Element, SubElement, tostring -from xml.dom import minidom +"""LocalSiteMap - generate standard-compliant sitemaps from local files. -def is_excluded(path, root_path, excluded_paths): - """ - Checks if a path is excluded based on a list of excluded paths/directories. - Only direct matches or matching path components are excluded, preventing partial substring matches. - """ - if not excluded_paths: - return False +The package crawls a directory tree that mirrors a website and writes a +``sitemap.xml`` file (or a set of numbered sitemaps plus a ``sitemapindex`` +when the URL limit is exceeded). - # Get absolute path and split into segments - abs_path = os.path.abspath(path) - path_segments = abs_path.split(os.sep) +Only the Python standard library is used, so the package has no runtime +dependencies. +""" - for excl in excluded_paths: - if not excl: - continue - # Standardize separators in excl - excl_norm = excl.replace('/', os.sep).replace('\\', os.sep) +import datetime +import fnmatch +import math +import os +import re +import tempfile +import warnings +from urllib.parse import quote + +from ._version import __version__ + +__all__ = [ + "__version__", + "generate_sitemap", + "get_priority", + "get_utc_lastmod", + "is_excluded", + "write_sitemap_file", + "write_sitemap_index_file", +] + +SITEMAP_NS = "http://www.sitemaps.org/schemas/sitemap/0.9" +XSI_NS = "http://www.w3.org/2001/XMLSchema-instance" +SITEMAP_SCHEMA = SITEMAP_NS + "/sitemap.xsd" + +#: Hard limit enforced by the sitemap protocol (and Google) per sitemap file. +MAX_URLS_PER_SITEMAP = 50000 + +DEFAULT_EXTENSIONS = (".html", ".htm") + +_GLOB_CHARS = re.compile(r"[*?\[]") +_SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*://") +_WINDOWS = os.name == "nt" + + +# --------------------------------------------------------------------------- +# Small helpers +# --------------------------------------------------------------------------- + +def _as_path_string(path): + """Return *path* as a ``str`` (accepts ``os.PathLike`` objects).""" + if hasattr(os, "fspath"): + path = os.fspath(path) + if isinstance(path, bytes): + path = os.fsdecode(path) + return path + + +def _segment_matches(segment, pattern): + """Compare one path segment with an exclusion pattern (glob aware).""" + if _GLOB_CHARS.search(pattern): + if fnmatch.fnmatchcase(segment, pattern): + return True + if _WINDOWS: + return fnmatch.fnmatchcase(segment.lower(), pattern.lower()) + return False + if _WINDOWS: + return segment.lower() == pattern.lower() + return segment == pattern - # If it's a simple name (no separators) - if os.sep not in excl_norm: - if excl_norm in path_segments: - return True - continue - # If it's a path (contains separators), we resolve it - # Try both assuming it's an absolute path or relative to root_path - if os.path.isabs(excl_norm): - abs_excl = os.path.abspath(excl_norm) - else: - abs_excl = os.path.abspath(os.path.join(root_path, excl_norm)) +def _normalise_extension(extension): + extension = str(extension).strip() + if not extension: + return "" + if not extension.startswith("."): + extension = "." + extension + return extension.lower() + - excl_segments = abs_excl.split(os.sep) - # Check if excl_segments is a prefix of path_segments - if len(path_segments) >= len(excl_segments): - if path_segments[:len(excl_segments)] == excl_segments: - return True +def _format_priority(value, default=None): + """Format a priority in the ``0.00`` - ``1.00`` range used by sitemaps.""" + try: + priority = float(value) + except (TypeError, ValueError) as error: + if default is None: + raise ValueError( + "priority values must be numbers, got %r" % (value,) + ) from error + priority = float(default) + if math.isnan(priority): + priority = float(default if default is not None else 0.5) + # The sitemap protocol only allows values between 0.0 and 1.0. + priority = min(1.0, max(0.0, priority)) + return "%.2f" % priority - return False def get_utc_lastmod(path): - """ - Returns the UTC last modified time of the path in YYYY-MM-DDThh:mm:ss+00:00 format. + """Return the UTC modification time of *path* formatted for ````. + + The value is an ISO-8601 date/time in UTC (``YYYY-MM-DDThh:mm:ss+00:00``). + If the timestamp cannot be read (for example because the file vanished + while the tree was being walked) the current UTC time is used instead. """ try: mtime = os.path.getmtime(path) - dt = datetime.datetime.fromtimestamp(mtime, tz=datetime.timezone.utc) - return dt.strftime("%Y-%m-%dT%H:%M:%S+00:00") - except Exception: - return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S+00:00") + moment = datetime.datetime.fromtimestamp(mtime, tz=datetime.timezone.utc) + except (OSError, OverflowError, ValueError): + moment = datetime.datetime.now(datetime.timezone.utc) + return moment.strftime("%Y-%m-%dT%H:%M:%S+00:00") -def get_priority(relative_path, is_dir, default_priority, priority_mapping=None): + +# --------------------------------------------------------------------------- +# Exclusions +# --------------------------------------------------------------------------- + +class _PathFilter(object): + """Decides whether a path inside the site root is excluded. + + Rules are resolved relative to the site root, so an exclusion can never + accidentally match a *parent* directory of the site (for example a site + checked out in a folder called ``media`` while ``media`` is excluded). """ - Computes priority based on the relative path, directory status, default, and mapping. + + def __init__(self, root_path, excluded_paths=None, exclude_hidden=False): + self.root_path = os.path.abspath(_as_path_string(root_path)) + self.exclude_hidden = bool(exclude_hidden) + if isinstance(excluded_paths, str): + excluded_paths = [excluded_paths] + self.rules = self._build_rules(excluded_paths or []) + + def _build_rules(self, excluded_paths): + rules = [] + seen = set() + for raw in excluded_paths: + if raw is None: + continue + text = str(raw).strip() + if not text: + continue + candidate = os.path.abspath( + text if os.path.isabs(text) + else os.path.join(self.root_path, text.replace("\\", "/")) + ) + try: + relative = os.path.relpath(candidate, self.root_path) + except ValueError: # pragma: no cover - different drives on Windows + continue + if relative == os.curdir: + # Excluding the root itself is never useful: the site root is + # always kept so the homepage is still emitted. + continue + parts = [part for part in relative.split(os.sep) if part not in ("", os.curdir)] + if not parts or parts[0] == os.pardir: + # Outside of the site root, therefore irrelevant. + continue + # A single relative name matches at any depth (e.g. "node_modules"); + # multi-segment patterns are anchored at the site root. + anchor = len(parts) > 1 or os.path.isabs(text) + key = (tuple(parts), anchor) + if key in seen: + continue + seen.add(key) + rules.append((parts, anchor)) + return rules + + def _segments(self, path): + absolute = os.path.abspath(_as_path_string(path)) + try: + relative = os.path.relpath(absolute, self.root_path) + except ValueError: # pragma: no cover - different drives on Windows + return None + if relative == os.curdir: + return [] + parts = [part for part in relative.split(os.sep) if part not in ("", os.curdir)] + if parts and parts[0] == os.pardir: + return None + return parts + + def excludes(self, path): + segments = self._segments(path) + if segments is None: + return False + if not segments: + return False + + if self.exclude_hidden: + for segment in segments: + if segment.startswith("."): + return True + + for pattern, anchor in self.rules: + if anchor: + if len(segments) < len(pattern): + continue + if all( + _segment_matches(segment, part) + for segment, part in zip(segments, pattern) + ): + return True + elif len(pattern) == 1: + if any(_segment_matches(segment, pattern[0]) for segment in segments): + return True + return False + + +def is_excluded(path, root_path, excluded_paths): + """Return ``True`` when *path* should be skipped. + + ``path`` and ``root_path`` may be absolute or relative. Exclusions are + interpreted relative to ``root_path``: + + * a plain name (``"node_modules"``) matches a file or folder with that + name at any depth, + * a relative path (``"blog/posts"``) matches that path and everything + below it, + * an absolute path matches only that path (and its contents) when it is + inside ``root_path``, + * shell-style wildcards (``"draft-*"``, ``"*.bak"``) are supported. """ + if not excluded_paths: + return False + return _PathFilter(root_path, excluded_paths).excludes(path) + + +# --------------------------------------------------------------------------- +# Priorities +# --------------------------------------------------------------------------- + +def _normalise_priority_mapping(mapping): + normalised = {} + for key, value in mapping.items(): + if key is None: + continue + text = str(key).strip().replace("\\", "/").strip("/") + if not text: + continue + normalised[text] = value + if text.startswith(".") and "/" not in text: + # Extension keys are matched case-insensitively. + normalised.setdefault(text.lower(), value) + return normalised + + +def _lookup_mapping_priority(mapping, relative_path, is_dir): + """Look up ``relative_path`` in an already normalised mapping.""" + rel = str(relative_path).replace("\\", "/").strip("/") + if not rel: + return None, None + basename = rel.rsplit("/", 1)[-1] + _, extension = os.path.splitext(basename) + candidates = [rel] + if is_dir or not extension: + candidates.append(rel + "/") + candidates.extend([basename, extension]) + if extension: + candidates.append(extension.lower()) + candidates.append(extension.lstrip(".").lower()) + for candidate in candidates: + if candidate and candidate in mapping: + return mapping[candidate], candidate + return None, None + + +def _resolve_priority(relative_path, is_dir, default_priority, priority_mapping): + """Resolve a priority; *priority_mapping* must already be normalised.""" if priority_mapping: if callable(priority_mapping): + value = None try: - p = priority_mapping(relative_path, is_dir) - if p is not None: - return f"{float(p):.2f}" - except Exception: - pass + value = priority_mapping(relative_path, is_dir) + except Exception: # noqa: BLE001 - user supplied callables may fail + value = None + if value is not None: + return _format_priority(value, default=default_priority) elif isinstance(priority_mapping, dict): - # Standardize relative_path to use forward slashes - norm_rel = relative_path.replace("\\", "/") - if norm_rel in priority_mapping: - return f"{float(priority_mapping[norm_rel]):.2f}" - # Also try matching file/dir name (basename) - basename = os.path.basename(relative_path) - if basename in priority_mapping: - return f"{float(priority_mapping[basename]):.2f}" - # Try matching by extension - _, ext = os.path.splitext(basename) - if ext in priority_mapping: - return f"{float(priority_mapping[ext]):.2f}" - return f"{float(default_priority):.2f}" + value, _key = _lookup_mapping_priority(priority_mapping, relative_path, is_dir) + if value is not None: + return _format_priority(value, default=default_priority) + return _format_priority(default_priority) -def write_sitemap_file(file_path, entries): - """ - Writes a list of entries to a sitemap XML file. - """ - urlset = Element("urlset") - urlset.set("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9") - urlset.set("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance") - urlset.set("xsi:schemaLocation", "http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd") +def get_priority(relative_path, is_dir, default_priority, priority_mapping=None): + """Return the formatted priority for *relative_path*. + + ``priority_mapping`` may be a callable (``f(relative_path, is_dir)``) or a + dict. Dict keys may be a relative path (``"about/index.html"``), a folder + name, a file name, or an extension (``".php"``); folder keys may be + written with or without a trailing slash and Windows separators are + accepted. Priorities are clamped to the ``0.00`` - ``1.00`` range allowed + by the sitemap protocol. + """ + if isinstance(priority_mapping, dict): + priority_mapping = _normalise_priority_mapping(priority_mapping) + return _resolve_priority(relative_path, is_dir, default_priority, priority_mapping) + + +# --------------------------------------------------------------------------- +# URL helpers +# --------------------------------------------------------------------------- + +def _normalise_base_url(base_url): + base = str(base_url).strip() + if not _SCHEME_RE.match(base): + raise ValueError( + "base_url must be an absolute URL including its scheme " + "(for example 'https://example.com'), got %r" % (base_url,) + ) + return base.rstrip("/") + + +def _quote_segment(segment): + # ``quote`` never escapes unreserved characters, so "index.html" is left + # untouched while spaces, "#", "&" and non-ASCII characters are encoded. + return quote(segment, safe="") + + +def _url_for(base, relative_path, is_dir): + """Build an absolute, percent-encoded URL for a relative filesystem path.""" + posix = str(relative_path).replace("\\", "/") + parts = [part for part in posix.split("/") if part not in ("", ".")] + encoded = "/".join(_quote_segment(part) for part in parts) + if not encoded: + return base + url = base + "/" + encoded + if is_dir: + url += "/" + return url + + +# --------------------------------------------------------------------------- +# XML writers +# --------------------------------------------------------------------------- + +def _xml_escape(value): + return ( + str(value) + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) + + +def _atomic_write(file_path, content): + """Write *content* to *file_path* atomically (create parent directories).""" + target = os.path.abspath(_as_path_string(file_path)) + directory = os.path.dirname(target) or os.curdir + os.makedirs(directory, exist_ok=True) + handle_fd, temporary = tempfile.mkstemp( + prefix=".localsitemap-", suffix=".tmp", dir=directory + ) + try: + with os.fdopen(handle_fd, "w", encoding="utf-8", newline="\n") as handle: + handle.write(content) + os.replace(temporary, target) + except BaseException: + try: + os.remove(temporary) + except OSError: + pass + raise + + +def _render_urlset(entries): + lines = [ + '', + '' + % (SITEMAP_NS, XSI_NS, SITEMAP_NS, SITEMAP_SCHEMA), + ] for entry in entries: - url_entry = SubElement(urlset, "url") - SubElement(url_entry, "loc").text = entry["loc"] - SubElement(url_entry, "lastmod").text = entry["lastmod"] - SubElement(url_entry, "priority").text = entry["priority"] + lines.append(" ") + lines.append(" %s" % _xml_escape(entry["loc"])) + if entry.get("lastmod"): + lines.append(" %s" % _xml_escape(entry["lastmod"])) + if entry.get("priority") is not None: + lines.append(" %s" % _xml_escape(entry["priority"])) + lines.append(" ") + lines.append("") + return "\n".join(lines) + "\n" + + +def _render_sitemapindex(sitemaps_info): + lines = [ + '', + '' % SITEMAP_NS, + ] + for info in sitemaps_info: + lines.append(" ") + lines.append(" %s" % _xml_escape(info["loc"])) + if info.get("lastmod"): + lines.append(" %s" % _xml_escape(info["lastmod"])) + lines.append(" ") + lines.append("") + return "\n".join(lines) + "\n" + - xml_string = tostring(urlset, "utf-8") - reparsed = minidom.parseString(xml_string) - pretty_xml = reparsed.toprettyxml(indent=" ") +def write_sitemap_file(file_path, entries): + """Write *entries* to *file_path* as a sitemap ````.""" + _atomic_write(file_path, _render_urlset(entries)) - with open(file_path, "w", encoding="utf-8") as f: - f.write(pretty_xml) def write_sitemap_index_file(file_path, sitemaps_info): - """ - Writes a list of sitemap files to a sitemap index XML file. - """ - sitemapindex = Element("sitemapindex") - sitemapindex.set("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9") + """Write *sitemaps_info* to *file_path* as a ````.""" + _atomic_write(file_path, _render_sitemapindex(sitemaps_info)) + + +# --------------------------------------------------------------------------- +# Sitemap generation +# --------------------------------------------------------------------------- + +def _walk_entries(root_path, base_url, path_filter, extensions, priority_mapping, + default_priority, show_progress): + entries = [] + seen_locations = set() + + def add(loc, source_path, is_dir, priority=None): + if loc in seen_locations: + return + seen_locations.add(loc) + entries.append({ + "loc": loc, + "lastmod": get_utc_lastmod(source_path), + "priority": priority if priority is not None else _resolve_priority( + _relative_posix(source_path, root_path), + is_dir=is_dir, + default_priority=default_priority, + priority_mapping=priority_mapping, + ), + }) + + for current_root, dirnames, filenames in os.walk(root_path): + # Keep the walk deterministic and prune excluded directories in place + # so their contents are never visited. + dirnames[:] = sorted( + name for name in dirnames + if not path_filter.excludes(os.path.join(current_root, name)) + ) + filenames = sorted(filenames) + + for dir_name in dirnames: + full_path = os.path.join(current_root, dir_name) + if show_progress: + print("Mapping directory: %s" % full_path) + add(_url_for(base_url, _relative_posix(full_path, root_path), is_dir=True), + full_path, True) + + for file_name in filenames: + full_path = os.path.join(current_root, file_name) + if path_filter.excludes(full_path): + continue + if not _has_allowed_extension(file_name, extensions): + continue + if show_progress: + print("Found file: %s" % full_path) + add(_url_for(base_url, _relative_posix(full_path, root_path), is_dir=False), + full_path, False) + + return entries + + +def _relative_posix(path, root_path): + relative = os.path.relpath(path, root_path) + return relative.replace(os.sep, "/") + + +def _has_allowed_extension(file_name, extensions): + lowered = file_name.lower() + return any(lowered.endswith(extension) for extension in extensions) - for info in sitemaps_info: - sitemap_entry = SubElement(sitemapindex, "sitemap") - SubElement(sitemap_entry, "loc").text = info["loc"] - SubElement(sitemap_entry, "lastmod").text = info["lastmod"] - xml_string = tostring(sitemapindex, "utf-8") - reparsed = minidom.parseString(xml_string) - pretty_xml = reparsed.toprettyxml(indent=" ") +def _split_sitemaps(entries, output_file, base_url, max_urls, show_progress): + """Write numbered sitemaps plus an index; return the created file paths.""" + output_file = _as_path_string(output_file) + directory = os.path.dirname(output_file) + base_name = os.path.basename(output_file) + name, extension = os.path.splitext(base_name) + + part_count = int(math.ceil(len(entries) / float(max_urls))) + created = [] + sitemaps_info = [] + + for part_number in range(1, part_count + 1): + chunk = entries[(part_number - 1) * max_urls:part_number * max_urls] + part_filename = "%s_%d%s" % (name, part_number, extension) + part_path = os.path.join(directory, part_filename) if directory else part_filename + if show_progress: + print("Saving split sitemap part %d to %s..." % (part_number, part_path)) + write_sitemap_file(part_path, chunk) + created.append(part_path) + sitemaps_info.append({ + "loc": base_url + "/" + _quote_segment(part_filename), + "lastmod": get_utc_lastmod(part_path), + }) + + # Remove leftovers from a previous, larger run so stale sitemaps are not + # left behind next to the freshly written index. + stale_pattern = re.compile(r"^%s_([0-9]+)%s$" % (re.escape(name), re.escape(extension))) + search_directory = directory or os.curdir + if os.path.isdir(search_directory): + for existing in sorted(os.listdir(search_directory)): + match = stale_pattern.match(existing) + if not match or int(match.group(1)) <= part_count: + continue + stale_path = os.path.join(search_directory, existing) + try: + os.remove(stale_path) + except OSError: + continue + if show_progress: + print("Removed stale sitemap part: %s" % stale_path) + + if show_progress: + print("Saving sitemap index to %s..." % output_file) + write_sitemap_index_file(output_file, sitemaps_info) + created.append(output_file) + return created - with open(file_path, "w", encoding="utf-8") as f: - f.write(pretty_xml) def generate_sitemap( root_path, @@ -131,126 +528,123 @@ def generate_sitemap( default_priority=0.80, homepage_priority=1.00, priority_mapping=None, - max_urls=50000 + max_urls=50000, + exclude_hidden=False, + include_homepage=True, + sort_entries=True, ): + """Generate a sitemap for a local website directory. + + Parameters + ---------- + root_path: + Root directory of the website. + base_url: + Public base URL of the website (for example ``https://example.com``). + output_file: + Sitemap path to write. When the URL limit is exceeded numbered parts + (``sitemap_1.xml``, ...) plus a ``sitemap.xml`` index are written. + excluded_paths: + File/folder names, relative paths, absolute paths or glob patterns to + skip. Matching is relative to ``root_path``. + show_progress: + Print the traversal progress. + allowed_extensions: + Extensions to include (default: ``[".html", ".htm"]``). Matching is + case-insensitive and the leading dot is optional. + default_priority: + Priority used when no mapping matches (default ``0.80``). + homepage_priority: + Priority for the site root (default ``1.00``). + priority_mapping: + Dict or callable mapping paths/folders/extensions to priorities. + max_urls: + Maximum number of URLs per sitemap file before splitting (max 50000). + exclude_hidden: + Skip files and folders starting with a dot. + include_homepage: + Emit the site root URL (default ``True``). + sort_entries: + Sort entries by URL for reproducible output (default ``True``). + + Returns + ------- + list of str + The paths that were written, in the order they were written. When the + sitemap is split, the numbered parts come first and the index last. """ - Generates a sitemap XML file or multiple files plus an index if URLs exceed max_urls. - - Parameters: - root_path: The root directory of your website. - base_url: The base URL of your website (e.g., "https://example.com"). - output_file: The name of the output XML file (default: "sitemap.xml"). - excluded_paths: A list of file paths or directories to exclude from the sitemap. - show_progress: Boolean to print mapping progress. - allowed_extensions: List of allowed file extensions (default: [".html", ".htm"]). - default_priority: Default priority for paths (default: 0.80). - homepage_priority: Priority for the root directory (default: 1.00). - priority_mapping: Dict or callable to map specific paths to priorities. - max_urls: Max URLs per single sitemap file before splitting (default: 50000). - """ - if excluded_paths is None: - excluded_paths = [] + root_path = _as_path_string(root_path) + if not os.path.isdir(root_path): + raise NotADirectoryError( + "root_path must point to an existing directory, got %r" % (root_path,) + ) + + base_url = _normalise_base_url(base_url) + + max_urls = int(max_urls) + if max_urls < 1: + raise ValueError("max_urls must be a positive integer, got %r" % (max_urls,)) + if max_urls > MAX_URLS_PER_SITEMAP: + warnings.warn( + "max_urls=%d exceeds the sitemap protocol limit of %d; " + "the limit will be used instead." % (max_urls, MAX_URLS_PER_SITEMAP), + RuntimeWarning, + stacklevel=2, + ) + max_urls = MAX_URLS_PER_SITEMAP if allowed_extensions is None: - allowed_extensions = [".html", ".htm"] + extensions = list(DEFAULT_EXTENSIONS) else: - # Normalize allowed_extensions to start with "." - allowed_extensions = [ext if ext.startswith(".") else f".{ext}" for ext in allowed_extensions] - - entries = [] - - for root, dirs, files in os.walk(root_path): - # Optimize by pruning excluded directories in-place - dirs[:] = [d for d in dirs if not is_excluded(os.path.join(root, d), root_path, excluded_paths)] - - # Add directories to entries - for dir_name in dirs: - full_dir_path = os.path.join(root, dir_name) - if show_progress: - print(f"Mapping directory: {full_dir_path}") - - relative_dir_path = os.path.relpath(full_dir_path, root_path) - base = base_url.rstrip("/") - url = (base + "/" + relative_dir_path.replace("\\", "/")).rstrip("/") + "/" - - lastmod = get_utc_lastmod(full_dir_path) - priority = get_priority(relative_dir_path, is_dir=True, default_priority=default_priority, priority_mapping=priority_mapping) - - entries.append({ - "loc": url, - "lastmod": lastmod, - "priority": priority - }) - - for file in files: - if file.endswith(tuple(allowed_extensions)): - full_path = os.path.join(root, file) - - if is_excluded(full_path, root_path, excluded_paths): - continue - - if show_progress: - print(f"Found file: {full_path}") - - relative_path = os.path.relpath(full_path, root_path) - base = base_url.rstrip("/") - url = base + "/" + relative_path.replace("\\", "/") - - lastmod = get_utc_lastmod(full_path) - priority = get_priority(relative_path, is_dir=False, default_priority=default_priority, priority_mapping=priority_mapping) + if isinstance(allowed_extensions, str): + allowed_extensions = [ + item.strip() for item in allowed_extensions.split(",") if item.strip() + ] + extensions = [ext for ext in (_normalise_extension(e) for e in allowed_extensions) if ext] + if not extensions: + raise ValueError("allowed_extensions must contain at least one extension") + + if isinstance(priority_mapping, dict): + priority_mapping = _normalise_priority_mapping(priority_mapping) + + path_filter = _PathFilter(root_path, excluded_paths, exclude_hidden=exclude_hidden) + + entries = _walk_entries( + root_path=root_path, + base_url=base_url, + path_filter=path_filter, + extensions=extensions, + priority_mapping=priority_mapping, + default_priority=default_priority, + show_progress=show_progress, + ) + + if include_homepage: + homepage = { + "loc": base_url, + "lastmod": get_utc_lastmod(root_path), + "priority": _format_priority(homepage_priority), + } + if any(entry["loc"] == homepage["loc"] for entry in entries): + for entry in entries: + if entry["loc"] == homepage["loc"]: + entry.update(homepage) + break + else: + entries.append(homepage) - entries.append({ - "loc": url, - "lastmod": lastmod, - "priority": priority - }) + if sort_entries: + entries.sort(key=lambda entry: entry["loc"]) - # Add homepage to entries - homepage_url = base_url - homepage_lastmod = get_utc_lastmod(root_path) - entries.append({ - "loc": homepage_url, - "lastmod": homepage_lastmod, - "priority": f"{float(homepage_priority):.2f}" - }) + if show_progress: + print("Total pages mapped: %d" % len(entries)) - # Write the XML output if len(entries) <= max_urls: if show_progress: - print(f"Total pages mapped: {len(entries)}") - print(f"Saving sitemap to {output_file}...") + print("Saving sitemap to %s..." % output_file) write_sitemap_file(output_file, entries) - else: - if show_progress: - print(f"Total pages mapped: {len(entries)}. Limit is {max_urls}. Splitting into multiple sitemaps...") - - dir_name = os.path.dirname(output_file) - base_name = os.path.basename(output_file) - name, ext = os.path.splitext(base_name) + return [_as_path_string(output_file)] - sitemaps_info = [] - - # Split into chunks of size max_urls - for i in range(0, len(entries), max_urls): - chunk = entries[i:i + max_urls] - part_num = (i // max_urls) + 1 - part_filename = f"{name}_{part_num}{ext}" - part_file_path = os.path.join(dir_name, part_filename) if dir_name else part_filename - - if show_progress: - print(f"Saving split sitemap part {part_num} to {part_file_path}...") - - write_sitemap_file(part_file_path, chunk) - - base = base_url.rstrip("/") - part_url = f"{base}/{part_filename}" - part_lastmod = get_utc_lastmod(part_file_path) - - sitemaps_info.append({ - "loc": part_url, - "lastmod": part_lastmod - }) - - if show_progress: - print(f"Saving sitemap index to {output_file}...") - write_sitemap_index_file(output_file, sitemaps_info) + if show_progress: + print("URL limit is %d. Splitting into multiple sitemaps..." % max_urls) + return _split_sitemaps(entries, output_file, base_url, max_urls, show_progress) diff --git a/localsitemap/__main__.py b/localsitemap/__main__.py new file mode 100644 index 0000000..63321a6 --- /dev/null +++ b/localsitemap/__main__.py @@ -0,0 +1,8 @@ +"""Allow ``python -m localsitemap`` to behave like the ``localsitemap`` CLI.""" + +import sys + +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/localsitemap/_version.py b/localsitemap/_version.py new file mode 100644 index 0000000..dd446de --- /dev/null +++ b/localsitemap/_version.py @@ -0,0 +1,7 @@ +"""Single source of truth for the LocalSiteMap version. + +The release workflow rewrites this file from the pushed git tag, so the +version published to PyPI always matches the tag. +""" + +__version__ = "2.1.0" diff --git a/localsitemap/cli.py b/localsitemap/cli.py new file mode 100644 index 0000000..64292a1 --- /dev/null +++ b/localsitemap/cli.py @@ -0,0 +1,135 @@ +"""Command line interface for LocalSiteMap. + +Usage:: + + localsitemap path/to/website https://example.com -o public/sitemap.xml + python -m localsitemap path/to/website https://example.com --exclude auth +""" + +import argparse +import json +import sys + +from . import __version__, generate_sitemap + + +def _comma_separated(value): + if value is None: + return None + return [item.strip() for item in value.split(",") if item.strip()] + + +def build_parser(): + parser = argparse.ArgumentParser( + prog="localsitemap", + description=( + "Generate a sitemap (and sitemap index when needed) from a local " + "directory that mirrors a website." + ), + ) + parser.add_argument("root_path", help="root directory of the website") + parser.add_argument( + "base_url", + help="public base URL of the website, e.g. https://example.com", + ) + parser.add_argument( + "-o", "--output", default="sitemap.xml", + help="sitemap file to write (default: sitemap.xml)", + ) + parser.add_argument( + "-e", "--exclude", action="append", default=[], metavar="PATTERN", + help=( + "name, relative/absolute path or glob pattern to exclude; " + "repeat for multiple exclusions" + ), + ) + parser.add_argument( + "--extensions", default=".html,.htm", metavar="LIST", + help="comma separated extensions to include (default: .html,.htm)", + ) + parser.add_argument( + "--default-priority", type=float, default=0.80, + help="priority used when no mapping matches (default: 0.80)", + ) + parser.add_argument( + "--homepage-priority", type=float, default=1.00, + help="priority for the site root (default: 1.00)", + ) + parser.add_argument( + "--max-urls", type=int, default=50000, + help="maximum URLs per sitemap file before splitting (default: 50000)", + ) + parser.add_argument( + "--priority-map", metavar="FILE", + help="JSON file containing a priority mapping object", + ) + parser.add_argument( + "--exclude-hidden", action="store_true", + help="skip files and folders whose name starts with a dot", + ) + parser.add_argument( + "--no-homepage", action="store_true", + help="do not emit the site root URL", + ) + parser.add_argument( + "--no-sort", action="store_true", + help="keep filesystem order instead of sorting URLs", + ) + parser.add_argument( + "-p", "--progress", action="store_true", + help="print the traversal progress", + ) + parser.add_argument( + "--version", action="version", version="%(prog)s " + __version__, + ) + return parser + + +def _load_priority_map(parser, path): + try: + with open(path, "r", encoding="utf-8") as handle: + mapping = json.load(handle) + except OSError as error: + parser.error("could not read priority map %s: %s" % (path, error)) + except ValueError as error: + parser.error("priority map %s is not valid JSON: %s" % (path, error)) + if not isinstance(mapping, dict): + parser.error("priority map %s must contain a JSON object" % path) + return mapping + + +def main(argv=None): + parser = build_parser() + args = parser.parse_args(argv) + + priority_mapping = None + if args.priority_map: + priority_mapping = _load_priority_map(parser, args.priority_map) + + try: + created = generate_sitemap( + root_path=args.root_path, + base_url=args.base_url, + output_file=args.output, + excluded_paths=args.exclude, + show_progress=args.progress, + allowed_extensions=_comma_separated(args.extensions), + default_priority=args.default_priority, + homepage_priority=args.homepage_priority, + priority_mapping=priority_mapping, + max_urls=args.max_urls, + exclude_hidden=args.exclude_hidden, + include_homepage=not args.no_homepage, + sort_entries=not args.no_sort, + ) + except (OSError, ValueError) as error: + print("error: %s" % error, file=sys.stderr) + return 1 + + for path in created: + print("Wrote %s" % path) + return 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b3c59f5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,66 @@ +[build-system] +requires = ["setuptools>=77.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "localsitemap" +dynamic = ["version"] +description = "Generate sitemaps and sitemap indexes from local files by crawling directories recursively." +readme = "README.md" +requires-python = ">=3.9" +license = "MIT" +license-files = ["LICENSE"] +authors = [ + { name = "Infinitode Pty Ltd", email = "infinitode.ltd@gmail.com" }, +] +keywords = [ + "sitemap", + "sitemap-index", + "seo", + "static-site", + "xml", + "crawler", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Internet :: WWW/HTTP :: Site Management", + "Topic :: Text Processing :: Markup :: XML", +] +dependencies = [] + +[project.urls] +Homepage = "https://github.com/Infinitode/LocalSiteMap" +Repository = "https://github.com/Infinitode/LocalSiteMap" +Issues = "https://github.com/Infinitode/LocalSiteMap/issues" +Changelog = "https://github.com/Infinitode/LocalSiteMap/releases" +PyPI = "https://pypi.org/project/localsitemap/" + +[project.scripts] +localsitemap = "localsitemap.cli:main" + +[tool.setuptools.dynamic] +version = { attr = "localsitemap._version.__version__" } + +[tool.setuptools.packages.find] +include = ["localsitemap*"] +namespaces = false + +[tool.ruff] +line-length = 100 +target-version = "py39" +exclude = ["build", "dist"] + +[tool.ruff.lint] +select = ["E", "F", "W", "B"] +ignore = ["B905"] # zip(strict=...) requires Python 3.10 diff --git a/setup.py b/setup.py index f73a221..a8a5213 100644 --- a/setup.py +++ b/setup.py @@ -1,30 +1,9 @@ -from setuptools import setup, find_packages +"""Compatibility shim. -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() +All project metadata lives in ``pyproject.toml``; this file only exists so +that legacy ``python setup.py ...`` invocations keep working. +""" -setup( - name='localsitemap', - version='{{VERSION_PLACEHOLDER}}', - author='Infinitode Pty Ltd', - author_email='infinitode.ltd@gmail.com', - description="LocalSiteMap is an open-source Python package designed for generating sitemaps and sitemap indexes from local files by crawling directories recursively.", - long_description=long_description, - long_description_content_type='text/markdown', - url='https://github.com/infinitode/localsitemap', - packages=find_packages(), - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - ], - python_requires='>=3.6', -) +from setuptools import setup + +setup() diff --git a/test.py b/test.py index 34ecb6b..4ee6a48 100644 --- a/test.py +++ b/test.py @@ -1,14 +1,30 @@ +"""Small manual demo of LocalSiteMap. + +Point ``root_directory`` at any local website checkout and run: + + python test.py + +For automated checks use ``python -m unittest -v test_localsitemap`` instead. +""" + from localsitemap import generate_sitemap -# Example usage +# Root site directory root_directory = r"path/to/your/website/directory" # Domain of your website (where it is hosted) base_url_of_your_website = "https://example.com" -# List of file paths or directories to exclude from the sitemap -excluded = ["auth", "forms", "template.html", "media", ".git", ".vscode", "node_modules"] # Example exclusions +# Only exact folder names, relative paths, absolute paths or glob patterns +# will be skipped +excluded = ["auth", "forms", "template.html", "media", ".git", ".vscode", "node_modules"] # Generate the sitemap -generate_sitemap(root_directory, base_url_of_your_website, "sitemap.xml", excluded) -print("Sitemap generated in sitemap.xml") \ No newline at end of file +written = generate_sitemap( + root_path=root_directory, + base_url=base_url_of_your_website, + output_file="sitemap.xml", + excluded_paths=excluded, + show_progress=True, +) +print("Sitemap generated:", ", ".join(written)) diff --git a/test_localsitemap.py b/test_localsitemap.py index db2521e..46d1b01 100644 --- a/test_localsitemap.py +++ b/test_localsitemap.py @@ -1,9 +1,23 @@ +import json import os import shutil +import subprocess +import sys import tempfile import unittest import xml.etree.ElementTree as ET -from localsitemap import generate_sitemap + +from localsitemap import ( + __version__, + generate_sitemap, + get_priority, + is_excluded, + write_sitemap_file, +) + +SITEMAP_NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} +REPO_ROOT = os.path.dirname(os.path.abspath(__file__)) + class TestLocalSitemap(unittest.TestCase): def setUp(self): @@ -31,46 +45,58 @@ def setUp(self): self.output_file = os.path.join(self.test_dir, "sitemap.xml") def tearDown(self): - shutil.rmtree(self.test_dir) + shutil.rmtree(self.test_dir, ignore_errors=True) - def write_dummy_file(self, path): + # -- helpers --------------------------------------------------------- + + def write_dummy_file(self, path, content="Test"): + directory = os.path.dirname(path) + if directory and not os.path.isdir(directory): + os.makedirs(directory) with open(path, "w", encoding="utf-8") as f: - f.write("Test") + f.write(content) + + def urls(self, path=None): + tree = ET.parse(path or self.output_file) + return [loc.text for loc in tree.getroot().findall("sm:url/sm:loc", SITEMAP_NS)] + + def entries(self, path=None): + tree = ET.parse(path or self.output_file) + result = {} + for url_node in tree.getroot().findall("sm:url", SITEMAP_NS): + loc = url_node.find("sm:loc", SITEMAP_NS).text + priority_node = url_node.find("sm:priority", SITEMAP_NS) + lastmod_node = url_node.find("sm:lastmod", SITEMAP_NS) + result[loc] = { + "priority": priority_node.text if priority_node is not None else None, + "lastmod": lastmod_node.text if lastmod_node is not None else None, + } + return result + + # -- existing behaviour ---------------------------------------------- def test_basic_sitemap_generation(self): - # Generate with defaults generate_sitemap( root_path=self.test_dir, base_url="https://example.com", output_file=self.output_file ) - # Parse XML and verify entries - tree = ET.parse(self.output_file) - root = tree.getroot() - - # Standard sitemap namespace - ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} - urls = [loc.text for loc in root.findall("sm:url/sm:loc", ns)] - - # We expect root directory, subdirectories, html/htm files - # Directories included: about, blog, auth, authority, blog/posts - # Files included: index.html, about/index.html, blog/first-post.html, blog/second-post.htm, blog/posts/post1.html, auth/login.html, authority/page.html - # Plus the homepage itself + urls = self.urls() self.assertIn("https://example.com", urls) self.assertIn("https://example.com/about/", urls) self.assertIn("https://example.com/about/index.html", urls) self.assertIn("https://example.com/blog/second-post.htm", urls) - # Custom.php and custom.txt should NOT be included by default + # custom.php and custom.txt should NOT be included by default self.assertNotIn("https://example.com/custom.php", urls) self.assertNotIn("https://example.com/custom.txt", urls) + # No duplicate entries are allowed in a sitemap + self.assertEqual(len(urls), len(set(urls))) + def test_exclusion_exact_match(self): - # Exclude "auth" only. - # "authority" and "authority/page.html" should NOT be excluded. - # "auth" directory and "auth/login.html" SHOULD be excluded. generate_sitemap( root_path=self.test_dir, base_url="https://example.com", @@ -78,21 +104,15 @@ def test_exclusion_exact_match(self): excluded_paths=["auth"] ) - tree = ET.parse(self.output_file) - root = tree.getroot() - ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} - urls = [loc.text for loc in root.findall("sm:url/sm:loc", ns)] + urls = self.urls() - # "auth" segments should be excluded self.assertNotIn("https://example.com/auth/", urls) self.assertNotIn("https://example.com/auth/login.html", urls) - # "authority" segments should remain self.assertIn("https://example.com/authority/", urls) self.assertIn("https://example.com/authority/page.html", urls) def test_custom_extensions(self): - # Generate with custom extensions [".php", "txt"] generate_sitemap( root_path=self.test_dir, base_url="https://example.com", @@ -100,18 +120,13 @@ def test_custom_extensions(self): allowed_extensions=[".php", "txt"] ) - tree = ET.parse(self.output_file) - root = tree.getroot() - ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} - urls = [loc.text for loc in root.findall("sm:url/sm:loc", ns)] + urls = self.urls() self.assertIn("https://example.com/custom.php", urls) self.assertIn("https://example.com/custom.txt", urls) - # HTML files should not be present now self.assertNotIn("https://example.com/index.html", urls) def test_custom_priority_and_mapping(self): - # Test default values and priority mapping as dict pm = { "about/index.html": 0.95, "blog/second-post.htm": 0.40, @@ -127,21 +142,13 @@ def test_custom_priority_and_mapping(self): priority_mapping=pm ) - tree = ET.parse(self.output_file) - root = tree.getroot() - ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} - - entries = {} - for url_node in root.findall("sm:url", ns): - loc = url_node.find("sm:loc", ns).text - prio = url_node.find("sm:priority", ns).text - entries[loc] = prio + entries = self.entries() - self.assertEqual(entries["https://example.com"], "0.99") - self.assertEqual(entries["https://example.com/about/index.html"], "0.95") - self.assertEqual(entries["https://example.com/blog/second-post.htm"], "0.40") - self.assertEqual(entries["https://example.com/blog/posts/"], "0.15") - self.assertEqual(entries["https://example.com/index.html"], "0.55") + self.assertEqual(entries["https://example.com"]["priority"], "0.99") + self.assertEqual(entries["https://example.com/about/index.html"]["priority"], "0.95") + self.assertEqual(entries["https://example.com/blog/second-post.htm"]["priority"], "0.40") + self.assertEqual(entries["https://example.com/blog/posts/"]["priority"], "0.15") + self.assertEqual(entries["https://example.com/index.html"]["priority"], "0.55") def test_custom_priority_callable(self): def my_priority_mapper(relative_path, is_dir): @@ -159,24 +166,13 @@ def my_priority_mapper(relative_path, is_dir): priority_mapping=my_priority_mapper ) - tree = ET.parse(self.output_file) - root = tree.getroot() - ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} + entries = self.entries() - entries = {} - for url_node in root.findall("sm:url", ns): - loc = url_node.find("sm:loc", ns).text - prio = url_node.find("sm:priority", ns).text - entries[loc] = prio - - self.assertEqual(entries["https://example.com/about/"], "0.11") - self.assertEqual(entries["https://example.com/blog/first-post.html"], "0.99") - self.assertEqual(entries["https://example.com/index.html"], "0.75") + self.assertEqual(entries["https://example.com/about/"]["priority"], "0.11") + self.assertEqual(entries["https://example.com/blog/first-post.html"]["priority"], "0.99") + self.assertEqual(entries["https://example.com/index.html"]["priority"], "0.75") def test_sitemap_splitting_and_index(self): - # Set max_urls to 3. Since there are more than 3 entries total, - # it should generate split sitemaps (sitemap_1.xml, sitemap_2.xml, etc) - # and a sitemap.xml index. generate_sitemap( root_path=self.test_dir, base_url="https://example.com", @@ -184,51 +180,461 @@ def test_sitemap_splitting_and_index(self): max_urls=3 ) - # Verify index sitemap.xml is created self.assertTrue(os.path.exists(self.output_file)) - tree = ET.parse(self.output_file) - root = tree.getroot() - - # It should be a sitemapindex root + root = ET.parse(self.output_file).getroot() self.assertEqual(root.tag, "{http://www.sitemaps.org/schemas/sitemap/0.9}sitemapindex") - ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} - sitemap_locs = [loc.text for loc in root.findall("sm:sitemap/sm:loc", ns)] + sitemap_locs = [loc.text for loc in root.findall("sm:sitemap/sm:loc", SITEMAP_NS)] self.assertGreater(len(sitemap_locs), 1) self.assertIn("https://example.com/sitemap_1.xml", sitemap_locs) - # Check that part 1 contains url entries part1_path = os.path.join(self.test_dir, "sitemap_1.xml") self.assertTrue(os.path.exists(part1_path)) - part_tree = ET.parse(part1_path) - part_root = part_tree.getroot() + part_root = ET.parse(part1_path).getroot() self.assertEqual(part_root.tag, "{http://www.sitemaps.org/schemas/sitemap/0.9}urlset") - part_urls = [loc.text for loc in part_root.findall("sm:url/sm:loc", ns)] + part_urls = [loc.text for loc in part_root.findall("sm:url/sm:loc", SITEMAP_NS)] self.assertEqual(len(part_urls), 3) def test_utc_timezone_format(self): - # Ensure that timestamps end in +00:00 and are in correct ISO 8601 format generate_sitemap( root_path=self.test_dir, base_url="https://example.com", output_file=self.output_file ) - tree = ET.parse(self.output_file) - root = tree.getroot() - ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} - - lastmods = [lm.text for lm in root.findall("sm:url/sm:lastmod", ns)] + lastmods = [value["lastmod"] for value in self.entries().values()] for lm in lastmods: self.assertTrue(lm.endswith("+00:00")) - # Format: YYYY-MM-DDThh:mm:ss+00:00 self.assertEqual(len(lm), 25) self.assertEqual(lm[10], 'T') self.assertEqual(lm[19], '+') + # -- bug fixes / new behaviour --------------------------------------- + + def test_exclusion_does_not_match_parent_directories(self): + """A site checked out below a folder named like an exclusion still works.""" + parent = tempfile.mkdtemp() + try: + site = os.path.join(parent, "media", "website") + os.makedirs(site) + self.write_dummy_file(os.path.join(site, "index.html")) + output = os.path.join(site, "sitemap.xml") + + generate_sitemap( + root_path=site, + base_url="https://example.com", + output_file=output, + excluded_paths=["media"], + ) + + urls = self.urls(output) + self.assertIn("https://example.com/index.html", urls) + finally: + shutil.rmtree(parent, ignore_errors=True) + + def test_exclusion_of_nested_relative_path(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + excluded_paths=[os.path.join("blog", "posts")], + ) + + urls = self.urls() + self.assertNotIn("https://example.com/blog/posts/", urls) + self.assertNotIn("https://example.com/blog/posts/post1.html", urls) + self.assertIn("https://example.com/blog/", urls) + + def test_exclusion_of_file_name(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + excluded_paths=["second-post.htm"], + ) + self.assertNotIn("https://example.com/blog/second-post.htm", self.urls()) + + def test_absolute_exclusion_path(self): + excluded_dir = os.path.join(self.test_dir, "blog", "posts") + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + excluded_paths=[excluded_dir], + ) + self.assertNotIn("https://example.com/blog/posts/post1.html", self.urls()) + + def test_glob_exclusions(self): + self.write_dummy_file(os.path.join(self.test_dir, "draft-page.html")) + self.write_dummy_file(os.path.join(self.test_dir, "blog", "draft-note.html")) + self.write_dummy_file(os.path.join(self.test_dir, "notes.tmp")) + + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + excluded_paths=["draft-*", "*.tmp"], + allowed_extensions=[".html", ".tmp"], + ) + + urls = self.urls() + self.assertNotIn("https://example.com/draft-page.html", urls) + self.assertNotIn("https://example.com/blog/draft-note.html", urls) + self.assertNotIn("https://example.com/notes.tmp", urls) + self.assertIn("https://example.com/index.html", urls) + + def test_is_excluded_helper(self): + self.assertTrue(is_excluded(os.path.join(self.test_dir, "auth"), self.test_dir, ["auth"])) + self.assertFalse( + is_excluded(os.path.join(self.test_dir, "authority"), self.test_dir, ["auth"]) + ) + self.assertFalse(is_excluded(self.test_dir, self.test_dir, ["website"])) + + def test_extensions_are_case_insensitive(self): + self.write_dummy_file(os.path.join(self.test_dir, "UPPER.HTML")) + self.write_dummy_file(os.path.join(self.test_dir, "upper.PHP")) + self.write_dummy_file(os.path.join(self.test_dir, "page.html.bak")) + + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + allowed_extensions=[".HTML", "PHP"], + ) + + urls = self.urls() + self.assertIn("https://example.com/UPPER.HTML", urls) + self.assertIn("https://example.com/upper.PHP", urls) + self.assertIn("https://example.com/custom.php", urls) + self.assertNotIn("https://example.com/page.html.bak", urls) + + def test_urls_are_percent_encoded(self): + self.write_dummy_file(os.path.join(self.test_dir, "my page.html")) + self.write_dummy_file(os.path.join(self.test_dir, "a&b.html")) + self.write_dummy_file(os.path.join(self.test_dir, "hash#1.html")) + self.write_dummy_file(os.path.join(self.test_dir, "café", "index.html")) + + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + ) + + urls = self.urls() + self.assertIn("https://example.com/my%20page.html", urls) + self.assertIn("https://example.com/a%26b.html", urls) + self.assertIn("https://example.com/hash%231.html", urls) + self.assertIn("https://example.com/caf%C3%A9/", urls) + self.assertIn("https://example.com/caf%C3%A9/index.html", urls) + for url in urls: + self.assertNotIn(" ", url) + self.assertNotIn("#", url) + + def test_priority_mapping_accepts_folder_and_extension_variants(self): + mapping = { + "blog/": 0.70, # documented trailing slash folder key + "posts\\": 0.20, # Windows separator + ".PHP": 0.35, # upper case extension + "about/index.html": 0.90, + } + + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + priority_mapping=mapping, + allowed_extensions=[".html", ".php"], + ) + + entries = self.entries() + self.assertEqual(entries["https://example.com/blog/"]["priority"], "0.70") + self.assertEqual(entries["https://example.com/blog/posts/"]["priority"], "0.20") + self.assertEqual(entries["https://example.com/custom.php"]["priority"], "0.35") + self.assertEqual(entries["https://example.com/about/index.html"]["priority"], "0.90") + + def test_priorities_are_clamped_to_the_protocol_range(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + default_priority=0.5, + priority_mapping={"about/index.html": 5, "index.html": -2}, + ) + entries = self.entries() + self.assertEqual(entries["https://example.com/about/index.html"]["priority"], "1.00") + self.assertEqual(entries["https://example.com/index.html"]["priority"], "0.00") + + def test_get_priority_public_helper(self): + self.assertEqual( + get_priority("blog/posts", True, 0.5, {"posts": 0.3}), "0.30" + ) + self.assertEqual(get_priority("blog/posts", True, 0.5), "0.50") + self.assertEqual( + get_priority("blog/posts", True, 0.5, lambda rel, is_dir: 0.25), "0.25" + ) + + def test_missing_root_raises(self): + with self.assertRaises(NotADirectoryError): + generate_sitemap( + os.path.join(self.test_dir, "does-not-exist"), + "https://example.com", + self.output_file, + ) + + def test_invalid_base_url_raises(self): + with self.assertRaises(ValueError): + generate_sitemap(self.test_dir, "example.com", self.output_file) + + def test_invalid_max_urls_raises(self): + for value in (0, -5): + with self.assertRaises(ValueError): + generate_sitemap( + self.test_dir, "https://example.com", self.output_file, max_urls=value + ) + + def test_max_urls_above_protocol_limit_warns(self): + with self.assertWarns(RuntimeWarning): + generate_sitemap( + self.test_dir, "https://example.com", self.output_file, max_urls=60000 + ) + self.assertEqual( + ET.parse(self.output_file).getroot().tag, + "{http://www.sitemaps.org/schemas/sitemap/0.9}urlset", + ) + + def test_output_directory_is_created(self): + nested = os.path.join(self.test_dir, "public", "seo", "sitemap.xml") + created = generate_sitemap(self.test_dir, "https://example.com", nested) + self.assertTrue(os.path.exists(nested)) + self.assertEqual(created, [nested]) + + def test_string_arguments_are_accepted(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + excluded_paths="auth", + allowed_extensions="html,php", + ) + urls = self.urls() + self.assertNotIn("https://example.com/auth/login.html", urls) + self.assertIn("https://example.com/custom.php", urls) + self.assertIn("https://example.com/index.html", urls) + + def test_exclude_hidden(self): + self.write_dummy_file(os.path.join(self.test_dir, ".hidden.html")) + self.write_dummy_file(os.path.join(self.test_dir, ".git", "config.html")) + + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + exclude_hidden=True, + ) + + urls = self.urls() + for url in urls: + self.assertNotIn("/.", url) + self.assertIn("https://example.com/index.html", urls) + + def test_include_homepage_false(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com/", + output_file=self.output_file, + include_homepage=False, + ) + self.assertNotIn("https://example.com", self.urls()) + + def test_base_url_trailing_slash_is_normalised(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com/", + output_file=self.output_file, + ) + urls = self.urls() + self.assertIn("https://example.com", urls) + self.assertIn("https://example.com/about/", urls) + self.assertEqual(len(urls), len(set(urls))) + + def test_output_is_deterministic(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + ) + with open(self.output_file, "rb") as handle: + first = handle.read() + + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + ) + with open(self.output_file, "rb") as handle: + second = handle.read() + + self.assertEqual(first, second) + + def test_output_is_pretty_and_parsable(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + ) + with open(self.output_file, "r", encoding="utf-8") as handle: + content = handle.read() + + self.assertTrue(content.startswith('')) + self.assertNotIn("\n\n", content) + self.assertIn('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"', content) + ET.fromstring(content) # must be well-formed + + def collect_urls(self, paths): + collected = [] + for path in paths: + collected.extend(self.urls(path)) + return collected + + def reference_urls(self): + """Generate an unsplit sitemap elsewhere and return its URLs.""" + reference_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, reference_dir, ignore_errors=True) + reference = os.path.join(reference_dir, "sitemap.xml") + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=reference, + ) + return set(self.urls(reference)) + + def test_split_covers_every_url_exactly_once(self): + created = generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + max_urls=2, + ) + + self.assertEqual(created[-1], self.output_file) + parts = [path for path in created if path != self.output_file] + self.assertGreater(len(parts), 1) + self.assertTrue(all(os.path.exists(part) for part in parts)) + + collected = self.collect_urls(parts) + self.assertEqual(len(collected), len(set(collected))) + self.assertEqual(set(collected), self.reference_urls()) + + def test_split_index_lists_every_part(self): + created = generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + max_urls=2, + ) + parts = [path for path in created if path != self.output_file] + + root = ET.parse(self.output_file).getroot() + index_locs = [loc.text for loc in root.findall("sm:sitemap/sm:loc", SITEMAP_NS)] + + expected = ["https://example.com/" + os.path.basename(part) for part in parts] + self.assertEqual(index_locs, expected) + self.assertEqual(len(index_locs), len(parts)) + + def test_stale_split_parts_are_removed(self): + stale = os.path.join(self.test_dir, "sitemap_9.xml") + self.write_dummy_file(stale, content="") + + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + max_urls=3, + ) + + self.assertFalse(os.path.exists(stale)) + self.assertTrue(os.path.exists(os.path.join(self.test_dir, "sitemap_1.xml"))) + + def test_write_sitemap_file_helper(self): + target = os.path.join(self.test_dir, "manual.xml") + write_sitemap_file( + target, + [ + { + "loc": "https://example.com/a?b=1&c=2", + "lastmod": "2024-01-01T00:00:00+00:00", + "priority": "0.50", + } + ], + ) + urls = self.urls(target) + self.assertEqual(urls, ["https://example.com/a?b=1&c=2"]) + + def test_version_is_exposed(self): + self.assertRegex(__version__, r"^\d+\.\d+\.\d+") + + # -- CLI ------------------------------------------------------------- + + def run_cli(self, *args): + environment = os.environ.copy() + environment["PYTHONPATH"] = os.pathsep.join( + [REPO_ROOT, environment.get("PYTHONPATH", "")] + ).strip(os.pathsep) + return subprocess.run( + [sys.executable, "-m", "localsitemap"] + list(args), + cwd=self.test_dir, + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + + def test_cli_generates_sitemap(self): + result = self.run_cli( + self.test_dir, + "https://example.com", + "-o", + "cli-sitemap.xml", + "--exclude", + "auth", + ) + self.assertEqual(result.returncode, 0, result.stderr) + target = os.path.join(self.test_dir, "cli-sitemap.xml") + self.assertTrue(os.path.exists(target)) + urls = self.urls(target) + self.assertIn("https://example.com/blog/first-post.html", urls) + self.assertNotIn("https://example.com/auth/login.html", urls) + + def test_cli_priority_map_and_version(self): + mapping_path = os.path.join(self.test_dir, "priorities.json") + with open(mapping_path, "w", encoding="utf-8") as handle: + json.dump({".html": 0.42}, handle) + + result = self.run_cli( + self.test_dir, + "https://example.com", + "-o", + "cli-sitemap.xml", + "--priority-map", + mapping_path, + ) + self.assertEqual(result.returncode, 0, result.stderr) + entries = self.entries(os.path.join(self.test_dir, "cli-sitemap.xml")) + self.assertEqual(entries["https://example.com/index.html"]["priority"], "0.42") + + version_result = self.run_cli("--version") + self.assertEqual(version_result.returncode, 0) + self.assertIn(__version__, version_result.stdout) + + def test_cli_reports_errors(self): + result = self.run_cli("missing-directory", "https://example.com") + self.assertEqual(result.returncode, 1) + self.assertIn("error:", result.stderr) + + if __name__ == "__main__": unittest.main()