From 0a5a53ded8c7a8aed69fa32855720a7650168fa8 Mon Sep 17 00:00:00 2001 From: pawank925 <229111348+pawank925@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:57:11 +0530 Subject: [PATCH] Split python-libzim into reader and writer parts Cython cannot conditionally parse `cdef extern` blocks, so declare the writer API through a local shim header that only includes libzim's writer headers when the library was built with writer support. Guard the writer implementations (C++ wrappers, writer callbacks, Hint, Creator and the libzim.writer registration) with LIBZIM_WITHOUT_WRITER, and let the build select that path explicitly through the LIBZIM_WITHOUT_WRITER env var. On a writer-less libzim the extension now builds and `import libzim.writer` raises ImportError, which is what Android/reader-only wheels need (#283). The reader-only build is covered by a dedicated CI job. Adds tests for lazy submodule loading and creator guards. Fixes #287 --- .github/workflows/Tests.yaml | 24 + README.md | 2 + libzim/__init__.pyi | 10 +- libzim/libwrapper.cpp | 7 + libzim/libwrapper.h | 10 +- libzim/libzim.pyx | 704 +++++++++++++++-------------- libzim/writer_headers.h | 31 ++ libzim/zim.pxd | 8 +- pyproject.toml | 2 +- setup.py | 22 +- tests/test_illustration_archive.py | 10 +- tests/test_libzim_creator.py | 71 +-- tests/test_libzim_modules.py | 12 + tests/test_libzim_reader.py | 14 +- 14 files changed, 533 insertions(+), 394 deletions(-) create mode 100644 libzim/writer_headers.h diff --git a/.github/workflows/Tests.yaml b/.github/workflows/Tests.yaml index 68287cc4..fef16bf8 100644 --- a/.github/workflows/Tests.yaml +++ b/.github/workflows/Tests.yaml @@ -49,6 +49,30 @@ jobs: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} + test-reader-only: + runs-on: ubuntu-26.04 + env: + LIBZIM_WITHOUT_WRITER: "1" + + steps: + - uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.14" + + - name: Display Python version + run: python -VV + + - name: Install dependencies (and project) + run: | + pip install -U pip + pip install -e .[test,scripts] + + - name: Run the tests + run: inv coverage --args "-vvv" + build-docs: # validate that doc can be built runs-on: ubuntu-26.04 diff --git a/README.md b/README.md index f9f83b11..9c515953 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,8 @@ Free-threaded CPython is supported: the extension declares itself free-threading Users on other platforms can install the source distribution (see [Building](#Building) below). +Writer support is decided when the extension is built, not when it is installed: there is no install-time extra for it. Set `LIBZIM_WITHOUT_WRITER=1` when building against a libzim compiled without writer support; such a build exposes the reader, search, suggestion, illustration and version modules only. Import `libzim.writer` in a `try`/`except ImportError` block to detect writer support at runtime. + ## Contributions diff --git a/libzim/__init__.pyi b/libzim/__init__.pyi index 812fae1b..4cbc2789 100644 --- a/libzim/__init__.pyi +++ b/libzim/__init__.pyi @@ -4,7 +4,13 @@ from libzim import ( search, # pyright: ignore[reportUnusedImport] suggestion, # pyright: ignore[reportUnusedImport] version, # pyright: ignore[reportUnusedImport] - writer, # pyright: ignore[reportUnusedImport] ) -__all__ = ["illustration", "reader", "search", "suggestion", "version", "writer"] +__all__ = ["illustration", "reader", "search", "suggestion", "version"] + +try: + from libzim import writer as writer + + __all__.append("writer") +except ImportError: + pass diff --git a/libzim/libwrapper.cpp b/libzim/libwrapper.cpp index a763f94d..d01b804f 100644 --- a/libzim/libwrapper.cpp +++ b/libzim/libwrapper.cpp @@ -28,7 +28,10 @@ #include #include #include +#include +#ifndef LIBZIM_WITHOUT_WRITER #include +#endif ObjWrapper::ObjWrapper(PyObject* obj) @@ -96,6 +99,7 @@ zim::Blob _callMethodOnObj(PyObject *obj, const std::string& methodName, std::st return blob_cy_call_fct(obj, methodName, &error); } +#ifndef LIBZIM_WITHOUT_WRITER template<> std::unique_ptr _callMethodOnObj(PyObject *obj, const std::string& methodName, std::string& error) { @@ -119,6 +123,7 @@ zim::writer::IndexData::GeoPosition _callMethodOnObj(PyObject *obj, const std::string& methodName, std::string& error) { return geoposition_cy_call_fct(obj, methodName, &error); } +#endif // LIBZIM_WITHOUT_WRITER // This cpp function call a python method on a python object. // It checks that we are in a valid state and handle any potential error coming from python. @@ -136,6 +141,7 @@ Output callMethodOnObj(PyObject* obj, const std::string& methodName) { } +#ifndef LIBZIM_WITHOUT_WRITER /* ################################ # Content Provider Wrapper # @@ -238,6 +244,7 @@ zim::writer::Hints WriterItemWrapper::getHints() const { return callMethodOnObj(m_obj, "get_hints"); } +#endif // LIBZIM_WITHOUT_WRITER zim::Compression comp_from_int(int compValue) { diff --git a/libzim/libwrapper.h b/libzim/libwrapper.h index 22692e77..92a89144 100644 --- a/libzim/libwrapper.h +++ b/libzim/libwrapper.h @@ -27,10 +27,14 @@ #include #include #include -#include -#include #include #include +#include + +#ifndef LIBZIM_WITHOUT_WRITER +#include +#include +#endif struct _object; typedef _object PyObject; @@ -313,6 +317,7 @@ class ObjWrapper PyObject* m_obj; }; +#ifndef LIBZIM_WITHOUT_WRITER class WriterItemWrapper : public zim::writer::Item, private ObjWrapper { public: @@ -347,6 +352,7 @@ class IndexDataWrapper: public zim::writer::IndexData, private ObjWrapper uint32_t getWordCount() const override; IndexData::GeoPosition getGeoPosition() const override; }; +#endif // LIBZIM_WITHOUT_WRITER // Small helpers diff --git a/libzim/libzim.pyx b/libzim/libzim.pyx index 75720761..79af2316 100644 --- a/libzim/libzim.pyx +++ b/libzim/libzim.pyx @@ -125,30 +125,31 @@ cdef public api: return move(zim.Blob()) - zim.ContentProvider* contentprovider_cy_call_fct(object obj, string method, string *error) with gil: - """Lookup and execute a pure virtual method on object returning a ContentProvider""" - try: - contentProvider = call_method(obj, method) - if not contentProvider: - raise RuntimeError("ContentProvider is None") - return new zim.ContentProviderWrapper(contentProvider) - except Exception as e: - error[0] = traceback.format_exc().encode('UTF-8') - - return NULL - - zim.IndexData* indexdata_cy_call_fct(object obj, string method, string *error) with gil: - """Lookup and execute a pure virtual method on object returning a IndexData""" - try: - indexData = call_method(obj, method) - if not indexData: - # indexData is none - return NULL; - return new zim.IndexDataWrapper(indexData) - except Exception as e: - error[0] = traceback.format_exc().encode('UTF-8') - - return NULL + IF not LIBZIM_WITHOUT_WRITER: + zim.ContentProvider* contentprovider_cy_call_fct(object obj, string method, string *error) with gil: + """Lookup and execute a pure virtual method on object returning a ContentProvider""" + try: + contentProvider = call_method(obj, method) + if not contentProvider: + raise RuntimeError("ContentProvider is None") + return new zim.ContentProviderWrapper(contentProvider) + except Exception as e: + error[0] = traceback.format_exc().encode('UTF-8') + + return NULL + + zim.IndexData* indexdata_cy_call_fct(object obj, string method, string *error) with gil: + """Lookup and execute a pure virtual method on object returning a IndexData""" + try: + indexData = call_method(obj, method) + if not indexData: + # indexData is none + return NULL; + return new zim.IndexDataWrapper(indexData) + except Exception as e: + error[0] = traceback.format_exc().encode('UTF-8') + + return NULL bool bool_cy_call_fct(object obj, string method, string *error) with gil: """Lookup and execute a pure virtual method on object returning a bool""" @@ -177,39 +178,40 @@ cdef public api: return 0 - zim.GeoPosition geoposition_cy_call_fct(object obj, string method, string *error) with gil: - """Lookup and execute a pure virtual method on object returning a GeoPosition""" - try: - geoPosition = call_method(obj, method) - if geoPosition: - return zim.GeoPosition(True, geoPosition[0], geoPosition[1]); - except Exception as e: - error[0] = traceback.format_exc().encode('UTF-8') - - return zim.GeoPosition(False, 0, 0) - - map[zim.HintKeys, uint64_t] convertToCppHints(dict hintsDict): - """C++ Hints from Python dict""" - cdef map[zim.HintKeys, uint64_t] ret; - for key, value in hintsDict.items(): - ret[key.value] = value - return ret - - map[zim.HintKeys, uint64_t] hints_cy_call_fct(object obj, string method, string* error) with gil: - """Lookup and execute a pure virtual method on object returning Hints""" - cdef map[zim.HintKeys, uint64_t] ret; - try: - func = getattr(obj, method.decode('UTF-8')) - hintsDict = {k: pybool(v) for k, v in func().items() if isinstance(k, Hint)} - return convertToCppHints(hintsDict) - except Exception as e: - error[0] = traceback.format_exc().encode('UTF-8') - - return ret + IF not LIBZIM_WITHOUT_WRITER: + zim.GeoPosition geoposition_cy_call_fct(object obj, string method, string *error) with gil: + """Lookup and execute a pure virtual method on object returning a GeoPosition""" + try: + geoPosition = call_method(obj, method) + if geoPosition: + return zim.GeoPosition(True, geoPosition[0], geoPosition[1]); + except Exception as e: + error[0] = traceback.format_exc().encode('UTF-8') + + return zim.GeoPosition(False, 0, 0) + + map[zim.HintKeys, uint64_t] convertToCppHints(dict hintsDict): + """C++ Hints from Python dict""" + cdef map[zim.HintKeys, uint64_t] ret; + for key, value in hintsDict.items(): + ret[key.value] = value + return ret + + map[zim.HintKeys, uint64_t] hints_cy_call_fct(object obj, string method, string* error) with gil: + """Lookup and execute a pure virtual method on object returning Hints""" + cdef map[zim.HintKeys, uint64_t] ret; + try: + func = getattr(obj, method.decode('UTF-8')) + hintsDict = {k: pybool(v) for k, v in func().items() if isinstance(k, Hint)} + return convertToCppHints(hintsDict) + except Exception as e: + error[0] = traceback.format_exc().encode('UTF-8') + + return ret ############################################################################### -#  Creator module # +# Creator module # ############################################################################### writer_module_name = f"{__name__}.writer" @@ -251,11 +253,12 @@ class Compression(enum.Enum): zstd = 1 -class Hint(enum.Enum): - """Generic way to pass information to the creator on how to handle item/redirection.""" - __module__ = writer_module_name - COMPRESS = zim.HintKeys.COMPRESS - FRONT_ARTICLE = zim.HintKeys.FRONT_ARTICLE +IF not LIBZIM_WITHOUT_WRITER: + class Hint(enum.Enum): + """Generic way to pass information to the creator on how to handle item/redirection.""" + __module__ = writer_module_name + COMPRESS = zim.HintKeys.COMPRESS + FRONT_ARTICLE = zim.HintKeys.FRONT_ARTICLE class ContentProvider: @@ -364,296 +367,297 @@ class BaseWritingItem: f"title={self.get_title()})" ) -cdef class _Creator: - """ZIM Creator. - - Args: - filename: Full path to a zim file. - - Attributes: - *c_creator (zim.ZimCreator): a pointer to the C++ Creator object - _filename (pathlib.Path): path to create the ZIM file at. - _started (bool): flag if the creator has started. - """ - __module__ = writer_module_name - - cdef zim.ZimCreator c_creator - cdef object _filename - cdef object _started - - def __cinit__(self, object filename: pathlib.Path, *args, **kwargs): - self._filename = pathlib.Path(filename) - self._started = False - # fail early if destination is not writable - parent = self._filename.expanduser().resolve().parent - if not os.access(parent, mode=os.W_OK, effective_ids=(os.access in os.supports_effective_ids)): - raise IOError(f"Unable to write ZIM file at {self._filename}") - - def __init__(self, filename: pathlib.Path): - pass - - def config_verbose(self, bool verbose: bool) -> Self: - """Set creator verbosity inside libzim (default: off). +IF not LIBZIM_WITHOUT_WRITER: + cdef class _Creator: + """ZIM Creator. Args: - verbose (bool): Whether to enable verbosity. + filename: Full path to a zim file. - Returns: - The Creator instance with updated verbosity settings. + Attributes: + *c_creator (zim.ZimCreator): a pointer to the C++ Creator object + _filename (pathlib.Path): path to create the ZIM file at. + _started (bool): flag if the creator has started. """ - if self._started: - raise RuntimeError("Creator started") - self.c_creator.configVerbose(verbose) - return self - - def config_compression(self, compression: Compression) -> Self: - """Set compression algorithm to use. - - Check libzim for default setting. (Fall 2021 default: zstd). - - Args: - compression: The compression algorithm to set. + __module__ = writer_module_name - Returns: - The Creator instance with updated compression settings. - """ - if self._started: - raise RuntimeError("Creator started") - self.c_creator.configCompression(zim.comp_from_int(compression.value)) - return self + cdef zim.ZimCreator c_creator + cdef object _filename + cdef object _started - def config_clustersize(self, int size: pyint) -> Self: - """Set size of created clusters. + def __cinit__(self, object filename: pathlib.Path, *args, **kwargs): + self._filename = pathlib.Path(filename) + self._started = False + # fail early if destination is not writable + parent = self._filename.expanduser().resolve().parent + if not os.access(parent, mode=os.W_OK, effective_ids=(os.access in os.supports_effective_ids)): + raise IOError(f"Unable to write ZIM file at {self._filename}") - Check libzim for default setting. (Fall 2021 default: 2Mib). + def __init__(self, filename: pathlib.Path): + pass - libzim will store at most this value per cluster before creating - another one. + def config_verbose(self, bool verbose: bool) -> Self: + """Set creator verbosity inside libzim (default: off). - Args: - size (int): The maximum size (in bytes) for each cluster. + Args: + verbose (bool): Whether to enable verbosity. - Returns: - The Creator instance with updated cluster size settings. - """ - if self._started: - raise RuntimeError("Creator started") - self.c_creator.configClusterSize(size) - return self + Returns: + The Creator instance with updated verbosity settings. + """ + if self._started: + raise RuntimeError("Creator started") + self.c_creator.configVerbose(verbose) + return self - def config_indexing(self, bool indexing: bool, str language: str) -> Self: - """Configures the full-text indexing feature. + def config_compression(self, compression: Compression) -> Self: + """Set compression algorithm to use. - Args: - indexing (bool): whether to create a full-text index of the content - language (str): language (ISO-639-3 code) to assume content in during indexation. + Check libzim for default setting. (Fall 2021 default: zstd). - Returns: - The Creator instance with updated indexing settings. - """ - if self._started: - raise RuntimeError("Creator started") - self.c_creator.configIndexing(indexing, language.encode('UTF-8')) - return self + Args: + compression: The compression algorithm to set. - def config_nbworkers(self, int nb_workers: pyint) -> Self: - """Configures the number of threads to use for internal workers (default: 4). + Returns: + The Creator instance with updated compression settings. + """ + if self._started: + raise RuntimeError("Creator started") + self.c_creator.configCompression(zim.comp_from_int(compression.value)) + return self - Args: - nb_workers (int): The number of threads to allocate. + def config_clustersize(self, int size: pyint) -> Self: + """Set size of created clusters. - Returns: - The Creator instance with updated worker thread settings. - """ - if self._started: - raise RuntimeError("Creator started") - self.c_creator.configNbWorkers(nb_workers) - return self + Check libzim for default setting. (Fall 2021 default: 2Mib). - def set_mainpath(self, str main_path: str) -> Self: - """Set path of the main entry. + libzim will store at most this value per cluster before creating + another one. - Args: - main_path (str): The path of the main entry. + Args: + size (int): The maximum size (in bytes) for each cluster. - Returns: - The Creator instance with the updated main entry path. - """ - self.c_creator.setMainPath(main_path.encode('UTF-8')) - return self + Returns: + The Creator instance with updated cluster size settings. + """ + if self._started: + raise RuntimeError("Creator started") + self.c_creator.configClusterSize(size) + return self - def add_illustration( - self, - content: bytes, - width: pyint, - height: pyint | None = None, - scale: float = 1.0, - ): - """Add a PNG illustration to Archive. + def config_indexing(self, bool indexing: bool, str language: str) -> Self: + """Configures the full-text indexing feature. - Shortcut building an IllustrationInfo from width/height/scale and - calling :meth:`add_illustration_info`. + Args: + indexing (bool): whether to create a full-text index of the content + language (str): language (ISO-639-3 code) to assume content in during indexation. - Refer to https://wiki.openzim.org/wiki/Metadata for more details. + Returns: + The Creator instance with updated indexing settings. + """ + if self._started: + raise RuntimeError("Creator started") + self.c_creator.configIndexing(indexing, language.encode('UTF-8')) + return self - Args: - content (bytes): The binary content of the PNG illustration. - width (int): Width in CSS pixels. - height (int): Height in CSS pixels. Defaults to width (square - illustration). - scale (float): Device pixel ratio. Defaults to 1.0. + def config_nbworkers(self, int nb_workers: pyint) -> Self: + """Configures the number of threads to use for internal workers (default: 4). - Raises: - RuntimeError: If an illustration with the same attributes already exists. + Args: + nb_workers (int): The number of threads to allocate. - Examples: - # Square illustration at scale 1 - creator.add_illustration(png_data, 48) + Returns: + The Creator instance with updated worker thread settings. + """ + if self._started: + raise RuntimeError("Creator started") + self.c_creator.configNbWorkers(nb_workers) + return self - # Explicit width, height and scale - creator.add_illustration(png_data, 48, 48, 2.0) - """ - if height is None: - height = width - self.add_illustration_info(content, IllustrationInfo(width, height, scale)) + def set_mainpath(self, str main_path: str) -> Self: + """Set path of the main entry. - def add_illustration_info(self, content: bytes, info: IllustrationInfo): - """Add a PNG illustration to Archive from an IllustrationInfo. + Args: + main_path (str): The path of the main entry. - Mirrors libzim's own addIllustration(IllustrationInfo, content), for - full control over the illustration (e.g. extra attributes). + Returns: + The Creator instance with the updated main entry path. + """ + self.c_creator.setMainPath(main_path.encode('UTF-8')) + return self - Refer to https://wiki.openzim.org/wiki/Metadata for more details. + def add_illustration( + self, + content: bytes, + width: pyint, + height: pyint | None = None, + scale: float = 1.0, + ): + """Add a PNG illustration to Archive. - Args: - content (bytes): The binary content of the PNG illustration. - info (IllustrationInfo): The illustration's metadata. + Shortcut building an IllustrationInfo from width/height/scale and + calling :meth:`add_illustration_info`. - Raises: - RuntimeError: If an illustration with the same attributes already exists. + Refer to https://wiki.openzim.org/wiki/Metadata for more details. - Examples: - info = IllustrationInfo(48, 48, 2.0) - creator.add_illustration_info(png_data, info) - """ - cdef string _content = content - cdef zim.IllustrationInfo c_ii = _illustration_info_to_c(info) - self.c_creator.addIllustration(c_ii, _content) + Args: + content (bytes): The binary content of the PNG illustration. + width (int): Width in CSS pixels. + height (int): Height in CSS pixels. Defaults to width (square + illustration). + scale (float): Device pixel ratio. Defaults to 1.0. -# def set_uuid(self, uuid) -> Self: -# self.c_creator.setUuid(uuid) + Raises: + RuntimeError: If an illustration with the same attributes already exists. - def add_item(self, writer_item not None: BaseWritingItem): - """Add an item to the Creator object. + Examples: + # Square illustration at scale 1 + creator.add_illustration(png_data, 48) - Args: - item (WriterItem): The item to add to the archive. + # Explicit width, height and scale + creator.add_illustration(png_data, 48, 48, 2.0) + """ + if height is None: + height = width + self.add_illustration_info(content, IllustrationInfo(width, height, scale)) - Raises: - RuntimeError: If an item with the same path already exists. - RuntimeError: If the ZimCreator has already been finalized. - """ - if not self._started: - raise RuntimeError("Creator not started") + def add_illustration_info(self, content: bytes, info: IllustrationInfo): + """Add a PNG illustration to Archive from an IllustrationInfo. - # Make a shared pointer to ZimArticleWrapper from the ZimArticle object - cdef shared_ptr[zim.WriterItem] item = shared_ptr[zim.WriterItem]( - new zim.WriterItemWrapper(writer_item)); - with nogil: - self.c_creator.addItem(item) + Mirrors libzim's own addIllustration(IllustrationInfo, content), for + full control over the illustration (e.g. extra attributes). - def add_metadata(self, str name: str, bytes content: bytes, str mimetype: str): - """Adds a metadata entry to the archive. + Refer to https://wiki.openzim.org/wiki/Metadata for more details. - Refer to https://wiki.openzim.org/wiki/Metadata for more details. + Args: + content (bytes): The binary content of the PNG illustration. + info (IllustrationInfo): The illustration's metadata. - Args: - name (str): The name of the metadata entry. - content (bytes): The binary content of the metadata entry. - mimetype (str): The MIME type of the metadata entry. + Raises: + RuntimeError: If an illustration with the same attributes already exists. - Raises: - RuntimeError: If a metadata entry with the same name already exists. - """ + Examples: + info = IllustrationInfo(48, 48, 2.0) + creator.add_illustration_info(png_data, info) + """ + cdef string _content = content + cdef zim.IllustrationInfo c_ii = _illustration_info_to_c(info) + self.c_creator.addIllustration(c_ii, _content) - if not self._started: - raise RuntimeError("Creator not started") + # def set_uuid(self, uuid) -> Self: + # self.c_creator.setUuid(uuid) - cdef string _name = name.encode('UTF-8') - cdef string _content = content - cdef string _mimetype = mimetype.encode('UTF-8') - with nogil: - self.c_creator.addMetadata(_name, _content, _mimetype) + def add_item(self, writer_item not None: BaseWritingItem): + """Add an item to the Creator object. - def add_redirection(self, str path: str, str title: str, str target_path: str, dict hints: dict[Hint, pyint]): - """Add redirection entry to the archive. + Args: + item (WriterItem): The item to add to the archive. + + Raises: + RuntimeError: If an item with the same path already exists. + RuntimeError: If the ZimCreator has already been finalized. + """ + if not self._started: + raise RuntimeError("Creator not started") + + # Make a shared pointer to ZimArticleWrapper from the ZimArticle object + cdef shared_ptr[zim.WriterItem] item = shared_ptr[zim.WriterItem]( + new zim.WriterItemWrapper(writer_item)); + with nogil: + self.c_creator.addItem(item) - Refer to https://wiki.openzim.org/wiki/ZIM_file_format#Redirect_Entry for more details. + def add_metadata(self, str name: str, bytes content: bytes, str mimetype: str): + """Adds a metadata entry to the archive. - Args: - path (str): The path of the redirection entry. - title (str): The title associated with the redirection. - target_path (str): The target path for the redirection. - hints (dict[Hint, int]): A dictionary of hints for the redirection. + Refer to https://wiki.openzim.org/wiki/Metadata for more details. - Raises: - RuntimeError: If a redirection entry exists with the same path. - """ + Args: + name (str): The name of the metadata entry. + content (bytes): The binary content of the metadata entry. + mimetype (str): The MIME type of the metadata entry. - if not self._started: - raise RuntimeError("Creator not started") + Raises: + RuntimeError: If a metadata entry with the same name already exists. + """ - cdef string _path = path.encode('UTF-8') - cdef string _title = title.encode('UTF-8') - cdef string _targetPath = target_path.encode('UTF-8') - cdef map[zim.HintKeys, uint64_t] _hints = convertToCppHints(hints) - with nogil: - self.c_creator.addRedirection(_path, _title, _targetPath, _hints) + if not self._started: + raise RuntimeError("Creator not started") - def add_alias(self, str path: str, str title: str, str target_path: str, dict hints: dict[Hint, pyint]): - """Alias the (existing) entry `target_path` as a new entry `path`. + cdef string _name = name.encode('UTF-8') + cdef string _content = content + cdef string _mimetype = mimetype.encode('UTF-8') + with nogil: + self.c_creator.addMetadata(_name, _content, _mimetype) - Args: - path (str): The path for the new alias. - title (str): The title associated with the alias. - target_path (str): The existing entry to be aliased. - hints (dict[Hint, int]): A dictionary of hints for the alias. + def add_redirection(self, str path: str, str title: str, str target_path: str, dict hints: dict[Hint, pyint]): + """Add redirection entry to the archive. - Raises: - RuntimeError: If the `target_path` entry doesn't exist. - """ + Refer to https://wiki.openzim.org/wiki/ZIM_file_format#Redirect_Entry for more details. - if not self._started: - raise RuntimeError("Creator not started") + Args: + path (str): The path of the redirection entry. + title (str): The title associated with the redirection. + target_path (str): The target path for the redirection. + hints (dict[Hint, int]): A dictionary of hints for the redirection. + + Raises: + RuntimeError: If a redirection entry exists with the same path. + """ + + if not self._started: + raise RuntimeError("Creator not started") + + cdef string _path = path.encode('UTF-8') + cdef string _title = title.encode('UTF-8') + cdef string _targetPath = target_path.encode('UTF-8') + cdef map[zim.HintKeys, uint64_t] _hints = convertToCppHints(hints) + with nogil: + self.c_creator.addRedirection(_path, _title, _targetPath, _hints) - cdef string _path = path.encode('UTF-8') - cdef string _title = title.encode('UTF-8') - cdef string _targetPath = target_path.encode('UTF-8') - cdef map[zim.HintKeys, uint64_t] _hints = convertToCppHints(hints) - with nogil: - self.c_creator.addAlias(_path, _title, _targetPath, _hints) + def add_alias(self, str path: str, str title: str, str target_path: str, dict hints: dict[Hint, pyint]): + """Alias the (existing) entry `target_path` as a new entry `path`. - def __enter__(self): - cdef string _path = str(self._filename).encode('UTF-8') - with nogil: - self.c_creator.startZimCreation(_path) - self._started = True - return self + Args: + path (str): The path for the new alias. + title (str): The title associated with the alias. + target_path (str): The existing entry to be aliased. + hints (dict[Hint, int]): A dictionary of hints for the alias. + + Raises: + RuntimeError: If the `target_path` entry doesn't exist. + """ + + if not self._started: + raise RuntimeError("Creator not started") + + cdef string _path = path.encode('UTF-8') + cdef string _title = title.encode('UTF-8') + cdef string _targetPath = target_path.encode('UTF-8') + cdef map[zim.HintKeys, uint64_t] _hints = convertToCppHints(hints) + with nogil: + self.c_creator.addAlias(_path, _title, _targetPath, _hints) - def __exit__(self, exc_type, exc_val, exc_tb): - if True or exc_type is None: + def __enter__(self): + cdef string _path = str(self._filename).encode('UTF-8') with nogil: - self.c_creator.finishZimCreation() - self._started = False + self.c_creator.startZimCreation(_path) + self._started = True + return self - @property - def filename(self) -> pathlib.Path: - """Path of the ZIM Archive on the filesystem. + def __exit__(self, exc_type, exc_val, exc_tb): + if True or exc_type is None: + with nogil: + self.c_creator.finishZimCreation() + self._started = False - Returns: - (pathlib.Path): Path of the ZIM Archive on the filesystem. - """ - return self._filename + @property + def filename(self) -> pathlib.Path: + """Path of the ZIM Archive on the filesystem. + + Returns: + (pathlib.Path): Path of the ZIM Archive on the filesystem. + """ + return self._filename class StringProvider(ContentProvider): @@ -753,58 +757,60 @@ class IndexData: return None -class Creator(_Creator): - """Creator to create ZIM files.""" - __module__ = writer_module_name - def config_compression(self, compression: Compression | str): - if not isinstance(compression, Compression): - compression = getattr(Compression, compression.lower()) - return super().config_compression(compression) - - def add_metadata( - self, name: str, content: str | bytes, - mimetype: str = "text/plain;charset=UTF-8" - ): - if isinstance(content, str): - content = content.encode("UTF-8") - super().add_metadata(name=name, content=content, mimetype=mimetype) - - def __repr__(self) -> str: - return f"Creator(filename={self.filename})" - -writer_module_doc = """libzim writer module - -- Creator to create ZIM files -- Item to store ZIM articles metadata -- ContentProvider to store an Item's content -- Blob to store actual content -- StringProvider to store an Item's content from a string -- FileProvider to store an Item's content from a file path -- Compression to select the algorithm to compress ZIM archive with - -Usage: - -```python -with Creator(pathlib.Path("myfile.zim")) as creator: - creator.config_verbose(False) - creator.add_metadata("Name", b"my name") - # example - creator.add_item(MyItemSubclass(path, title, mimetype, content) - creator.set_mainpath(path) - creator.add_illustration(png_data, 48, 48, 2.0) -```""" -writer_public_objects = [ - Creator, - Compression, - ('Blob', WritingBlob), - Hint, - ('Item', BaseWritingItem), - ContentProvider, - FileProvider, - StringProvider, - IndexData -] -writer = create_module(writer_module_name, writer_module_doc, writer_public_objects) +IF not LIBZIM_WITHOUT_WRITER: + class Creator(_Creator): + """Creator to create ZIM files.""" + __module__ = writer_module_name + def config_compression(self, compression: Compression | str): + if not isinstance(compression, Compression): + compression = getattr(Compression, compression.lower()) + return super().config_compression(compression) + + def add_metadata( + self, name: str, content: str | bytes, + mimetype: str = "text/plain;charset=UTF-8" + ): + if isinstance(content, str): + content = content.encode("UTF-8") + super().add_metadata(name=name, content=content, mimetype=mimetype) + + def __repr__(self) -> str: + return f"Creator(filename={self.filename})" + +IF not LIBZIM_WITHOUT_WRITER: + writer_module_doc = """libzim writer module + + - Creator to create ZIM files + - Item to store ZIM articles metadata + - ContentProvider to store an Item's content + - Blob to store actual content + - StringProvider to store an Item's content from a string + - FileProvider to store an Item's content from a file path + - Compression to select the algorithm to compress ZIM archive with + + Usage: + + ```python + with Creator(pathlib.Path("myfile.zim")) as creator: + creator.config_verbose(False) + creator.add_metadata("Name", b"my name") + # example + creator.add_item(MyItemSubclass(path, title, mimetype, content) + creator.set_mainpath(path) + creator.add_illustration(png_data, 48, 48, 2.0) + ```""" + writer_public_objects = [ + Creator, + Compression, + ('Blob', WritingBlob), + Hint, + ('Item', BaseWritingItem), + ContentProvider, + FileProvider, + StringProvider, + IndexData + ] + writer = create_module(writer_module_name, writer_module_doc, writer_public_objects) ############################################################################### @@ -2049,14 +2055,18 @@ class ModuleLoader(importlib.abc.Loader): # Create our module. Easy, just return the created module @staticmethod def create_module(spec): - return { + modules = { 'libzim.illustration': illustration, - 'libzim.writer': writer, 'libzim.reader': reader, 'libzim.search': search, 'libzim.suggestion': suggestion, 'libzim.version': version - }.get(spec.name, None) + } + IF not LIBZIM_WITHOUT_WRITER: + modules['libzim.writer'] = writer + if spec.name not in modules: + raise ImportError(f"No module named '{spec.name}'", name=spec.name) + return modules[spec.name] @staticmethod def exec_module(module): @@ -2074,4 +2084,6 @@ class ModuleFinder(importlib.abc.MetaPathFinder): # register finder for our submodules sys.meta_path.insert(0, ModuleFinder()) -__all__ = ["illustration", "writer", "reader", "search", "suggestion", "version"] +__all__ = ["illustration", "reader", "search", "suggestion", "version"] +IF not LIBZIM_WITHOUT_WRITER: + __all__.append("writer") diff --git a/libzim/writer_headers.h b/libzim/writer_headers.h new file mode 100644 index 00000000..cbc012ba --- /dev/null +++ b/libzim/writer_headers.h @@ -0,0 +1,31 @@ +// -*- c++ -*- +/* + * This file is part of python-libzim + * (see https://github.com/libzim/python-libzim) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef LIBZIM_WRITER_HEADERS_H +#define LIBZIM_WRITER_HEADERS_H + +#include + +#ifndef LIBZIM_WITHOUT_WRITER +#include +#include +#include +#endif + +#endif // LIBZIM_WRITER_HEADERS_H diff --git a/libzim/zim.pxd b/libzim/zim.pxd index 238b9efc..26bbb68c 100644 --- a/libzim/zim.pxd +++ b/libzim/zim.pxd @@ -54,7 +54,7 @@ cdef extern from "zim/zim.h" namespace "zim": Zstd "zim::Compression::Zstd" -cdef extern from "zim/writer/item.h" namespace "zim::writer": +cdef extern from "writer_headers.h" namespace "zim::writer": cdef cppclass WriterItem "zim::writer::Item": pass ctypedef enum HintKeys: @@ -64,17 +64,17 @@ cdef extern from "zim/writer/item.h" namespace "zim::writer": cdef cppclass IndexData: pass -cdef extern from "zim/writer/item.h" namespace "zim::writer::IndexData": +cdef extern from "writer_headers.h" namespace "zim::writer::IndexData": cppclass GeoPosition: GeoPosition() GeoPosition(bool, double, double) -cdef extern from "zim/writer/contentProvider.h" namespace "zim::writer": +cdef extern from "writer_headers.h" namespace "zim::writer": cdef cppclass ContentProvider: pass -cdef extern from "zim/writer/creator.h" namespace "zim::writer": +cdef extern from "writer_headers.h" namespace "zim::writer": cdef cppclass ZimCreator "zim::writer::Creator": void configVerbose(bint verbose) void configCompression(Compression compression) diff --git a/pyproject.toml b/pyproject.toml index 8625fa11..a6b0256b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -300,7 +300,7 @@ ban-relative-imports = "all" [tool.ruff.lint.per-file-ignores] # Tests can use magic values, assertions, and relative imports, and print -"tests/**/*" = ["PLR2004", "S101", "TID252", "T201", "ARG001", "ARG002", "N803"] +"tests/**/*" = ["PLR2004", "S101", "TID252", "T201", "ARG001", "ARG002", "N803", "E402"] "setup.py" = ["T201"] "libzim/*.pyi" = ["FBT001"] diff --git a/setup.py b/setup.py index 3198a0aa..5a8acf21 100755 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ class Config: libzim_dl_version: str = os.getenv("LIBZIM_DL_VERSION", "9.8.2-1") use_system_libzim: bool = bool(os.getenv("USE_SYSTEM_LIBZIM") or False) download_libzim: bool = not bool(os.getenv("DONT_DOWNLOAD_LIBZIM") or False) + without_writer: bool = bool(os.getenv("LIBZIM_WITHOUT_WRITER") or False) # toggle profiling for coverage report in Cython profiling: bool = os.getenv("PROFILE", "0") == "1" @@ -395,10 +396,21 @@ def can_use_limited_api(self) -> bool: def get_cython_extension() -> list[Extension]: define_macros: list[tuple[str, str | None]] = [] compiler_directives = {"language_level": "3"} + compile_time_env = {} if getattr(sys, "_is_gil_enabled", None) is not None and not sys._is_gil_enabled(): define_macros.append(("Py_GIL_DISABLED", "1")) + if not config.use_system_libzim and config.download_libzim: + print("Downloading libzim. Set `DONT_DOWNLOAD_LIBZIM` not to.") + config.download_to_dest() + + without_writer = config.without_writer + if without_writer: + define_macros.append(("LIBZIM_WITHOUT_WRITER", "1")) + print("Building without writer support.") + compile_time_env["LIBZIM_WITHOUT_WRITER"] = int(without_writer) + if config.profiling: define_macros += [ ("CYTHON_TRACE", "1"), @@ -426,10 +438,6 @@ def get_cython_extension() -> list[Extension]: "Adjust CFLAGS/LDFLAGS if needed" ) else: - if config.download_libzim: - print("Downloading libzim. Set `DONT_DOWNLOAD_LIBZIM` not to.") - config.download_to_dest() - # Check for the CPP Libzim library headers in expected directory if not config.header_file.exists() or not config.dylib_file.exists(): raise OSError( @@ -467,7 +475,11 @@ def get_cython_extension() -> list[Extension]: define_macros=define_macros, py_limited_api=config.can_use_limited_api, ) - return cythonize([wrapper_extension], compiler_directives=compiler_directives) + return cythonize( + [wrapper_extension], + compiler_directives=compiler_directives, + compile_time_env=compile_time_env, + ) class LibzimBuildExt(build_ext): diff --git a/tests/test_illustration_archive.py b/tests/test_illustration_archive.py index 3519e339..8c6954aa 100644 --- a/tests/test_illustration_archive.py +++ b/tests/test_illustration_archive.py @@ -1,10 +1,18 @@ import pytest +try: + from libzim.writer import Creator # pyright: ignore [reportMissingModuleSource] + + HAS_WRITER = True +except ImportError: + HAS_WRITER = False + +pytestmark = pytest.mark.skipif(not HAS_WRITER, reason="writer support not available") + from libzim.illustration import ( # pyright: ignore [reportMissingModuleSource] IllustrationInfo, ) from libzim.reader import Archive # pyright: ignore [reportMissingModuleSource] -from libzim.writer import Creator # pyright: ignore [reportMissingModuleSource] @pytest.fixture(scope="function") diff --git a/tests/test_libzim_creator.py b/tests/test_libzim_creator.py index a398f872..55ea44e9 100644 --- a/tests/test_libzim_creator.py +++ b/tests/test_libzim_creator.py @@ -1,5 +1,25 @@ from __future__ import annotations +import pytest + +try: + from libzim.writer import ( # pyright: ignore [reportMissingModuleSource] + Blob, + ContentProvider, + Creator, + FileProvider, + Hint, + IndexData, + Item, + StringProvider, + ) + + HAS_WRITER = True +except ImportError: + HAS_WRITER = False + +pytestmark = pytest.mark.skipif(not HAS_WRITER, reason="writer support not available") + import itertools import os import pathlib @@ -7,9 +27,7 @@ import subprocess import sys -import pytest - -import libzim.writer # pyright: ignore [reportMissingModuleSource] +import libzim from libzim.illustration import ( # pyright: ignore [reportMissingModuleSource] IllustrationInfo, ) @@ -22,42 +40,33 @@ from libzim.suggestion import ( # pyright: ignore [reportMissingModuleSource] SuggestionSearcher, ) -from libzim.writer import ( # pyright: ignore [reportMissingModuleSource] - Blob, - ContentProvider, - Creator, - FileProvider, - Hint, - IndexData, - Item, - StringProvider, -) HOME_PATH = "lorem_ipsum" +if HAS_WRITER: -class StaticItem(libzim.writer.Item): - def __init__(self, **kwargs): - super().__init__() - for k, v in kwargs.items(): - setattr(self, k, v) + class StaticItem(Item): + def __init__(self, **kwargs): + super().__init__() + for k, v in kwargs.items(): + setattr(self, k, v) - def get_path(self) -> str: - return getattr(self, "path", "") + def get_path(self) -> str: + return getattr(self, "path", "") - def get_title(self) -> str: - return getattr(self, "title", "") + def get_title(self) -> str: + return getattr(self, "title", "") - def get_mimetype(self) -> str: - return getattr(self, "mimetype", "") + def get_mimetype(self) -> str: + return getattr(self, "mimetype", "") - def get_contentprovider(self) -> libzim.writer.ContentProvider: - if getattr(self, "filepath", None): - return FileProvider(filepath=getattr(self, "filepath", "")) - return StringProvider(content=getattr(self, "content", "")) + def get_contentprovider(self) -> ContentProvider: + if getattr(self, "filepath", None): + return FileProvider(filepath=getattr(self, "filepath", "")) + return StringProvider(content=getattr(self, "content", "")) - def get_hints(self) -> dict[Hint, int]: - return getattr(self, "hints", {Hint.FRONT_ARTICLE: True}) + def get_hints(self) -> dict[Hint, int]: + return getattr(self, "hints", {Hint.FRONT_ARTICLE: True}) @pytest.fixture(scope="function") @@ -564,6 +573,8 @@ def test_creator_alias(fpath, lipsum_item): c = Creator(fpath) with pytest.raises(RuntimeError, match="not started"): c.add_redirection("home", "hello", HOME_PATH, {Hint.FRONT_ARTICLE: True}) + with pytest.raises(RuntimeError, match="not started"): + c.add_alias("home", "hello", HOME_PATH, {Hint.FRONT_ARTICLE: True}) del c with Creator(fpath) as c: diff --git a/tests/test_libzim_modules.py b/tests/test_libzim_modules.py index 8b4fb8e5..60204442 100644 --- a/tests/test_libzim_modules.py +++ b/tests/test_libzim_modules.py @@ -14,8 +14,20 @@ def test_submodule_exports(module_name: str): assert getattr(module, name) is not None +@pytest.mark.parametrize("module_name", libzim.__all__) +def test_submodule_reimport_uses_loader(monkeypatch, module_name: str): + monkeypatch.delitem(sys.modules, f"libzim.{module_name}", raising=False) + module = importlib.import_module(f"libzim.{module_name}") + assert module is importlib.import_module(f"libzim.{module_name}") + + def test_submodules_not_registered_under_member_names(): for module_name in libzim.__all__: module = importlib.import_module(f"libzim.{module_name}") for name in module.__all__: assert sys.modules.get(name) is not module + + +def test_unknown_submodule_import_fails(): + with pytest.raises(ImportError, match=r"libzim\.unknown"): + importlib.import_module("libzim.unknown") diff --git a/tests/test_libzim_reader.py b/tests/test_libzim_reader.py index dd015356..b813be2c 100644 --- a/tests/test_libzim_reader.py +++ b/tests/test_libzim_reader.py @@ -1,12 +1,18 @@ +import pytest + +try: + import libzim.writer # pyright: ignore [reportMissingModuleSource] + + HAS_WRITER = True +except ImportError: + HAS_WRITER = False + import gc import os import pathlib import uuid from urllib.request import urlretrieve -import pytest - -import libzim.writer # pyright: ignore [reportMissingModuleSource] from libzim.reader import ( # pyright: ignore [reportMissingModuleSource] Archive, Entry, @@ -287,6 +293,8 @@ def all_zims(tmpdir_factory): urlretrieve(url, temp_dir / os.path.basename(url)) # noqa: S310 # nosec # create blank using pylibzim + if not HAS_WRITER: + pytest.skip("writer support not available") creator = libzim.writer.Creator(temp_dir / "blank.zim") with creator: pass