diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dd1dbd5..4f5a61c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ Instructions: Add a subsection under `[Unreleased]` for additions, fixes, change - `pdf` targets accept a `method` attribute selecting the route to the PDF: `latex` (the default) or `fo`, the LaTeX-free route through XSL-FO and Apache FOP. - `lualatex` is now a usable `latex-engine`, with a matching entry in `executables.ptx`. - `latex-engine` is now permitted by the schema on every target, not just `pdf` ones; it has always been the engine that compiles `latex-image` assets for all formats. +- Naming a journal in the publication file now also selects that journal's bibliography and citation style: references are formatted in that style, in every output format, with nothing else to set. A `citation-stylesheet-language/@style` of your own takes precedence. +- Formatted references are now a generated asset like any other, built when needed and regenerated when the bibliography, the citations, or the chosen style changes. `pretext generate references` is no longer a debugging-only step. ### Fixed diff --git a/pretext/__init__.py b/pretext/__init__.py index c7ec159e..2792fcf8 100644 --- a/pretext/__init__.py +++ b/pretext/__init__.py @@ -18,7 +18,7 @@ VERSION = get_version("pretext", Path(__file__).parent.parent) -CORE_COMMIT = "2c8806b9988f855e94d185fb145226bf6c0a5b20" +CORE_COMMIT = "d2fb936e501f50a7e2fe12a1f0e012278da91251" def activate() -> None: diff --git a/pretext/constants.py b/pretext/constants.py index fbea270e..de120829 100644 --- a/pretext/constants.py +++ b/pretext/constants.py @@ -29,6 +29,7 @@ "datafile", "myopenmath", "dynamic-subs", + "references", "qrcode", "gdscript", ], @@ -45,6 +46,7 @@ "mermaid", "myopenmath", "dynamic-subs", + "references", ], "latex": [ "webwork", @@ -59,6 +61,7 @@ "mermaid", "myopenmath", "dynamic-subs", + "references", ], "epub": [ "webwork", @@ -73,6 +76,7 @@ "mermaid", "myopenmath", "dynamic-subs", + "references", ], "epub_nozip": [ "webwork", @@ -87,6 +91,7 @@ "mermaid", "myopenmath", "dynamic-subs", + "references", ], "kindle": [ "webwork", @@ -101,6 +106,7 @@ "mermaid", "myopenmath", "dynamic-subs", + "references", ], "braille": [ "webwork", @@ -114,9 +120,11 @@ "mermaid", "myopenmath", "dynamic-subs", + "references", ], "revealjs": [ "webwork", + "references", "latex-image", "sageplot", "asymptote", @@ -139,6 +147,7 @@ "mermaid", "myopenmath", "dynamic-subs", + "references", ], "webwork": [ "webwork", @@ -157,6 +166,7 @@ "mermaid", "myopenmath", "dynamic-subs", + "references", "gdscript", ], } @@ -175,7 +185,7 @@ "mermaid": ".//mermaid", "myopenmath": ".//myopenmath", "dynamic-subs": ".//statement[.//fillin and ancestor::exercise/evaluation]", - "references": ".//biblio", + "references": ".//biblio|.//xref", "stack": ".//stack", "gdscript": ".//program[@pck]", } diff --git a/pretext/project/__init__.py b/pretext/project/__init__.py index 95817c83..b085c61f 100644 --- a/pretext/project/__init__.py +++ b/pretext/project/__init__.py @@ -465,7 +465,6 @@ def original_source_element(self) -> ET._Element: source_doc = ET.parse(self.source_abspath()) for _ in range(25): source_doc.xinclude() - print("Type of source_doc: ", type(source_doc)) return source_doc.getroot() def source_element(self) -> ET._Element: @@ -520,6 +519,61 @@ def source_element_with_ids(self) -> ET._Element: def publication_abspath(self) -> Path: return self._project.publication_abspath() / self.publication + def publication_csl_settings(self) -> t.Tuple[t.Optional[str], t.Optional[str]]: + """ + The journal name and the CSL style named in the publication file, + either of which may be absent. Between them they decide whether + references and citations are rendered by CSL at all, and which + style does it. + + Read from the publication file directly: answering from there + saves a publisher variable report -- a full XSL pass over the + source -- on every build that could not possibly need one. + """ + try: + publication = ET.parse(self.publication_abspath()) + except Exception: + # a malformed publication file is reported, loudly, elsewhere + return (None, None) + + def first(xpath: str) -> t.Optional[str]: + values = publication.xpath(xpath) + assert isinstance(values, t.List) + return str(values[0]) if values else None + + return ( + first("/publication/common/journal/@name"), + first("/publication/common/citation-stylesheet-language/@style"), + ) + + def note_csl_file_state(self) -> None: + """ + Record whether the generated file of formatted references and + citations is on disk, for the stylesheets to consult. + + XSLT cannot ask, and libxslt evaluates every global variable at + the start of a transformation -- including the one that opens + this file. So any pass at all, a syntax check as much as a + conversion, aborts outright on a project that has opted into CSL + styles but has yet to generate its references. Saying plainly + that the file is absent turns that into a warning and a fall back + to default bibliography handling. + + Cheap enough (one stat) to repeat whenever the answer may have + changed, which it does the moment references are generated. + """ + csl_file = self.generated_dir_abspath() / "references" / "csl-bibliography.xml" + self.stringparams["csl.file.missing"] = "" if csl_file.exists() else "yes" + + def publication_uses_csl(self) -> bool: + """ + Whether the publication file opts into CSL styles for references + and citations, by naming a style outright or a journal that + implies one. + """ + journal, csl_style = self.publication_csl_settings() + return (journal is not None) or (csl_style is not None) + def output_dir_abspath(self) -> Path: if self.is_standalone() and self.output_dir is None: if self.format == Format.PDF or self.compression == Compression.SCORM: @@ -699,6 +753,16 @@ def generate_asset_table(self) -> pt.AssetTable: base_url = self._read_publication_file_subset().baseurl if base_url is not None: hash.update(base_url.encode("utf-8")) + # For references, the style is as much an input as the + # bibliography itself: changing journals must regenerate. Both + # publication file entries are hashed, since either can pick the + # style. (A style that changes inside core, for a journal whose + # name stays put, is caught instead by the assembly-time check + # against the style stamped on the generated file.) + if asset == "references": + for setting in self.publication_csl_settings(): + if setting is not None: + hash.update(setting.encode("utf-8")) # Finally, we store the hash as a string in the dictionary. asset_hash_dict[asset] = hash.hexdigest() return asset_hash_dict @@ -965,6 +1029,9 @@ def build( # Add cli.version to stringparams. Use only the major and minor version numbers. self.stringparams["cli.version"] = VERSION[: VERSION.rfind(".")] + # Before the source is assembled for the first time below. + self.note_csl_file_state() + # Check for xml syntax errors and quit if xml invalid: try: # Access the source_element to trigger assembly if it hasn't been done yet. @@ -991,6 +1058,31 @@ def build( if generate: self.generate_assets(xmlid=xmlid, clean_tmp_dirs=clean_tmp_dirs) + # A journal named in the publication file supplies the CSL style for + # bibliographies and citations. Core resolves that into a stringparam + # here, once, so every format's conversion agrees with the generated + # references about which style is in force. A style named in the + # publication file outranks the journal's, and core says so. + # + # This must follow generation, not precede it. Resolving the style + # also records whether the generated references file is on disk, and + # generation is what puts it there: asked any earlier, the answer + # would be "absent" and every conversion below would fall back to + # default bibliography handling despite the file now existing. + self.note_csl_file_state() + if self.publication_uses_csl(): + try: + core.get_csl_style( + xml=self.source_abspath(), + pub_file=self.publication_abspath().as_posix(), + stringparams=self.stringparams, + ) + except Exception as e: + log.error( + f"Unable to determine the journal's bibliography style:\n {e}" + ) + log.debug(e, exc_info=True) + # Ensure the output directories exist. self.ensure_output_directory() @@ -1226,25 +1318,22 @@ def generate_assets( """ log.info("Generating any needed assets.") + # Generation runs on its own as well as from a build, and the schema + # check just below assembles the source. + self.note_csl_file_state() + # Warn about schema errors here too, since assets are often generated # without a build. A no-op when a build already ran the check. self.check_schema() - # To help with debugging, we are temporarily adding a reference generation step here. The only way this will be called is if `pretext generate references` is called explicitly. - if requested_asset_types == ("references",): - try: - core.references( - xml_source=self.source_abspath(), - pub_file=self.publication_abspath().as_posix(), - stringparams=self.stringparams.copy(), - xmlid_root=xmlid, - dest_dir=self.generated_dir_abspath() / "references", - ) - except Exception as e: - log.error(f"Unable to generate some references:\n {e}") - log.debug(e, exc_info=True) - finally: - return + # Whether references were asked for by name, which changes how a + # project that has not opted into CSL styles is answered below. + references_requested = ( + requested_asset_types is not None + and "references" in requested_asset_types + and "ALL" not in requested_asset_types + ) + # To help with debugging, we are temporarily adding a stack generation step here. The only way this will be called is if `pretext generate stack` is called explicitly. if requested_asset_types == ("stack",): try: @@ -1294,6 +1383,18 @@ def generate_assets( log.debug( f"Based on format {self.format}, assets to be generated are: {requested_asset_types}." ) + # Rendering references and citations with CSL is opt-in: without a + # journal or a style named in the publication file there is nothing + # to render them with, and core would rightly object. + if "references" in requested_asset_types and not self.publication_uses_csl(): + if references_requested: + log.warning( + "References are rendered with a Citation Style Language (CSL) style, " + "which this project has not asked for. Name a style as " + "`citation-stylesheet-language/@style`, or a journal as `journal/@name`, " + "in the publication file's `common` element. No references will be generated." + ) + requested_asset_types.remove("references") # We always build the asset hash table, even if only_changed=True: this tells us which assets need to be built, and how to update the saved asset hash table at the end of the method. # utils.clean_asset_table purges any asset types from the loaded table that are no longer in the target. source_asset_table = self.generate_asset_table() @@ -1585,19 +1686,19 @@ def generate_assets( log.error(f"Unable to generate some datafiles:\n {e}") log.debug(e, exc_info=True) # The following code will eventually be needed, but for now, we leave as a placeholder. - # if "references" in assets_to_generate and debug_references: - # try: - # core.references( - # xml_source=self.source_abspath(), - # pub_file=self.publication_abspath().as_posix(), - # stringparams=stringparams_copy, - # xmlid_root=xmlid, - # dest_dir=self.generated_dir_abspath() / "references", - # ) - # successful_assets.append("references") - # except Exception as e: - # log.error(f"Unable to generate some references:\n {e}") - # log.debug(e, exc_info=True) + if "references" in assets_to_generate: + try: + core.references( + xml_source=self.source_abspath(), + pub_file=self.publication_abspath().as_posix(), + stringparams=stringparams_copy, + xmlid_root=xmlid, + dest_dir=self.generated_dir_abspath() / "references", + ) + successful_assets.append("references") + except Exception as e: + log.error(f"Unable to generate some references:\n {e}") + log.debug(e, exc_info=True) # Delete temporary directories left behind by core: try: core.release_temporary_directories(any_log_level=clean_tmp_dirs) diff --git a/tests/examples/projects/journal-bibliography/project.ptx b/tests/examples/projects/journal-bibliography/project.ptx new file mode 100644 index 00000000..c67a9c45 --- /dev/null +++ b/tests/examples/projects/journal-bibliography/project.ptx @@ -0,0 +1,22 @@ + + + + + html + source/main.ptx + publication.xml + output/web + + + + latex + pdflatex + xelatex + asy + sage + pdftops + pdf-crop-margins + pageres + node + + diff --git a/tests/examples/projects/journal-bibliography/publication.xml b/tests/examples/projects/journal-bibliography/publication.xml new file mode 100644 index 00000000..1ea81491 --- /dev/null +++ b/tests/examples/projects/journal-bibliography/publication.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/tests/examples/projects/journal-bibliography/source/main.ptx b/tests/examples/projects/journal-bibliography/source/main.ptx new file mode 100644 index 00000000..c44f8444 --- /dev/null +++ b/tests/examples/projects/journal-bibliography/source/main.ptx @@ -0,0 +1,23 @@ + + +
+ An Article For A Journal +
+ One +

A result appears in and also in .

+
+ + + References + + ThomasJudson + Abstract Algebra: Theory and Applications + + + David C.Lay + Subspaces and Echelon Forms + + + +
+ diff --git a/tests/test_project.py b/tests/test_project.py index 97eee449..5f5b7a58 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -24,6 +24,7 @@ import pydantic import pytest +from pretext import core from pretext import project as pr from pretext import utils from pretext.resources import resource_base_path @@ -979,3 +980,166 @@ def test_stage(tmp_path: Path) -> None: assert "foobar" in f.read() assert (project.stage_abspath() / "web2" / "article-id.html").exists() shutil.rmtree(project.stage_abspath()) + + +# The journal bibliography feature needs a core that can resolve a journal +# name into a Citation Style Language style. Until that core is the one +# vendored here, these skip rather than fail. +requires_core_csl = pytest.mark.skipif( + not hasattr(core, "get_csl_style"), + reason="vendored core does not resolve journal bibliography styles yet", +) + + +def test_publication_csl_settings(tmp_path: Path) -> None: + """A publication file's journal name and CSL style are read back, and + either one on its own counts as opting into CSL styles.""" + prj_path = tmp_path / "journal-bibliography" + shutil.copytree(EXAMPLES_DIR / "projects" / "journal-bibliography", prj_path) + publication = prj_path / "publication.xml" + with utils.working_directory(prj_path): + target = pr.Project.parse().get_target("web") + assert target.publication_csl_settings() == ("bull-amer-math-soc", None) + assert target.publication_uses_csl() + + # a style of the publisher's own, and no journal + publication.write_text( + publication.read_text().replace( + '', + '', + ) + ) + target = pr.Project.parse().get_target("web") + assert target.publication_csl_settings() == (None, "harvard1") + assert target.publication_uses_csl() + + # neither: the feature stays out of the way + publication.write_text( + publication.read_text().replace( + '', "" + ) + ) + target = pr.Project.parse().get_target("web") + assert target.publication_csl_settings() == (None, None) + assert not target.publication_uses_csl() + + +def test_references_asset_hash(tmp_path: Path) -> None: + """The references hash covers the bibliography, the citations, and the + style that formats them: each changes it, and nothing else does.""" + prj_path = tmp_path / "journal-bibliography" + shutil.copytree(EXAMPLES_DIR / "projects" / "journal-bibliography", prj_path) + source = prj_path / "source" / "main.ptx" + publication = prj_path / "publication.xml" + source_text, publication_text = source.read_text(), publication.read_text() + + def references_hash() -> Any: + with utils.working_directory(prj_path): + return ( + pr.Project.parse() + .get_target("web") + .generate_asset_table()["references"] + ) + + original = references_hash() + assert references_hash() == original + + # an edited bibliography entry + source.write_text( + source_text.replace("Judson", "Judsen") + ) + assert references_hash() != original + source.write_text(source_text) + + # an added citation, with every "biblio" untouched + source.write_text( + source_text.replace( + "References", + 'References

', + ) + ) + assert references_hash() != original + source.write_text(source_text) + + # a different journal, with the source untouched + publication.write_text( + publication_text.replace("bull-amer-math-soc", "ann-pure-appl-logic") + ) + assert references_hash() != original + publication.write_text(publication_text) + + assert references_hash() == original + + +def test_references_need_opting_in(tmp_path: Path) -> None: + """Without a journal or a style in the publication file, references are + left alone: nothing is generated and the build is undisturbed.""" + prj_path = tmp_path / "journal-bibliography" + shutil.copytree(EXAMPLES_DIR / "projects" / "journal-bibliography", prj_path) + publication = prj_path / "publication.xml" + publication.write_text( + publication.read_text().replace('', "") + ) + with utils.working_directory(prj_path): + target = pr.Project.parse().get_target("web") + target.generate_assets(requested_asset_types=["references"]) + assert not (target.generated_dir_abspath() / "references").exists() + + +@requires_core_csl +def test_journal_selects_bibliography_style(tmp_path: Path) -> None: + """Naming a journal formats the bibliography in that journal's style, + with no other setting: the generated file records the style, and the + built HTML carries the formatted entries and citations.""" + prj_path = tmp_path / "journal-bibliography" + shutil.copytree(EXAMPLES_DIR / "projects" / "journal-bibliography", prj_path) + with utils.working_directory(prj_path): + target = pr.Project.parse().get_target("web") + target.build() + + # "bull-amer-math-soc" has no style of its own; it inherits the one + # belonging to the AMS texstyle file it extends. + generated = ( + target.generated_dir_abspath() / "references" / "csl-bibliography.xml" + ) + assert 'csl-style-file="american-mathematical-society-numeric"' in ( + generated.read_text() + ) + + # the style file itself is fetched once and kept + assert ( + target.generated_dir_abspath() + / "csl" + / "american-mathematical-society-numeric.csl" + ).exists() + + # numeric citations, in a style that orders entries by citation + references = ( + target.output_dir_abspath() / "references-backmatter.html" + ).read_text() + assert "Lay, David C." in references + section = (target.output_dir_abspath() / "sec-one.html").read_text() + assert "[1]" in section + + +@requires_core_csl +def test_publisher_style_outranks_journal(tmp_path: Path) -> None: + """A style named in the publication file is used in place of the one the + journal would supply.""" + prj_path = tmp_path / "journal-bibliography" + shutil.copytree(EXAMPLES_DIR / "projects" / "journal-bibliography", prj_path) + publication = prj_path / "publication.xml" + publication.write_text( + publication.read_text().replace( + '', + '' + '', + ) + ) + with utils.working_directory(prj_path): + target = pr.Project.parse().get_target("web") + target.build() + generated = ( + target.generated_dir_abspath() / "references" / "csl-bibliography.xml" + ) + assert 'csl-style-file="harvard1"' in generated.read_text()