Add portable CHM and PDF Markdown export

.github/workflows/markdown-export.yml

@@ -0,0 +1,221 @@
+name: Markdown Export
+
+on:
+  pull_request:
+  push:
+    branches: [develop]
+    tags: ['FieldWorks*']
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: 'Build and report, but do not publish'
+        type: boolean
+        default: false
+
+concurrency:
+  group: markdown-export
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  PANDOC_VERSION: '3.9.0.2'
+  PANDOC_SHA256: ce4ac48f48aa7eadc1f5dbdf3449a1739f188ecb8c5421c5adc070fe7479e567
+  EXPORT_BRANCH: markdown-export
+
+jobs:
+  validate:
+    name: Validate markdown export
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      # The runner context is not available to job-level env, so the build
+      # paths are resolved here from $RUNNER_TEMP for every later step.
+      - name: Resolve build paths
+        run: |
+          set -euo pipefail
+          {
+            echo "EXPORT_DIR=${RUNNER_TEMP}/markdown-export"
+            echo "WORK_DIR=${RUNNER_TEMP}/markdown-export-work"
+            echo "DIAGNOSTICS=${RUNNER_TEMP}/markdown-export-diagnostics.json"
+          } >> "$GITHUB_ENV"
+
+      - name: Checkout source
+        uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09
+        with:
+          path: src
+          fetch-depth: 1
+
+      - name: Install Pandoc and CHM extractor
+        run: |
+          set -euo pipefail
+          curl -fsSL -o /tmp/pandoc.deb \
+            "https://github.com/jgm/pandoc/releases/download/${PANDOC_VERSION}/pandoc-${PANDOC_VERSION}-1-amd64.deb"
+          echo "${PANDOC_SHA256}  /tmp/pandoc.deb" | sha256sum --check --strict
+          sudo dpkg -i /tmp/pandoc.deb
+          # p7zip-full remains a runner-image package dependency for CHM extraction.
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq p7zip-full
+          pandoc --version | head -2
+          7z i > /dev/null && echo "7z ok"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e
+        with:
+          version: '0.9.4'
+
+      - name: Provision locked Python and dependencies
+        working-directory: src
+        run: |
+          set -euo pipefail
+          uv python install
+          uv lock --check
+          uv sync --frozen
+
+      - name: Lint converters
+        working-directory: src
+        run: uv run --frozen ruff check tools
+
+      - name: Test converters
+        working-directory: src
+        run: uv run --frozen python -m unittest discover -s tools -p 'test_*.py' -v
+
+      - name: Convert
+        run: |
+          set -euo pipefail
+          rm -rf -- "${EXPORT_DIR}" "${WORK_DIR}"
+          # --directory already puts uv in the checkout, so --repo is relative to it.
+          uv run --frozen --directory src python tools/convert.py \
+            --repo . \
+            --out "${EXPORT_DIR}" \
+            --work "${WORK_DIR}" \
+            --diagnostics "${DIAGNOSTICS}" \
+            --source-ref "${GITHUB_SHA::7}"
+
+      - name: Summarise quality report
+        if: always()
+        run: |
+          uv run --frozen --directory src python -c '
+          import json, os
+          from pathlib import Path
+          diagnostics = Path(os.environ["DIAGNOSTICS"])
+          if not diagnostics.exists():
+              print("## Markdown Export\n\nNo diagnostics produced — validation failed early.")
+              raise SystemExit(0)
+          r = json.loads(diagnostics.read_text(encoding="utf-8"))
+          corpus = r.get("corpus", {})
+          summary = r.get("summary", {})
+          source_ref = corpus.get("source_ref", "?")
+          topic_count = corpus.get("topic_count", 0)
+          pdf_count = corpus.get("pdf_count", 0)
+          fatal_count = summary.get("fatal", 0)
+          advisory_count = summary.get("advisory", 0)
+          print(f"## Markdown Export — ref {source_ref}\n")
+          print(f"**{topic_count:,} topics and {pdf_count:,} PDFs converted**\n")
+          print("| Check | Count |\n| --- | ---: |")
+          print(f"| fatal | {fatal_count} |")
+          print(f"| advisory | {advisory_count} |")
+          for code, count in sorted(summary.get("by_code", {}).items()):
+              label = code.replace("_", " ")
+              print(f"| {label} | {count} |")
+          for code in ("source_missing_link", "source_missing_image", "missing_link", "missing_image", "not_in_toc"):
+              items = [item for item in r.get("issues", []) if item.get("code") == code]
+              if items:
+                  label = items[0].get("label", code.replace("_", " "))
+                  print(f"\n<details><summary>{label} ({len(items)})</summary>\n")
+                  for item in items[:50]:
+                      item_label = item.get("label", label)
+                      item_path = item.get("path", "")
+                      item_message = item.get("message", "")
+                      print(f"- **{item_label}** `{item_path}`: {item_message}")
+                  print("\n</details>")
+          ' >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload diagnostics
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: markdown-export-diagnostics
+          path: ${{ runner.temp }}/markdown-export-diagnostics.json
+          if-no-files-found: ignore
+          retention-days: 14
+
+      - name: Upload completed export
+        if: success()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: markdown-export
+          path: ${{ runner.temp }}/markdown-export
+          if-no-files-found: error
+          include-hidden-files: true
+          retention-days: 14
+
+  publish:
+    name: Publish markdown export
+    needs: validate
+    if: >-
+      needs.validate.result == 'success' &&
+      (github.event_name == 'push' ||
+       (github.event_name == 'workflow_dispatch' && !inputs.dry_run))
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Download completed export
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0
+        with:
+          name: markdown-export
+          path: ${{ runner.temp }}/markdown-export
+
+      - name: Publish incremental markdown-export history
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          EXPORT_DIR: ${{ runner.temp }}/markdown-export
+          PUBLISH_DIR: ${{ runner.temp }}/markdown-export-publish
+        run: |
+          set -euo pipefail
+          rm -rf -- "${PUBLISH_DIR}"
+          mkdir -p "${PUBLISH_DIR}"
+          git -C "${PUBLISH_DIR}" init -q -b "${EXPORT_BRANCH}"
+          git -C "${PUBLISH_DIR}" config user.name "github-actions[bot]"
+          git -C "${PUBLISH_DIR}" config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git -C "${PUBLISH_DIR}" remote add origin \
+            "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
+          if git -C "${PUBLISH_DIR}" fetch --no-tags origin \
+              "refs/heads/${EXPORT_BRANCH}:refs/remotes/origin/${EXPORT_BRANCH}"; then
+            git -C "${PUBLISH_DIR}" checkout -q -B "${EXPORT_BRANCH}" \
+              "refs/remotes/origin/${EXPORT_BRANCH}"
+          else
+            git -C "${PUBLISH_DIR}" checkout -q -B "${EXPORT_BRANCH}"
+          fi
+          git -C "${PUBLISH_DIR}" rm -r -q --ignore-unmatch -- .
+          git -C "${PUBLISH_DIR}" clean -fdx -q
+          cp -a "${EXPORT_DIR}/." "${PUBLISH_DIR}/"
+          git -C "${PUBLISH_DIR}" add -A
+          if git -C "${PUBLISH_DIR}" diff --cached --quiet; then
+            echo "${EXPORT_BRANCH} is already up to date"
+          else
+            git -C "${PUBLISH_DIR}" commit -q -m "Markdown export of help and PDFs ${GITHUB_SHA::7}"
+            git -C "${PUBLISH_DIR}" push origin "HEAD:${EXPORT_BRANCH}"
+            echo "published to ${EXPORT_BRANCH}"
+          fi
+          echo "PUBLISH_DIR=${PUBLISH_DIR}" >> "$GITHUB_ENV"
+
+      - name: Tag the export to match the release
+        if: startsWith(github.ref, 'refs/tags/FieldWorks') &&
+            (github.event_name != 'workflow_dispatch' || !inputs.dry_run)
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+          cd "${PUBLISH_DIR}"
+          TAG="${EXPORT_BRANCH}/${GITHUB_REF_NAME}"
+          if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" > /dev/null 2>&1; then
+            echo "tag already exists: ${TAG}"
+          else
+            git tag -a "${TAG}" -m "Markdown export for ${GITHUB_REF_NAME}"
+            git push origin "refs/tags/${TAG}"
+            echo "tagged ${TAG}"
+          fi

.gitignore

@@ -0,0 +1,5 @@
+.chm-work/
+*.pyc
+__pycache__/
+tools/out/
+.fwhelps-export-*.lock

.python-version

@@ -0,0 +1 @@
+3.13.5

README.md

@@ -0,0 +1,131 @@
+# FwHelps
+
+Documentation for [FieldWorks Language Explorer](https://software.sil.org/fieldworks/)
+(FLEx). Help content is authored in Adobe RoboHelp and committed here as a
+compiled CHM, alongside training and technical-note PDFs.
+
+| File | Contents |
+| --- | --- |
+| `FieldWorks_Language_Explorer_Help.chm` | The main help system — 1,599 topics |
+| `Language Explorer/Training/` | Technical notes: Send-Receive, imports, Word export |
+| `Language Explorer/Utilities/` | AlloGen, PcPatr, ToneParsFLEx, VarGen documentation |
+| `WW-ConceptualIntro/` | Conceptual Introduction to FLEx |
+
+The FieldWorks installer consumes this repo directly: `patch-installer-cd.yml`
+in [sillsdev/FieldWorks](https://github.com/sillsdev/FieldWorks) checks it out
+via a `helps_ref` input, and `Build/releaseTagger.py` tags it `FieldWorks<version>`
+at release time.
+
+## Markdown export
+
+The CHM is also published as markdown on the
+[**`markdown-export`**](../../tree/markdown-export) branch — one file per help
+topic, with images, YAML frontmatter, and a full table of contents.
+
+It exists for two reasons:
+
+- **AI retrieval.** The FieldWorks AI bot previously ingested raw RoboHelp
+  HTML, where roughly two thirds of every topic is markup rather than
+  documentation. The markdown corpus is about 65% smaller in tokens
+  (~2.14M → ~759K) with the prose intact.
+- **Reviewable diffs.** A help change is otherwise a 5&nbsp;MB opaque binary.
+  On the export branch it is a readable text diff, one changed file per
+  edited topic.
+
+Built automatically by
+[`markdown-export.yml`](.github/workflows/markdown-export.yml) on every push to
+`develop`. Nothing there is hand-edited — edit the help in RoboHelp and commit
+the CHM.
+
+Each build also produces `author-report.md` for RoboHelp/PDF authors and
+`author-report.json` for automation. Both cover broken links, topics missing
+from the table of contents, and other source/export quality findings. The
+Markdown report preserves the exact source path and evidence for every finding
+and gives issue-specific repair guidance; JSON retains the stable
+`{corpus, summary, issues}` schema.
+
+Exporter mutation boundaries use a non-blocking native advisory lock on a
+deterministic sibling lockfile, so cooperating invocations targeting the same
+destination fail clearly instead of racing. The lock is a coordination aid,
+not protection against a local process that deliberately ignores file locks;
+stale lockfiles are harmless because ownership is held by the OS handle.
+
+### Versions
+
+Each FieldWorks release is tagged here by `releaseTagger.py`; the matching
+export is tagged `markdown-export/<tag>`.
+
+| FieldWorks | Released | Markdown export |
+| --- | --- | --- |
+| 9.3.7-beta | 2026-02-25 | [`markdown-export/FieldWorks9.3.7-beta`](../../tree/markdown-export/FieldWorks9.3.7-beta) |
+| 9.3.6-beta | 2026-01-29 | [`markdown-export/FieldWorks9.3.6-beta`](../../tree/markdown-export/FieldWorks9.3.6-beta) |
+| 9.3.4 | 2025-10-30 | [`markdown-export/FieldWorks9.3.4`](../../tree/markdown-export/FieldWorks9.3.4) |
+| 9.3.1 | 2025-07-25 | [`markdown-export/FieldWorks9.3.1`](../../tree/markdown-export/FieldWorks9.3.1) |
+| 9.3.0 | 2025-06-17 | [`markdown-export/FieldWorks9.3.0`](../../tree/markdown-export/FieldWorks9.3.0) |
+
+> [!NOTE]
+> Export tags are created going forward, as each release is tagged. Rows above
+> that have no corresponding export tag yet can be backfilled by re-running the
+> workflow against that tag.
+
+## Tools
+
+| Script | Purpose |
+| --- | --- |
+| [`tools/convert.py`](tools/convert.py) | CHM → markdown corpus (the build) |
+| [`tools/fwhelp.lua`](tools/fwhelp.lua) | Pandoc filter: RoboHelp semantics → clean GFM |
+| [`tools/chm_extract.py`](tools/chm_extract.py) | Cross-platform CHM extraction, with validation |
+| [`tools/pdf_convert.py`](tools/pdf_convert.py) | PDF → markdown (bookmarks or font inference) |
+| [`tools/pdf_outlines.json`](tools/pdf_outlines.json) | Pinned PDF outlines; drift fails the build |
+| [`tools/survey.py`](tools/survey.py) | Read-only census of the corpus |
+
+Local build (needs `pandoc` 3.x, and `7z` or Windows' built-in `hh.exe`):
+
+```sh
+uv run --frozen tools/convert.py --repo . --out export
+```
+
+### Reproducible local setup
+
+The converter uses uv with the exact Python version in
+[`.python-version`](.python-version). `uv.lock` is the authoritative dependency
+file; `requirements.txt` and `requirements-dev.txt` are deterministic
+lock-derived compatibility exports for older tooling and should not be edited
+independently. The CI workflow installs only from the frozen lock:
+
+```sh
+uv python install
+uv lock --check
+uv sync --frozen
+uv run --frozen \
+  -m unittest discover -s tools -p 'test_*.py' -v
+uv run --frozen ruff check tools
+uv run --frozen \
+  tools/convert.py --repo . --out export
+uv run --frozen \
+  tools/pdf_convert.py --repo . --out export --update-outlines
+```
+
+To regenerate the compatibility exports after changing project dependencies:
+
+```sh
+uv export --frozen --no-dev --no-hashes --format requirements.txt \
+  --output-file requirements.txt
+uv export --frozen --only-group dev --no-hashes --format requirements.txt \
+  --output-file requirements-dev.txt
+```
+
+On PowerShell, the same `uv run --frozen` commands work unchanged. Updating
+outline locks is intentional and should be reviewed with the resulting
+`tools/pdf_outlines.json` change.
+
+The workflow still installs the runner-image package `p7zip-full` for CHM
+extraction; it is intentionally outside the Python lock. Pandoc is downloaded
+as the pinned 3.9.0.2 amd64 package and its SHA-256 is checked before install.
+
+> [!IMPORTANT]
+> `hh.exe -decompile` silently truncates filenames when the output path exceeds
+> Windows' 260-character limit — no error, no non-zero exit. `chm_extract.py`
+> refuses to run it into a too-long path and validates every extraction against
+> the CHM's own table of contents. Use a short `--work` directory on Windows,
+> or install 7-Zip.