diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 0000000..d87f9a5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,25 @@ +name: Framework improvement +description: Propose a reusable SecondBrain framework change +title: "[Feature]: " +labels: [enhancement] +body: + - type: textarea + id: problem + attributes: + label: Problem + description: Describe the reusable problem without including personal vault data. + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposed change + validations: + required: true + - type: checkboxes + id: privacy + attributes: + label: Privacy + options: + - label: I removed personal notes, paths, identities, credentials, and private links. + required: true diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..83a3298 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,12 @@ +## What changed + +## Why + +## User impact + +## Privacy review + +- [ ] No personal content, credentials, host paths, or private links are present. +- [ ] Publication allowlist and denylist behaviour remains fail-closed. + +## Validation diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 0000000..609baa0 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,28 @@ +name: Validate framework + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Compile tools + run: python3 -m py_compile framework/tools/*.py install.py + - name: Run framework regression tests + run: python3 -m unittest discover -s tests + - name: Install and lint sanitized example + run: | + vault="$(mktemp -d)" + python3 install.py --vault "$vault" + python3 "$vault/tools/wiki.py" lint + python3 "$vault/tools/dtm.py" lint diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4a5bb25 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +*.py[cod] +.DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..7254ce2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +This project follows Keep a Changelog principles and uses semantic versioning +for tagged releases. + +## [Unreleased] + +### Added + +- Initial Knowledge Agent and Digital TeamMate framework. +- Allowlist-only publication boundary and repeatable installer. +- Sanitized automation definitions and empty-vault example. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..bb0e5d9 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,9 @@ +# Code of Conduct + +Be respectful, constructive, and considerate. Discuss ideas and implementations +without attacking people. Harassment, discrimination, disclosure of private +information, and deliberately unsafe contributions are not acceptable. + +Maintainers may edit or remove contributions and participation that violate +these expectations. Serious or sensitive concerns should be reported privately +through the repository's security reporting channel. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..db51d7e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,26 @@ +# Contributing + +Thank you for improving SecondBrain Framework. + +## Before opening a pull request + +1. Keep framework behaviour separate from personal vault content. +2. Never include real Daily Notes, wiki pages, sources, activity logs, project + details, credentials, usernames, home-directory paths, or private links. +3. Use synthetic examples with unmistakably fictional data. +4. Update documentation and automation definitions alongside behavioural + changes. +5. Run the validation commands below. + +```sh +python3 -m py_compile framework/tools/*.py install.py +python3 -m unittest discover -s tests +tmpdir="$(mktemp -d)" +python3 install.py --vault "$tmpdir" +python3 "$tmpdir/tools/wiki.py" lint +python3 "$tmpdir/tools/dtm.py" lint +``` + +Pull requests should explain the motivation, user impact, privacy implications, +and validation performed. Prefer focused changes that can be rolled back +independently. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..46c5a01 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 SecondBrain Framework contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index f71291d..a13d4c5 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,53 @@ -# secondbrain-framework -An Obsidian-native LLM Knowledge Agent and Digital TeamMate framework +# SecondBrain Framework + +An Obsidian-native personal knowledge and operational system maintained by two +deliberately separate LLM roles: + +- the **LLM Knowledge Agent**, which compiles curated sources into a persistent, + cited wiki; and +- the **Digital TeamMate (DTM)**, which maintains day-to-day continuity across + notes, tasks, projects, decisions, and follow-ups. + +This repository contains only the reusable framework. It contains no personal +vault data, source material, Daily Notes, project content, or activity history. + +## Why this exists + +Most document assistants rediscover context at query time. SecondBrain instead +maintains a durable intermediate knowledge layer and a separate operational +memory. The division keeps long-term evidence synthesis disciplined while still +supporting practical daily collaboration. + +## Quick start + +Requirements: Python 3.9+ and an existing or new Obsidian vault. + +```sh +python3 install.py --vault /path/to/your/obsidian-vault +``` + +The installer copies only framework files and never copies this repository's +`.git` directory. Existing files are preserved unless `--force` is supplied. +After installation: + +1. Put a source directly in `raw/` and ask the Knowledge Agent to ingest it. +2. Ask the DTM to open or update today's Daily Note. +3. Review `framework/automation-definitions/` and recreate only the automations + appropriate for your host. + +See [Installation](docs/installation.md), [Architecture](docs/architecture.md), +and [Operations](docs/operations.md) for the full model. + +## Privacy model + +The public framework is produced through an allowlist-only exporter. Personal +content paths are denied independently of the allowlist, and the export fails +closed when it detects live-vault Git metadata, unsafe destinations, or likely +personal identifiers. See [Publication and versioning](docs/publication.md). + +## Status + +The framework is evolving and should be reviewed before use with sensitive or +high-stakes information. Contributions are welcome through pull requests. + +Licensed under the [MIT License](LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..8cd942d --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,13 @@ +# Security and privacy + +Please do not report a vulnerability by opening a public issue when it could +expose personal data or credentials. Use GitHub's private vulnerability +reporting feature for this repository. + +The primary safety boundary is architectural: a live personal vault is not a +Git repository, and publication uses an explicit allowlist into a separate +checkout. Treat any path-bypass, unintended-file export, credential exposure, +or publication of personal content as a security issue. + +Before reporting, remove personal material from screenshots, logs, examples, +and reproduction cases. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..56484f1 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,35 @@ +# Architecture + +SecondBrain separates operational collaboration from durable knowledge work. + +## Runtime layers + +1. **Raw sources** are human-curated evidence. Top-level files await ingestion; + completed sources move unchanged to `raw/processed/`. +2. **Wiki** pages are agent-maintained syntheses with provenance, citations, + cross-links, an index, and an append-only operation log. +3. **Daily operations** contain Daily Notes, tasks, decisions, project plans, + working documents, and recurrence state maintained by the DTM. +4. **Schema and tools** define role boundaries, formats, workflows, validation, + templates, and automations. + +## Role boundary + +The Knowledge Agent owns systematic source ingestion and wiki maintenance. The +DTM owns Daily Notes and operational continuity. The DTM does not scan the raw +inbox, and the Knowledge Agent never writes the Daily Note activity section. + +## Production boundary + +The live Obsidian vault may be synchronized by a file service such as iCloud, +but it is never a Git checkout. A separate repository contains a sanitized +product representation of the framework. Publication is strictly one-way: + +```text +live vault -- allowlist + privacy checks --> staged framework --> Git checkout +``` + +No branch, merge, worktree, or repository operation runs against the live +vault. Rollback means restoring a reviewed framework version through the +installer or manually applying a known-good change—not checking out history in +production. diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..fe53cc5 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,37 @@ +# Installation + +## Requirements + +- Python 3.9 or later +- An Obsidian vault +- An LLM coding agent capable of reading a vault-level `AGENTS.md` + +## Install + +Clone this repository somewhere outside the vault, then run: + +```sh +python3 install.py --vault /path/to/vault +``` + +The installer creates missing operational directories and copies framework +contracts, templates, tools, safe Obsidian settings, recurrence defaults, and +sanitized seed pages. It does not copy `.git` metadata and does not overwrite +existing files by default. Preview with `--dry-run`; deliberately replace +framework files with `--force` after reviewing the changes. + +## Configure + +1. Review `AGENTS.md` and `DTM.md` for local preferences. +2. Keep `scratch.md` human-only or remove that convention explicitly. +3. Edit `dtm/recurring-tasks.json`; example rules are disabled by default. +4. Configure automation definitions for the local agent host. Replace every + `{{PLACEHOLDER}}`; never commit the resulting machine-specific values. +5. Open the directory as an Obsidian vault. + +## Validate + +```sh +python3 tools/wiki.py lint +python3 tools/dtm.py lint +``` diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..8d58cd0 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,28 @@ +# Operations + +## Knowledge ingestion + +Place new sources directly under `raw/`. The Knowledge Agent processes each +source, integrates its evidence across the wiki, updates the index and log, +validates the result, then moves the unchanged source to `raw/processed/`. +Incomplete work remains visible in the inbox. + +## Daily collaboration + +The DTM opens and closes Daily Notes, carries only incomplete tasks and active +questions forward, instantiates enabled recurrence, captures decisions, and +maintains project continuity. Durable knowledge can graduate to the wiki while +remaining clearly distinguished from externally sourced evidence. + +## Maintenance + +Run both structural linters after framework changes. Periodically perform a +semantic wiki review for contradictions, stale claims, unsupported assertions, +orphans, and evidence gaps. + +## Automation + +Definitions under `framework/automation-definitions/` are parameterized +documentation, not credentials or host configuration. Recreate them in the +automation system used on the always-on host and keep machine-specific values +out of this repository. diff --git a/docs/publication.md b/docs/publication.md new file mode 100644 index 0000000..3055d6e --- /dev/null +++ b/docs/publication.md @@ -0,0 +1,40 @@ +# Publication and versioning + +## Safety properties + +- The live vault is never a Git repository. +- Export starts from an explicit manifest, not an ignore file. +- Personal content paths are denied even if accidentally allowlisted. +- Staged text is scanned for home paths, email addresses, and live-vault + identifiers before touching the public checkout. +- The target must be a separate Git checkout outside iCloud. +- A no-change export creates no branch, commit, or pull request. + +## Weekly flow + +The publisher fetches and fast-forwards the public repository's default branch, +builds a sanitized staging tree, and compares its content fingerprint. When a +meaningful diff exists it creates a fingerprinted weekly branch, validates the +export, commits, pushes, and opens an assigned draft pull request. Re-running +the same export reuses the branch or existing pull request. If an earlier +framework publication PR is still open, the run defers rather than creating a +stack of noisy or competing reviews. + +## Review and promotion + +Review the generated PR with special attention to the privacy boundary, +contracts, automation behaviour, and install output. Merge only intentional +changes. Release tags should use semantic versioning and summarize migration +requirements in the changelog. + +## Rollback + +Do not use Git directly in a live synchronized vault. Instead: + +1. identify the last known-good tag or commit in the public repository; +2. check it out in the separate repository clone; +3. preview restoration with `python3 install.py --vault /path --dry-run`; +4. restore only reviewed framework files, using `--force` deliberately; +5. validate the live system and record the rollback in its system log. + +Personal knowledge remains outside this version history and is unaffected. diff --git a/examples/empty-vault/README.md b/examples/empty-vault/README.md new file mode 100644 index 0000000..bc2a069 --- /dev/null +++ b/examples/empty-vault/README.md @@ -0,0 +1,4 @@ +# Empty SecondBrain vault example + +This directory contains sanitized seed scaffolding only. It contains no real +Daily Notes, sources, projects, wiki knowledge, or activity history. diff --git a/examples/empty-vault/daily/README.md b/examples/empty-vault/daily/README.md new file mode 100644 index 0000000..b32bd1d --- /dev/null +++ b/examples/empty-vault/daily/README.md @@ -0,0 +1,4 @@ +# Daily Notes + +The DTM creates `YYYY-MM-DD.md` Daily Notes here. This public example contains +no Daily Notes. diff --git a/examples/empty-vault/projects/README.md b/examples/empty-vault/projects/README.md new file mode 100644 index 0000000..67f2b39 --- /dev/null +++ b/examples/empty-vault/projects/README.md @@ -0,0 +1,4 @@ +# Projects + +DTM-maintained project plans live here. This public example contains no project +information. diff --git a/examples/empty-vault/raw/README.md b/examples/empty-vault/raw/README.md new file mode 100644 index 0000000..bfb03e2 --- /dev/null +++ b/examples/empty-vault/raw/README.md @@ -0,0 +1,5 @@ +# Raw source inbox + +Place new sources directly in this directory. After a fully successful ingest, +the Knowledge Agent moves the unchanged source to `processed/`. This public +example contains no source material. diff --git a/examples/empty-vault/raw/assets/.gitkeep b/examples/empty-vault/raw/assets/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/empty-vault/raw/assets/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/empty-vault/raw/processed/README.md b/examples/empty-vault/raw/processed/README.md new file mode 100644 index 0000000..c5236b3 --- /dev/null +++ b/examples/empty-vault/raw/processed/README.md @@ -0,0 +1,4 @@ +# Processed sources + +Successfully ingested immutable sources live here. This public example contains +no source material. diff --git a/examples/empty-vault/wiki/index.md b/examples/empty-vault/wiki/index.md new file mode 100644 index 0000000..b7dd98e --- /dev/null +++ b/examples/empty-vault/wiki/index.md @@ -0,0 +1,28 @@ +# Wiki Index + +This sanitized seed index is empty. The Knowledge Agent adds each generated +content page exactly once during ingestion and maintenance. + +## Overview + +- [[wiki/overview|Wiki Overview]] — Empty-vault starting point. `seed` + +## Topics + +_No topic pages yet._ + +## Concepts + +_No concept pages yet._ + +## Entities + +_No entity pages yet._ + +## Analyses + +_No analysis pages yet._ + +## Sources + +_No sources ingested yet._ diff --git a/examples/empty-vault/wiki/log.md b/examples/empty-vault/wiki/log.md new file mode 100644 index 0000000..26ef13b --- /dev/null +++ b/examples/empty-vault/wiki/log.md @@ -0,0 +1,8 @@ +# Wiki Log + +Append-only system operation history. This public seed contains no personal +activity. + +## [1970-01-01] setup | Empty framework initialized + +Created sanitized framework scaffolding with no personal content. diff --git a/examples/empty-vault/wiki/overview.md b/examples/empty-vault/wiki/overview.md new file mode 100644 index 0000000..b526c19 --- /dev/null +++ b/examples/empty-vault/wiki/overview.md @@ -0,0 +1,17 @@ +--- +title: Wiki Overview +type: overview +status: seed +created: 1970-01-01 +updated: 1970-01-01 +tags: + - wiki + - navigation +--- + +# Wiki Overview + +This is a deliberately empty, sanitized wiki seed. Add a source to `raw/` and +ask the Knowledge Agent to ingest it. + +Browse [[wiki/index|Wiki Index]] or [[wiki/log|Wiki Log]]. diff --git a/examples/empty-vault/work/README.md b/examples/empty-vault/work/README.md new file mode 100644 index 0000000..5234bac --- /dev/null +++ b/examples/empty-vault/work/README.md @@ -0,0 +1,4 @@ +# Working documents + +Temporary DTM working documents live here. This public example contains no +working material. diff --git a/framework/AGENTS.md b/framework/AGENTS.md new file mode 100644 index 0000000..b222b56 --- /dev/null +++ b/framework/AGENTS.md @@ -0,0 +1,287 @@ +# SecondBrain agent operating contract + +This vault has two distinct agent roles. Follow this contract for every task in +this directory and keep their responsibilities visibly separate. + +## Role routing + +### LLM Knowledge Agent + +Use the Knowledge Agent role for source ingestion, wiki synthesis, provenance, +index maintenance, and wiki health. It owns the long-term knowledge workflow +defined below and may inspect `raw/` when doing that work. + +### Digital TeamMate (DTM) + +Use the DTM role for daily collaboration: Daily Notes, tasks, projects, +decisions, follow-ups, working documents, assigned research, and operational +continuity. Before acting as the DTM, read and follow `DTM.md`. + +The user can select either role explicitly. Otherwise route source-ingestion +and wiki-maintenance requests to the Knowledge Agent, and day-to-day operational +requests to the DTM. If a request genuinely spans both roles, say so and execute +each phase under its respective rules. + +The DTM must never scan or automatically process `raw/`. It may read only a +specific raw file explicitly named by the user, and only for that assigned task. +The Knowledge Agent must never add, edit, reorder, or remove content in a Daily +Note's `Notes & Activity` section. + +## Knowledge Agent mission + +Compile curated source material into a durable, navigable wiki. Integrate new +evidence into existing understanding instead of creating isolated summaries. +Preserve provenance, surface contradictions, and make useful discoveries +compound across sessions. + +## Layers and ownership + +### `scratch.md` — human-only, excluded + +- Ignore `scratch.md` completely in every role and workflow unless the user + explicitly names that file and directs the agent to use it for the current + task. +- Do not read, search, index, summarize, cite, link, edit, move, delete, lint, + monitor, or infer context from it during ordinary vault operations. +- Exclude it from broad file discovery and content searches. Its mere presence + is not pending work and must never trigger ingestion or DTM capture. +- Explicit access applies only to the task in which the user grants it; return + to ignoring the file afterwards. + +### `raw/` — human-owned inbox and immutable archive + +- Read any source necessary for the task. +- Treat top-level source documents in `raw/` as the pending ingestion queue. +- After—and only after—a completely successful Knowledge Agent ingest, move the + source unchanged into `raw/processed/` and update all provenance links to its + archived path. This one-time archival move is the only permitted agent move + within `raw/`. +- Never overwrite an existing processed file. If the destination filename is + occupied, stop and ask the user how to resolve it. +- Never archive a source when ingestion, integration, index maintenance, log + maintenance, or linting remains incomplete. +- Never edit, rename, normalize, delete, or subsequently move a file in + `raw/processed/`. +- Never silently replace a source with a downloaded or reformatted copy. +- Assets referenced by a source belong in `raw/assets/` and are equally + immutable once placed there. +- If a source is unreadable, incomplete, duplicated, or needs correction, tell + the user. Record the limitation in its source page. + +### `wiki/` — agent-owned, maintained + +- Create and revise pages freely when supported by the sources or by clearly + labelled analysis. +- Prefer updating an existing canonical page over creating a near-duplicate. +- Keep links, index entries, summaries, dates, and citations consistent in the + same change. +- Never present inference as sourced fact. + +### `AGENTS.md` — shared schema + +Change this contract only when the user asks, or when a task exposes a durable +workflow improvement. Explain material schema changes before applying them. + +### Publication boundary + +- The live iCloud vault is production and must never become a Git repository. + Never run `git init`, `git status`, `git add`, `git commit`, branch, merge, + worktree, or other repository operations against this directory, and never + create `.git` metadata anywhere inside it. +- `publication/` contains sanitized public-repository documents and an explicit + export manifest. `automation-definitions/` contains parameterized, reusable + descriptions of system automations. Both are framework-owned and must contain + no personal data. +- Public extraction is allowlist-only through `tools/publish_framework.py`. + Never publish by copying the vault, using a broad recursive sync, or deriving + an export from everything not ignored. +- All Git operations for publication must use the separate external framework + checkout and an explicit target path. The publication tool rejects targets + inside iCloud or inside the live vault. +- Never publish `scratch.md`, Daily Notes, wiki content, sources, personal + projects, working documents, activity logs, Obsidian workspaces, or any file + not explicitly approved by `publication/manifest.json`. +- When system architecture, templates, tools, Obsidian configuration, or + automations change, update their sanitized framework representation in the + same task where practical. + +## Wiki structure + +- `wiki/overview.md` — current high-level synthesis and navigation. +- `wiki/index.md` — complete content catalogue with one-line descriptions. +- `wiki/log.md` — append-only chronological activity record. +- `wiki/sources/` — one provenance and summary page per ingested source. +- `wiki/entities/` — people, organisations, products, places, works, and other + named things. +- `wiki/concepts/` — reusable ideas, terms, methods, and mechanisms. +- `wiki/topics/` — broader subject hubs that connect entities and concepts. +- `wiki/analyses/` — comparisons, answers, hypotheses, timelines, and other + synthesized outputs worth retaining. +- `templates/` — Obsidian templates for wiki and DTM documents; never list + these as content in the wiki index. + +Use lowercase kebab-case filenames. Choose the shortest unambiguous canonical +name. Page titles use normal title case. When names collide, qualify the +filename, for example `mercury-planet.md` and `mercury-element.md`. + +## Required frontmatter + +Every content page except `index.md` and `log.md` has: + +```yaml +--- +title: Human-readable title +type: overview | source | entity | concept | topic | analysis +status: current | seed | needs-review | disputed | superseded +created: YYYY-MM-DD +updated: YYYY-MM-DD +tags: + - wiki +--- +``` + +Additional fields by type: + +- Source: `source_path`, `source_kind`, and `ingested`. +- Analysis: `question` when it answers a specific query. +- Any page may use `aliases` as a YAML list. + +Dates use the user's local date. `created` never changes; update `updated` when +the body materially changes. `seed` means useful but thin. `needs-review` marks +uncertainty or incomplete processing. `disputed` means credible sources +conflict. `superseded` pages must link prominently to their replacement. + +## Writing and linking + +- Start each content page with a concise synthesis, not a table of metadata. +- Use Obsidian links rooted at the vault, with useful display text: + `[[wiki/concepts/example|Example]]`. +- Link the first meaningful mention of another wiki page. Avoid link spam. +- Link back from a newly created page to at least one hub or related page. +- Use headings that describe the domain. Do not force every page into an + identical outline. +- Distinguish facts, source claims, and analysis with prose. Use explicit labels + such as **Inference**, **Open question**, and **Conflict** where ambiguity is + possible. +- Do not copy long source passages. Summarize; quote only short language whose + exact wording matters. + +## Citations and provenance + +Claims that are specific, contestable, quantitative, or not common knowledge +need a citation to a wiki source page. Cite at sentence or paragraph level: + +```markdown +Claim being supported. [[wiki/sources/source-name#Relevant heading|Source Name]] +``` + +The linked source page must identify the immutable raw file via `source_path` +and a body link such as `[[raw/article.md|Open raw source]]`. Use a meaningful +heading, page, timestamp, section, row, or figure as a locator whenever the raw +format permits it. If several sources support a synthesis, cite each relevant +source. If sources disagree, preserve both claims and explain the conflict. + +Never cite an analysis page as though it were primary evidence. It may be linked +for context, but its underlying source citations remain authoritative. + +## Ingest workflow + +When asked to ingest one or more sources: + +1. Read `wiki/index.md`, `wiki/overview.md`, recent entries in `wiki/log.md`, and + the source. Search the wiki for related names and concepts before creating + pages. +2. For image-bearing material, read the text first, then inspect locally + available images that could change the interpretation. Note uninspected or + unavailable material. +3. Detect exact or likely duplicate ingestion. Do not create a second source + page for the same raw file. +4. Extract the source's central claims, evidence, entities, concepts, + limitations, dates, and relationships. Identify what is genuinely new to + the existing wiki. +5. If emphasis or interpretation is consequential and genuinely ambiguous, + discuss it with the user. Otherwise proceed and mark uncertainty explicitly. +6. Create or update the source page. Then integrate the evidence into every + materially affected entity, concept, topic, analysis, and overview page. + Do not stop at a standalone summary. +7. Reconcile contradictions: record both positions, date them where relevant, + assess source quality without inventing certainty, and use `disputed` or + `superseded` when appropriate. +8. Update `wiki/index.md` for every created, renamed, or materially re-scoped + page. Keep entries in their category and sorted by title. +9. Append one log entry describing the source and all important pages created + or revised. +10. Run `python3 tools/wiki.py lint`. Fix safe structural issues; report + substantive unresolved issues to the user. +11. Only when all preceding steps succeed, move a top-level source into + `raw/processed/`. Update its source page's `source_path` and body link to the + processed location, rerun lint, and confirm that the top-level inbox is + clear of that document. Sources already under `raw/processed/` are not moved. + +## Query workflow + +When asked a question: + +1. Read `wiki/index.md`, then search and read the relevant wiki pages. +2. Follow citations to source pages and raw material when the claim is + high-stakes, disputed, unclear, or requires precise wording. +3. Answer from the available evidence with inline wiki citations. State gaps + and confidence plainly; do not fill gaps from memory without saying so. +4. Use web research only when the user asks or when current external facts are + necessary. Save any source the user chooses to curate under `raw/` before + treating it as durable wiki evidence. +5. Offer to file a substantial reusable answer. If the user already asked to + file it, create or update an analysis/topic page, update the index and + related pages, and append a `query` log entry. + +Short answers and one-off operational chat do not need to become pages. + +## Lint workflow + +For a health check, run the local lint first, then do a semantic review that +automation cannot perform: + +- claims contradicted by newer or stronger sources; +- stale time-sensitive claims and superseded conclusions; +- near-duplicate pages or inconsistent names; +- important unlinked concepts that deserve pages; +- missing reciprocal/contextual links and weak hub pages; +- source pages whose evidence was never integrated elsewhere; +- unsupported assertions, vague citations, and unclear inference; +- unanswered questions and evidence gaps worth investigating. + +Fix mechanical issues directly. For substantive reinterpretations, preserve the +old view in the log and explain the change. Append a `lint` log entry listing +checks performed, fixes made, and unresolved recommendations. + +## Index contract + +`wiki/index.md` is curated navigation, not a raw file dump. Every non-template +content page appears exactly once with a link, one-line description, status, +and updated date. Organize it under Overview, Topics, Concepts, Entities, +Analyses, and Sources. Omit empty categories only if the structure remains +obvious. Keep descriptions concrete enough to route future queries. + +## Log contract + +`wiki/log.md` is append-only. Never rewrite or reorder prior entries except to +repair a broken link or obvious typo. Put newest entries at the bottom, using: + +```markdown +## [YYYY-MM-DD] ingest | Source title + +Summary of what changed, with links to affected pages. +``` + +Allowed operation labels are `setup`, `ingest`, `query`, `lint`, `schema`, and +`dtm`. DTM log entries are reserved for significant operational changes such as +project milestones, durable decisions, or created/updated wiki artefacts; routine +daily rollover does not need a wiki log entry. +Mention unresolved conflicts, gaps, or follow-ups in the same entry. + +## Completion standard + +A wiki task is complete only when the content, backlinks, index, dates, and log +agree; citations resolve to source pages; any newly ingested source is archived +unchanged under `raw/processed/`; and the local lint passes or its remaining +warnings are explained. diff --git a/framework/DTM.md b/framework/DTM.md new file mode 100644 index 0000000..8360f91 --- /dev/null +++ b/framework/DTM.md @@ -0,0 +1,139 @@ +# Digital TeamMate operating contract + +The Digital TeamMate (DTM) is the user's day-to-day operational partner. Read +`AGENTS.md` first; this file adds the DTM-specific contract. + +## Purpose + +Maintain continuity of work across conversations and days. Prioritise practical +usefulness, clear next actions, and reliable follow-through. Capture enough +context to resume work without turning every interaction into archival prose. + +## Boundaries + +- Never scan, monitor, triage, or batch-process `raw/`. +- Read a raw document only when the user explicitly identifies that document + and asks the DTM to process it. Ignore every other raw file during that task. +- DTM knowledge may come from conversations, Daily Notes, project artefacts, + decisions, outcomes, and explicitly assigned research. +- The DTM may create or update wiki pages when durable knowledge emerges. Apply + the wiki schema and citation rules in `AGENTS.md`, update `wiki/index.md`, and + log the material change. +- Do not treat conversational claims as externally verified facts. Attribute + personal decisions and observations to the user/context where useful. +- The Knowledge Agent owns systematic source ingestion. The DTM owns Daily + Notes and operational files. + +## DTM workspace + +- `daily/YYYY-MM-DD.md` — canonical Daily Notes. +- `projects/` — durable project plans, status, milestones, and next actions. +- `work/` — working documents, drafts, scratch analyses, and deliverables that + are not yet durable wiki knowledge. +- `dtm/recurring-tasks.json` — recurrence definitions used at day open. +- `templates/` — Obsidian templates for daily notes, projects, and wiki pages. + +Prefer links rooted at the vault, for example +`[[projects/example-project|Example project]]`. A project page should hold stable +context and current state; its day-specific activity belongs in the Daily Note. + +## Interaction capture + +For substantive DTM work, use today's Daily Note. If none exists, run +`python3 tools/dtm.py open` before recording activity. + +Add DTM interactions chronologically under `Notes & Activity` using: + +```markdown +- HH:MM — DTM — concise action, result, or relevant context. +``` + +Capture outcomes, created artefacts, material status changes, and commitments. +Do not transcribe routine chat. The Knowledge Agent must never write in this +section. + +Significant DTM actions also receive an append-only `wiki/log.md` entry using +the `dtm` operation label. Significant means a durable decision, project +milestone, or created/updated wiki artefact—not ordinary task edits or rollover. + +## Tasks + +- Put personal and professional tasks in their respective Daily Note sections. +- Use Obsidian task syntax: `- [ ] Action` and `- [x] Completed action`. +- Make tasks concrete and outcome-oriented. Link the relevant project when one + exists. +- At day open, carry incomplete tasks forward exactly once. Leave completed + tasks in the historical note and never carry them forward. +- Preserve useful completion context in the day's activity or project page. +- Recurring instances behave like ordinary tasks after creation; completing one + does not disable the recurrence definition. + +## Recurring and scheduled work + +`dtm/recurring-tasks.json` is the recurrence source of truth. Each rule has a +stable ID, task text, area (`personal`, `professional`, or `schedule`), enabled +state, and a schedule. Supported schedules are: + +- `daily`; +- `weekly` with one or more weekday names; +- `monthly` with a calendar day, or `last` for the month's last day. + +The rollover tool instantiates each enabled rule only on applicable dates and +marks it with `` to prevent duplicates. Edit the rule, +not generated historical instances, to change future behaviour. + +## Decisions + +Record meaningful decisions in today's `Decisions` section. Use one heading per +decision and capture the decision, context, rationale, and resulting actions: + +```markdown +### DEC-YYYY-MM-DD-NN — Short title + +- **Decision:** … +- **Context:** … +- **Rationale:** … +- **Actions:** … +``` + +Update an affected project page immediately. If the decision has value beyond +the project/day, create or update an appropriate wiki page and add a `dtm` log +entry. + +## References and questions + +- `References` contains only documents created or updated by the DTM that day. + Do not list Knowledge-Agent-only changes. +- Add a reference once, with a short description of the change. +- `Open Questions` contains active uncertainties as plain bullets. Carry them + forward until resolved. When answered, remove the question from the current + note immediately; preserve the resolution in activity, a decision, project, + or wiki page as appropriate. Historical notes remain historical. + +## Day close and day open + +The scheduled lifecycle runs at 00:01 in the user's local timezone. It should: + +1. Run `python3 tools/dtm.py rollover` to safely close yesterday and create + today. The tool is idempotent and handles mechanical carry-forward. +2. Finalise yesterday's activity and decision records without inventing events. +3. Write today's `Previous Day` synthesis from yesterday's note: activities, + outcomes, meaningful developments, and unfinished threads. +4. Refine today's `Focus` into a short ranked recommendation based on carried + work, active projects, questions, deadlines, and outcomes. +5. Keep the mechanically carried personal/professional tasks and open questions; + correct duplicates or categorisation errors if necessary. +6. Confirm scheduled and recurring instances are relevant for the date. +7. Link relevant active projects and workstreams. +8. Add a DTM activity entry to today's note. Update `wiki/log.md` only if the + rollover surfaced a significant durable change. + +When there is no prior note, say so plainly and initialise the day without +fabricating a previous-day summary. + +## Completion standard + +DTM work is complete when today's note reflects the material action, tasks and +questions have correct current state, affected project/working/wiki documents +are linked under `References`, durable changes are logged where appropriate, +and `python3 tools/dtm.py lint` passes. diff --git a/framework/automation-definitions/dtm-daily-rollover.json b/framework/automation-definitions/dtm-daily-rollover.json new file mode 100644 index 0000000..d9fc8ff --- /dev/null +++ b/framework/automation-definitions/dtm-daily-rollover.json @@ -0,0 +1,8 @@ +{ + "name": "DTM Daily Rollover", + "role": "Digital TeamMate", + "schedule": "00:01 daily in the user's local timezone", + "execution_environment": "local", + "workspace": "{{VAULT_PATH}}", + "prompt": "Read AGENTS.md and DTM.md, run python3 tools/dtm.py rollover, finalise the previous note from recorded evidence, populate today's Previous Day and Focus, preserve valid carried tasks and questions, instantiate recurrence, link active projects, record DTM activity, log only significant durable changes, and run DTM and wiki lint checks." +} diff --git a/framework/automation-definitions/framework-weekly-publication.json b/framework/automation-definitions/framework-weekly-publication.json new file mode 100644 index 0000000..c7c21c7 --- /dev/null +++ b/framework/automation-definitions/framework-weekly-publication.json @@ -0,0 +1,10 @@ +{ + "name": "SecondBrain Framework Publication", + "role": "Publisher", + "schedule": "03:00 every Sunday in the user's local timezone", + "execution_environment": "local", + "workspace": "{{VAULT_PATH}}", + "public_repository_checkout": "{{PUBLIC_REPO_PATH}}", + "github_repository": "{{GITHUB_OWNER}}/secondbrain-framework", + "prompt": "Run python3 tools/publish_framework.py --target {{PUBLIC_REPO_PATH}} --repo {{GITHUB_OWNER}}/secondbrain-framework --assignee {{GITHUB_LOGIN}}. The tool must export only the allowlisted sanitized framework, create no commit when there is no meaningful diff, and otherwise create or reuse an idempotent publication branch, push it, and open an assigned draft pull request. Never run Git against the live iCloud vault." +} diff --git a/framework/automation-definitions/knowledge-agent-raw-inbox.json b/framework/automation-definitions/knowledge-agent-raw-inbox.json new file mode 100644 index 0000000..b69ad98 --- /dev/null +++ b/framework/automation-definitions/knowledge-agent-raw-inbox.json @@ -0,0 +1,8 @@ +{ + "name": "Knowledge Agent Raw Inbox", + "role": "LLM Knowledge Agent", + "schedule": "02:00 daily in the user's local timezone", + "execution_environment": "local", + "workspace": "{{VAULT_PATH}}", + "prompt": "Read AGENTS.md and inspect only top-level pending files in raw/. Process each independently through the full ingestion workflow. Archive a source unchanged under raw/processed/ only after integration, index, log and lint all succeed. Leave blocked or failed sources visible and report them." +} diff --git a/framework/defaults/dtm/README.md b/framework/defaults/dtm/README.md new file mode 100644 index 0000000..e2c0029 --- /dev/null +++ b/framework/defaults/dtm/README.md @@ -0,0 +1,27 @@ +# DTM configuration + +The Digital TeamMate's behaviour is defined in [[DTM|DTM.md]]. This folder +contains machine-readable operational configuration. + +## Recurring tasks + +Edit `recurring-tasks.json` to manage routines. The included examples are +disabled so the DTM does not silently create commitments you have not chosen. +Set `enabled` to `true` when a rule is wanted. + +Supported schedule examples: + +```json +{"frequency": "daily"} +{"frequency": "weekly", "weekdays": ["Monday", "Friday"]} +{"frequency": "monthly", "day": 1} +{"frequency": "monthly", "day": "last"} +``` + +Areas determine the destination section: `personal`, `professional`, or +`schedule`. Rule IDs must remain stable and unique. + +The DTM validates the complete recurrence file before rollover. Invalid rules +produce an explicit error before either Daily Note is changed; they are not +silently skipped, because a missed recurring commitment could otherwise go +unnoticed. diff --git a/framework/defaults/dtm/recurring-tasks.json b/framework/defaults/dtm/recurring-tasks.json new file mode 100644 index 0000000..c4cfed6 --- /dev/null +++ b/framework/defaults/dtm/recurring-tasks.json @@ -0,0 +1,35 @@ +{ + "version": 1, + "tasks": [ + { + "id": "weekly-backlog-prioritisation", + "task": "Prioritise the weekly backlog", + "area": "professional", + "enabled": false, + "schedule": { + "frequency": "weekly", + "weekdays": ["Monday"] + } + }, + { + "id": "monthly-review", + "task": "Complete the monthly review", + "area": "schedule", + "enabled": false, + "schedule": { + "frequency": "monthly", + "day": "last" + } + }, + { + "id": "weekly-admin", + "task": "Complete the weekly administrative routine", + "area": "personal", + "enabled": false, + "schedule": { + "frequency": "weekly", + "weekdays": ["Friday"] + } + } + ] +} diff --git a/framework/obsidian/app.json b/framework/obsidian/app.json new file mode 100644 index 0000000..1b330c3 --- /dev/null +++ b/framework/obsidian/app.json @@ -0,0 +1,8 @@ +{ + "promptDelete": false, + "alwaysUpdateLinks": true, + "newFileLocation": "folder", + "newFileFolderPath": "raw", + "attachmentFolderPath": "raw/assets", + "vimMode": true +} \ No newline at end of file diff --git a/framework/obsidian/daily-notes.json b/framework/obsidian/daily-notes.json new file mode 100644 index 0000000..7f261ef --- /dev/null +++ b/framework/obsidian/daily-notes.json @@ -0,0 +1,5 @@ +{ + "folder": "daily", + "format": "YYYY-MM-DD", + "template": "templates/daily-note" +} diff --git a/framework/obsidian/templates.json b/framework/obsidian/templates.json new file mode 100644 index 0000000..c50a8f7 --- /dev/null +++ b/framework/obsidian/templates.json @@ -0,0 +1,3 @@ +{ + "folder": "templates" +} diff --git a/framework/publication-manifest.json b/framework/publication-manifest.json new file mode 100644 index 0000000..4e5dcf4 --- /dev/null +++ b/framework/publication-manifest.json @@ -0,0 +1,35 @@ +{ + "version": 1, + "entries": [ + {"source": "publication/repository", "destination": "."}, + {"source": "AGENTS.md", "destination": "framework/AGENTS.md"}, + {"source": "DTM.md", "destination": "framework/DTM.md"}, + {"source": "templates", "destination": "framework/templates"}, + {"source": "tools/wiki.py", "destination": "framework/tools/wiki.py"}, + {"source": "tools/dtm.py", "destination": "framework/tools/dtm.py"}, + {"source": "tools/export_framework.py", "destination": "framework/tools/export_framework.py"}, + {"source": "tools/publish_framework.py", "destination": "framework/tools/publish_framework.py"}, + {"source": ".obsidian/app.json", "destination": "framework/obsidian/app.json"}, + {"source": ".obsidian/daily-notes.json", "destination": "framework/obsidian/daily-notes.json"}, + {"source": ".obsidian/templates.json", "destination": "framework/obsidian/templates.json"}, + {"source": "dtm/README.md", "destination": "framework/defaults/dtm/README.md"}, + {"source": "dtm/recurring-tasks.json", "destination": "framework/defaults/dtm/recurring-tasks.json"}, + {"source": "automation-definitions", "destination": "framework/automation-definitions"}, + {"source": "publication/manifest.json", "destination": "framework/publication-manifest.json"} + ], + "forbidden_live_prefixes": [ + "daily", + "wiki", + "raw", + "projects", + "work", + "scratch.md", + ".obsidian/workspace.json", + ".obsidian/workspace-mobile.json" + ], + "forbidden_export_path_patterns": [ + "(^|/)daily/[0-9]{4}-[0-9]{2}-[0-9]{2}\\.md$", + "(^|/)wiki/(sources|entities|concepts|topics|analyses)/", + "^raw/processed/[^/]+$" + ] +} diff --git a/framework/templates/analysis.md b/framework/templates/analysis.md new file mode 100644 index 0000000..2653bc0 --- /dev/null +++ b/framework/templates/analysis.md @@ -0,0 +1,29 @@ +--- +title: Analysis title +type: analysis +status: current +created: YYYY-MM-DD +updated: YYYY-MM-DD +question: The question this analysis answers +tags: + - wiki + - analysis +--- + +# Analysis title + +## Answer + +Direct synthesis with inline links to source pages. + +## Reasoning + +How the evidence fits together, including competing interpretations. + +## Evidence gaps + +- Missing evidence or unresolved question. + +## Related pages + +- Related page link diff --git a/framework/templates/daily-note.md b/framework/templates/daily-note.md new file mode 100644 index 0000000..63ad074 --- /dev/null +++ b/framework/templates/daily-note.md @@ -0,0 +1,31 @@ +--- +title: Daily Note — {{date:YYYY-MM-DD}} +type: daily-note +date: {{date:YYYY-MM-DD}} +status: open +previous: +next: +tags: + - daily-note + - dtm +--- + +# {{date:dddd, D MMMM YYYY}} + +## Previous Day + +## Focus + +## Personal To-Do + +## Professional To-Do + +## Schedule & Recurring + +## Notes & Activity + +## Decisions + +## References + +## Open Questions diff --git a/framework/templates/knowledge-page.md b/framework/templates/knowledge-page.md new file mode 100644 index 0000000..353faf5 --- /dev/null +++ b/framework/templates/knowledge-page.md @@ -0,0 +1,29 @@ +--- +title: Page title +type: concept +status: seed +created: YYYY-MM-DD +updated: YYYY-MM-DD +tags: + - wiki +--- + +# Page title + +Concise current synthesis of the subject. + +## What we know + +Evidence-backed claims with sentence- or paragraph-level source links. + +## Connections + +- Related page link — relationship. + +## Tensions and uncertainty + +Conflicting evidence, limitations, and explicit inference. + +## Open questions + +- What evidence would improve this page? diff --git a/framework/templates/project.md b/framework/templates/project.md new file mode 100644 index 0000000..85ea7a7 --- /dev/null +++ b/framework/templates/project.md @@ -0,0 +1,36 @@ +--- +title: Project title +type: project +status: active +created: YYYY-MM-DD +updated: YYYY-MM-DD +owner: +tags: + - project +--- + +# Project title + +## Outcome + +What successful completion looks like. + +## Current state + +Where the project stands now. + +## Plan + +- [ ] Next meaningful action + +## Decisions + +Links to relevant Daily Note decision records. + +## Risks and open questions + +- Active uncertainty. + +## Activity + +- YYYY-MM-DD — Material change with a link to the Daily Note. diff --git a/framework/templates/source.md b/framework/templates/source.md new file mode 100644 index 0000000..b8c750d --- /dev/null +++ b/framework/templates/source.md @@ -0,0 +1,41 @@ +--- +title: Source title +type: source +status: current +created: YYYY-MM-DD +updated: YYYY-MM-DD +source_path: raw/path-to-source.ext +source_kind: article +ingested: YYYY-MM-DD +tags: + - wiki + - source +--- + +# Source title + + + +## Summary + +What the source says and why it matters. + +## Key claims + +- Claim, with the most precise locator available in the heading or prose. + +## Evidence and method + +How the source supports its claims, including relevant data or examples. + +## Limitations and perspective + +Scope, uncertainty, incentives, omissions, and likely bias. + +## Connections + +- Related page link — relationship. + +## Questions raised + +- Open question. diff --git a/framework/tools/dtm.py b/framework/tools/dtm.py new file mode 100644 index 0000000..1700153 --- /dev/null +++ b/framework/tools/dtm.py @@ -0,0 +1,418 @@ +#!/usr/bin/env python3 +"""Daily Note lifecycle and structural checks for the Digital TeamMate.""" + +from __future__ import annotations + +import argparse +import calendar +import json +import re +import sys +from datetime import date, datetime, timedelta +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +DAILY = ROOT / "daily" +RECURRENCE = ROOT / "dtm" / "recurring-tasks.json" + +SECTIONS = [ + "Previous Day", + "Focus", + "Personal To-Do", + "Professional To-Do", + "Schedule & Recurring", + "Notes & Activity", + "Decisions", + "References", + "Open Questions", +] +AREA_SECTION = { + "personal": "Personal To-Do", + "professional": "Professional To-Do", + "schedule": "Schedule & Recurring", +} +TASK_RE = re.compile(r"^- \[ \] .+$", re.M) +QUESTION_RE = re.compile(r"^- (?!\[[ xX]\] )(?!' + ) + return result + + +def unique(lines: list[str]) -> list[str]: + seen: set[str] = set() + result: list[str] = [] + for line in lines: + key = re.sub(r"\s*\s*$", "", line).casefold() + if key not in seen: + seen.add(key) + result.append(line) + return result + + +def block(lines: list[str], empty_hint: str | None = None) -> str: + if lines: + return "\n".join(lines) + return f"" if empty_hint else "" + + +def render_note(day: date, previous_text: str | None) -> str: + previous_day = day - timedelta(days=1) + recurrence = recurrence_instances(day) + personal: list[str] = [] + professional: list[str] = [] + schedule: list[str] = [] + questions: list[str] = [] + + if previous_text: + personal = incomplete_tasks(previous_text, "Personal To-Do") + professional = incomplete_tasks(previous_text, "Professional To-Do") + schedule = incomplete_tasks(previous_text, "Schedule & Recurring") + questions = open_questions(previous_text) + + personal = unique(personal + recurrence["Personal To-Do"]) + professional = unique(professional + recurrence["Professional To-Do"]) + schedule = unique(schedule + recurrence["Schedule & Recurring"]) + previous_link = ( + f"[[daily/{previous_day.isoformat()}|{previous_day.isoformat()}]]" + if previous_text + else "No Daily Note exists for the previous calendar day." + ) + focus = ( + "1. Review and prioritise carried work, active projects, and open questions.\n" + "" + if previous_text + else "1. Establish today's priorities and active workstreams." + ) + previous_summary = ( + f"Previous note: {previous_link}\n\n" + "" + if previous_text + else previous_link + ) + now = datetime.now().strftime("%H:%M") + + return f"""--- +title: Daily Note — {day.isoformat()} +type: daily-note +date: {day.isoformat()} +status: open +previous: {f'"{previous_link}"' if previous_text else ''} +next: +tags: + - daily-note + - dtm +--- + +# {day.strftime('%A, %d %B %Y').replace(' 0', ' ')} + +## Previous Day + +{previous_summary} + +## Focus + +{focus} + +## Personal To-Do + +{block(personal, 'No carried personal tasks.')} + +## Professional To-Do + +{block(professional, 'No carried professional tasks.')} + +## Schedule & Recurring + +{block(schedule, 'No scheduled or recurring tasks apply today.')} + +## Notes & Activity + +- {now} — DTM — Opened the day and carried forward active work. + +## Decisions + + + +## References + + + +## Open Questions + +{block(questions, 'No active open questions.')} +""" + + +def replace_frontmatter_value(text: str, field: str, value: str) -> str: + pattern = re.compile(rf"^{re.escape(field)}:.*$", re.M) + replacement = f"{field}: {value}" + if pattern.search(text): + return pattern.sub(replacement, text, count=1) + end = text.find("\n---\n", 4) + if end < 0: + return text + return text[:end] + f"\n{replacement}" + text[end:] + + +def close_day(day: date, next_day: date | None = None) -> bool: + path = note_path(day) + if not path.exists(): + print(f"No Daily Note to close: {path.relative_to(ROOT)}") + return False + text = path.read_text(encoding="utf-8") + changed = replace_frontmatter_value(text, "status", "closed") + if next_day: + changed = replace_frontmatter_value( + changed, "next", f'"[[daily/{next_day.isoformat()}|{next_day.isoformat()}]]"' + ) + if changed != text: + path.write_text(changed, encoding="utf-8") + print(f"Closed {path.relative_to(ROOT)}") + else: + print(f"Already closed {path.relative_to(ROOT)}") + return True + + +def open_day(day: date) -> bool: + path = note_path(day) + if path.exists(): + print(f"Already open/present: {path.relative_to(ROOT)}") + return False + previous_text = read_note(day - timedelta(days=1)) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(render_note(day, previous_text), encoding="utf-8") + print(f"Created {path.relative_to(ROOT)}") + return True + + +def rollover(day: date) -> int: + # Validate before closing yesterday so bad configuration cannot leave the + # lifecycle half-applied with no new Daily Note. + require_valid_recurrence() + previous_day = day - timedelta(days=1) + close_day(previous_day, day) + open_day(day) + return 0 + + +def validate_recurrence() -> tuple[list[str], list[str]]: + errors: list[str] = [] + warnings: list[str] = [] + try: + config = load_recurrence() + except (OSError, json.JSONDecodeError) as exc: + return [f"dtm/recurring-tasks.json: {exc}"], warnings + if not isinstance(config, dict): + errors.append("dtm/recurring-tasks.json: top level must be an object") + return errors, warnings + if config.get("version") != 1 or not isinstance(config.get("tasks"), list): + errors.append("dtm/recurring-tasks.json: expected version 1 and a tasks list") + return errors, warnings + ids: set[str] = set() + weekdays = set(calendar.day_name) + for position, rule in enumerate(config["tasks"], start=1): + label = f"recurrence rule {position}" + if not isinstance(rule, dict): + errors.append(f"{label}: must be an object") + continue + for field in ("id", "task", "area", "enabled", "schedule"): + if field not in rule: + errors.append(f"{label}: missing {field}") + rule_id = rule.get("id") + if not isinstance(rule_id, str) or not re.fullmatch(r"[a-z0-9-]+", rule_id): + errors.append(f"{label}: id must use lowercase kebab-case") + elif rule_id in ids: + errors.append(f"{label}: duplicate id {rule_id}") + else: + ids.add(rule_id) + task = rule.get("task") + if not isinstance(task, str) or not task.strip(): + errors.append(f"{label}: task must be a non-empty string") + if rule.get("area") not in AREA_SECTION: + errors.append(f"{label}: invalid area {rule.get('area')!r}") + if not isinstance(rule.get("enabled"), bool): + errors.append(f"{label}: enabled must be true or false") + schedule = rule.get("schedule") + if not isinstance(schedule, dict): + errors.append(f"{label}: schedule must be an object") + continue + frequency = schedule.get("frequency") + if frequency not in {"daily", "weekly", "monthly"}: + errors.append(f"{label}: unsupported frequency {frequency!r}") + if frequency == "weekly": + values = schedule.get("weekdays") + if not isinstance(values, list) or not values or any(v not in weekdays for v in values): + errors.append(f"{label}: weekly schedule needs valid weekday names") + if frequency == "monthly": + value = schedule.get("day") + if value != "last" and (not isinstance(value, int) or not 1 <= value <= 31): + errors.append(f"{label}: monthly day must be 1..31 or 'last'") + elif isinstance(value, int) and value > 28: + warnings.append(f"{label}: will not occur in months without day {value}") + return errors, warnings + + +def require_valid_recurrence() -> None: + errors, _ = validate_recurrence() + if errors: + details = "\n".join(f"- {error}" for error in errors) + raise DTMError(f"Invalid recurrence configuration:\n{details}") + + +def lint() -> int: + errors, warnings = validate_recurrence() + note_files = sorted(path for path in DAILY.glob("*.md") if path.name != "README.md") + for path in note_files: + if not DATE_RE.fullmatch(path.stem): + warnings.append(f"{path.relative_to(ROOT)}: filename is not an ISO date") + continue + text = path.read_text(encoding="utf-8") + for heading in SECTIONS: + count = len(re.findall(rf"^## {re.escape(heading)}\s*$", text, re.M)) + if count != 1: + errors.append( + f"{path.relative_to(ROOT)}: section {heading!r} occurs {count} times" + ) + positions = [text.find(f"## {heading}") for heading in SECTIONS] + if all(position >= 0 for position in positions) and positions != sorted(positions): + errors.append(f"{path.relative_to(ROOT)}: required sections are out of order") + for field in ("type: daily-note", f"date: {path.stem}", "status:"): + if field not in text[: text.find("\n---\n", 4) + 5]: + errors.append(f"{path.relative_to(ROOT)}: missing frontmatter value {field}") + + for item in sorted(set(errors)): + print(f"ERROR {item}") + for item in sorted(set(warnings)): + print(f"WARNING {item}") + print( + f"Checked {len(note_files)} Daily Note(s): " + f"{len(set(errors))} error(s), {len(set(warnings))} warning(s)." + ) + return 1 if errors else 0 + + +def status() -> int: + require_valid_recurrence() + notes = sorted(DAILY.glob("*.md")) + if not notes: + print("No Daily Notes.") + return 0 + latest = notes[-1] + text = latest.read_text(encoding="utf-8") + task_count = sum( + len(incomplete_tasks(text, heading)) + for heading in ("Personal To-Do", "Professional To-Do", "Schedule & Recurring") + ) + question_count = len(open_questions(text)) + status_match = re.search(r"^status:\s*(\S+)", text, re.M) + current_status = status_match.group(1) if status_match else "unknown" + enabled = sum(1 for rule in load_recurrence().get("tasks", []) if rule.get("enabled")) + print(f"Latest note: {latest.stem} ({current_status})") + print(f"Outstanding tasks: {task_count}") + print(f"Open questions: {question_count}") + print(f"Enabled recurrence rules: {enabled}") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + for command in ("open", "close", "rollover"): + command_parser = sub.add_parser(command) + command_parser.add_argument("--date", type=parse_date, default=date.today()) + sub.add_parser("lint") + sub.add_parser("status") + args = parser.parse_args() + try: + if args.command == "open": + open_day(args.date) + return 0 + if args.command == "close": + close_day(args.date) + return 0 + if args.command == "rollover": + return rollover(args.date) + if args.command == "lint": + return lint() + return status() + except DTMError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/framework/tools/export_framework.py b/framework/tools/export_framework.py new file mode 100644 index 0000000..8dbeaa5 --- /dev/null +++ b/framework/tools/export_framework.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Build a fail-closed, sanitized SecondBrain framework export.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import shutil +import sys +from pathlib import Path + +LIVE_ROOT = Path(__file__).resolve().parent.parent +MANIFEST = LIVE_ROOT / "publication" / "manifest.json" + + +class ExportError(RuntimeError): + pass + + +def inside(path: Path, parent: Path) -> bool: + return path == parent or parent in path.parents + + +def safe_relative(value: str, label: str) -> Path: + path = Path(value) + if path.is_absolute() or ".." in path.parts: + raise ExportError(f"Unsafe {label}: {value}") + return path + + +def load_manifest() -> dict[str, object]: + try: + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ExportError(f"Cannot read publication manifest: {exc}") from exc + if manifest.get("version") != 1 or not isinstance(manifest.get("entries"), list): + raise ExportError("Publication manifest must have version 1 and an entries list") + return manifest + + +def assert_live_boundary() -> None: + if (LIVE_ROOT / ".git").exists(): + raise ExportError("Refusing export: live vault contains forbidden .git metadata") + if LIVE_ROOT.is_symlink(): + raise ExportError("Refusing export from a symlinked live root") + + +def denied_source(source: Path, prefixes: list[str]) -> bool: + relative = source.relative_to(LIVE_ROOT).as_posix() + return any(relative == prefix or relative.startswith(prefix.rstrip("/") + "/") for prefix in prefixes) + + +def copy_entry(source: Path, destination: Path) -> None: + if source.is_symlink(): + raise ExportError(f"Symlinks are not publishable: {source.relative_to(LIVE_ROOT)}") + if source.is_file(): + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + return + if not source.is_dir(): + raise ExportError(f"Manifest source does not exist: {source.relative_to(LIVE_ROOT)}") + for item in sorted(source.rglob("*")): + if item.is_symlink(): + raise ExportError(f"Symlinks are not publishable: {item.relative_to(LIVE_ROOT)}") + if item.is_file(): + target = destination / item.relative_to(source) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(item, target) + + +def staged_files(root: Path) -> list[Path]: + return sorted(path for path in root.rglob("*") if path.is_file()) + + +def privacy_scan(root: Path, manifest: dict[str, object]) -> None: + failures: list[str] = [] + path_patterns = [ + re.compile(pattern) for pattern in manifest.get("forbidden_export_path_patterns", []) + ] + home = Path.home().resolve() + personal_literals = { + str(LIVE_ROOT), + str(home), + home.name, + } + email_pattern = re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.I) + + for path in staged_files(root): + relative = path.relative_to(root).as_posix() + if any(pattern.search(relative) for pattern in path_patterns): + failures.append(f"forbidden export path: {relative}") + try: + text = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + failures.append(f"unexpected binary file: {relative}") + continue + folded = text.casefold() + for literal in personal_literals: + if len(literal) >= 4 and literal.casefold() in folded: + failures.append(f"personal identifier in {relative}: {literal!r}") + if email_pattern.search(text): + failures.append(f"email address in {relative}") + + if failures: + details = "\n".join(f"- {failure}" for failure in sorted(set(failures))) + raise ExportError(f"Privacy scan failed:\n{details}") + + +def tree_digest(root: Path) -> str: + digest = hashlib.sha256() + for path in staged_files(root): + relative = path.relative_to(root).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(4, "big")) + digest.update(relative) + content = path.read_bytes() + digest.update(len(content).to_bytes(8, "big")) + digest.update(content) + return digest.hexdigest() + + +def clear_output(output: Path) -> None: + if output.exists(): + for child in output.iterdir(): + if child.is_dir() and not child.is_symlink(): + shutil.rmtree(child) + else: + child.unlink() + else: + output.mkdir(parents=True) + + +def export_framework(output: Path) -> str: + assert_live_boundary() + output = output.expanduser().resolve() + if output in {Path("/"), Path.home().resolve()} or inside(output, LIVE_ROOT) or inside(LIVE_ROOT, output): + raise ExportError(f"Unsafe export destination: {output}") + if (output / ".git").exists(): + raise ExportError("Export destination is a Git checkout; stage elsewhere and publish explicitly") + manifest = load_manifest() + prefixes = list(manifest.get("forbidden_live_prefixes", [])) + clear_output(output) + + destinations: set[str] = set() + for entry in manifest["entries"]: + if not isinstance(entry, dict) or set(entry) != {"source", "destination"}: + raise ExportError(f"Malformed manifest entry: {entry!r}") + source_relative = safe_relative(str(entry["source"]), "source") + destination_relative = safe_relative(str(entry["destination"]), "destination") + source = (LIVE_ROOT / source_relative).resolve() + if not inside(source, LIVE_ROOT): + raise ExportError(f"Source escapes live root: {source_relative}") + if denied_source(source, prefixes): + raise ExportError(f"Manifest attempts to publish denied source: {source_relative}") + destination_key = destination_relative.as_posix() + if destination_key in destinations: + raise ExportError(f"Duplicate manifest destination: {destination_relative}") + destinations.add(destination_key) + copy_entry(source, output / destination_relative) + + privacy_scan(output, manifest) + return tree_digest(output) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + fingerprint = export_framework(args.output) + except ExportError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + print(f"Exported sanitized framework: {fingerprint[:12]}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/framework/tools/publish_framework.py b/framework/tools/publish_framework.py new file mode 100644 index 0000000..cc78e0e --- /dev/null +++ b/framework/tools/publish_framework.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Publish a sanitized framework branch and assigned draft pull request.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +from export_framework import ExportError, export_framework, staged_files, tree_digest + +LIVE_ROOT = Path(__file__).resolve().parent.parent + + +class PublishError(RuntimeError): + pass + + +def run(command: list[str], check: bool = True) -> subprocess.CompletedProcess[str]: + result = subprocess.run(command, text=True, capture_output=True) + if check and result.returncode: + detail = result.stderr.strip() or result.stdout.strip() + raise PublishError(f"Command failed ({' '.join(command[:3])}): {detail}") + return result + + +def git(target: Path, *arguments: str, check: bool = True) -> subprocess.CompletedProcess[str]: + return run(["git", "-C", str(target), *arguments], check=check) + + +def assert_target(target: Path) -> None: + target_text = str(target) + if target == LIVE_ROOT or LIVE_ROOT in target.parents or target in LIVE_ROOT.parents: + raise PublishError("Public checkout must be separate from the live vault") + if "Mobile Documents" in target.parts or "iCloud~md~obsidian" in target_text: + raise PublishError("Public checkout must not be inside an iCloud-synchronized path") + if (LIVE_ROOT / ".git").exists(): + raise PublishError("Live vault contains forbidden .git metadata") + if not (target / ".git").exists(): + raise PublishError(f"Target is not a Git checkout: {target}") + top = Path(git(target, "rev-parse", "--show-toplevel").stdout.strip()).resolve() + if top != target: + raise PublishError(f"Target is not the repository root: {target}") + + +def repository_digest(target: Path) -> str: + with tempfile.TemporaryDirectory(prefix="secondbrain-current-") as temporary: + snapshot = Path(temporary) + for source in staged_files(target): + relative = source.relative_to(target) + if ".git" in relative.parts or source.name in {".DS_Store"} or "__pycache__" in relative.parts: + continue + destination = snapshot / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + return tree_digest(snapshot) + + +def replace_worktree(target: Path, staging: Path) -> None: + for child in target.iterdir(): + if child.name == ".git": + continue + if child.is_dir() and not child.is_symlink(): + shutil.rmtree(child) + else: + child.unlink() + for source in staged_files(staging): + destination = target / source.relative_to(staging) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + + +def validate_export(target: Path) -> None: + python_files = [str(target / "install.py")] + python_files.extend(str(path) for path in sorted((target / "framework" / "tools").glob("*.py"))) + run([sys.executable, "-m", "py_compile", *python_files]) + with tempfile.TemporaryDirectory(prefix="secondbrain-install-") as temporary: + vault = Path(temporary) / "vault" + run([sys.executable, str(target / "install.py"), "--vault", str(vault)]) + run([sys.executable, str(vault / "tools" / "wiki.py"), "lint"]) + run([sys.executable, str(vault / "tools" / "dtm.py"), "lint"]) + + +def existing_pr(repo: str, branch: str) -> str: + result = run( + ["gh", "pr", "list", "--repo", repo, "--head", branch, "--state", "open", "--json", "url", "--jq", ".[0].url"], + ) + return result.stdout.strip() + + +def open_publication_pr(repo: str) -> str: + result = run( + [ + "gh", "pr", "list", "--repo", repo, "--state", "open", + "--json", "url,headRefName", + "--jq", '.[] | select(.headRefName == "codex/initial-secondbrain-framework" or (.headRefName | startswith("automation/framework-sync-"))) | .url', + ], + ) + return result.stdout.splitlines()[0].strip() if result.stdout.strip() else "" + + +def create_pr(repo: str, branch: str, base: str, assignee: str, week: str) -> str: + body = ( + "## What changed\n\n" + "Automated allowlist-only synchronization of the reusable SecondBrain framework.\n\n" + "## Privacy boundary\n\n" + "The exporter denied personal vault paths and scanned staged text for host paths, " + "identifiers, and email addresses before publication.\n\n" + "## Validation\n\n" + "- Python tools compiled\n" + "- Clean-vault installation completed\n" + "- Wiki and DTM structural lint passed\n" + ) + result = run( + [ + "gh", "pr", "create", "--repo", repo, "--draft", "--base", base, + "--head", branch, "--assignee", assignee, + "--title", f"[automation] Sync SecondBrain framework ({week})", + "--body", body, + ] + ) + return result.stdout.strip() + + +def publish(target: Path, repo: str, assignee: str, base: str) -> int: + target = target.expanduser().resolve() + assert_target(target) + if git(target, "status", "--porcelain").stdout.strip(): + raise PublishError("Public checkout has uncommitted changes; refusing automated publication") + + git(target, "fetch", "origin", "--prune") + git(target, "switch", base) + git(target, "pull", "--ff-only", "origin", base) + + with tempfile.TemporaryDirectory(prefix="secondbrain-export-") as temporary: + staging = Path(temporary) + fingerprint = export_framework(staging) + if fingerprint == repository_digest(target): + print("No meaningful framework changes; no branch, commit, or PR created.") + return 0 + + pending = open_publication_pr(repo) + if pending: + print(f"Publication review already open; deferring a new branch: {pending}") + return 0 + + today = dt.date.today() + week = f"{today.isocalendar().year}-W{today.isocalendar().week:02d}" + branch = f"automation/framework-sync-{week.lower()}-{fingerprint[:8]}" + remote = git( + target, "ls-remote", "--exit-code", "--heads", "origin", f"refs/heads/{branch}", check=False + ) + if remote.returncode == 0: + url = existing_pr(repo, branch) + if url: + print(f"Publication already exists: {url}") + return 0 + url = create_pr(repo, branch, base, assignee, week) + print(f"Created missing draft PR for existing branch: {url}") + return 0 + + git(target, "switch", "-c", branch) + replace_worktree(target, staging) + validate_export(target) + git(target, "add", "-A") + if git(target, "diff", "--cached", "--quiet", check=False).returncode == 0: + print("No meaningful staged changes; no commit or PR created.") + return 0 + git(target, "commit", "-m", f"Sync SecondBrain framework ({week})") + git(target, "push", "-u", "origin", branch) + url = create_pr(repo, branch, base, assignee, week) + print(f"Published draft PR: {url}") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--target", type=Path, required=True) + parser.add_argument("--repo", required=True, help="GitHub owner/repository") + parser.add_argument("--assignee", required=True) + parser.add_argument("--base", default="main") + args = parser.parse_args() + try: + return publish(args.target, args.repo, args.assignee, args.base) + except (ExportError, PublishError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/framework/tools/wiki.py b/framework/tools/wiki.py new file mode 100644 index 0000000..83bb78e --- /dev/null +++ b/framework/tools/wiki.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Dependency-free structural diagnostics for the LLM Wiki.""" + +from __future__ import annotations + +import argparse +import re +import sys +from collections import Counter +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +WIKI = ROOT / "wiki" +RAW = ROOT / "raw" +SPECIAL = {WIKI / "index.md", WIKI / "log.md"} +REQUIRED = {"title", "type", "status", "created", "updated", "tags"} +VALID_TYPES = {"overview", "source", "entity", "concept", "topic", "analysis"} +VALID_STATUSES = {"current", "seed", "needs-review", "disputed", "superseded"} +LINK_RE = re.compile(r"\[\[([^\]|#]+)(?:#[^\]|]+)?(?:\|[^\]]+)?\]\]") +LOG_RE = re.compile(r"^## \[(\d{4}-\d{2}-\d{2})\] ([a-z-]+) \| (.+)$", re.M) + + +def pages() -> list[Path]: + return sorted( + path for path in WIKI.rglob("*.md") if "_templates" not in path.parts + ) + + +def content_pages() -> list[Path]: + return [path for path in pages() if path not in SPECIAL] + + +def relative(path: Path) -> str: + return path.relative_to(ROOT).as_posix() + + +def frontmatter(text: str) -> dict[str, object] | None: + if not text.startswith("---\n"): + return None + end = text.find("\n---\n", 4) + if end < 0: + return None + result: dict[str, object] = {} + active_list: str | None = None + for line in text[4:end].splitlines(): + if re.match(r"^\s+-\s+", line) and active_list: + value = re.sub(r"^\s+-\s+", "", line).strip().strip('"\'') + assert isinstance(result[active_list], list) + result[active_list].append(value) + continue + match = re.match(r"^([A-Za-z_][\w-]*):(?:\s*(.*))?$", line) + if not match: + active_list = None + continue + key, value = match.groups() + if value: + result[key] = value.strip().strip('"\'') + active_list = None + else: + result[key] = [] + active_list = key + return result + + +def normalize_target(raw_target: str) -> str: + target = raw_target.strip().replace("\\", "/") + return target[:-3] if target.endswith(".md") else target + + +def target_exists(target: str, known: set[str]) -> bool: + normalized = normalize_target(target) + if normalized in known: + return True + path = ROOT / target + if path.exists() or path.with_suffix(".md").exists(): + return True + if "/" not in normalized: + return sum(item.rsplit("/", 1)[-1] == normalized for item in known) == 1 + return False + + +def collect_links(path: Path) -> list[str]: + return LINK_RE.findall(path.read_text(encoding="utf-8")) + + +def lint() -> int: + errors: list[str] = [] + warnings: list[str] = [] + all_pages = pages() + contents = content_pages() + known = {relative(path)[:-3] for path in all_pages} + known.update( + relative(path)[:-3] if path.suffix == ".md" else relative(path) + for path in RAW.rglob("*") if path.is_file() + ) + titles: list[str] = [] + inbound: Counter[str] = Counter() + + for path in contents: + text = path.read_text(encoding="utf-8") + meta = frontmatter(text) + if meta is None: + errors.append(f"{relative(path)}: missing or malformed frontmatter") + else: + missing = REQUIRED - meta.keys() + if missing: + errors.append( + f"{relative(path)}: missing fields {', '.join(sorted(missing))}" + ) + if meta.get("type") not in VALID_TYPES: + errors.append(f"{relative(path)}: invalid type {meta.get('type')!r}") + if meta.get("status") not in VALID_STATUSES: + errors.append(f"{relative(path)}: invalid status {meta.get('status')!r}") + if meta.get("title"): + titles.append(str(meta["title"]).casefold()) + if meta.get("type") == "source": + for field in ("source_path", "source_kind", "ingested"): + if not meta.get(field): + errors.append(f"{relative(path)}: source missing {field}") + source_path = meta.get("source_path") + if isinstance(source_path, str) and not (ROOT / source_path).exists(): + errors.append( + f"{relative(path)}: source_path does not exist: {source_path}" + ) + + for target in LINK_RE.findall(text): + normalized = normalize_target(target) + if not target_exists(target, known): + errors.append(f"{relative(path)}: broken link [[{target}]]") + if normalized in known: + inbound[normalized] += 1 + + duplicates = [title for title, count in Counter(titles).items() if count > 1] + for title in duplicates: + warnings.append(f"duplicate page title: {title}") + + index_text = (WIKI / "index.md").read_text(encoding="utf-8") + index_links = [normalize_target(link) for link in LINK_RE.findall(index_text)] + for path in contents: + key = relative(path)[:-3] + count = sum(link == key for link in index_links) + if count == 0: + errors.append(f"{relative(path)}: missing from wiki/index.md") + elif count > 1: + errors.append(f"{relative(path)}: listed {count} times in wiki/index.md") + if path != WIKI / "overview.md" and inbound[key] == 0: + warnings.append(f"{relative(path)}: orphan page (no inbound wiki links)") + + for path in all_pages: + for target in collect_links(path): + if not target_exists(target, known): + errors.append(f"{relative(path)}: broken link [[{target}]]") + + log_text = (WIKI / "log.md").read_text(encoding="utf-8") + if not LOG_RE.search(log_text): + errors.append("wiki/log.md: no parseable log entries") + + errors = sorted(set(errors)) + warnings = sorted(set(warnings)) + for item in errors: + print(f"ERROR {item}") + for item in warnings: + print(f"WARNING {item}") + print( + f"Checked {len(contents)} content pages: " + f"{len(errors)} error(s), {len(warnings)} warning(s)." + ) + return 1 if errors else 0 + + +def status() -> int: + counts: Counter[str] = Counter() + states: Counter[str] = Counter() + for path in content_pages(): + meta = frontmatter(path.read_text(encoding="utf-8")) or {} + counts[str(meta.get("type", "unknown"))] += 1 + states[str(meta.get("status", "unknown"))] += 1 + pending = sorted( + path for path in RAW.iterdir() + if path.is_file() and path.name not in {".gitkeep", "README.md"} + ) + processed = sum( + 1 for path in (RAW / "processed").rglob("*") + if path.is_file() and path.name not in {".gitkeep", "README.md"} + ) + print(f"Pending raw sources: {len(pending)}") + print(f"Processed raw sources: {processed}") + print(f"Wiki pages: {sum(counts.values())}") + print("By type: " + (", ".join(f"{k}={v}" for k, v in sorted(counts.items())) or "none")) + print("By status: " + (", ".join(f"{k}={v}" for k, v in sorted(states.items())) or "none")) + return 0 + + +def pending() -> int: + sources = sorted( + path for path in RAW.iterdir() + if path.is_file() and path.name not in {".gitkeep", "README.md"} + ) + if not sources: + print("No pending raw sources.") + return 0 + for path in sources: + print(relative(path)) + return 0 + + +def recent(limit: int) -> int: + entries = LOG_RE.findall((WIKI / "log.md").read_text(encoding="utf-8")) + for date, operation, title in entries[-limit:]: + print(f"{date} {operation:<7} {title}") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("lint", help="check structure, metadata, links, index, and log") + sub.add_parser("status", help="summarize raw files and wiki pages") + sub.add_parser("pending", help="list top-level raw sources awaiting ingest") + recent_parser = sub.add_parser("recent", help="show recent log entries") + recent_parser.add_argument("count", nargs="?", type=int, default=5) + args = parser.parse_args() + if args.command == "lint": + return lint() + if args.command == "status": + return status() + if args.command == "pending": + return pending() + return recent(max(0, args.count)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/install.py b/install.py new file mode 100644 index 0000000..d58ff27 --- /dev/null +++ b/install.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Install SecondBrain framework files into an Obsidian vault without Git.""" + +from __future__ import annotations + +import argparse +import shutil +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent +FRAMEWORK = REPO / "framework" +EXAMPLE = REPO / "examples" / "empty-vault" + + +def files_under(root: Path): + for path in sorted(root.rglob("*")): + if path.is_file() and ".git" not in path.parts: + yield path + + +def copy_file(source: Path, destination: Path, force: bool, dry_run: bool) -> str: + if destination.exists() and not force: + return f"preserve {destination}" + if not dry_run: + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + return f"{'would write' if dry_run else 'wrote'} {destination}" + + +def install(vault: Path, force: bool, dry_run: bool) -> int: + vault = vault.expanduser().resolve() + if vault == REPO or REPO in vault.parents: + print("Refusing to install inside the framework repository.", file=sys.stderr) + return 2 + mappings = [ + (FRAMEWORK / "AGENTS.md", vault / "AGENTS.md"), + (FRAMEWORK / "DTM.md", vault / "DTM.md"), + ] + for folder, target in ( + (FRAMEWORK / "templates", vault / "templates"), + (FRAMEWORK / "tools", vault / "tools"), + (FRAMEWORK / "obsidian", vault / ".obsidian"), + (FRAMEWORK / "defaults" / "dtm", vault / "dtm"), + (FRAMEWORK / "automation-definitions", vault / "automation-definitions"), + (EXAMPLE, vault), + ): + for source in files_under(folder): + mappings.append((source, target / source.relative_to(folder))) + for source, destination in mappings: + print(copy_file(source, destination, force, dry_run)) + print("Installation preview complete." if dry_run else "Installation complete.") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--vault", type=Path, required=True) + parser.add_argument("--force", action="store_true", help="replace existing framework files") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + return install(args.vault, args.force, args.dry_run) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_framework.py b/tests/test_framework.py new file mode 100644 index 0000000..405c571 --- /dev/null +++ b/tests/test_framework.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import importlib.util +import json +import tempfile +import unittest +from datetime import date, timedelta +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +def load_dtm_module(): + path = ROOT / "framework" / "tools" / "dtm.py" + spec = importlib.util.spec_from_file_location("secondbrain_dtm", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class ObsidianConfigurationTests(unittest.TestCase): + def test_templates_and_daily_notes_match_installed_layout(self): + obsidian = ROOT / "framework" / "obsidian" + templates = json.loads((obsidian / "templates.json").read_text()) + daily = json.loads((obsidian / "daily-notes.json").read_text()) + + self.assertEqual(templates["folder"], "templates") + self.assertEqual(daily["folder"], "daily") + self.assertEqual(daily["format"], "YYYY-MM-DD") + self.assertEqual(daily["template"], "templates/daily-note") + + +class RecurrenceSafetyTests(unittest.TestCase): + def test_invalid_rule_fails_before_rollover_mutates_notes(self): + dtm = load_dtm_module() + today = date(2030, 1, 2) + yesterday = today - timedelta(days=1) + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + daily = root / "daily" + daily.mkdir() + previous = daily / f"{yesterday.isoformat()}.md" + previous.write_text("---\nstatus: open\n---\n", encoding="utf-8") + recurrence = root / "recurring-tasks.json" + recurrence.write_text( + json.dumps( + { + "version": 1, + "tasks": [ + { + "id": "missing-task", + "area": "personal", + "enabled": True, + "schedule": {"frequency": "daily"}, + } + ], + } + ), + encoding="utf-8", + ) + dtm.DAILY = daily + dtm.RECURRENCE = recurrence + + with self.assertRaises(dtm.DTMError): + dtm.rollover(today) + + self.assertIn("status: open", previous.read_text(encoding="utf-8")) + self.assertFalse((daily / f"{today.isoformat()}.md").exists()) + + +if __name__ == "__main__": + unittest.main()