Repository navigation
Expand file tree
/
Copy pathsourcecode_snapshot.py
More file actions
360 lines (318 loc) · 13.3 KB
/
Copy pathsourcecode_snapshot.py
File metadata and controls
360 lines (318 loc) · 13.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
sourcecode_snapshot.py
Create a Markdown snapshot of a repository (code-only) and optionally package it as a ZIP.
Outputs:
- <out_md> (default: your_sourcecode_snapshot.md)
- optional ZIP archive containing the Markdown and the selected source files
Features:
- Includes only selected file extensions and specific filenames.
- Skips common large/irrelevant directories and very large files.
- Produces a short directory tree excerpt (configurable depth).
- Attempts UTF-8 read, falls back to latin-1.
- Non-destructive: reads files and writes outputs only.
- CLI with flags: --src, --dst, --out, --zip, --max-size, --depth, --dry-run, --verbose
Usage:
python3 sourcecode_snapshot.py [--src /path/to/src] [--dst /path/to/outdir] \
[--out my_snapshot.md] [--zip my_snapshot.zip] [--dry-run]
Examples:
python3 sourcecode_snapshot.py --out project_snapshot.md --zip project_snapshot.zip
python3 sourcecode_snapshot.py --src .. --dst ./artifacts --out repo.md --zip repo.zip
"""
from __future__ import annotations
import argparse
import os
import sys
import time
import zipfile
import stat
from pathlib import Path
from typing import List, Iterable, Tuple, Set
# --------------------
# Default configuration
# --------------------
DEFAULT_OUT_MD = "your_sourcecode_snapshot.md"
DEFAULT_ALLOWED_EXTS: Set[str] = {
".py", ".html", ".yml", ".yaml", ".sh", ".ps1", ".txt", ".service"
}
DEFAULT_ALLOWED_NAMES: Set[str] = {
".env.example", ".gitignore", ".dockerignore", "Dockerfile", "Makefile"
}
DEFAULT_EXCLUDE_DIRS: Set[str] = {
".venv", "venv", "__pycache__", ".git", ".pytest_cache", ".mypy_cache",
"models", "outputs", "recordings", "node_modules", "dist", "build"
}
DEFAULT_MAX_FILE_SIZE = 30 * 1024 * 1024 # 30 MiB
DEFAULT_TREE_DEPTH = 2
SCRIPT_BASENAME = Path(__file__).name # used to ensure self-exclusion
# --------------------
# Utility functions
# --------------------
def is_allowed_file(path: Path, allowed_exts: Set[str], allowed_names: Set[str]) -> bool:
name = path.name
if name in allowed_names:
return True
if path.suffix.lower() in allowed_exts:
return True
return False
def should_skip_dir(dirpath: Path, exclude_dirs: Set[str]) -> bool:
return dirpath.name in exclude_dirs
def gather_files(
root: Path,
allowed_exts: Set[str],
allowed_names: Set[str],
exclude_dirs: Set[str],
max_file_size: int,
out_md: Path,
script_paths_to_exclude: Set[Path],
verbose: bool=False
) -> List[Path]:
files: List[Path] = []
for dirpath, dirnames, filenames in os.walk(root):
# filter dirs in-place to avoid walking excluded dirs
dirnames[:] = [d for d in dirnames if not should_skip_dir(Path(dirpath) / d, exclude_dirs)]
for fn in filenames:
p = Path(dirpath) / fn
# Always skip the running script file by name or resolved path
if fn == SCRIPT_BASENAME:
if verbose:
print(f"Skipping script itself by name: {p}")
continue
try:
pres = p.resolve()
if pres in script_paths_to_exclude:
if verbose:
print(f"Skipping script itself by path: {p}")
continue
except Exception:
# best-effort; if resolve fails, keep other checks
pass
# skip the output markdown if it exists or is targeted in this run
try:
if p.resolve() == out_md.resolve():
if verbose:
print(f"Skipping output file itself: {p}")
continue
except Exception:
# path resolution may fail for broken symlinks; just continue
pass
# skip symlinks (do not follow)
try:
if p.is_symlink():
if verbose:
print(f"Skipping symlink: {p}")
continue
except Exception:
continue
# skip very large files
try:
st = p.stat()
if st.st_size > max_file_size:
if verbose:
print(f"Skipping large file: {p} ({st.st_size} bytes)")
continue
# Skip sockets, fifos, device files
mode = st.st_mode
if not stat.S_ISREG(mode):
if verbose:
print(f"Skipping non-regular file: {p}")
continue
except Exception:
# skip unreadable files
if verbose:
print(f"Skipping unreadable file: {p}")
continue
if is_allowed_file(p, allowed_exts, allowed_names):
files.append(p)
files.sort()
return files
def make_tree_listing(root: Path, depth: int = DEFAULT_TREE_DEPTH) -> str:
lines: List[str] = []
def walk(p: Path, prefix: str, current_depth: int):
if current_depth > depth:
return
try:
entries = sorted([e for e in p.iterdir()], key=lambda x: (not x.is_dir(), x.name.lower()))
except Exception:
return
for i, e in enumerate(entries):
connector = "└── " if i == len(entries)-1 else "├── "
lines.append(f"{prefix}{connector}{e.name}")
if e.is_dir():
ext_prefix = prefix + (" " if i == len(entries)-1 else "│ ")
walk(e, ext_prefix, current_depth + 1)
lines.append(f"{root.name}/")
walk(root, "", 1)
return "\n".join(lines)
def read_text_file(path: Path) -> str:
try:
return path.read_text(encoding="utf-8")
except UnicodeDecodeError:
try:
return path.read_text(encoding="latin-1")
except Exception:
return f"# Could not read file {path}\n"
except Exception:
return f"# Could not read file {path}\n"
def write_markdown(tree: str, files: List[Path], out_md: Path, project_name: str, timestamp: str, src_root: Path) -> None:
header = f"# {project_name} - snapshot\n\n"
header += f"Created: {timestamp}\n\n"
header += f"Source root: {src_root}\n\n"
header += "## Directory tree (excerpt)\n\n"
header += "```\n" + tree + "\n```\n\n"
header += "## Included source files (code-only snapshot)\n\n"
header += ("The following files are included in the snapshot. "
"Each filename is followed by its content.\n\n")
out_md.parent.mkdir(parents=True, exist_ok=True)
with out_md.open("w", encoding="utf-8") as f:
f.write(header)
for p in files:
try:
rel = p.relative_to(src_root)
except ValueError:
# If file isn't under src_root (shouldn't happen), fall back to name
rel = p.name
f.write(f"\n---\n\n### {rel}\n\n")
lang = p.suffix.lstrip(".")
f.write(f"```{lang}\n")
content = read_text_file(p)
f.write(content)
if not content.endswith("\n"):
f.write("\n")
f.write("```\n")
print(f"Wrote markdown snapshot: {out_md}")
def add_file_to_zip(z: zipfile.ZipFile, path: Path, arcname: str) -> None:
zi = zipfile.ZipInfo(arcname)
st = path.stat()
# preserve permission bits
zi.external_attr = (st.st_mode & 0xFFFF) << 16
zi.date_time = time.localtime(st.st_mtime)[:6]
with path.open("rb") as f:
data = f.read()
z.writestr(zi, data)
def create_zip_archive(zip_path: Path, md_path: Path, files: List[Path], src_root: Path, verbose: bool=False) -> None:
zip_path.parent.mkdir(parents=True, exist_ok=True)
if zip_path.exists():
raise FileExistsError(f"{zip_path} already exists")
with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as z:
# add markdown first
add_file_to_zip(z, md_path, md_path.name)
for p in files:
try:
rel = p.relative_to(src_root)
arcname = str(rel).replace(os.sep, "/")
except ValueError:
# fallback if not inside src_root
arcname = p.name
if verbose:
print(f"Adding to zip: {arcname}")
add_file_to_zip(z, p, arcname)
print(f"Created zip archive: {zip_path}")
# --------------------
# CLI
# --------------------
def parse_args() -> argparse.Namespace:
ap = argparse.ArgumentParser(description="Create a code-only Markdown snapshot and optional ZIP")
ap.add_argument("--src", default=".", help="Source directory to scan (default: current working directory)")
ap.add_argument("--dst", default=".", help="Output directory for generated files (default: current working directory)")
ap.add_argument("--out", "-o", default=DEFAULT_OUT_MD, help="Output markdown filename (default: %(default)s)")
ap.add_argument("--zip", "-z", dest="zipout", default=None, help="Optional ZIP output filename or path relative to --dst")
ap.add_argument("--max-size", type=int, default=DEFAULT_MAX_FILE_SIZE, help="Maximum file size (bytes) to include (default: %(default)s)")
ap.add_argument("--depth", type=int, default=DEFAULT_TREE_DEPTH, help="Directory tree excerpt depth (default: %(default)s)")
ap.add_argument("--dry-run", action="store_true", help="Dry run: show what would be included, do not write files")
ap.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
ap.add_argument("--project-name", default=None, help="Project name to show in header (defaults to source directory name)")
ap.add_argument("--allow-ext", action="append", help="Additional allowed extension (e.g. --allow-ext .md). Can be passed multiple times.")
ap.add_argument("--allow-name", action="append", help="Additional allowed filename (e.g. --allow-name README.md). Can be passed multiple times.")
ap.add_argument("--exclude-dir", action="append", help="Additional directory name to exclude (folder basename). Can be passed multiple times.")
return ap.parse_args()
def main() -> int:
args = parse_args()
# Resolve source and destination roots
cwd = Path.cwd()
src_root = (cwd / args.src).resolve()
dst_root = (cwd / args.dst).resolve()
if not src_root.exists() or not src_root.is_dir():
print(f"Source directory does not exist or is not a directory: {src_root}", file=sys.stderr)
return 2
# Prepare output paths
out_md = (dst_root / args.out).resolve()
zip_path = (dst_root / args.zipout).resolve() if args.zipout else None
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
project_name = args.project_name or src_root.name
allowed_exts = set(DEFAULT_ALLOWED_EXTS)
allowed_names = set(DEFAULT_ALLOWED_NAMES)
exclude_dirs = set(DEFAULT_EXCLUDE_DIRS)
if args.exclude_dir:
exclude_dirs.update(set(args.exclude_dir))
max_file_size = args.max_size
depth = args.depth
dry_run = args.dry_run
verbose = args.verbose
if args.allow_ext:
for e in args.allow_ext:
if not e.startswith("."):
e = "." + e
allowed_exts.add(e.lower())
if args.allow_name:
for n in args.allow_name:
allowed_names.add(n)
# Determine absolute paths to exclude for the script itself
script_paths_to_exclude: Set[Path] = set()
try:
script_paths_to_exclude.add(Path(__file__).resolve())
except Exception:
pass
if verbose:
print(f"Source root: {src_root}")
print(f"Destination root: {dst_root}")
print(f"Out markdown: {out_md}")
if zip_path:
print(f"ZIP output: {zip_path}")
print(f"Allowed extensions: {sorted(allowed_exts)}")
print(f"Allowed names: {sorted(allowed_names)}")
print(f"Exclude dirs: {sorted(exclude_dirs)}")
print(f"Max file size: {max_file_size}")
print(f"Tree depth: {depth}")
print(f"Dry run: {dry_run}")
print(f"Script basename (excluded): {SCRIPT_BASENAME}")
files = gather_files(
src_root, allowed_exts, allowed_names, exclude_dirs,
max_file_size, out_md, script_paths_to_exclude, verbose=verbose
)
if not files:
print("No source files found with the allowed patterns.", file=sys.stderr)
return 3
tree = make_tree_listing(src_root, depth)
# Dry-run: print summary and exit
if dry_run:
print("\nDRY RUN: files that would be included:")
for p in files:
try:
rel = p.relative_to(src_root)
except ValueError:
rel = p.name
print(f" - {rel}")
print("\nDirectory tree excerpt:\n")
print(tree)
return 0
# Ensure destination exists
dst_root.mkdir(parents=True, exist_ok=True)
# Write markdown
try:
write_markdown(tree, files, out_md, project_name, timestamp, src_root)
except Exception as e:
print(f"Failed to write markdown: {e}", file=sys.stderr)
return 4
# Optionally create zip
if zip_path:
try:
create_zip_archive(zip_path, out_md, files, src_root, verbose=verbose)
except Exception as e:
print(f"Failed to create zip: {e}", file=sys.stderr)
return 5
print("\nDone.")
return 0
if __name__ == "__main__":
raise SystemExit(main())