From b1104ba7c932dddc8a96d80b99d28623a40dd459 Mon Sep 17 00:00:00 2001 From: Takahiro Yamashita <921820+gaupon@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:07:47 +0900 Subject: [PATCH] Use the Markdown API workaround for GitHub App tokens and repo-scoped image URLs GitHub App installation tokens get the same 404 as fine-grained PATs when downloading attachments from private repositories (#477), but the workaround was only used with -f. Older github.com/{owner}/{repo}/assets/* image URLs also 404 with both token types and render to the same JWT-signed URLs. Move the check into needs_jwt_workaround() so it can be tested directly, and pass the current token from get_auth() so App tokens are refreshed. The README now also notes that the workaround needs read access to contents: without it, POST /markdown returns 200 but leaves the image URLs unsigned. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JefeUY9THfz2WzRtBGEP3U --- README.rst | 2 +- github_backup/github_backup.py | 45 ++++++++++------ tests/test_attachments.py | 94 ++++++++++++++++++++++++---------- 3 files changed, 95 insertions(+), 46 deletions(-) diff --git a/README.rst b/README.rst index 56a2e96..872e02b 100644 --- a/README.rst +++ b/README.rst @@ -314,7 +314,7 @@ The tool automatically extracts file extensions from HTTP headers to ensure file **Repository filtering** for repo files/assets handles renamed and transferred repositories gracefully. URLs are included if they either match the current repository name directly, or redirect to it (e.g., ``willmcgugan/rich`` redirects to ``Textualize/rich`` after transfer). -**Fine-grained token limitation:** Due to a GitHub platform limitation, fine-grained personal access tokens (``github_pat_...``) cannot download attachments from private repositories directly. This affects both ``/assets/`` (images) and ``/files/`` (documents) URLs. The tool implements a workaround for image attachments using GitHub's Markdown API, which converts URLs to temporary JWT-signed URLs that can be downloaded. However, this workaround only works for images - document attachments (PDFs, text files, etc.) will fail with 404 errors when using fine-grained tokens on private repos. For full attachment support on private repositories, use a classic token (``-t``) instead of a fine-grained token (``-f``). See `#477 `_ for details. +**Fine-grained and GitHub App token limitation:** Due to a GitHub platform limitation, fine-grained personal access tokens (``github_pat_...``) and GitHub App installation tokens (``--as-app`` or ``--app-id``) cannot download attachments from private repositories directly. This affects ``/assets/`` (images) and ``/files/`` (documents) URLs, in both the ``github.com/user-attachments/`` and the older ``github.com/{owner}/{repo}/`` format. The tool implements a workaround for image attachments using GitHub's Markdown API, which converts URLs to temporary JWT-signed URLs that can be downloaded. The token needs read access to contents for this (included in the permissions above); without it, the Markdown API still responds but does not sign the image URLs. However, this workaround only works for images - document attachments (PDFs, text files, etc.) will fail with 404 errors when using these tokens on private repos. For full attachment support on private repositories, use a classic token (``-t``) instead. See `#477 `_ for details. About Discussions diff --git a/github_backup/github_backup.py b/github_backup/github_backup.py index 64c08c3..004e140 100644 --- a/github_backup/github_backup.py +++ b/github_backup/github_backup.py @@ -1420,6 +1420,25 @@ def download_attachment_file(url, path, auth, as_app=False, fine=False): return metadata +def needs_jwt_workaround(args, repository, url): + """Whether an attachment must be fetched through a JWT-signed URL. + + Fine-grained PATs (#477) and GitHub App installation tokens get a 404 when + downloading attachments from private repositories directly, while classic + tokens do not. Image attachments, in both the modern + github.com/user-attachments/assets/* and the older + github.com/{owner}/{repo}/assets/* format, can be fetched through the + JWT-signed URL that the Markdown API renders for them. /files/ URLs cannot. + """ + if args.token_fine is None and not args.as_app: + return False + if not repository.get("private", False): + return False + return "github.com/user-attachments/assets/" in url or bool( + re.match(r"https://github\.com/[^/]+/[^/]+/assets/\d+/", url) + ) + + def get_jwt_signed_url_via_markdown_api(url, token, repo_context): """Convert a user-attachments/assets URL to a JWT-signed URL via Markdown API. @@ -1427,8 +1446,9 @@ def get_jwt_signed_url_via_markdown_api(url, token, repo_context): JWT-signed private-user-images.githubusercontent.com URLs that work without token authentication. - This is a workaround for issue #477 where fine-grained PATs cannot - download user-attachments URLs from private repos directly. + This is a workaround for issue #477 where fine-grained PATs (and GitHub + App installation tokens) cannot download attachments from private repos + directly. Limitations: - Only works for /assets/ URLs (images) @@ -1436,8 +1456,8 @@ def get_jwt_signed_url_via_markdown_api(url, token, repo_context): - JWT URLs expire after ~5 minutes Args: - url: The github.com/user-attachments/assets/UUID URL - token: Raw fine-grained PAT (github_pat_...) + url: The image attachment URL (user-attachments/assets or repo assets) + token: Raw fine-grained PAT (github_pat_...) or App installation token repo_context: Repository context as "owner/repo" Returns: @@ -1832,16 +1852,7 @@ def download_attachments( filename = get_attachment_filename(url) filepath = os.path.join(attachments_dir, filename) - # Issue #477: Fine-grained PATs cannot download user-attachments/assets - # from private repos directly (404). Use Markdown API workaround to get - # a JWT-signed URL. Only works for /assets/ (images), not /files/. - needs_jwt = ( - args.token_fine is not None - and repository.get("private", False) - and "github.com/user-attachments/assets/" in url - ) - - if not needs_jwt: + if not needs_jwt_workaround(args, repository, url): # NORMAL download path metadata = download_attachment_file( url, @@ -1851,7 +1862,7 @@ def download_attachments( fine=args.token_fine is not None, ) elif jwt_url := get_jwt_signed_url_via_markdown_api( - url, args.token_fine, repository["full_name"] + url, get_auth(args, encode=False), repository["full_name"] ): # JWT needed and extracted, download via JWT metadata = download_attachment_file( @@ -1865,8 +1876,8 @@ def download_attachments( "url": url, "success": False, "skipped_at": datetime.now(timezone.utc).isoformat(), - "error": "Fine-grained token cannot download private repo attachments. " - "Markdown API workaround failed. Use --token-classic instead.", + "error": "Fine-grained and GitHub App tokens cannot download private repo " + "attachments. Markdown API workaround failed. Use --token-classic instead.", } logger.warning( "Skipping attachment {0}: {1}".format(url, metadata["error"]) diff --git a/tests/test_attachments.py b/tests/test_attachments.py index 241a08f..45c0237 100644 --- a/tests/test_attachments.py +++ b/tests/test_attachments.py @@ -398,37 +398,75 @@ def test_markdown_api_returns_none_when_no_jwt_url(self): assert result is None - def test_needs_jwt_only_for_fine_grained_private_assets(self): - """needs_jwt is True only for fine-grained + private + /assets/ URL.""" - assets_url = "https://github.com/user-attachments/assets/abc123" - files_url = "https://github.com/user-attachments/files/123/doc.pdf" - token_fine = "github_pat_test" - private = True - public = False - - # Fine-grained + private + assets = True - needs_jwt = ( - token_fine is not None - and private - and "github.com/user-attachments/assets/" in assets_url - ) - assert needs_jwt is True + @pytest.mark.parametrize( + "token, private, url, expected", + [ + # Fine-grained PAT (#477) + ("fine", True, "https://github.com/user-attachments/assets/abc123", True), + ("fine", True, "https://github.com/user-attachments/files/123/doc.pdf", False), + ("fine", False, "https://github.com/user-attachments/assets/abc123", False), + # GitHub App installation token gets the same 404 as a fine-grained PAT + ("app", True, "https://github.com/user-attachments/assets/abc123", True), + ("app", True, "https://github.com/user-attachments/files/123/doc.pdf", False), + ("app", False, "https://github.com/user-attachments/assets/abc123", False), + # Older repo-scoped image URLs render to the same JWT-signed URLs + ("fine", True, "https://github.com/testuser/testrepo/assets/123/abc-def", True), + ("app", True, "https://github.com/testuser/testrepo/assets/123/abc-def", True), + ("app", True, "https://github.com/testuser/testrepo/files/123/doc.pdf", False), + # Classic tokens download private attachments directly + ("classic", True, "https://github.com/user-attachments/assets/abc123", False), + ("classic", True, "https://github.com/testuser/testrepo/assets/123/abc-def", False), + ], + ) + def test_needs_jwt_workaround(self, create_args, token, private, url, expected): + """Only fine-grained and App tokens on private repos use the workaround, and only for images.""" + args = create_args() + if token == "fine": + args.token_fine = "github_pat_test" + elif token == "app": + args.as_app = True + args.token_classic = "ghs_test" + else: + args.token_classic = "ghp_test" + repository = {"full_name": "testuser/testrepo", "private": private} + + assert github_backup.needs_jwt_workaround(args, repository, url) is expected + + def test_jwt_workaround_uses_app_installation_token(self, attachment_test_setup): + """With --as-app, the Markdown API is called with the installation token.""" + setup = attachment_test_setup + setup["args"].as_app = True + setup["args"].token_classic = "ghs_test" + setup["repository"]["private"] = True - # Fine-grained + private + files = False - needs_jwt = ( - token_fine is not None - and private - and "github.com/user-attachments/assets/" in files_url - ) - assert needs_jwt is False + issue_data = {"body": "https://github.com/user-attachments/assets/abc123"} + jwt_url = "https://private-user-images.githubusercontent.com/123/abc.png?jwt=token" + + with patch( + "github_backup.github_backup.get_jwt_signed_url_via_markdown_api", + return_value=jwt_url, + ) as markdown_api, patch( + "github_backup.github_backup.download_attachment_file", + return_value={"success": True, "http_status": 200, "url": jwt_url}, + ) as download: + github_backup.download_attachments( + setup["args"], setup["issue_cwd"], issue_data, 123, setup["repository"] + ) - # Fine-grained + public + assets = False - needs_jwt = ( - token_fine is not None - and public - and "github.com/user-attachments/assets/" in assets_url + markdown_api.assert_called_once_with( + "https://github.com/user-attachments/assets/abc123", + "ghs_test", + "testuser/testrepo", ) - assert needs_jwt is False + # The JWT-signed URL is downloaded without the installation token + download.assert_called_once() + assert download.call_args.args[0] == jwt_url + assert download.call_args.kwargs["auth"] is None + + manifest_path = os.path.join(setup["issue_cwd"], "attachments", "123", "manifest.json") + with open(manifest_path) as f: + manifest = json.load(f) + assert manifest["attachments"][0]["jwt_workaround"] is True def test_jwt_workaround_sets_manifest_flag(self, attachment_test_setup): """Successful JWT workaround sets jwt_workaround flag in manifest."""