From d2e4c808e6897fafbf4ab0716d1ded2a1e872e4d Mon Sep 17 00:00:00 2001 From: Jakub Kuczys Date: Sat, 14 Nov 2026 17:31:09 +0100 Subject: [PATCH] Fix handling of temporary private forks --- github_backup/github_backup.py | 78 ++++++++++++++++++++++++++++---- github_backup/graphql_queries.py | 67 +++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 9 deletions(-) diff --git a/github_backup/github_backup.py b/github_backup/github_backup.py index 64c08c3..f3d1e8a 100644 --- a/github_backup/github_backup.py +++ b/github_backup/github_backup.py @@ -42,7 +42,9 @@ DISCUSSION_PAGE_SIZE, DISCUSSION_REPLIES_QUERY, ISSUE_CROSS_REFERENCE_COUNT_QUERY, + PULL_COMMENTS_QUERY, PULL_CROSS_REFERENCE_COUNT_QUERY, + PULL_PAGE_SIZE, ) FILE_URI_PREFIX = "file://" @@ -2367,7 +2369,7 @@ def backup_repositories(args, output_directory, repositories): lfs_clone=args.lfs_clone, no_prune=args.no_prune, ) - if args.include_issues or args.include_everything: + if repository["has_issues"] and (args.include_issues or args.include_everything): incremental_resource_work_attempted = True issue_cwd = os.path.join(repo_cwd, "issues") args.since = read_resource_last_update( @@ -2376,7 +2378,7 @@ def backup_repositories(args, output_directory, repositories): backup_issues(args, repo_cwd, repository, repos_template) write_resource_last_update(args, issue_cwd, repository) - if args.include_pulls or args.include_everything: + if repository["has_pull_requests"] and (args.include_pulls or args.include_everything): incremental_resource_work_attempted = True pulls_cwd = os.path.join(repo_cwd, "pulls") args.since = read_resource_last_update( @@ -2385,7 +2387,7 @@ def backup_repositories(args, output_directory, repositories): backup_pulls(args, repo_cwd, repository, repos_template) write_resource_last_update(args, pulls_cwd, repository) - if args.include_discussions or args.include_everything: + if repository["has_discussions"] and (args.include_discussions or args.include_everything): backup_discussions(args, repo_cwd, repository) if args.include_milestones or args.include_everything: @@ -3144,7 +3146,6 @@ def pull_is_due_for_repository_checkpoint(pull): # regular comments need to be fetched via issue API. # For backwards compatibility with versions <= 0.41.0 # keep name "comment_data" for review comments - comments_regular_template = _issue_template + "/{0}/comments" comments_template = _pulls_template + "/{0}/comments" commits_template = _pulls_template + "/{0}/commits" reviews_template = _pulls_template + "/{0}/reviews" @@ -3175,8 +3176,49 @@ def pull_is_due_for_repository_checkpoint(pull): if ( args.include_pull_comments or args.include_everything ) and should_fetch_non_review_data: - template = comments_regular_template.format(number) - pulls[number]["comment_regular_data"] = retrieve_data(args, template) + after = None + comments_page = 1 + comments = [] + comment_total_count = 0 + owner, name = _repository_owner_name(repository) + while True: + data = retrieve_graphql_data( + args, + PULL_COMMENTS_QUERY, + { + "owner": owner, + "name": name, + "number": number, + "after": after, + "pageSize": PULL_PAGE_SIZE, + }, + log_context="pull request {0}#{1} comments page {2}".format( + repository["full_name"], number, comments_page + ), + ) + repository_data = data.get("repository") or {} + pr_data = repository_data.get("pullRequest") + if pr_data is None: + raise Exception( + "Pull request #{0} not found in {1}".format( + number, repository["full_name"] + ) + ) + + comments_connection = pr_data.get("comments") or {} + comment_total_count = comments_connection.get( + "totalCount", comment_total_count + ) + comments.extend(_connection_nodes(comments_connection)) + + page_info = comments_connection.get("pageInfo") or {} + if not page_info.get("hasNextPage"): + break + + after = page_info.get("endCursor") + comments_page += 1 + pulls[number]["comment_regular_data"] = comments + template = comments_template.format(number) pulls[number]["comment_data"] = retrieve_data(args, template) if include_pull_reviews: @@ -3230,7 +3272,13 @@ def backup_milestones(args, repo_cwd, repository, repos_template): query_args = {"state": "all"} - _milestones = retrieve_data(args, template, query_args=query_args) + try: + _milestones = retrieve_data(args, template, query_args=query_args) + except HTTPError as e: + if e.code == 404: + logger.info("Milestones are not available for this repository, skipping") + return + raise milestones = {} for milestone in _milestones: @@ -3303,7 +3351,13 @@ def backup_labels(args, repo_cwd, repository, repos_template): label_cwd = os.path.join(repo_cwd, "labels") output_file = "{0}/labels.json".format(label_cwd) template = "{0}/{1}/labels".format(repos_template, repository["full_name"]) - _backup_data(args, "labels", template, output_file, label_cwd) + try: + _backup_data(args, "labels", template, output_file, label_cwd) + except HTTPError as e: + if e.code == 404: + logger.info("Labels are not available for this repository, skipping") + return + raise def backup_hooks(args, repo_cwd, repository, repos_template): @@ -3348,7 +3402,13 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F query_args = {} release_template = "{0}/{1}/releases".format(repos_template, repository_fullname) - releases = retrieve_data(args, release_template, query_args=query_args) + try: + releases = retrieve_data(args, release_template, query_args=query_args) + except HTTPError as e: + if e.code == 404: + logger.info("Releases are not available for this repository, skipping") + return + raise if args.skip_prerelease: releases = [r for r in releases if not r["prerelease"] and not r["draft"]] diff --git a/github_backup/graphql_queries.py b/github_backup/graphql_queries.py index a80d131..ed2ac4d 100644 --- a/github_backup/graphql_queries.py +++ b/github_backup/graphql_queries.py @@ -1,6 +1,7 @@ """GraphQL query templates used by github-backup.""" DISCUSSION_PAGE_SIZE = 100 +PULL_PAGE_SIZE = 100 CROSS_REFERENCE_PAGE_SIZE = 100 @@ -62,6 +63,72 @@ } """ +PULL_COMMENTS_QUERY = """ +query($owner: String!, $name: String!, $number: Int!, $after: String, $pageSize: Int!) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + comments(first: $pageSize, after: $after) { + totalCount + nodes { + ...IssueCommentFields + } + pageInfo { + hasNextPage + endCursor + } + } + } + } +} + +fragment ActorFields on Actor { + avatarUrl + login + resourcePath + url +} + +fragment ReactionGroupFields on ReactionGroup { + content + reactors { + totalCount + } +} + +fragment IssueCommentFields on IssueComment { + author { + ...ActorFields + } + authorAssociation + body + bodyHTML + bodyText + createdAt + createdViaEmail + databaseId + editor { + ...ActorFields + } + id + includesCreatedEdit + isMinimized + isPinned + lastEditedAt + minimizedReason + pinnedAt + pinnedBy { + ...ActorFields + } + publishedAt + reactionGroups { + ...ReactionGroupFields + } + resourcePath + updatedAt + url +} +""" + DISCUSSION_LIST_QUERY = """ query($owner: String!, $name: String!, $after: String, $pageSize: Int!) { repository(owner: $owner, name: $name) {