Skip to content

Fix handling of temporary private forks #548

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Jackenmen wants to merge 1 commit into
base: master
Choose a base branch
from
+136 −9
Open

Fix handling of temporary private forks #548

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 69 additions & 9 deletions github_backup/github_backup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,9 @@
DISCUSSION_PAGE_SIZE,
DISCUSSION_REPLIES_QUERY,
ISSUE_CROSS_REFERENCE_COUNT_QUERY,
PULL_COMMENTS_QUERY,
PULL_CROSS_REFERENCE_COUNT_QUERY,
PULL_PAGE_SIZE,
)

FILE_URI_PREFIX = "file://"
Expand Down Expand Up @@ -2367,7 +2369,7 @@ def backup_repositories(args, output_directory, repositories):
lfs_clone=args.lfs_clone,
no_prune=args.no_prune,
)
if args.include_issues or args.include_everything:
if repository["has_issues"] and (args.include_issues or args.include_everything):
incremental_resource_work_attempted = True
issue_cwd = os.path.join(repo_cwd, "issues")
args.since = read_resource_last_update(
Expand All @@ -2376,7 +2378,7 @@ def backup_repositories(args, output_directory, repositories):
backup_issues(args, repo_cwd, repository, repos_template)
write_resource_last_update(args, issue_cwd, repository)

if args.include_pulls or args.include_everything:
if repository["has_pull_requests"] and (args.include_pulls or args.include_everything):
incremental_resource_work_attempted = True
pulls_cwd = os.path.join(repo_cwd, "pulls")
args.since = read_resource_last_update(
Expand All @@ -2385,7 +2387,7 @@ def backup_repositories(args, output_directory, repositories):
backup_pulls(args, repo_cwd, repository, repos_template)
write_resource_last_update(args, pulls_cwd, repository)

if args.include_discussions or args.include_everything:
if repository["has_discussions"] and (args.include_discussions or args.include_everything):
backup_discussions(args, repo_cwd, repository)

if args.include_milestones or args.include_everything:
Expand Down Expand Up @@ -3144,7 +3146,6 @@ def pull_is_due_for_repository_checkpoint(pull):
# regular comments need to be fetched via issue API.
# For backwards compatibility with versions <= 0.41.0
# keep name "comment_data" for review comments
comments_regular_template = _issue_template + "/{0}/comments"
comments_template = _pulls_template + "/{0}/comments"
commits_template = _pulls_template + "/{0}/commits"
reviews_template = _pulls_template + "/{0}/reviews"
Expand Down Expand Up @@ -3175,8 +3176,49 @@ def pull_is_due_for_repository_checkpoint(pull):
if (
args.include_pull_comments or args.include_everything
) and should_fetch_non_review_data:
template = comments_regular_template.format(number)
pulls[number]["comment_regular_data"] = retrieve_data(args, template)
after = None
comments_page = 1
comments = []
comment_total_count = 0
owner, name = _repository_owner_name(repository)
while True:
data = retrieve_graphql_data(
args,
PULL_COMMENTS_QUERY,
{
"owner": owner,
"name": name,
"number": number,
"after": after,
"pageSize": PULL_PAGE_SIZE,
},
log_context="pull request {0}#{1} comments page {2}".format(
repository["full_name"], number, comments_page
),
)
repository_data = data.get("repository") or {}
pr_data = repository_data.get("pullRequest")
if pr_data is None:
raise Exception(
"Pull request #{0} not found in {1}".format(
number, repository["full_name"]
)
)

comments_connection = pr_data.get("comments") or {}
comment_total_count = comments_connection.get(
"totalCount", comment_total_count
)
comments.extend(_connection_nodes(comments_connection))

page_info = comments_connection.get("pageInfo") or {}
if not page_info.get("hasNextPage"):
break

after = page_info.get("endCursor")
comments_page += 1
pulls[number]["comment_regular_data"] = comments

template = comments_template.format(number)
pulls[number]["comment_data"] = retrieve_data(args, template)
if include_pull_reviews:
Expand Down Expand Up @@ -3230,7 +3272,13 @@ def backup_milestones(args, repo_cwd, repository, repos_template):

query_args = {"state": "all"}

_milestones = retrieve_data(args, template, query_args=query_args)
try:
_milestones = retrieve_data(args, template, query_args=query_args)
except HTTPError as e:
if e.code == 404:
logger.info("Milestones are not available for this repository, skipping")
return
raise

milestones = {}
for milestone in _milestones:
Expand Down Expand Up @@ -3303,7 +3351,13 @@ def backup_labels(args, repo_cwd, repository, repos_template):
label_cwd = os.path.join(repo_cwd, "labels")
output_file = "{0}/labels.json".format(label_cwd)
template = "{0}/{1}/labels".format(repos_template, repository["full_name"])
_backup_data(args, "labels", template, output_file, label_cwd)
try:
_backup_data(args, "labels", template, output_file, label_cwd)
except HTTPError as e:
if e.code == 404:
logger.info("Labels are not available for this repository, skipping")
return
raise


def backup_hooks(args, repo_cwd, repository, repos_template):
Expand Down Expand Up @@ -3348,7 +3402,13 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
query_args = {}

release_template = "{0}/{1}/releases".format(repos_template, repository_fullname)
releases = retrieve_data(args, release_template, query_args=query_args)
try:
releases = retrieve_data(args, release_template, query_args=query_args)
except HTTPError as e:
if e.code == 404:
logger.info("Releases are not available for this repository, skipping")
return
raise

if args.skip_prerelease:
releases = [r for r in releases if not r["prerelease"] and not r["draft"]]
Expand Down
67 changes: 67 additions & 0 deletions github_backup/graphql_queries.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""GraphQL query templates used by github-backup."""

DISCUSSION_PAGE_SIZE = 100
PULL_PAGE_SIZE = 100

CROSS_REFERENCE_PAGE_SIZE = 100

Expand Down Expand Up @@ -62,6 +63,72 @@
}
"""

PULL_COMMENTS_QUERY = """
query($owner: String!, $name: String!, $number: Int!, $after: String, $pageSize: Int!) {
repository(owner: $owner, name: $name) {
pullRequest(number: $number) {
comments(first: $pageSize, after: $after) {
totalCount
nodes {
...IssueCommentFields
}
pageInfo {
hasNextPage
endCursor
}
}
}
}
}

fragment ActorFields on Actor {
avatarUrl
login
resourcePath
url
}

fragment ReactionGroupFields on ReactionGroup {
content
reactors {
totalCount
}
}

fragment IssueCommentFields on IssueComment {
author {
...ActorFields
}
authorAssociation
body
bodyHTML
bodyText
createdAt
createdViaEmail
databaseId
editor {
...ActorFields
}
id
includesCreatedEdit
isMinimized
isPinned
lastEditedAt
minimizedReason
pinnedAt
pinnedBy {
...ActorFields
}
publishedAt
reactionGroups {
...ReactionGroupFields
}
resourcePath
updatedAt
url
}
"""

DISCUSSION_LIST_QUERY = """
query($owner: String!, $name: String!, $after: String, $pageSize: Int!) {
repository(owner: $owner, name: $name) {
Expand Down