Merge pull request #545 from capehart/github-app-private-key

Create GitHub App installation tokens from the app's private key

Author: josegonzalez

Dockerfile

@@ -24,6 +24,7 @@ RUN apk add --no-cache \
     ca-certificates \
     git \
     git-lfs \
+    openssl \
     && addgroup -g 1000 appuser \
     && adduser -D -u 1000 -G appuser appuser
 

README.rst

@@ -37,7 +37,10 @@ Show the CLI help output::
 CLI Help output::
 
     github-backup [-h] [-t TOKEN_CLASSIC] [-f TOKEN_FINE] [--token-from-gh]
-                  [-q] [--as-app] [-o OUTPUT_DIRECTORY] [-l LOG_LEVEL] [-i]
+                  [-q] [--as-app] [--app-id APP_ID]
+                  [--app-private-key APP_PRIVATE_KEY]
+                  [--app-installation-id APP_INSTALLATION_ID]
+                  [-o OUTPUT_DIRECTORY] [-l LOG_LEVEL] [-i]
                   [--incremental-by-files]
                   [--starred] [--all-starred] [--starred-skip-size-over MB]
                   [--watched] [--followers] [--following] [--all]
@@ -78,6 +81,14 @@ CLI Help output::
       -q, --quiet           supress log messages less severe than warning, e.g.
                             info
       --as-app              authenticate as github app instead of as a user.
+      --app-id APP_ID       github app id or client id; creates and refreshes
+                            installation tokens (implies --as-app)
+      --app-private-key APP_PRIVATE_KEY
+                            path to the github app private key (.pem), optionally
+                            as file://...
+      --app-installation-id APP_INSTALLATION_ID
+                            id of the github app installation to back up
+                            (default: looked up from the user or organization)
       -o, --output-directory OUTPUT_DIRECTORY
                             directory at which to backup the repositories
       -l, --log-level LOG_LEVEL
@@ -212,9 +223,19 @@ For a user account (no ``--organization``), ``--as-app`` lists the repositories
 7. Go to *Install App* in your app's settings
 8. Select the account/organization and which repositories to back up
 
-**CI/CD usage with GitHub Actions:**
+**Using the app's private key:**
 
-Store the App ID as a repository variable and the private key contents as a secret, then use ``actions/create-github-app-token``::
+Pass the App ID and private key file, and github-backup creates installation tokens itself, refreshing them before they expire so long-running backups keep working::
+
+    github-backup myorg --organization --app-id 12345 --app-private-key /path/to/app.private-key.pem -o ./backup --all
+
+The app's installation on the user or organization being backed up is looked up automatically. To skip the lookup, pass ``--app-installation-id`` with the number at the end of the installation's settings URL (``.../settings/installations/<id>``).
+
+``--app-id`` implies ``--as-app`` and cannot be combined with ``-t``, ``-f``, ``--token-from-gh`` or the keychain options. The ``iss`` claim of the signed JWT is set to the value of ``--app-id``, so the app's client ID works there too. The JWT is signed by the ``openssl`` command line tool, which must be on your ``PATH`` (the Docker image includes it).
+
+**Using an existing installation token:**
+
+If something else already creates the installation token, pass it with ``-t`` and ``--as-app``. In GitHub Actions, store the App ID as a repository variable and the private key contents as a secret, then use ``actions/create-github-app-token``::
 
     - uses: actions/create-github-app-token@v1
       id: app-token
@@ -224,7 +245,7 @@ Store the App ID as a repository variable and the private key contents as a secr
 
     - run: github-backup myorg -t ${{ steps.app-token.outputs.token }} --as-app -o ./backup --all
 
-Note: Installation tokens expire after 1 hour. For long-running backups, use a fine-grained personal access token instead.
+Note: Installation tokens passed with ``-t`` expire after 1 hour and are not refreshed. For backups that may run longer, use ``--app-private-key`` instead.
 
 
 Prefer SSH

github_backup/github_backup.py

@@ -12,6 +12,7 @@
 import platform
 import random
 import re
+import shutil
 import socket
 import ssl
 import subprocess
@@ -45,6 +46,7 @@
 )
 
 FILE_URI_PREFIX = "file://"
+APP_TOKEN_REFRESH_MARGIN = 10 * 60  # seconds
 logger = logging.getLogger(__name__)
 
 
@@ -195,6 +197,21 @@ def parse_args(args=None):
         dest="as_app",
         help="authenticate as github app instead of as a user.",
     )
+    parser.add_argument(
+        "--app-id",
+        dest="app_id",
+        help="github app id or client id; creates and refreshes installation tokens (implies --as-app)",
+    )
+    parser.add_argument(
+        "--app-private-key",
+        dest="app_private_key",
+        help="path to the github app private key (.pem), optionally as file://...",
+    )
+    parser.add_argument(
+        "--app-installation-id",
+        dest="app_installation_id",
+        help="id of the github app installation to back up (default: looked up from the user or organization)",
+    )
     parser.add_argument(
         "-o",
         "--output-directory",
@@ -522,12 +539,39 @@ def parse_args(args=None):
         default=5,
         help="maximum number of retries for API calls (default: 5)",
     )
-    return parser.parse_args(args)
+    args = parser.parse_args(args)
+
+    if args.app_id or args.app_private_key or args.app_installation_id:
+        if not (args.app_id and args.app_private_key):
+            parser.error("--app-id and --app-private-key must be used together")
+        if (
+            args.token_classic
+            or args.token_fine
+            or args.token_from_gh
+            or args.osx_keychain_item_name
+            or args.osx_keychain_item_account
+        ):
+            parser.error(
+                "--app-id cannot be combined with -t, -f, --token-from-gh or keychain options"
+            )
+        args.app_private_key = args.app_private_key.removeprefix(FILE_URI_PREFIX)
+        if not os.path.isfile(args.app_private_key):
+            parser.error(
+                "--app-private-key file not found: {0}".format(args.app_private_key)
+            )
+        if not shutil.which("openssl"):
+            parser.error("--app-private-key requires the 'openssl' command on PATH")
+        args.as_app = True
+
+    return args
 
 
 def get_auth(args, encode=True, for_git_cli=False):
     auth = None
 
+    if args.app_id:
+        args.token_classic = get_app_installation_token(args)
+
     if args.osx_keychain_item_name:
         if not args.osx_keychain_item_account:
             raise Exception(
@@ -676,6 +720,110 @@ def read_token_from_gh_cli(args):
     return token
 
 
+def build_app_jwt(private_key_path, claims):
+    """Build an RS256 JWT for authenticating as a GitHub App.
+
+    Signs with the openssl CLI so the project keeps no runtime dependencies.
+    """
+
+    def b64url(data):
+        return base64.urlsafe_b64encode(data).rstrip(b"=")
+
+    header = b64url(json.dumps({"alg": "RS256", "typ": "JWT"}).encode("ascii"))
+    signing_input = header + b"." + b64url(json.dumps(claims).encode("ascii"))
+
+    try:
+        signature = subprocess.run(
+            ["openssl", "dgst", "-sha256", "-sign", private_key_path],
+            input=signing_input,
+            capture_output=True,
+            check=True,
+        ).stdout
+    except subprocess.CalledProcessError as e:
+        stderr = e.stderr.decode("utf-8", errors="replace").strip()
+        raise Exception("Unable to sign GitHub App JWT: {0}".format(stderr))
+
+    return (signing_input + b"." + b64url(signature)).decode("ascii")
+
+
+def get_app_installation_token(args):
+    """Return a GitHub App installation token, creating a new one when needed.
+
+    Installation tokens expire after an hour, so a new one is created once the
+    cached token is within APP_TOKEN_REFRESH_MARGIN seconds of expiring.
+    """
+    cached_token = getattr(args, "_app_token_value", None)
+    if cached_token and time.time() < (
+        args._app_token_expires_at - APP_TOKEN_REFRESH_MARGIN
+    ):
+        return cached_token
+
+    # GitHub rejects an exp more than 10 minutes out; backdate iat to tolerate
+    # clock drift.
+    now = int(time.time())
+    app_jwt = build_app_jwt(
+        args.app_private_key, {"iat": now - 60, "exp": now + 540, "iss": args.app_id}
+    )
+
+    if not args.app_installation_id:
+        owner_type = "orgs" if args.organization else "users"
+        installation = request_as_app(
+            args,
+            app_jwt,
+            "/{0}/{1}/installation".format(owner_type, args.user),
+            "Unable to find the GitHub App installation for {0}".format(args.user),
+        )
+        args.app_installation_id = str(installation["id"])
+        logger.info(
+            "Using GitHub App installation {0} for {1}".format(
+                args.app_installation_id, args.user
+            )
+        )
+
+    response = request_as_app(
+        args,
+        app_jwt,
+        "/app/installations/{0}/access_tokens".format(args.app_installation_id),
+        "Unable to create GitHub App installation token",
+        method="POST",
+    )
+
+    args._app_token_value = response["token"]
+    args._app_token_expires_at = calendar.timegm(
+        time.strptime(response["expires_at"], "%Y-%m-%dT%H:%M:%SZ")
+    )
+    logger.info(
+        "Created GitHub App installation token (expires {0})".format(
+            response["expires_at"]
+        )
+    )
+    return args._app_token_value
+
+
+def request_as_app(args, app_jwt, path, error_message, method="GET"):
+    """Call the GitHub API authenticated as the app itself, with its JWT."""
+    request = Request(
+        "https://{0}{1}".format(get_github_api_host(args), path),
+        data=b"" if method == "POST" else None,
+        method=method,
+    )
+    request.add_header("Accept", "application/vnd.github+json")
+    request.add_header("Authorization", "Bearer " + app_jwt)
+
+    try:
+        http_response = make_request_with_retry(request, app_jwt, args.max_retries)
+    except HTTPError as exc:
+        raise Exception(
+            "{0}: HTTP {1} {2}: {3}".format(
+                error_message,
+                exc.code,
+                exc.reason,
+                exc.read().decode("utf-8", errors="replace"),
+            )
+        )
+    return json.loads(http_response.read().decode("utf-8"))
+
+
 def get_github_repo_url(args, repository):
     if repository.get("is_gist"):
         if args.prefer_ssh:
@@ -734,7 +882,6 @@ def retrieve_data(args, template, query_args=None, paginated=True, lazy=False):
     Handles throttling, retries, read errors, and DMCA takedowns.
     """
     query_args = query_args or {}
-    auth = get_auth(args, encode=not args.as_app)
     per_page = 100
 
     def _extract_next_page_url(link_header):
@@ -758,6 +905,8 @@ def _extract_legal_url(response_body_bytes):
             # FIRST: Fetch response
 
             for attempt in range(args.max_retries + 1):
+                # Resolved per request so long runs pick up refreshed app tokens
+                auth = get_auth(args, encode=not args.as_app)
                 request = _construct_request(
                     per_page=per_page if paginated else None,
                     query_args=query_args,
@@ -2210,7 +2359,8 @@ def backup_repositories(args, output_directory, repositories):
             if repository["has_wiki"] and download_wiki:
                 fetch_repository(
                     repository["name"],
-                    repo_url.replace(".git", ".wiki.git"),
+                    # Rebuilt so a long repository clone can't leave it an expired app token
+                    get_github_repo_url(args, repository).replace(".git", ".wiki.git"),
                     os.path.join(repo_cwd, "wiki"),
                     skip_existing=args.skip_existing,
                     bare_clone=args.bare_clone,