Source OS release information from (draft) GitHub release, not pre-prod [DI-693]

.github/python/add-release/add_release.py

@@ -9,7 +9,7 @@
 from pathlib import Path
 
 import semver
-from version_metadata import VersionMetadata
+from version_metadata import EEVersionMetadata, OSVersionMetadata, VersionMetadata
 
 
 def main() -> None:
@@ -24,19 +24,21 @@ def main() -> None:
     parser.add_argument("--should-build-ee", required=True)
     args = parser.parse_args()
 
-    version_metadata = VersionMetadata(
-        args.version,
-        args.github_org,
-        args.ee_release_repo_name,
-        args.jfrog_preprod_files_repo,
-    )
-    logging.debug("version=%s", version_metadata.version)
-
     if parse_boolean(args.should_build_oss):
+        version_metadata = OSVersionMetadata(
+            version=args.version, github_org=args.github_org
+        )
+
         update_hazelcast_open_source_metadata(version_metadata)
         update_imdg_clients_metadata(version_metadata)
 
     if parse_boolean(args.should_build_ee):
+        version_metadata = EEVersionMetadata(
+            version=args.version,
+            release_repo_name=args.ee_release_repo_name,
+            jfrog_preprod_files_repo=args.jfrog_preprod_files_repo,
+        )
+
         update_hazelcast_enterprise_metadata(version_metadata)
 
 
@@ -50,18 +52,18 @@ def update_hazelcast_open_source_metadata(version_metadata: VersionMetadata):
     version_block = f"""---
 Version: {version_metadata.version}
 Date: ${{TBC_RELEASE_DATE}}
-Download_ZIP_URL: {version_metadata.os_downloads.full_zip.public_url}
-Download_ZIP_Size: {version_metadata.os_downloads.full_zip.size}
-Download_slim_ZIP_URL: {version_metadata.os_downloads.slim_zip.public_url}
-Download_slim_ZIP_Size: {version_metadata.os_downloads.slim_zip.size}
-Download_TAR_URL: {version_metadata.os_downloads.full_tar.public_url}
-Download_TAR_Size: {version_metadata.os_downloads.full_tar.size}
-Download_slim_TAR_URL: {version_metadata.os_downloads.slim_tar.public_url}
-Download_slim_TAR_Size: {version_metadata.os_downloads.slim_tar.size}
+Download_ZIP_URL: {version_metadata.downloads.full_zip.public_url}
+Download_ZIP_Size: {version_metadata.downloads.full_zip.size}
+Download_slim_ZIP_URL: {version_metadata.downloads.slim_zip.public_url}
+Download_slim_ZIP_Size: {version_metadata.downloads.slim_zip.size}
+Download_TAR_URL: {version_metadata.downloads.full_tar.public_url}
+Download_TAR_Size: {version_metadata.downloads.full_tar.size}
+Download_slim_TAR_URL: {version_metadata.downloads.slim_tar.public_url}
+Download_slim_TAR_Size: {version_metadata.downloads.slim_tar.size}
 Docs_HTML: {version_metadata.docs_url}
 Docs_PDF:
-APIDocs: {version_metadata.os_apidocs_url}
-ReleaseNotes: {version_metadata.os_release_notes_url}
+APIDocs: {version_metadata.apidocs_url}
+ReleaseNotes: {version_metadata.release_notes_url}
 CodeSamples_URL: {version_metadata.code_samples_url}
 CodeSamples_Size:
 Github: {version_metadata.sources_url}"""
@@ -79,18 +81,18 @@ def update_hazelcast_enterprise_metadata(version_metadata: VersionMetadata):
     version_block = f"""---
 Version: {version_metadata.version}
 Date: ${{TBC_RELEASE_DATE}}
-Download_ZIP_URL: {version_metadata.ee_downloads.full_zip.public_url}
-Download_ZIP_Size: {version_metadata.ee_downloads.full_zip.size}
-Download_slim_ZIP_URL: {version_metadata.ee_downloads.slim_zip.public_url}
-Download_slim_ZIP_Size: {version_metadata.ee_downloads.slim_zip.size}
-Download_TAR_URL: {version_metadata.ee_downloads.full_tar.public_url}
-Download_TAR_Size: {version_metadata.ee_downloads.full_tar.size}
-Download_slim_TAR_URL: {version_metadata.ee_downloads.slim_tar.public_url}
-Download_slim_TAR_Size: {version_metadata.ee_downloads.slim_tar.size}
+Download_ZIP_URL: {version_metadata.downloads.full_zip.public_url}
+Download_ZIP_Size: {version_metadata.downloads.full_zip.size}
+Download_slim_ZIP_URL: {version_metadata.downloads.slim_zip.public_url}
+Download_slim_ZIP_Size: {version_metadata.downloads.slim_zip.size}
+Download_TAR_URL: {version_metadata.downloads.full_tar.public_url}
+Download_TAR_Size: {version_metadata.downloads.full_tar.size}
+Download_slim_TAR_URL: {version_metadata.downloads.slim_tar.public_url}
+Download_slim_TAR_Size: {version_metadata.downloads.slim_tar.size}
 Docs_HTML: {version_metadata.docs_url}
 Docs_PDF:
-APIDocs: {version_metadata.ee_apidocs_url}
-ReleaseNotes: {version_metadata.ee_release_notes_url}
+APIDocs: {version_metadata.apidocs_url}
+ReleaseNotes: {version_metadata.release_notes_url}
 CodeSamples_URL:
 CodeSamples_Size:
 Github:"""
@@ -101,8 +103,6 @@ def update_hazelcast_enterprise_metadata(version_metadata: VersionMetadata):
 def update_hazelcast_metadata(
     version_metadata: VersionMetadata, file_path, version_block
 ):
-    """Updates `hazelcast-xxx_txt`"""
-
     header = "========== "
     current_stable_header = header + "Current Stable"
     previous_stable_header = header + "Previous Stable"
@@ -169,12 +169,12 @@ def update_imdg_clients_metadata(version_metadata: VersionMetadata):
     version_block = f"""---
 Version: {version_metadata.version}
 Date: ${{TBC_RELEASE_DATE}}
-Download: {version_metadata.os_downloads.slim_zip.public_url}
-Download_Size: {version_metadata.os_downloads.slim_zip.size}
+Download: {version_metadata.downloads.slim_zip.public_url}
+Download_Size: {version_metadata.downloads.slim_zip.size}
 Github: {version_metadata.sources_url}
 Docs: {version_metadata.docs_url}
-APIDocs: {version_metadata.os_apidocs_url}
-ReleaseNotes: {version_metadata.os_release_notes_url}"""
+APIDocs: {version_metadata.apidocs_url}
+ReleaseNotes: {version_metadata.release_notes_url}"""
 
     insert_version_block_below_header(
         file_path, file_path.read_text(), header, version_block

.github/python/add-release/requirements.txt

@@ -1,3 +1,4 @@
 humanize==4.15.0
+PyGithub==2.10.0
 requests==2.33.1
 semver==3.0.4

.github/python/add-release/tests/test_version_metadata.py

@@ -10,8 +10,22 @@
 vm = pytest.importorskip("version_metadata")
 
 
+@pytest.fixture(autouse=True)
+def github_release_download_base_url(monkeypatch):
+    def fake_download_base_url(self, github_org):
+        return (
+            f"https://github.com/{github_org}/hazelcast/releases/download/untagged-test"
+        )
+
+    monkeypatch.setattr(
+        vm.OSVersionMetadata,
+        "_get_github_release_download_base_url",
+        fake_download_base_url,
+    )
+
+
 def test_build_downloads():
-    version_metadata = vm.VersionMetadata("5.4.1", "hazelcast", "download", "preprod")
+    version_metadata = vm.EEVersionMetadata("5.4.1", "download", "preprod")
     downloads = version_metadata._build_downloads(
         "https://example.com/foo", "https://example.com/bar"
     )
@@ -22,9 +36,18 @@ def test_build_downloads():
     assert downloads.slim_tar.public_url == "https://example.com/foo-slim.tar.gz"
 
 
-def test_download_size():
-    version_metadata = vm.VersionMetadata("5.6.0", "hazelcast", "download", "preprod")
-    slim_zip = version_metadata.os_downloads.slim_zip
+def test_download_size(monkeypatch):
+    def fake_get_size(url):
+        assert (
+            url
+            == "https://github.com/hazelcast/hazelcast/releases/download/untagged-test/hazelcast-5.6.0-slim.zip"
+        )
+        return "41 MB"
+
+    monkeypatch.setattr(vm.DownloadUrl, "_get_size", staticmethod(fake_get_size))
+
+    version_metadata = vm.OSVersionMetadata("5.6.0", "hazelcast")
+    slim_zip = version_metadata.downloads.slim_zip
 
     assert (
         slim_zip.public_url

.github/python/add-release/version_metadata.py

@@ -1,12 +1,23 @@
 import logging
 import os
+from abc import ABC, abstractmethod
 from dataclasses import InitVar, dataclass, field
+from posixpath import dirname
+from urllib.parse import urlparse
 
 import requests
 import semver
+from github import Auth, Github
 from humanize import naturalsize
 from requests.auth import HTTPBasicAuth
-from urllib.parse import urlparse
+
+
+def _github_token() -> str | None:
+    return os.getenv("GH_TOKEN")
+
+
+def _github_client() -> Github:
+    return Github(auth=Auth.Token(_github_token()))
 
 
 @dataclass
@@ -21,8 +32,6 @@ class DownloadUrl:
     @staticmethod
     def _get_size(url: str) -> str:
         """Fetch and cache the artifact size without _actually_ downloading it."""
-        # Lazily evaluates
-        # So in the case of (say) an EE-only release we don't query (non-existent) OS artifacts
         logging.debug("Getting size of %s", url)
 
         username = os.getenv("RELEASE_REPO_USER")
@@ -35,28 +44,33 @@ def _get_size(url: str) -> str:
             and password
             else None
         )
+        headers = (
+            {"Authorization": f"Bearer {_github_token()}"}
+            if urlparse(url).hostname == "github.com" and _github_token()
+            else None
+        )
 
-        response = requests.head(
+        with requests.get(
             url,
             allow_redirects=True,
             auth=auth,
-        )
-        response.raise_for_status()
+            headers=headers,
+            stream=True,
+        ) as response:
+            response.raise_for_status()
+
+            content_length = response.headers.get("Content-Length")
 
-        content_length = response.headers["Content-Length"]
+            if content_length:
+                return naturalsize(int(content_length), format="%.0f")
 
-        if content_length:
-            return naturalsize(int(content_length), format="%.0f")
-        else:
             raise Exception(f"{url} did not return a size")
 
     @property
     def size(self) -> str:
         if self._size is None:
-            try:
-                self._size = self._get_size(self.preprod_url)
-            except requests.RequestException:
-                self._size = self._get_size(self.public_url)
+            # We only need to resolve size during `package` phase, before promotion - hence it will only ever be in preprod
+            self._size = self._get_size(self.preprod_url)
 
         return self._size
 
@@ -80,48 +94,113 @@ class Downloads:
 
 
 @dataclass
-class VersionMetadata:
+class VersionMetadata(ABC):
     """Release metadata derived from a semantic version."""
 
     version: semver.Version | str
-    github_org: InitVar[str]
-    ee_release_repo_name: InitVar[str]
-    jfrog_preprod_files_repo: InitVar[str]
 
-    def __post_init__(self, github_org, ee_release_repo_name, jfrog_preprod_files_repo):
-        """Normalize version and populate derived URLs and metadata."""
+    def __post_init__(self):
         if isinstance(self.version, str):
             self.version = semver.Version.parse(self.version)
 
-        self.os_downloads = self._build_downloads(
-            f"https://github.com/{github_org}/hazelcast/releases/download/v{self.version}/hazelcast-{self.version}",
-            f"https://repository.hazelcast.com/{jfrog_preprod_files_repo}/hazelcast/hazelcast-{self.version}",
+        self.docs_url = f"https://docs.hazelcast.com/hazelcast/{self.version.major}.{self.version.minor}/getting-started/quickstart.html"
+
+    @property
+    @abstractmethod
+    def downloads(self):
+        """Download metadata for this release."""
+
+    @property
+    @abstractmethod
+    def apidocs_url(self) -> str:
+        """API documentation URL for this release."""
+
+    def _build_downloads(self, live_base_url: str, preprod_base_url: str) -> Downloads:
+        """Build a Downloads object for a given artifact base live URL."""
+        return Downloads(
+            full_zip=DownloadUrl(live_base_url, preprod_base_url, ".zip"),
+            slim_zip=DownloadUrl(live_base_url, preprod_base_url, "-slim.zip"),
+            full_tar=DownloadUrl(live_base_url, preprod_base_url, ".tar.gz"),
+            slim_tar=DownloadUrl(live_base_url, preprod_base_url, "-slim.tar.gz"),
         )
-        self.ee_downloads = self._build_downloads(
-            f"https://repository.hazelcast.com/{ee_release_repo_name}/hazelcast-enterprise/hazelcast-enterprise-{self.version}",
-            f"https://repository.hazelcast.com/{jfrog_preprod_files_repo}/hazelcast-enterprise/hazelcast-enterprise-{self.version}",
+
+
+@dataclass
+class OSVersionMetadata(VersionMetadata):
+    github_org: InitVar[str]
+
+    def __post_init__(self, github_org):
+        super().__post_init__()
+
+        self._downloads = self._build_downloads(
+            f"https://github.com/{github_org}/hazelcast/releases/download/v{self.version}/hazelcast-{self.version}",
+            f"{self._get_github_release_download_base_url(github_org)}/hazelcast-{self.version}",
         )
 
+        self.release_notes_url = f"https://docs.hazelcast.com/hazelcast/{self.version.major}.{self.version.minor}/release-notes/community#{self.version.major}-{self.version.minor}-{self.version.patch}"
+
         self.sources_url = (
             f"https://github.com/{github_org}/hazelcast/tree/v{self.version}"
         )
         self.code_samples_url = (
             f"https://github.com/{github_org}/hazelcast-code-samples"
         )
 
-        self.docs_url = f"https://docs.hazelcast.com/hazelcast/{self.version.major}.{self.version.minor}/getting-started/quickstart.html"
-        self.os_release_notes_url = f"https://docs.hazelcast.com/hazelcast/{self.version.major}.{self.version.minor}/release-notes/community#{self.version.major}-{self.version.minor}-{self.version.patch}"
-        self.ee_release_notes_url = f"https://docs.hazelcast.com/hazelcast/{self.version.major}.{self.version.minor}/release-notes/enterprise#{self.version.major}-{self.version.minor}-{self.version.patch}"
-        self.os_apidocs_url = f"https://docs.hazelcast.org/docs/{self.version}/javadoc"
-        self.ee_apidocs_url = (
-            f"https://docs.hazelcast.org/hazelcast-ee-docs/{self.version}/javadoc"
-        )
+    @property
+    def downloads(self):
+        return self._downloads
 
-    def _build_downloads(self, live_base_url: str, preprod_base_url: str) -> Downloads:
-        """Build a Downloads object for a given artifact base live URL."""
-        return Downloads(
-            full_zip=DownloadUrl(live_base_url, preprod_base_url, ".zip"),
-            slim_zip=DownloadUrl(live_base_url, preprod_base_url, "-slim.zip"),
-            full_tar=DownloadUrl(live_base_url, preprod_base_url, ".tar.gz"),
-            slim_tar=DownloadUrl(live_base_url, preprod_base_url, "-slim.tar.gz"),
+    @property
+    def apidocs_url(self) -> str:
+        return f"https://docs.hazelcast.org/docs/{self.version}/javadoc"
+
+    def _get_github_release_download_base_url(self, github_org: str) -> str:
+        """Find the release and derive its asset download base URL - for draft releases, this is non-trivial - e.g. could be https://github.com/hazelcast/hazelcast/releases/download/untagged-ae7bc91fc31113ff3102/hazelcast-1.2.3.zip"""
+
+        with _github_client() as github_client:
+            repo = github_client.get_repo(f"{github_org}/hazelcast")
+            release_name = f"v{self.version}"
+
+            try:
+                # have to iterate as get_release(releasename) doesn't find drafts
+                releases = repo.get_releases()
+                release = next(
+                    release for release in releases if release.name == release_name
+                )
+
+                asset = next(iter(release.get_assets()))
+
+                if asset:
+                    # Now we have an asset (e.g. https://github.com/hazelcast/hazelcast/releases/download/untagged-ae7bc91fc31113ff3102/hazelcast-1.2.3.zip), extract the base URL of the releases' assets (e.g. https://github.com/hazelcast/hazelcast/releases/download/untagged-ae7bc91fc31113ff3102)
+                    parsed_url = urlparse(asset.browser_download_url)
+                    return parsed_url._replace(path=dirname(parsed_url.path)).geturl()
+                else:
+                    raise ValueError(f"No assets found for version {release}")
+            except StopIteration:
+                raise ValueError(
+                    f"Release `{release_name}` not found in {repo} - found {[release.name for release in releases]}"
+                ) from None
+
+
+@dataclass
+class EEVersionMetadata(VersionMetadata):
+    release_repo_name: InitVar[str]
+    jfrog_preprod_files_repo: InitVar[str]
+
+    def __post_init__(self, release_repo_name, jfrog_preprod_files_repo):
+        super().__post_init__()
+
+        self._downloads = self._build_downloads(
+            f"https://repository.hazelcast.com/{release_repo_name}/hazelcast-enterprise/hazelcast-enterprise-{self.version}",
+            f"https://repository.hazelcast.com/{jfrog_preprod_files_repo}/hazelcast-enterprise/hazelcast-enterprise-{self.version}",
         )
+
+        self.release_notes_url = f"https://docs.hazelcast.com/hazelcast/{self.version.major}.{self.version.minor}/release-notes/enterprise#{self.version.major}-{self.version.minor}-{self.version.patch}"
+
+    @property
+    def downloads(self):
+        return self._downloads
+
+    @property
+    def apidocs_url(self) -> str:
+        return f"https://docs.hazelcast.org/hazelcast-ee-docs/{self.version}/javadoc"