diff --git a/.github/python/add-release/add_release.py b/.github/python/add-release/add_release.py index 4198916..53bace2 100644 --- a/.github/python/add-release/add_release.py +++ b/.github/python/add-release/add_release.py @@ -9,7 +9,7 @@ from pathlib import Path import semver -from version_metadata import VersionMetadata +from version_metadata import EEVersionMetadata, OSVersionMetadata, VersionMetadata def main() -> None: @@ -24,19 +24,21 @@ def main() -> None: parser.add_argument("--should-build-ee", required=True) args = parser.parse_args() - version_metadata = VersionMetadata( - args.version, - args.github_org, - args.ee_release_repo_name, - args.jfrog_preprod_files_repo, - ) - logging.debug("version=%s", version_metadata.version) - if parse_boolean(args.should_build_oss): + version_metadata = OSVersionMetadata( + version=args.version, github_org=args.github_org + ) + update_hazelcast_open_source_metadata(version_metadata) update_imdg_clients_metadata(version_metadata) if parse_boolean(args.should_build_ee): + version_metadata = EEVersionMetadata( + version=args.version, + release_repo_name=args.ee_release_repo_name, + jfrog_preprod_files_repo=args.jfrog_preprod_files_repo, + ) + update_hazelcast_enterprise_metadata(version_metadata) @@ -50,18 +52,18 @@ def update_hazelcast_open_source_metadata(version_metadata: VersionMetadata): version_block = f"""--- Version: {version_metadata.version} Date: ${{TBC_RELEASE_DATE}} -Download_ZIP_URL: {version_metadata.os_downloads.full_zip.public_url} -Download_ZIP_Size: {version_metadata.os_downloads.full_zip.size} -Download_slim_ZIP_URL: {version_metadata.os_downloads.slim_zip.public_url} -Download_slim_ZIP_Size: {version_metadata.os_downloads.slim_zip.size} -Download_TAR_URL: {version_metadata.os_downloads.full_tar.public_url} -Download_TAR_Size: {version_metadata.os_downloads.full_tar.size} -Download_slim_TAR_URL: {version_metadata.os_downloads.slim_tar.public_url} -Download_slim_TAR_Size: {version_metadata.os_downloads.slim_tar.size} +Download_ZIP_URL: {version_metadata.downloads.full_zip.public_url} +Download_ZIP_Size: {version_metadata.downloads.full_zip.size} +Download_slim_ZIP_URL: {version_metadata.downloads.slim_zip.public_url} +Download_slim_ZIP_Size: {version_metadata.downloads.slim_zip.size} +Download_TAR_URL: {version_metadata.downloads.full_tar.public_url} +Download_TAR_Size: {version_metadata.downloads.full_tar.size} +Download_slim_TAR_URL: {version_metadata.downloads.slim_tar.public_url} +Download_slim_TAR_Size: {version_metadata.downloads.slim_tar.size} Docs_HTML: {version_metadata.docs_url} Docs_PDF: -APIDocs: {version_metadata.os_apidocs_url} -ReleaseNotes: {version_metadata.os_release_notes_url} +APIDocs: {version_metadata.apidocs_url} +ReleaseNotes: {version_metadata.release_notes_url} CodeSamples_URL: {version_metadata.code_samples_url} CodeSamples_Size: Github: {version_metadata.sources_url}""" @@ -79,18 +81,18 @@ def update_hazelcast_enterprise_metadata(version_metadata: VersionMetadata): version_block = f"""--- Version: {version_metadata.version} Date: ${{TBC_RELEASE_DATE}} -Download_ZIP_URL: {version_metadata.ee_downloads.full_zip.public_url} -Download_ZIP_Size: {version_metadata.ee_downloads.full_zip.size} -Download_slim_ZIP_URL: {version_metadata.ee_downloads.slim_zip.public_url} -Download_slim_ZIP_Size: {version_metadata.ee_downloads.slim_zip.size} -Download_TAR_URL: {version_metadata.ee_downloads.full_tar.public_url} -Download_TAR_Size: {version_metadata.ee_downloads.full_tar.size} -Download_slim_TAR_URL: {version_metadata.ee_downloads.slim_tar.public_url} -Download_slim_TAR_Size: {version_metadata.ee_downloads.slim_tar.size} +Download_ZIP_URL: {version_metadata.downloads.full_zip.public_url} +Download_ZIP_Size: {version_metadata.downloads.full_zip.size} +Download_slim_ZIP_URL: {version_metadata.downloads.slim_zip.public_url} +Download_slim_ZIP_Size: {version_metadata.downloads.slim_zip.size} +Download_TAR_URL: {version_metadata.downloads.full_tar.public_url} +Download_TAR_Size: {version_metadata.downloads.full_tar.size} +Download_slim_TAR_URL: {version_metadata.downloads.slim_tar.public_url} +Download_slim_TAR_Size: {version_metadata.downloads.slim_tar.size} Docs_HTML: {version_metadata.docs_url} Docs_PDF: -APIDocs: {version_metadata.ee_apidocs_url} -ReleaseNotes: {version_metadata.ee_release_notes_url} +APIDocs: {version_metadata.apidocs_url} +ReleaseNotes: {version_metadata.release_notes_url} CodeSamples_URL: CodeSamples_Size: Github:""" @@ -101,8 +103,6 @@ def update_hazelcast_enterprise_metadata(version_metadata: VersionMetadata): def update_hazelcast_metadata( version_metadata: VersionMetadata, file_path, version_block ): - """Updates `hazelcast-xxx_txt`""" - header = "========== " current_stable_header = header + "Current Stable" previous_stable_header = header + "Previous Stable" @@ -169,12 +169,12 @@ def update_imdg_clients_metadata(version_metadata: VersionMetadata): version_block = f"""--- Version: {version_metadata.version} Date: ${{TBC_RELEASE_DATE}} -Download: {version_metadata.os_downloads.slim_zip.public_url} -Download_Size: {version_metadata.os_downloads.slim_zip.size} +Download: {version_metadata.downloads.slim_zip.public_url} +Download_Size: {version_metadata.downloads.slim_zip.size} Github: {version_metadata.sources_url} Docs: {version_metadata.docs_url} -APIDocs: {version_metadata.os_apidocs_url} -ReleaseNotes: {version_metadata.os_release_notes_url}""" +APIDocs: {version_metadata.apidocs_url} +ReleaseNotes: {version_metadata.release_notes_url}""" insert_version_block_below_header( file_path, file_path.read_text(), header, version_block diff --git a/.github/python/add-release/requirements.txt b/.github/python/add-release/requirements.txt index 6c69481..85475bb 100644 --- a/.github/python/add-release/requirements.txt +++ b/.github/python/add-release/requirements.txt @@ -1,3 +1,4 @@ humanize==4.15.0 +PyGithub==2.10.0 requests==2.33.1 semver==3.0.4 diff --git a/.github/python/add-release/tests/test_version_metadata.py b/.github/python/add-release/tests/test_version_metadata.py index d0fd33a..e33c5d7 100644 --- a/.github/python/add-release/tests/test_version_metadata.py +++ b/.github/python/add-release/tests/test_version_metadata.py @@ -10,8 +10,22 @@ vm = pytest.importorskip("version_metadata") +@pytest.fixture(autouse=True) +def github_release_download_base_url(monkeypatch): + def fake_download_base_url(self, github_org): + return ( + f"https://github.com/{github_org}/hazelcast/releases/download/untagged-test" + ) + + monkeypatch.setattr( + vm.OSVersionMetadata, + "_get_github_release_download_base_url", + fake_download_base_url, + ) + + def test_build_downloads(): - version_metadata = vm.VersionMetadata("5.4.1", "hazelcast", "download", "preprod") + version_metadata = vm.EEVersionMetadata("5.4.1", "download", "preprod") downloads = version_metadata._build_downloads( "https://example.com/foo", "https://example.com/bar" ) @@ -22,9 +36,18 @@ def test_build_downloads(): assert downloads.slim_tar.public_url == "https://example.com/foo-slim.tar.gz" -def test_download_size(): - version_metadata = vm.VersionMetadata("5.6.0", "hazelcast", "download", "preprod") - slim_zip = version_metadata.os_downloads.slim_zip +def test_download_size(monkeypatch): + def fake_get_size(url): + assert ( + url + == "https://github.com/hazelcast/hazelcast/releases/download/untagged-test/hazelcast-5.6.0-slim.zip" + ) + return "41 MB" + + monkeypatch.setattr(vm.DownloadUrl, "_get_size", staticmethod(fake_get_size)) + + version_metadata = vm.OSVersionMetadata("5.6.0", "hazelcast") + slim_zip = version_metadata.downloads.slim_zip assert ( slim_zip.public_url diff --git a/.github/python/add-release/version_metadata.py b/.github/python/add-release/version_metadata.py index 5643554..c1cc407 100644 --- a/.github/python/add-release/version_metadata.py +++ b/.github/python/add-release/version_metadata.py @@ -1,12 +1,23 @@ import logging import os +from abc import ABC, abstractmethod from dataclasses import InitVar, dataclass, field +from posixpath import dirname +from urllib.parse import urlparse import requests import semver +from github import Auth, Github from humanize import naturalsize from requests.auth import HTTPBasicAuth -from urllib.parse import urlparse + + +def _github_token() -> str | None: + return os.getenv("GH_TOKEN") + + +def _github_client() -> Github: + return Github(auth=Auth.Token(_github_token())) @dataclass @@ -21,8 +32,6 @@ class DownloadUrl: @staticmethod def _get_size(url: str) -> str: """Fetch and cache the artifact size without _actually_ downloading it.""" - # Lazily evaluates - # So in the case of (say) an EE-only release we don't query (non-existent) OS artifacts logging.debug("Getting size of %s", url) username = os.getenv("RELEASE_REPO_USER") @@ -35,28 +44,33 @@ def _get_size(url: str) -> str: and password else None ) + headers = ( + {"Authorization": f"Bearer {_github_token()}"} + if urlparse(url).hostname == "github.com" and _github_token() + else None + ) - response = requests.head( + with requests.get( url, allow_redirects=True, auth=auth, - ) - response.raise_for_status() + headers=headers, + stream=True, + ) as response: + response.raise_for_status() + + content_length = response.headers.get("Content-Length") - content_length = response.headers["Content-Length"] + if content_length: + return naturalsize(int(content_length), format="%.0f") - if content_length: - return naturalsize(int(content_length), format="%.0f") - else: raise Exception(f"{url} did not return a size") @property def size(self) -> str: if self._size is None: - try: - self._size = self._get_size(self.preprod_url) - except requests.RequestException: - self._size = self._get_size(self.public_url) + # We only need to resolve size during `package` phase, before promotion - hence it will only ever be in preprod + self._size = self._get_size(self.preprod_url) return self._size @@ -80,28 +94,51 @@ class Downloads: @dataclass -class VersionMetadata: +class VersionMetadata(ABC): """Release metadata derived from a semantic version.""" version: semver.Version | str - github_org: InitVar[str] - ee_release_repo_name: InitVar[str] - jfrog_preprod_files_repo: InitVar[str] - def __post_init__(self, github_org, ee_release_repo_name, jfrog_preprod_files_repo): - """Normalize version and populate derived URLs and metadata.""" + def __post_init__(self): if isinstance(self.version, str): self.version = semver.Version.parse(self.version) - self.os_downloads = self._build_downloads( - f"https://github.com/{github_org}/hazelcast/releases/download/v{self.version}/hazelcast-{self.version}", - f"https://repository.hazelcast.com/{jfrog_preprod_files_repo}/hazelcast/hazelcast-{self.version}", + self.docs_url = f"https://docs.hazelcast.com/hazelcast/{self.version.major}.{self.version.minor}/getting-started/quickstart.html" + + @property + @abstractmethod + def downloads(self): + """Download metadata for this release.""" + + @property + @abstractmethod + def apidocs_url(self) -> str: + """API documentation URL for this release.""" + + def _build_downloads(self, live_base_url: str, preprod_base_url: str) -> Downloads: + """Build a Downloads object for a given artifact base live URL.""" + return Downloads( + full_zip=DownloadUrl(live_base_url, preprod_base_url, ".zip"), + slim_zip=DownloadUrl(live_base_url, preprod_base_url, "-slim.zip"), + full_tar=DownloadUrl(live_base_url, preprod_base_url, ".tar.gz"), + slim_tar=DownloadUrl(live_base_url, preprod_base_url, "-slim.tar.gz"), ) - self.ee_downloads = self._build_downloads( - f"https://repository.hazelcast.com/{ee_release_repo_name}/hazelcast-enterprise/hazelcast-enterprise-{self.version}", - f"https://repository.hazelcast.com/{jfrog_preprod_files_repo}/hazelcast-enterprise/hazelcast-enterprise-{self.version}", + + +@dataclass +class OSVersionMetadata(VersionMetadata): + github_org: InitVar[str] + + def __post_init__(self, github_org): + super().__post_init__() + + self._downloads = self._build_downloads( + f"https://github.com/{github_org}/hazelcast/releases/download/v{self.version}/hazelcast-{self.version}", + f"{self._get_github_release_download_base_url(github_org)}/hazelcast-{self.version}", ) + self.release_notes_url = f"https://docs.hazelcast.com/hazelcast/{self.version.major}.{self.version.minor}/release-notes/community#{self.version.major}-{self.version.minor}-{self.version.patch}" + self.sources_url = ( f"https://github.com/{github_org}/hazelcast/tree/v{self.version}" ) @@ -109,19 +146,61 @@ def __post_init__(self, github_org, ee_release_repo_name, jfrog_preprod_files_re f"https://github.com/{github_org}/hazelcast-code-samples" ) - self.docs_url = f"https://docs.hazelcast.com/hazelcast/{self.version.major}.{self.version.minor}/getting-started/quickstart.html" - self.os_release_notes_url = f"https://docs.hazelcast.com/hazelcast/{self.version.major}.{self.version.minor}/release-notes/community#{self.version.major}-{self.version.minor}-{self.version.patch}" - self.ee_release_notes_url = f"https://docs.hazelcast.com/hazelcast/{self.version.major}.{self.version.minor}/release-notes/enterprise#{self.version.major}-{self.version.minor}-{self.version.patch}" - self.os_apidocs_url = f"https://docs.hazelcast.org/docs/{self.version}/javadoc" - self.ee_apidocs_url = ( - f"https://docs.hazelcast.org/hazelcast-ee-docs/{self.version}/javadoc" - ) + @property + def downloads(self): + return self._downloads - def _build_downloads(self, live_base_url: str, preprod_base_url: str) -> Downloads: - """Build a Downloads object for a given artifact base live URL.""" - return Downloads( - full_zip=DownloadUrl(live_base_url, preprod_base_url, ".zip"), - slim_zip=DownloadUrl(live_base_url, preprod_base_url, "-slim.zip"), - full_tar=DownloadUrl(live_base_url, preprod_base_url, ".tar.gz"), - slim_tar=DownloadUrl(live_base_url, preprod_base_url, "-slim.tar.gz"), + @property + def apidocs_url(self) -> str: + return f"https://docs.hazelcast.org/docs/{self.version}/javadoc" + + def _get_github_release_download_base_url(self, github_org: str) -> str: + """Find the release and derive its asset download base URL - for draft releases, this is non-trivial - e.g. could be https://github.com/hazelcast/hazelcast/releases/download/untagged-ae7bc91fc31113ff3102/hazelcast-1.2.3.zip""" + + with _github_client() as github_client: + repo = github_client.get_repo(f"{github_org}/hazelcast") + release_name = f"v{self.version}" + + try: + # have to iterate as get_release(releasename) doesn't find drafts + releases = repo.get_releases() + release = next( + release for release in releases if release.name == release_name + ) + + asset = next(iter(release.get_assets())) + + if asset: + # Now we have an asset (e.g. https://github.com/hazelcast/hazelcast/releases/download/untagged-ae7bc91fc31113ff3102/hazelcast-1.2.3.zip), extract the base URL of the releases' assets (e.g. https://github.com/hazelcast/hazelcast/releases/download/untagged-ae7bc91fc31113ff3102) + parsed_url = urlparse(asset.browser_download_url) + return parsed_url._replace(path=dirname(parsed_url.path)).geturl() + else: + raise ValueError(f"No assets found for version {release}") + except StopIteration: + raise ValueError( + f"Release `{release_name}` not found in {repo} - found {[release.name for release in releases]}" + ) from None + + +@dataclass +class EEVersionMetadata(VersionMetadata): + release_repo_name: InitVar[str] + jfrog_preprod_files_repo: InitVar[str] + + def __post_init__(self, release_repo_name, jfrog_preprod_files_repo): + super().__post_init__() + + self._downloads = self._build_downloads( + f"https://repository.hazelcast.com/{release_repo_name}/hazelcast-enterprise/hazelcast-enterprise-{self.version}", + f"https://repository.hazelcast.com/{jfrog_preprod_files_repo}/hazelcast-enterprise/hazelcast-enterprise-{self.version}", ) + + self.release_notes_url = f"https://docs.hazelcast.com/hazelcast/{self.version.major}.{self.version.minor}/release-notes/enterprise#{self.version.major}-{self.version.minor}-{self.version.patch}" + + @property + def downloads(self): + return self._downloads + + @property + def apidocs_url(self) -> str: + return f"https://docs.hazelcast.org/hazelcast-ee-docs/{self.version}/javadoc" diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index eb1b41e..47cc47a 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -63,7 +63,7 @@ jobs: --should-build-oss=${{ steps.resolve-editions.outputs.should_build_oss }} \ --should-build-ee=${{ steps.resolve-editions.outputs.should_build_ee }} env: - VERSION: ${{ inputs.VERSION }} + GH_TOKEN: ${{ secrets.GH_TOKEN }} RELEASE_REPO_USER: ${{ steps.jfrog.outputs.user }} RELEASE_REPO_TOKEN: ${{ steps.jfrog.outputs.token }}