diff --git a/CHANGES/+hugging-face.feature b/CHANGES/+hugging-face.feature new file mode 100644 index 000000000..5290032b5 --- /dev/null +++ b/CHANGES/+hugging-face.feature @@ -0,0 +1 @@ +Added the `hugging-face` command group to manage Hugging Face remotes, repositories, and distributions for pull-through caching of Hugging Face Hub content. \ No newline at end of file diff --git a/CHANGES/pulp-glue/+hugging-face.feature b/CHANGES/pulp-glue/+hugging-face.feature new file mode 100644 index 000000000..86299f179 --- /dev/null +++ b/CHANGES/pulp-glue/+hugging-face.feature @@ -0,0 +1 @@ +Added entity contexts for the `hugging_face` plugin (remote, repository, repository version, and distribution). \ No newline at end of file diff --git a/pulp-glue/src/pulp_glue/hugging_face/__init__.py b/pulp-glue/src/pulp_glue/hugging_face/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pulp-glue/src/pulp_glue/hugging_face/context.py b/pulp-glue/src/pulp_glue/hugging_face/context.py new file mode 100644 index 000000000..4bede0340 --- /dev/null +++ b/pulp-glue/src/pulp_glue/hugging_face/context.py @@ -0,0 +1,56 @@ +from pulp_glue.common.context import ( + EntityDefinition, + PluginRequirement, + PulpDistributionContext, + PulpRemoteContext, + PulpRepositoryContext, + PulpRepositoryVersionContext, +) +from pulp_glue.common.i18n import get_translation + +translation = get_translation(__package__) +_ = translation.gettext + + +class PulpHuggingFaceRemoteContext(PulpRemoteContext): + PLUGIN = "hugging_face" + RESOURCE_TYPE = "hugging-face" + ENTITY = _("hugging face remote") + ENTITIES = _("hugging face remotes") + HREF = "hugging_face_hugging_face_remote_href" + ID_PREFIX = "remotes_hugging_face_hugging_face" + NEEDS_PLUGINS = [PluginRequirement("hugging_face", specifier=">=0.1.0")] + + +class PulpHuggingFaceDistributionContext(PulpDistributionContext): + PLUGIN = "hugging_face" + RESOURCE_TYPE = "hugging-face" + ENTITY = _("hugging face distribution") + ENTITIES = _("hugging face distributions") + HREF = "hugging_face_hugging_face_distribution_href" + ID_PREFIX = "distributions_hugging_face_hugging_face" + NEEDS_PLUGINS = [PluginRequirement("hugging_face", specifier=">=0.1.0")] + + def preprocess_entity(self, body: EntityDefinition, partial: bool = False) -> EntityDefinition: + body = super().preprocess_entity(body, partial=partial) + if not partial and self.pulp_ctx.has_plugin(PluginRequirement("core", specifier=">=3.16.0")): + body.setdefault("repository", None) + body.setdefault("remote", None) + return body + + +class PulpHuggingFaceRepositoryVersionContext(PulpRepositoryVersionContext): + HREF = "hugging_face_hugging_face_repository_version_href" + ID_PREFIX = "repositories_hugging_face_hugging_face_versions" + NEEDS_PLUGINS = [PluginRequirement("hugging_face", specifier=">=0.1.0")] + + +class PulpHuggingFaceRepositoryContext(PulpRepositoryContext): + PLUGIN = "hugging_face" + RESOURCE_TYPE = "hugging-face" + ENTITY = _("hugging face repository") + ENTITIES = _("hugging face repositories") + HREF = "hugging_face_hugging_face_repository_href" + ID_PREFIX = "repositories_hugging_face_hugging_face" + VERSION_CONTEXT = PulpHuggingFaceRepositoryVersionContext + NEEDS_PLUGINS = [PluginRequirement("hugging_face", specifier=">=0.1.0")] diff --git a/pulp-glue/src/pulp_glue/hugging_face/py.typed b/pulp-glue/src/pulp_glue/hugging_face/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/pyproject.toml b/pyproject.toml index 83c8ad9d2..7e669ed47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ common = "pulpcore.cli.common" container = "pulpcore.cli.container" core = "pulpcore.cli.core" file = "pulpcore.cli.file" +hugging-face = "pulpcore.cli.hugging_face" python = "pulpcore.cli.python" rpm = "pulpcore.cli.rpm" diff --git a/src/pulpcore/cli/hugging_face/__init__.py b/src/pulpcore/cli/hugging_face/__init__.py new file mode 100644 index 000000000..1639cea1a --- /dev/null +++ b/src/pulpcore/cli/hugging_face/__init__.py @@ -0,0 +1,27 @@ +import typing as t + +import click + +from pulp_glue.common.i18n import get_translation + +from pulp_cli.generic import pulp_group +from pulpcore.cli.hugging_face.distribution import distribution +from pulpcore.cli.hugging_face.remote import remote +from pulpcore.cli.hugging_face.repository import repository + +translation = get_translation(__package__) +_ = translation.gettext + +__version__ = "0.1.0" + + +@pulp_group(name="hugging-face") +def hugging_face() -> None: + """Manage Hugging Face Hub content via Pulp.""" + + +def mount(main: click.Group, **kwargs: t.Any) -> None: + hugging_face.add_command(remote) + hugging_face.add_command(distribution) + hugging_face.add_command(repository) + main.add_command(hugging_face) diff --git a/src/pulpcore/cli/hugging_face/distribution.py b/src/pulpcore/cli/hugging_face/distribution.py new file mode 100644 index 000000000..c047bb95a --- /dev/null +++ b/src/pulpcore/cli/hugging_face/distribution.py @@ -0,0 +1,81 @@ +import click + +from pulp_glue.common.context import PulpRemoteContext, PulpRepositoryContext +from pulp_glue.common.i18n import get_translation +from pulp_glue.hugging_face.context import ( + PulpHuggingFaceDistributionContext, + PulpHuggingFaceRemoteContext, + PulpHuggingFaceRepositoryContext, +) + +from pulp_cli.generic import ( + common_distribution_create_options, + create_command, + destroy_command, + distribution_filter_options, + distribution_lookup_option, + href_option, + label_command, + list_command, + name_option, + pulp_group, + pulp_labels_option, + resource_option, + show_command, + type_option, + update_command, +) + +translation = get_translation(__package__) +_ = translation.gettext + + +remote_option = resource_option( + "--remote", + default_plugin="hugging_face", + default_type="hugging-face", + context_table={"hugging_face:hugging-face": PulpHuggingFaceRemoteContext}, + href_pattern=PulpRemoteContext.HREF_PATTERN, + help=_( + "Hugging Face remote to use for pull-through caching." + " Specified as '[[:]:]' or as href." + " Pass an empty string to detach the remote." + ), +) +repository_option = resource_option( + "--repository", + default_plugin="hugging_face", + default_type="hugging-face", + context_table={"hugging_face:hugging-face": PulpHuggingFaceRepositoryContext}, + href_pattern=PulpRepositoryContext.HREF_PATTERN, + help=_( + "Hugging Face repository whose latest version is served." + " Specified as '[[:]:]' or as href." + " Pass an empty string to detach the repository." + ), +) + + +@pulp_group() +@type_option(choices={"hugging-face": PulpHuggingFaceDistributionContext}) +def distribution() -> None: + pass + + +lookup_options = [href_option, name_option, distribution_lookup_option] +nested_lookup_options = [distribution_lookup_option] +update_options = [ + remote_option, + repository_option, + pulp_labels_option, +] +create_options = common_distribution_create_options + update_options + +distribution.add_command(list_command(decorators=distribution_filter_options)) +distribution.add_command(show_command(decorators=lookup_options)) +distribution.add_command(create_command(decorators=create_options)) +distribution.add_command( + update_command(decorators=lookup_options + update_options + [click.option("--base-path")]) +) +distribution.add_command(destroy_command(decorators=lookup_options)) +distribution.add_command(label_command(decorators=nested_lookup_options)) diff --git a/src/pulpcore/cli/hugging_face/py.typed b/src/pulpcore/cli/hugging_face/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/src/pulpcore/cli/hugging_face/remote.py b/src/pulpcore/cli/hugging_face/remote.py new file mode 100644 index 000000000..1e5b6e601 --- /dev/null +++ b/src/pulpcore/cli/hugging_face/remote.py @@ -0,0 +1,62 @@ +import click + +from pulp_glue.common.i18n import get_translation +from pulp_glue.hugging_face.context import PulpHuggingFaceRemoteContext + +from pulp_cli.generic import ( + common_remote_create_options, + common_remote_update_options, + create_command, + destroy_command, + href_option, + label_command, + list_command, + name_option, + pulp_group, + remote_filter_options, + remote_lookup_option, + show_command, + type_option, + update_command, +) + +translation = get_translation(__package__) +_ = translation.gettext + + +@pulp_group() +@type_option(choices={"hugging-face": PulpHuggingFaceRemoteContext}) +def remote() -> None: + pass + + +lookup_options = [href_option, name_option, remote_lookup_option] +nested_lookup_options = [remote_lookup_option] +hugging_face_remote_options = [ + click.option( + "--hf-token", + "hf_token", + help=_( + "Hugging Face API token for accessing private repositories." + " This value is write-only and will not appear in API responses." + ), + ), + click.option( + "--policy", + type=click.Choice(["immediate", "on_demand", "streamed"], case_sensitive=False), + help=_("Policy for downloading content (use 'on_demand' for pull-through caching)."), + ), +] + +remote.add_command(list_command(decorators=remote_filter_options)) +remote.add_command(show_command(decorators=lookup_options)) +remote.add_command( + create_command(decorators=common_remote_create_options + hugging_face_remote_options) +) +remote.add_command( + update_command( + decorators=lookup_options + common_remote_update_options + hugging_face_remote_options + ) +) +remote.add_command(destroy_command(decorators=lookup_options)) +remote.add_command(label_command(decorators=nested_lookup_options)) diff --git a/src/pulpcore/cli/hugging_face/repository.py b/src/pulpcore/cli/hugging_face/repository.py new file mode 100644 index 000000000..90b4f2f05 --- /dev/null +++ b/src/pulpcore/cli/hugging_face/repository.py @@ -0,0 +1,57 @@ +import click + +from pulp_glue.common.i18n import get_translation +from pulp_glue.hugging_face.context import PulpHuggingFaceRepositoryContext + +from pulp_cli.generic import ( + create_command, + destroy_command, + href_option, + label_command, + label_select_option, + list_command, + name_option, + pulp_group, + pulp_labels_option, + repository_href_option, + repository_lookup_option, + retained_versions_option, + show_command, + type_option, + update_command, + version_command, +) + +translation = get_translation(__package__) +_ = translation.gettext + + +@pulp_group() +@type_option(choices={"hugging-face": PulpHuggingFaceRepositoryContext}) +def repository() -> None: + pass + + +lookup_options = [href_option, name_option, repository_lookup_option] +nested_lookup_options = [repository_href_option, repository_lookup_option] +update_options = [ + click.option("--description"), + retained_versions_option, + pulp_labels_option, +] +create_options = update_options + [click.option("--name", required=True)] + +repository.add_command( + list_command( + decorators=[ + label_select_option, + click.option("--name-contains", "name__contains"), + ] + ) +) +repository.add_command(show_command(decorators=lookup_options)) +repository.add_command(create_command(decorators=create_options)) +repository.add_command(update_command(decorators=lookup_options + update_options)) +repository.add_command(destroy_command(decorators=lookup_options)) +repository.add_command(version_command(decorators=nested_lookup_options)) +repository.add_command(label_command(decorators=nested_lookup_options)) diff --git a/tests/scripts/pulp_hugging_face/test_distribution.sh b/tests/scripts/pulp_hugging_face/test_distribution.sh new file mode 100755 index 000000000..38900f655 --- /dev/null +++ b/tests/scripts/pulp_hugging_face/test_distribution.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -eu +# shellcheck source=tests/scripts/config.source +. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source + +pulp debug has-plugin --name "hugging_face" || exit 23 + +cleanup() { + pulp hugging-face distribution destroy --name "cli_test_hugging_face_distro" || true + pulp hugging-face repository destroy --name "cli_test_hugging_face_distro_repo" || true + pulp hugging-face remote destroy --name "cli_test_hugging_face_distro_remote" || true +} +trap cleanup EXIT + +REMOTE_HREF="$(pulp hugging-face remote create --name "cli_test_hugging_face_distro_remote" --url "https://huggingface.co/" --policy "on_demand" | jq -r '.pulp_href')" +REPO_HREF="$(pulp hugging-face repository create --name "cli_test_hugging_face_distro_repo" | jq -r '.pulp_href')" + +expect_succ pulp hugging-face distribution create --name "cli_test_hugging_face_distro" --base-path "cli_test_hugging_face_distro" --remote "cli_test_hugging_face_distro_remote" +expect_succ pulp hugging-face distribution show --distribution "cli_test_hugging_face_distro" +test "$(echo "$OUTPUT" | jq -r '.remote')" = "$REMOTE_HREF" + +expect_succ pulp hugging-face distribution update --distribution "cli_test_hugging_face_distro" --remote "" --repository "cli_test_hugging_face_distro_repo" +expect_succ pulp hugging-face distribution show --distribution "cli_test_hugging_face_distro" +test "$(echo "$OUTPUT" | jq -r '.remote')" = "null" +test "$(echo "$OUTPUT" | jq -r '.repository')" = "$REPO_HREF" + +expect_succ pulp hugging-face distribution update --distribution "cli_test_hugging_face_distro" --repository "" +expect_succ pulp hugging-face distribution show --distribution "cli_test_hugging_face_distro" +test "$(echo "$OUTPUT" | jq -r '.repository')" = "null" + +expect_succ pulp hugging-face distribution list --base-path "cli_test_hugging_face_distro" +test "$(echo "$OUTPUT" | jq -r length)" -eq 1 + +expect_succ pulp hugging-face distribution destroy --distribution "cli_test_hugging_face_distro" diff --git a/tests/scripts/pulp_hugging_face/test_remote.sh b/tests/scripts/pulp_hugging_face/test_remote.sh new file mode 100755 index 000000000..3c1f7ab0d --- /dev/null +++ b/tests/scripts/pulp_hugging_face/test_remote.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -eu +# shellcheck source=tests/scripts/config.source +. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source + +pulp debug has-plugin --name "hugging_face" || exit 23 + +cleanup() { + pulp hugging-face remote destroy --name "cli_test_hugging_face_remote" || true +} +trap cleanup EXIT + +expect_succ pulp hugging-face remote list + +expect_succ pulp hugging-face remote create --name "cli_test_hugging_face_remote" --url "https://huggingface.co/" --policy "on_demand" --hf-token "s3cr3t" +expect_succ pulp hugging-face remote show --remote "cli_test_hugging_face_remote" +HREF="$(echo "$OUTPUT" | jq -r '.pulp_href')" +test "$(echo "$OUTPUT" | jq -r '.policy')" = "on_demand" +# hf_token is write-only and must never be returned in API responses. +test "$(echo "$OUTPUT" | jq -r '.hf_token')" = "null" + +expect_succ pulp hugging-face remote update --remote "$HREF" --policy "immediate" +expect_succ pulp hugging-face remote show --remote "cli_test_hugging_face_remote" +test "$(echo "$OUTPUT" | jq -r '.policy')" = "immediate" + +expect_succ pulp hugging-face remote list --name-contains "li_test_hugging_face_remot" +test "$(echo "$OUTPUT" | jq -r '.|length')" = "1" + +expect_succ pulp hugging-face remote destroy --remote "cli_test_hugging_face_remote" \ No newline at end of file diff --git a/tests/scripts/pulp_hugging_face/test_repository.sh b/tests/scripts/pulp_hugging_face/test_repository.sh new file mode 100755 index 000000000..bf0f4b0e0 --- /dev/null +++ b/tests/scripts/pulp_hugging_face/test_repository.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -eu +# shellcheck source=tests/scripts/config.source +. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source + +pulp debug has-plugin --name "hugging_face" || exit 23 + +cleanup() { + pulp hugging-face repository destroy --name "cli_test_hugging_face_repo" || true +} +trap cleanup EXIT + +expect_succ pulp hugging-face repository list + +expect_succ pulp hugging-face repository create --name "cli_test_hugging_face_repo" --description "Test repository for CLI tests" +HREF="$(echo "$OUTPUT" | jq -r '.pulp_href')" +expect_succ pulp hugging-face repository update --repository "cli_test_hugging_face_repo" --description "" +expect_succ pulp hugging-face repository show --repository "$HREF" +test "$(echo "$OUTPUT" | jq -r '.description')" = "null" + +expect_succ pulp hugging-face repository update --repository "cli_test_hugging_face_repo" --description $'Test\nrepository' +expect_succ pulp hugging-face repository show --repository "cli_test_hugging_face_repo" +test "$(echo "$OUTPUT" | jq '.description')" = '"Test\nrepository"' + +expect_succ pulp hugging-face repository version list --repository "cli_test_hugging_face_repo" +test "$(echo "$OUTPUT" | jq -r '.|length')" = "1" + +expect_succ pulp hugging-face repository list +test "$(echo "$OUTPUT" | jq -r '.|length')" != "0" +expect_succ pulp hugging-face repository list --name-contains "cli_test_hugging_face" +test "$(echo "$OUTPUT" | jq -r '.|length')" -ge "1" + +expect_succ pulp hugging-face repository destroy --repository "cli_test_hugging_face_repo" \ No newline at end of file diff --git a/tests/test_help_pages.py b/tests/test_help_pages.py index dbc4ea027..b0b83073f 100644 --- a/tests/test_help_pages.py +++ b/tests/test_help_pages.py @@ -105,6 +105,17 @@ def test_help_shows_all_available_commands(no_api: None) -> None: "42", ], ), + pytest.param( + [ + "hugging-face", + "repository", + "show", + ], + [ + "--repository", + "dummy", + ], + ), ], ) def test_deferred_context(