-
Notifications
You must be signed in to change notification settings - Fork 7
fix(skill): use FC temp OSS for skill code packages #18
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,10 +18,16 @@ | |
| """ | ||
|
|
||
| import base64 | ||
| import email.utils | ||
| import hashlib | ||
| import hmac | ||
| import io | ||
| import json | ||
| import os | ||
| import urllib.error | ||
| import urllib.request | ||
| import zipfile | ||
| from dataclasses import dataclass | ||
|
|
||
| import click | ||
|
|
||
|
|
@@ -58,16 +64,198 @@ def _serialize_tool(t) -> dict: | |
| } | ||
|
|
||
|
|
||
| def _zip_directory(dir_path: str) -> str: | ||
| """ZIP a directory and return base64-encoded content.""" | ||
| def _zip_skill_directory_bytes(dir_path: str) -> bytes: | ||
| """Package a Skill directory and inject a placeholder main.py when missing.""" | ||
| buf = io.BytesIO() | ||
| has_main_py = False | ||
| with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: | ||
| for root, _dirs, files in os.walk(dir_path): | ||
| for fname in files: | ||
| full_path = os.path.join(root, fname) | ||
| arcname = os.path.relpath(full_path, dir_path) | ||
| if arcname == "main.py": | ||
| has_main_py = True | ||
| zf.write(full_path, arcname) | ||
| return base64.b64encode(buf.getvalue()).decode("ascii") | ||
| if not has_main_py: | ||
| zf.writestr( | ||
| "main.py", | ||
| "def handler(event, context):\n" | ||
| " return {'status': 'ok'}\n\n" | ||
| "if __name__ == '__main__':\n" | ||
| " print('skill package placeholder')\n", | ||
| ) | ||
| return buf.getvalue() | ||
|
|
||
|
|
||
| def _reject_inline_code_fields(value: object) -> None: | ||
| """Reject legacy inline code fields in user-provided payloads.""" | ||
| if isinstance(value, dict): | ||
| for key, nested in value.items(): | ||
| if key in {"zipFile", "zip_file"}: | ||
| raise click.UsageError( | ||
| "Inline code base64 fields are not supported for Skill " | ||
| "creation; use --code-dir so the CLI uploads the package " | ||
| "to FC TempBucket OSS." | ||
| ) | ||
| _reject_inline_code_fields(nested) | ||
| elif isinstance(value, list): | ||
| for nested in value: | ||
| _reject_inline_code_fields(nested) | ||
|
|
||
|
|
||
| @dataclass(frozen=True) | ||
| class _CodePackageLocation: | ||
| oss_bucket_name: str | ||
| oss_object_name: str | ||
|
|
||
|
|
||
| def _upload_skill_archive_to_fc_temp_bucket( | ||
| zip_data: bytes, | ||
| *, | ||
| profile: str | None, | ||
| region: str | None, | ||
| ) -> _CodePackageLocation: | ||
| """Upload a Skill ZIP to FC TempBucket OSS and return its code location.""" | ||
| import oss2 | ||
|
|
||
| cfg = build_sdk_config(profile_name=profile, region=region) | ||
| ak = cfg.get_access_key_id() | ||
| sk = cfg.get_access_key_secret() | ||
| token = cfg.get_security_token() | ||
| try: | ||
| account_id = cfg.get_account_id() | ||
| except ValueError as exc: | ||
| raise click.ClickException( | ||
| "Creating a Skill requires access_key_id, access_key_secret, " | ||
| "account_id, and region." | ||
| ) from exc | ||
| region_id = cfg.get_region_id() | ||
| if not ak or not sk or not account_id or not region_id: | ||
| raise click.ClickException( | ||
| "Creating a Skill requires access_key_id, access_key_secret, " | ||
| "account_id, and region." | ||
| ) | ||
|
|
||
| payload = _get_fc_temp_bucket_token(ak, sk, token, account_id, region_id) | ||
| oss_region = _required_temp_bucket_field(payload, "ossRegion") | ||
| oss_bucket = _required_temp_bucket_field(payload, "ossBucket") | ||
| object_name = _temp_bucket_object_name( | ||
| account_id, _required_temp_bucket_field(payload, "objectName") | ||
| ) | ||
| credentials = payload.get("credentials") or payload.get("Credentials") or {} | ||
| temp_ak = _required_temp_bucket_field(credentials, "accessKeyId") | ||
| temp_sk = _required_temp_bucket_field(credentials, "accessKeySecret") | ||
| temp_token = _required_temp_bucket_field(credentials, "securityToken") | ||
|
|
||
| auth = oss2.StsAuth(temp_ak, temp_sk, temp_token) | ||
| bucket = oss2.Bucket(auth, _oss_endpoint_from_region(oss_region), oss_bucket) | ||
| bucket.put_object(object_name, zip_data) | ||
| return _CodePackageLocation(oss_bucket, object_name) | ||
|
|
||
|
|
||
| def _get_fc_temp_bucket_token( | ||
| access_key_id: str, | ||
| access_key_secret: str, | ||
| security_token: str | None, | ||
| account_id: str, | ||
| region_id: str, | ||
| ) -> dict: | ||
| """Call the FC 2016 API to fetch temporary OSS credentials.""" | ||
| host = f"{account_id}.{region_id}.fc.aliyuncs.com" | ||
| path = "/2016-08-15/tempBucketToken" | ||
| date = email.utils.formatdate(usegmt=True) | ||
| headers = { | ||
| "Host": host, | ||
| "Accept": "application/json", | ||
| "Date": date, | ||
| "User-Agent": "agentrun-cli", | ||
| "X-Fc-Account-Id": account_id, | ||
| } | ||
| if security_token: | ||
| headers["X-Fc-Security-Token"] = security_token | ||
| headers["Authorization"] = _fc_authorization( | ||
| access_key_id, access_key_secret, "GET", headers, path | ||
| ) | ||
| request = urllib.request.Request( | ||
| f"https://{host}{path}", headers=headers, method="GET" | ||
| ) | ||
| try: | ||
| with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 | ||
| raw = response.read() | ||
| except urllib.error.HTTPError as exc: | ||
| detail = exc.read().decode("utf-8", errors="replace") | ||
| message = f"Failed to get FC TempBucket token: {detail}" | ||
| raise click.ClickException(message) from exc | ||
| except urllib.error.URLError as exc: | ||
| message = f"Failed to get FC TempBucket token: {exc.reason}" | ||
| raise click.ClickException(message) from exc | ||
| try: | ||
| return json.loads(raw.decode("utf-8")) | ||
| except json.JSONDecodeError as exc: | ||
| preview = raw[:200].decode("utf-8", errors="replace") | ||
| message = f"Failed to parse FC TempBucket token response: {preview}" | ||
| raise click.ClickException(message) from exc | ||
|
|
||
|
|
||
| def _fc_authorization( | ||
| access_key_id: str, | ||
| access_key_secret: str, | ||
| method: str, | ||
| headers: dict[str, str], | ||
| resource: str, | ||
| ) -> str: | ||
| """Build the Authorization header for FC 2016 API requests.""" | ||
| lower_headers = {key.lower(): value for key, value in headers.items()} | ||
| fc_headers = "" | ||
| for key in sorted(k for k in lower_headers if k.startswith("x-fc-")): | ||
| fc_headers += f"{key}:{lower_headers[key]}\n" | ||
| string_to_sign = "\n".join( | ||
| [ | ||
| method, | ||
| lower_headers.get("content-md5", ""), | ||
| lower_headers.get("content-type", ""), | ||
| lower_headers.get("date", ""), | ||
| f"{fc_headers}{resource}", | ||
| ] | ||
| ) | ||
| digest = hmac.new( | ||
| access_key_secret.encode("utf-8"), | ||
| string_to_sign.encode("utf-8"), | ||
| hashlib.sha256, | ||
| ).digest() | ||
| signature = base64.b64encode(digest).decode("ascii") | ||
| return f"FC {access_key_id}:{signature}" | ||
|
|
||
|
|
||
| def _required_temp_bucket_field(data: dict, key: str) -> str: | ||
| """Read a required FC TempBucket field, accepting PascalCase variants.""" | ||
| value = data.get(key) | ||
| if value is None: | ||
| value = data.get(key[:1].upper() + key[1:]) | ||
| if not isinstance(value, str) or not value.strip(): | ||
| raise click.ClickException(f"FC TempBucket response is missing '{key}'") | ||
| return value.strip() | ||
|
|
||
|
|
||
| def _temp_bucket_object_name(account_id: str, object_name: str) -> str: | ||
| """Build the final FC TempBucket object name.""" | ||
| account_id = account_id.strip().strip("/") | ||
| object_name = object_name.strip().lstrip("/") | ||
| if not account_id: | ||
| return object_name | ||
| return f"{account_id}/{object_name}" | ||
|
|
||
|
|
||
| def _oss_endpoint_from_region(oss_region: str) -> str: | ||
| """Build an OSS endpoint from the FC-returned OSS region.""" | ||
| value = oss_region.strip() | ||
| if value.startswith(("http://", "https://")): | ||
| return value | ||
| if "." in value: | ||
| return f"https://{value}" | ||
| if value.startswith("oss-"): | ||
| return f"https://{value}.aliyuncs.com" | ||
| return f"https://oss-{value}.aliyuncs.com" | ||
|
|
||
|
|
||
| def _load_json_option(raw: str | None) -> dict | None: | ||
|
|
@@ -119,6 +307,7 @@ def skill_create(ctx, skill_name, code_dir, description, credential_name, from_f | |
|
|
||
| if from_file: | ||
| payload = _load_json_option(from_file) | ||
| _reject_inline_code_fields(payload) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| payload.setdefault("tool_type", "SKILL") | ||
| inp = models.CreateToolInputV2(**payload) | ||
| else: | ||
|
|
@@ -131,14 +320,24 @@ def skill_create(ctx, skill_name, code_dir, description, credential_name, from_f | |
| if not description: | ||
| description = _extract_description(skill_md) | ||
|
|
||
| # ZIP and base64 encode | ||
| zip_b64 = _zip_directory(code_dir) | ||
| location = _upload_skill_archive_to_fc_temp_bucket( | ||
| _zip_skill_directory_bytes(code_dir), | ||
| profile=profile, | ||
| region=region, | ||
| ) | ||
|
|
||
| code_cfg = models.CodeConfiguration(zip_file=zip_b64) | ||
| code_cfg = models.CodeConfiguration( | ||
| oss_bucket_name=location.oss_bucket_name, | ||
| oss_object_name=location.oss_object_name, | ||
| language="python3.12", | ||
| command=["python", "main.py"], | ||
| ) | ||
|
|
||
| inp = models.CreateToolInputV2( | ||
| tool_name=skill_name, | ||
| tool_type="SKILL", | ||
| create_method="CODE_PACKAGE", | ||
| artifact_type="Code", | ||
| description=description, | ||
| code_configuration=code_cfg, | ||
| credential_name=credential_name, | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
这里新增了对
oss2的运行时依赖,但agentrun.spec仍然把oss2放在EXCLUDES里。CI 的 smoke 只做make build,不会执行ar skill create,所以 standalone 二进制会构建成功,但运行到这里时会报ModuleNotFoundError: No module named 'oss2'。需要把oss2从 PyInstaller excludes 中移除(必要时加一个打包后二进制执行skill create/导入的 smoke),否则发布出来的dist/agentrun无法创建 skill。