-
Notifications
You must be signed in to change notification settings - Fork 77
CDTOOL-1649: Add Python Language Support #1811
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
posborne
wants to merge
9
commits into
main
Choose a base branch
from
posborne/python-language-support
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
9 commits
Select commit
Hold shift + click to select a range
d6c5dcf
Support merging existing Wasm metadata during build annotation
posborne c7b818f
Add support for the Python programming language
posborne 5fc35b2
Avoid panic when prompt language has empty starter kits
posborne 088b021
Address gosec file permission warnings in metadata unit tests
posborne dd0ac32
Update CHANGELOG.md to note Python language addition
posborne c9baae9
Update CHANGELOG.md
posborne ac87f1c
test(compute): add build test scenarios for Python language
posborne f3b8e2c
Install uv in CI for tests
posborne 1e54913
Merge branch 'main' into posborne/python-language-support
anthony-gomez-fastly File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -330,15 +330,21 @@ func (c *BuildCommand) AnnotateWasmBinaryLong(wasmtools string, args []string, l | |
| // Allow customer to specify their own env variables to be filtered. | ||
| ExtendStaticSecretEnvVars(c.MetadataFilterEnvVars) | ||
|
|
||
| dc := DataCollection{} | ||
|
|
||
| metadata := c.Globals.Config.WasmMetadata | ||
|
|
||
| // Only record basic data if user has disabled all other metadata collection. | ||
| if metadata.BuildInfo == "disable" && metadata.MachineInfo == "disable" && metadata.PackageInfo == "disable" && metadata.ScriptInfo == "disable" { | ||
| return c.AnnotateWasmBinaryShort(wasmtools, args) | ||
| } | ||
|
|
||
| // Seed from any fastly_data already embedded by the build tool (e.g. Python). | ||
| // This lets the build tool supply package_info while the CLI fills in the | ||
| // remaining fields it is responsible for. | ||
| dc := DataCollection{} | ||
| if existing := c.readExistingFastlyData(wasmtools); existing != nil { | ||
| dc = *existing | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. instead of overwriting the whole dc, do we only want to overwrite package_info?
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. also is this used for all languages or just python? |
||
| } | ||
|
|
||
| if metadata.BuildInfo == "enable" { | ||
| dc.BuildInfo = DataCollectionBuildInfo{ | ||
| MemoryHeapAlloc: bucketMB(bytesToMB(ms.HeapAlloc)) + "MB", | ||
|
|
@@ -354,9 +360,11 @@ func (c *BuildCommand) AnnotateWasmBinaryLong(wasmtools string, args []string, l | |
| } | ||
| } | ||
| if metadata.PackageInfo == "enable" { | ||
| dc.PackageInfo = DataCollectionPackageInfo{ | ||
| ClonedFrom: c.Globals.Manifest.File.ClonedFrom, | ||
| Packages: language.Dependencies(), | ||
| if dc.PackageInfo.Packages == nil { | ||
| dc.PackageInfo.Packages = language.Dependencies() | ||
| } | ||
| if dc.PackageInfo.ClonedFrom == "" { | ||
| dc.PackageInfo.ClonedFrom = c.Globals.Manifest.File.ClonedFrom | ||
| } | ||
| } | ||
| if metadata.ScriptInfo == "enable" { | ||
|
|
@@ -377,6 +385,63 @@ func (c *BuildCommand) AnnotateWasmBinaryLong(wasmtools string, args []string, l | |
| return c.Globals.ExecuteWasmTools(wasmtools, args, c.Globals) | ||
| } | ||
|
|
||
| // readExistingFastlyData reads any fastly_data already embedded in the Wasm | ||
| // binary by the build tool. Returns nil if absent or unparseable. | ||
| func (c *BuildCommand) readExistingFastlyData(wasmtools string) *DataCollection { | ||
| // #nosec G204 -- wasmtools path comes from trusted CLI config | ||
| out, err := exec.Command(wasmtools, "metadata", "show", "--json", binWasmPath).Output() | ||
| if err != nil { | ||
| return nil | ||
| } | ||
|
|
||
| // wasm-tools metadata show --json encodes producers differently for modules and components: | ||
| // - Wasm Modules: {"module":{"producers":[...]}} | ||
| // - Wasm Components: {"component":{"metadata":{"producers":[...]}}} | ||
| var meta struct { | ||
| Module struct { | ||
| Producers []json.RawMessage `json:"producers"` | ||
| } `json:"module"` | ||
| Component struct { | ||
| Metadata struct { | ||
| Producers []json.RawMessage `json:"producers"` | ||
| } `json:"metadata"` | ||
| } `json:"component"` | ||
| } | ||
| if err := json.Unmarshal(out, &meta); err != nil { | ||
| return nil | ||
| } | ||
|
|
||
| producers := meta.Component.Metadata.Producers | ||
| if len(producers) == 0 { | ||
| producers = meta.Module.Producers | ||
| } | ||
|
|
||
| for _, raw := range producers { | ||
| var pair [2]json.RawMessage | ||
| if err := json.Unmarshal(raw, &pair); err != nil { | ||
| continue | ||
| } | ||
| var field string | ||
| if err := json.Unmarshal(pair[0], &field); err != nil || field != "processed-by" { | ||
| continue | ||
| } | ||
| var entries map[string]string | ||
| if err := json.Unmarshal(pair[1], &entries); err != nil { | ||
| continue | ||
| } | ||
| val, ok := entries["fastly_data"] | ||
| if !ok { | ||
| continue | ||
| } | ||
| var dc DataCollection | ||
| if err := json.Unmarshal([]byte(val), &dc); err != nil { | ||
| return nil | ||
| } | ||
| return &dc | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // ShowMetadata displays the metadata attached to the Wasm binary. | ||
| func (c *BuildCommand) ShowMetadata(wasmtools string, out io.Writer) { | ||
| // gosec flagged this: | ||
|
|
@@ -715,6 +780,12 @@ func language(toolchain, manifestFilename string, c *BuildCommand, in io.Reader, | |
| SourceDirectory: JsSourceDirectory, | ||
| Toolchain: NewJavaScript(c, in, manifestFilename, out, spinner), | ||
| }) | ||
| case "python": | ||
| language = NewLanguage(&LanguageOptions{ | ||
| Name: "python", | ||
| SourceDirectory: PythonSourceDirectory, | ||
| Toolchain: NewPython(c, in, manifestFilename, out, spinner), | ||
| }) | ||
| case "rust": | ||
| language = NewLanguage(&LanguageOptions{ | ||
| Name: "rust", | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,121 @@ | ||
| package compute | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "os" | ||
| "path/filepath" | ||
| "testing" | ||
| ) | ||
|
|
||
| // mockWasmToolsScript generates a mock executable shell script for wasm-tools. | ||
| // | ||
| // Because the production code runs exec.Command under the hood, we mock it by writing | ||
| // a temporary executable bash script to disk that outputs the mock JSON we expect. | ||
| // We use a bash heredoc (cat << 'EOF') so that the JSON structure and inner quotes | ||
| // are written exactly as-is, avoiding platform-specific shell escape/echo issues. | ||
| func mockWasmToolsScript(staticOutput string) string { | ||
| return "#!/usr/bin/env bash\ncat << 'EOF'\n" + staticOutput + "\nEOF" | ||
| } | ||
|
|
||
| func TestReadExistingFastlyData(t *testing.T) { | ||
| // Create a temporary directory for our mock environment | ||
| rootdir, err := os.MkdirTemp("", "fastly-metadata-test-*") | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| defer os.RemoveAll(rootdir) | ||
|
|
||
| // Save original PWD and return to it later | ||
| pwd, err := os.Getwd() | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| if err := os.Chdir(rootdir); err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| defer func() { | ||
| _ = os.Chdir(pwd) | ||
| }() | ||
|
|
||
| // Ensure the bin directory and main.wasm file exist | ||
| // (binWasmPath points to "./bin/main.wasm") | ||
| if err := os.MkdirAll("bin", 0o755); err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| if err := os.WriteFile(binWasmPath, []byte("mock-wasm-binary"), 0o600); err != nil { | ||
| t.Fatal(err) | ||
| } | ||
|
|
||
| scenarios := []struct { | ||
| name string | ||
| jsonOutput string | ||
| expectedData *DataCollection | ||
| }{ | ||
| { | ||
| name: "extracts from component-based metadata structure", | ||
| jsonOutput: `{"component":{"metadata":{"producers":[["processed-by",{"fastly_data":"{\"package_info\":{\"packages\":{\"foo\":\"1.0.0\"}},\"script_info\":{\"build_script\":\"echo component\"}}"}]]}}}`, | ||
| expectedData: &DataCollection{ | ||
| PackageInfo: DataCollectionPackageInfo{ | ||
| Packages: map[string]string{"foo": "1.0.0"}, | ||
| }, | ||
| ScriptInfo: DataCollectionScriptInfo{ | ||
| BuildScript: "echo component", | ||
| }, | ||
| }, | ||
| }, | ||
| { | ||
| name: "extracts from module-based metadata structure", | ||
| jsonOutput: `{"module":{"producers":[["processed-by",{"fastly_data":"{\"package_info\":{\"packages\":{\"bar\":\"2.0.0\"}},\"script_info\":{\"build_script\":\"echo module\"}}"}]]}}`, | ||
| expectedData: &DataCollection{ | ||
| PackageInfo: DataCollectionPackageInfo{ | ||
| Packages: map[string]string{"bar": "2.0.0"}, | ||
| }, | ||
| ScriptInfo: DataCollectionScriptInfo{ | ||
| BuildScript: "echo module", | ||
| }, | ||
| }, | ||
| }, | ||
| { | ||
| name: "handles missing fastly_data gracefully", | ||
| jsonOutput: `{"component":{"metadata":{"producers":[["processed-by",{"other_tool":"1.0.0"}]]}}}`, | ||
| expectedData: nil, | ||
| }, | ||
| { | ||
| name: "handles invalid JSON from wasm-tools gracefully", | ||
| jsonOutput: `invalid-json`, | ||
| expectedData: nil, | ||
| }, | ||
| } | ||
|
|
||
| for _, tc := range scenarios { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| wasmtoolsBin := filepath.Join(rootdir, "mock-wasm-tools") | ||
| scriptContent := mockWasmToolsScript(tc.jsonOutput) | ||
| // #nosec G306 -- mock binary must be executable | ||
| if err := os.WriteFile(wasmtoolsBin, []byte(scriptContent), 0o700); err != nil { | ||
| t.Fatal(err) | ||
| } | ||
|
|
||
| cmd := &BuildCommand{} | ||
| actualData := cmd.readExistingFastlyData(wasmtoolsBin) | ||
|
|
||
| if tc.expectedData == nil { | ||
| if actualData != nil { | ||
| t.Fatalf("expected nil, got: %+v", actualData) | ||
| } | ||
| return | ||
| } | ||
|
|
||
| if actualData == nil { | ||
| t.Fatal("expected non-nil DataCollection, got nil") | ||
| } | ||
|
|
||
| // Validate values | ||
| expectedBytes, _ := json.Marshal(tc.expectedData) | ||
| actualBytes, _ := json.Marshal(actualData) | ||
| if string(expectedBytes) != string(actualBytes) { | ||
| t.Errorf("\nwant: %s\ngot: %s", expectedBytes, actualBytes) | ||
| } | ||
| }) | ||
| } | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Going to need to shift this to the right spot