From 2618297c6d3908132b8a302ad7db209c0e07aece Mon Sep 17 00:00:00 2001 From: Marius Arvinte Date: Wed, 15 Jul 2026 17:46:27 -0700 Subject: [PATCH] Release 2026.7 (#989) # :tada: Major Updates - **Faster C code parsing** - Significantly improved the speed of C code AST parsing and re-consolidation using parallel-TU. - **Added experimental C context selection** - To reduce translation costs and context sizes in large-scale projects, IDEAS now attempts to select the most informative C context. # :balloon: Minor Updates - Fixed an incorrect template for binary test generation agents. Disabled test generation agents by default. --------- Co-authored-by: Cory Cornelius --- AGENTS.mk | 10 +- IDEAS.mk | 64 +---- README.md | 129 ++++----- src/ideas/adapters.py | 3 +- src/ideas/agents/build.py | 7 +- src/ideas/agents/utils.py | 8 +- src/ideas/ast.py | 347 ++++++++++++++--------- src/ideas/ast_rust.py | 8 +- src/ideas/convert_tests.py | 2 +- src/ideas/evaluate.py | 2 +- src/ideas/init/build.py | 122 +++++---- src/ideas/init/consolidate.py | 440 +++++++++++------------------- src/ideas/init/crate.py | 28 +- src/ideas/test_symbol.py | 13 +- src/ideas/tools.py | 65 ++++- src/ideas/translate.py | 55 ++-- src/ideas/translate_recurrent.py | 22 +- src/ideas/translate_snippet.py | 46 ++-- src/ideas/wrapper.py | 73 +++-- test/test_cargo_test.py | 2 +- test/test_consolidate.py | 254 +++++++++-------- test/test_extract_code_from_tu.py | 26 ++ test/test_tools.py | 5 +- test/test_wrapper.py | 6 +- uv.lock | 12 +- 25 files changed, 871 insertions(+), 878 deletions(-) diff --git a/AGENTS.mk b/AGENTS.mk index 10f52f4..020f246 100644 --- a/AGENTS.mk +++ b/AGENTS.mk @@ -56,11 +56,12 @@ endif test_crates/%/Cargo.toml \ test_crates/%/src/lib.c \ test_crates/%/build.rs: | build-ninja/lib%.so.sources - uv run python -m ideas.init.crate crate_type=lib \ + uv run python -m ideas.init.crate cargo_toml=test_crates/$*/Cargo.toml \ + template=lib \ reexport_lib=false \ hydra.output_subdir=null \ hydra.run.dir=test_crates/$* - uv run python -m ideas.init.consolidate filename=build-ninja/compile_commands.json \ + uv run python -m ideas.init.consolidate compile_commands=build-ninja/compile_commands.json \ cargo_toml=test_crates/$*/Cargo.toml \ source_priority=build-ninja/lib$*.so.sources \ hydra.output_subdir=null \ @@ -73,10 +74,11 @@ test_crates/%/build.rs: | build-ninja/lib%.so.sources test_crates/%/Cargo.toml \ test_crates/%/src/main.c \ test_crates/%/build.rs: | build-ninja/%.sources - uv run python -m ideas.init.crate crate_type=bin \ + uv run python -m ideas.init.crate cargo_toml=test_crates/$*/Cargo.toml \ + template=bin \ hydra.output_subdir=null \ hydra.run.dir=test_crates/$* - uv run python -m ideas.init.consolidate filename=build-ninja/compile_commands.json \ + uv run python -m ideas.init.consolidate compile_commands=build-ninja/compile_commands.json \ cargo_toml=test_crates/$*/Cargo.toml \ source_priority=build-ninja/$*.sources \ hydra.output_subdir=null \ diff --git a/IDEAS.mk b/IDEAS.mk index b93f65f..0f1d5e7 100644 --- a/IDEAS.mk +++ b/IDEAS.mk @@ -21,29 +21,18 @@ endif RUSTFLAGS ?= -Awarnings## Ignore Rust compiler warnings CARGO_NET_OFFLINE ?= true## Cargo offline mode CFLAGS ?= -w## Ignore C compiler warnings -LARGE_PROJECT ?= 0## Disable translation-time tests and enable context compression -export EXTRACT_INFO_CMAKE CFLAGS LARGE_PROJECT +export EXTRACT_INFO_CMAKE CFLAGS VCS ?= git GIT_AUTHOR_NAME ?= ideas GIT_AUTHOR_EMAIL ?= ideas@localhost export GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL -ifeq ($(LARGE_PROJECT),1) TRANSLATION_TEST ?= smoke -else -TRANSLATION_TEST ?= test_assert -endif - EVALUATION_TEST ?= test_cases TEST_FILES := $(wildcard test_vectors/*.json) -ifeq ($(LARGE_PROJECT),1) -TARGETS_LIB ?= -TARGETS_BIN ?= $(shell [ -d build-ninja ] && find build-ninja -maxdepth 1 -name '*.sources' ! -name '*.so.sources' -exec basename {} .sources \; ) -else TARGETS_LIB ?= $(shell [ -d build-ninja ] && find build-ninja -maxdepth 1 -name 'lib*.so.sources' -exec basename {} .so.sources \; | sed -e "s/^lib//gi") TARGETS_BIN ?= $(shell [ -d build-ninja ] && find build-ninja -maxdepth 1 -name '*.sources' ! -name 'lib*.so.sources' -exec basename {} .sources \; ) -endif TARGETS ?= $(TARGETS_BIN) $(TARGETS_LIB) ifeq (${TARGETS},) ifeq ($(filter cmake clean,$(MAKECMDGOALS)),) @@ -88,17 +77,19 @@ ${TRANSLATION_DIR}/Cargo.toml: ${TRANSLATION_DIR}/%/Cargo.toml \ ${TRANSLATION_DIR}/%/src/lib.c \ ${TRANSLATION_DIR}/%/build.rs: | ${TRANSLATION_DIR}/Cargo.toml build-ninja/compile_commands.json build-ninja/lib%.so.sources - uv run python -m ideas.init.crate crate_type=lib \ + uv run python -m ideas.init.crate cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ + template=lib \ vcs=${VCS} \ hydra.output_subdir=.init.crate \ hydra.run.dir=${TRANSLATION_DIR}/$* - uv run python -m ideas.init.consolidate filename=build-ninja/compile_commands.json \ + uv run python -m ideas.init.consolidate compile_commands=build-ninja/compile_commands.json \ vcs=${VCS} \ cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ source_priority=build-ninja/lib$*.so.sources \ hydra.output_subdir=.init.consolidate \ hydra.run.dir=${TRANSLATION_DIR}/$* - uv run python -m ideas.init.build vcs=${VCS} \ + uv run python -m ideas.init.build cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ + vcs=${VCS} \ hydra.output_subdir=.init.build \ hydra.job.name=init.build \ hydra.run.dir=${TRANSLATION_DIR}/$* @@ -106,17 +97,19 @@ ${TRANSLATION_DIR}/%/build.rs: | ${TRANSLATION_DIR}/Cargo.toml build-ninja/compi ${TRANSLATION_DIR}/%/Cargo.toml \ ${TRANSLATION_DIR}/%/src/main.c \ ${TRANSLATION_DIR}/%/build.rs: | ${TRANSLATION_DIR}/Cargo.toml build-ninja/compile_commands.json build-ninja/%.sources - uv run python -m ideas.init.crate crate_type=bin \ + uv run python -m ideas.init.crate cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ + template=bin \ vcs=${VCS} \ hydra.output_subdir=.init.crate \ hydra.run.dir=${TRANSLATION_DIR}/$* - uv run python -m ideas.init.consolidate filename=build-ninja/compile_commands.json \ + uv run python -m ideas.init.consolidate compile_commands=build-ninja/compile_commands.json \ vcs=${VCS} \ cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ source_priority=build-ninja/$*.sources \ hydra.output_subdir=.init.consolidate \ hydra.run.dir=${TRANSLATION_DIR}/$* - uv run python -m ideas.init.build vcs=${VCS} \ + uv run python -m ideas.init.build cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ + vcs=${VCS} \ hydra.output_subdir=.init.build \ hydra.job.name=init.build \ hydra.run.dir=${TRANSLATION_DIR}/$* @@ -127,33 +120,6 @@ translate: $(patsubst %,${TRANSLATION_DIR}/%/translate,${TARGETS}) ; ${TRANSLATION_DIR}/%/translate: ${TRANSLATION_DIR}/%/src/lib.rs | build-ninja/lib%.so.sources ; ${TRANSLATION_DIR}/%/translate: ${TRANSLATION_DIR}/%/src/main.rs | build-ninja/%.sources ; -ifeq ($(LARGE_PROJECT),1) -.PRECIOUS: ${TRANSLATION_DIR}/%/src/lib.rs -${TRANSLATION_DIR}/%/src/lib.rs: | ${TRANSLATION_DIR}/%/Cargo.toml ${TRANSLATION_DIR}/%/tests/${TRANSLATION_TEST}.rs build-ninja/compile_commands.json - -uv run python -m ideas.translate model.name=${PROVIDER}/${MODEL} \ - filename=build-ninja/compile_commands.json \ - cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ - source_priority=build-ninja/lib$*.so.sources \ - tests=${TRANSLATION_TEST} \ - vcs=${VCS} \ - hydra.output_subdir=.translate \ - hydra.job.name=translate \ - hydra.run.dir=${TRANSLATION_DIR}/$* ${TRANSLATE_ARGS} - @touch $@ - -.PRECIOUS: ${TRANSLATION_DIR}/%/src/main.rs -${TRANSLATION_DIR}/%/src/main.rs: | ${TRANSLATION_DIR}/%/Cargo.toml ${TRANSLATION_DIR}/%/tests/${TRANSLATION_TEST}.rs build-ninja/compile_commands.json - -uv run python -m ideas.translate model.name=${PROVIDER}/${MODEL} \ - filename=build-ninja/compile_commands.json \ - cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ - source_priority=build-ninja/$*.sources \ - tests=${TRANSLATION_TEST} \ - vcs=${VCS} \ - hydra.output_subdir=.translate \ - hydra.job.name=translate \ - hydra.run.dir=${TRANSLATION_DIR}/$* ${TRANSLATE_ARGS} - @touch $@ -else .PRECIOUS: ${TRANSLATION_DIR}/%/src/lib.rs ${TRANSLATION_DIR}/%/src/lib.rs: ${TRANSLATION_DIR}/%/src/lib.c | ${TRANSLATION_DIR}/%/Cargo.toml ${TRANSLATION_DIR}/%/tests/${TRANSLATION_TEST}.rs -uv run python -m ideas.translate model.name=${PROVIDER}/${MODEL} \ @@ -164,6 +130,7 @@ ${TRANSLATION_DIR}/%/src/lib.rs: ${TRANSLATION_DIR}/%/src/lib.c | ${TRANSLATION_ hydra.output_subdir=.translate \ hydra.job.name=translate \ hydra.run.dir=${TRANSLATION_DIR}/$* ${TRANSLATE_ARGS} + @touch ${TRANSLATION_DIR}/$*/build.rs @touch $@ .PRECIOUS: ${TRANSLATION_DIR}/%/src/main.rs @@ -176,8 +143,8 @@ ${TRANSLATION_DIR}/%/src/main.rs: ${TRANSLATION_DIR}/%/src/main.c | ${TRANSLATIO hydra.output_subdir=.translate \ hydra.job.name=translate \ hydra.run.dir=${TRANSLATION_DIR}/$* ${TRANSLATE_ARGS} + @touch ${TRANSLATION_DIR}/$*/build.rs @touch $@ -endif # build .PHONY: build @@ -260,10 +227,7 @@ ${TRANSLATION_DIR}/%/tests/test_assert_wrapper.rs: test_crates/%/tests/test_asse .PRECIOUS: ${TRANSLATION_DIR}/%/tests/smoke.rs ${TRANSLATION_DIR}/%/tests/smoke.rs: mkdir -p $(dir $@) - echo "#[test]" >> $@ - echo "fn smoke() {" >> $@ - echo " assert_eq!(1, 1);" >> $@ - echo "}" >> $@ + printf '#[test]\nfn smoke() {\n assert_eq!(1, 1);\n}\n' > $@ # clean diff --git a/README.md b/README.md index e01759e..727c889 100644 --- a/README.md +++ b/README.md @@ -3,33 +3,44 @@ [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/IntelLabs/IDEAS/badge)](https://scorecard.dev/viewer/?uri=github.com/IntelLabs/IDEAS) > [!NOTE] -> IDEAS is a framework under active development which may go through major changes with each release. +> IDEAS is a framework under active development which may go through major changes with each release. > If you encounter any issues or have questions about how to run the framework, please do not hesitate to [open a GitHub issue](https://github.com/IntelLabs/IDEAS/issues/new). # Requirements -Tested on Ubuntu 24.04. +Developed and tested on Ubuntu 24.04. -To install the Python and Rust toolchain dependencies, run `make install` and follow the prompts if requested. -> [!TIP] -> This will install `uv@0.9.22` and `rust@1.88.0` for the current user. +IDEAS requires a specific version of `clang` and Rust toolchains to translate C-to-Rust. +A docker image with the user-specific name `ideas-${UID}` can be built and launched in an interactive session using: +```bash +make docker +``` + +We strongly recommend launching all runs in the Docker image. + +> [!NOTE] +> If the `OPENROUTER_API_KEY` or `OPENAI_API_KEY` environment variables are set on the host, they will be automatically passed to the interactive session. + +# Quickstart +To translate a single C project to a Rust workspace, ensure it uses Cmake as a build system, place it in the `examples` folder and generate an [OpenRouter API key](https://openrouter.ai/workspaces/default/keys). -# Quick start -To translate a single C project to a Rust workspace, ensure it uses Cmake as a build system, place it in the `examples` folder and run: +Then, build and launch the official Docker image: ```bash -make examples/C-project-name/translate \ - OPENROUTER_API_KEY="your-key" \ - MODEL="openai/gpt-5.1" \ - TRANSLATION_TEST=smoke +make docker +``` + +And trigger end-to-end translation: +```bash +make examples/C-project-name/translate OPENROUTER_API_KEY="your-key" ``` # Expected C project structure -To run translation on a C project folder, it must be placed in the top-level `examples` folder. -The toolkit assumes the [official DARPA TRACTOR folder structure](https://github.com/DARPA-TRACTOR-Program/PUBLIC-Test-Corpus#test-case-structure) for the C projects that will be translated: +To run translation on a C project folder, it must be copied to the top-level `examples` folder. +IDEAS requires the [official DARPA TRACTOR folder structure](https://github.com/DARPA-TRACTOR-Program/PUBLIC-Test-Corpus#test-case-structure) for the C projects that will be translated: ``` 📦IDEAS ┣ 📂src/ideas # Core library ┗ 📂examples # Project folders go here - ┣ 📂C-project-name # A single C project folder with an arbitrary name + ┣ 📂C-project-name ┃ ┣ 📂test_case ┃ ┃ ┣ 📂include ┃ ┃ ┣ 📂src @@ -39,42 +50,41 @@ The toolkit assumes the [official DARPA TRACTOR folder structure](https://github ┃ ┗ 📄other-name.json ┗ 📂other-C-project-name ``` -This follows the standard evaluation convention in the DARPA TRACTOR project, which can be consulted in more detail [here](https://github.com/DARPA-TRACTOR-Program/PUBLIC-Test-Corpus) (including many C projects in `examples`). +See the [`examples/templates`](examples/templates/) folder for minimal examples. # Translated Rust structure -The translation tool identifies each Cmake target (library or binary), and translates it to a separate, self-contained Rust [crate](https://doc.rust-lang.org/book/ch07-01-packages-and-crates.html#packages-and-crates). All crates are organized together in a Rust [workspace](https://doc.rust-lang.org/cargo/reference/workspaces.html) under the folder given by the `TRANSLATION_DIR` environment variable, alongside the original C `test_case` folder: - -``` -📂C-project-name # A single C project folder with an arbitrary name - ┣ 📂test_case - ┣ 📂test_vectors - ┗ 📂${TRANSLATION_DIR} - ┣ 📂target-1 - ┃ ┣📂src - ┃ ┃ ┣📄lib.rs / main.rs # For libraries / executables - ┃ ┃ ┣📄wrapper.rs # Wrapper module (libraries only) - ┃ ┃ ┗📂wrapper # Individual wrappers for every exported C symbol (libraries only) - ┃ ┗📄Cargo.toml # For crate (translated C target) - ┣ 📂target-2 - ┗ 📄Cargo.toml # For workspace -``` +The translation tool identifies each Cmake target (library or binary), and translates it to a separate, self-contained Rust [crate](https://doc.rust-lang.org/book/ch07-01-packages-and-crates.html#packages-and-crates). +Crates are organized together in a Rust [workspace](https://doc.rust-lang.org/cargo/reference/workspaces.html) under the folder given by the `TRANSLATION_DIR` environment variable, alongside the original C `test_case` folder. -Inside each translated crate, there are at most **two** distinct components of the Rust translation: -1. A consolidated `lib.rs` or `main.rs` file containing the **guaranteed** memory-safe, attempted Rust translation. -2. (only for library targets/crates) A `wrapper` directory containing C FFI wrappers for exact backwards compatibility with the original C library. - -The translation tool is also capable of testing the individual Rust crates in a workspace, but the DARPA TRACTOR evaluation schema must be followed by the `.json` files. See [here](https://github.com/DARPA-TRACTOR-Program/PUBLIC-Test-Corpus?tab=readme-ov-file#test-vector-schema-json) for more details and the exact specification. - -# Docker image -A docker image with the user-specific name `ideas-${UID}` can be built and directly entered using: +For example, running ```bash make docker +make examples/templates/hello_world_lib/translate TRANSLATION_DIR="translation.demo" OPENROUTER_API_KEY="your-key" ``` -This allows for isolated execution in a reproducible environment. +Should produce the following translated folder structure: +``` +📂examples/templates/hello_world_lib + ┣ 📂test_case + ┣ 📂test_vectors + ┗ 📂translation.demo + ┣ 📂hello_world_lib + ┃ ┣ 📂src + ┃ ┃ ┣ 📄lib.c # Stub C library + ┃ ┃ ┣ 📄lib.rs # Translated Rust library + ┃ ┃ ┣ 📄wrapper.rs # Wrapper module + ┃ ┃ ┗ 📂wrapper # Per-symbol C FFI compatibility wrappers for all symbols + ┃ ┣ 📂tests + ┃ ┃ ┗ 📄 smoke.rs # No tests by default + ┃ ┗ 📄Cargo.toml # Crate manifest + ┣ 📄build.rs # Hybrid build script + ┣ 🗄️cache.db # Resumable translation cache + ┣ 📄Cargo.lock # Workspace lockfile + ┗ 📄Cargo.toml # Workspace manifest +``` -> [!TIP] -> If the `OPENROUTER_API_KEY` environment variable is set on the host, it will be automatically passed to the interactive session. +IDEAS is capable of testing Rust translations with the DARPA TRACTOR evaluation schema. +See [here](https://github.com/DARPA-TRACTOR-Program/PUBLIC-Test-Corpus?tab=readme-ov-file#test-vector-schema-json) for more details and the exact specification for writing test vectors and `cando2` runners. # Usage with OpenRouter API Our translation framework treats [OpenRouter](https://openrouter.ai/) as the default provider, allowing easy switching between models. @@ -85,8 +95,6 @@ To run LLM-based memory-safe translation of a single project and save the transl make examples/C-project-name/translate \ TRANSLATION_DIR="translated_rust" \ OPENROUTER_API_KEY="your-key" \ - MODEL="openai/gpt-5.1" \ - TRANSLATION_TEST=smoke ``` If a project (library or executable) was not already found under `TRANSLATION_DIR`, our dependency chain will first trigger its memory-safe translation, followed by C FFI wrappers (only for libraries). @@ -98,10 +106,9 @@ make examples/C-project-name/translate \ TRANSLATION_DIR="translated_rust" \ ANTHROPIC_API_KEY="your-key" \ PROVIDER="anthropic" \ - MODEL="claude-sonnet-4.6" \ - TRANSLATION_TEST=smoke + MODEL="claude-sonnet-4.6" ``` -Note the prefix `anthropic` is missing from `MODEL`, and must instead be set as the `PROVIDER`. +Note the `anthropic` prefix is missing from `MODEL` and is instead set as the `PROVIDER`. # Usage with OpenAI API IDEAS can be used with any OpenAI model by setting the `PROVIDER`, `MODEL`, and `OPEN_API_KEY` variables: @@ -110,34 +117,30 @@ make examples/C-project-name/translate \ TRANSLATION_DIR="translated_rust" \ OPEN_API_KEY="your-key" \ PROVIDER="openai" \ - MODEL="gpt-5.5" \ - TRANSLATION_TEST=smoke + MODEL="gpt-5.4" ``` -Note the prefix `anthropic` is missing from `MODEL`, and must instead be set as the `PROVIDER`. +Note the `openai` prefix is missing from `MODEL` and is instead set as the `PROVIDER`. # Usage with other APIs IDEAS relies on [`litellm`](https://github.com/BerriAI/litellm), which supports many other model providers (e.g., Google Vertex, MS Azure, etc). The instructions at https://docs.litellm.ai/docs/providers inform which parameters should be set in `litellm` and IDEAS flows through the [`dspy.LM`](https://dspy.ai/api/models/LM/) instance. -We are working on writing a comprehensive guide for more providers, but developers can inspect [how the `dspy.LM` is instantiated by IDEAS](https://github.com/IntelLabs/IDEAS/blob/main/src/ideas/model.py) and infer any additional required parameters that need to be passed to `litellm`. +Developers can inspect [how a `dspy.LM` is instantiated by IDEAS](https://github.com/IntelLabs/IDEAS/blob/main/src/ideas/model.py) and infer any additional required parameters that need to be passed to `litellm`. + +# (Experimental) C/Rust equivalence test generation +IDEAS has a submodule that automatically generates portable Rust C FFI tests for libraries and binary targets. +To directly launch the test generation agent in an isolated enviroment run (from host): -# Running TRACTOR tests -To run all tests (if available and following the [DARPA TRACTOR evaluation schema](https://github.com/DARPA-TRACTOR-Program/PUBLIC-Test-Corpus#test-vector-schema-json)) on an existing translation, run: ```bash -make examples/C-project-name/test +make examples/templates/hello_world_lib/testgen_agent OPENROUTER_API_KEY="your-key" ``` -If a project was not already translated, this will trigger complete translation and testing, and can be used a **single-click** translation-and-evaluation command. +This launches a self-sufficient KISS agent that is tasked with generating unit and integration tests with high branch coverage. +You can consult the agent prompt (for libraries; similar for binaries) [here](src/ideas/agents/testgen.py). -To translate, test, and aggregate statistics about all projects contained in the `examples` folder, run: -```bash -make -j128 examples/test \ - VERBOSE=1 - ... -``` -> [!TIP] -> The 128-way parallelism will be CPU-intensive, and can be reduced if needed. +> [!NOTE] +> This behavior is disabled by default and may not be reliable on large codebases and/or weaker LLMs. ## Acknowledgments This material is based upon work supported by the Defense Advanced Research Projects Agency (DARPA) under Agreement No. HR00112590134. diff --git a/src/ideas/adapters.py b/src/ideas/adapters.py index 8f6398c..f9ed7a5 100644 --- a/src/ideas/adapters.py +++ b/src/ideas/adapters.py @@ -34,8 +34,7 @@ def __init__(self, code: str = "", **kwargs): kwargs["code"] = code super().__init__(**kwargs) - @property - def text(self) -> str: + def __str__(self): return self.code def __add__(self, other): diff --git a/src/ideas/agents/build.py b/src/ideas/agents/build.py index 6b8c756..20e7a26 100644 --- a/src/ideas/agents/build.py +++ b/src/ideas/agents/build.py @@ -115,7 +115,7 @@ def write_symbol_binding(crate: Crate, symbol_name: str): "\n\n".join( [ "#![allow(unused_attributes)]", - symbol_binding.text, + str(symbol_binding), ] ) ) @@ -182,10 +182,7 @@ def _main(cfg: BuildConfig) -> None: output_dir = Path(HydraConfig.get().runtime.output_dir) # Fetch crate - crate = Crate( - cargo_toml=output_dir / "Cargo.toml", - vcs=cfg.vcs, # type: ignore[reportArgumentType] - ) + crate = Crate(output_dir / "Cargo.toml", vcs=cfg.vcs) # type: ignore[reportArgumentType] # Get global symbol table tu = create_translation_unit(crate.c_src_path) diff --git a/src/ideas/agents/utils.py b/src/ideas/agents/utils.py index c3b3366..4b7f956 100644 --- a/src/ideas/agents/utils.py +++ b/src/ideas/agents/utils.py @@ -80,10 +80,10 @@ def write_collect_script(crate: Crate) -> Path: panic!("UBSAN detected during collection!"); } println!("{{"); - println!(" \"name\": \"{}\",", name); - println!(" \"stdout\": {},", serde_json::to_string(&*stdout).unwrap()); - println!(" \"stderr\": {},", serde_json::to_string(&*stderr).unwrap()); - println!(" \"exit_code\": {}", code); + println!(" \\"name\\": \\"{}\\",", name); + println!(" \\"stdout\\": {},", serde_json::to_string(&*stdout).unwrap()); + println!(" \\"stderr\\": {},", serde_json::to_string(&*stderr).unwrap()); + println!(" \\"exit_code\\": {}", code); println!("}}"); } """ diff --git a/src/ideas/ast.py b/src/ideas/ast.py index 0528f4e..75e7655 100644 --- a/src/ideas/ast.py +++ b/src/ideas/ast.py @@ -4,18 +4,18 @@ # SPDX-License-Identifier: Apache-2.0 # -import re import logging from pathlib import Path from collections import defaultdict from collections.abc import Iterable -from dataclasses import dataclass, field +from dataclasses import dataclass, field, fields, replace +from typing import get_args from clang.cindex import TranslationUnit, TranslationUnitLoadError, Diagnostic from clang.cindex import Cursor, CursorKind, SourceRange, TokenKind from clang.cindex import PrintingPolicy, PrintingPolicyProperty, LinkageKind, StorageClass -from clang.cindex import conf, SourceLocation -from ctypes import pointer, c_size_t, c_char_p +from clang.cindex import conf, SourceLocation, _CXString +from ctypes import byref, pointer, c_size_t, c_char_p, c_uint from .adapters import Code @@ -26,100 +26,190 @@ @dataclass(frozen=True) class Symbol: + # Symbol identity name: str - cursor: Cursor - parent: Cursor | None = None - decl: Cursor | None = None - - @property - def spelling(self) -> str: - return self.cursor.spelling - - @property - def kind(self) -> CursorKind: - return self.cursor.kind - - @property - def llm_context_declaration(self) -> str: - # Synthesize forward declaration from cursor - if self.cursor.kind == CursorKind.FUNCTION_DECL: - result_type = ( - self.cursor.result_type.spelling if self.cursor.result_type else "void" - ) - params = ", ".join( - p.type.spelling + (" " + p.spelling if p.spelling else "") # type: ignore[reportOptionalMemberAccess] - for p in self.cursor.get_arguments() - ) - return f"{result_type} {self.cursor.spelling}({params});" - elif self.cursor.kind in ( - CursorKind.STRUCT_DECL, - CursorKind.UNION_DECL, - CursorKind.ENUM_DECL, - ): - kind_name = { - CursorKind.STRUCT_DECL: "struct", - CursorKind.UNION_DECL: "union", - CursorKind.ENUM_DECL: "enum", - }[self.cursor.kind] - return f"{kind_name} {self.cursor.spelling};" - elif self.cursor.kind == CursorKind.TYPEDEF_DECL: - underlying = self.cursor.underlying_typedef_type.spelling - return f"typedef {underlying} {self.cursor.spelling};" - elif self.cursor.kind == CursorKind.VAR_DECL: - return f"{self.cursor.type.spelling} {self.cursor.spelling};" + spelling: str + kind: CursorKind + + # Rendered C snippets + llm_context_declaration: str + declaration: CodeC | None + code: CodeC + + # Symbol semantics + is_definition: bool + is_variable: bool + is_function: bool + is_global: bool + is_system: bool + is_top_level: bool + storage_class: StorageClass + + # Source and lexical metadata + tu_path: Path + presumed_path: Path | None + tu_preorder_index: int + line_directive: CodeC | None + declaration_line_directive: CodeC | None + + @classmethod + def from_cursor( + cls, + name: str, + cursor: Cursor, + parent: Cursor | None = None, + decl: Cursor | None = None, + tu_preorder_index: int = -1, + ) -> "Symbol": + parent_or_cursor = parent or cursor + code = get_cursor_code(parent_or_cursor, pretty_print=True) + presumed_location = clang_get_presumed_location(parent_or_cursor) + return cls( + name=name, + spelling=cursor.spelling, + kind=cursor.kind, + llm_context_declaration=_synthesize_llm_context_declaration( + cursor, fallback_code=code + ), + declaration=get_cursor_code(decl, pretty_print=True) if decl else None, + code=code, + is_definition=cursor.is_definition(), + is_variable=cursor.kind == CursorKind.VAR_DECL, + is_function=cursor.kind == CursorKind.FUNCTION_DECL, + is_global=cursor.linkage == LinkageKind.EXTERNAL, + is_system=cursor.location.is_in_system_header, + tu_path=Path(cursor.translation_unit.spelling).resolve(), + presumed_path=Path(presumed_location[0]) if presumed_location is not None else None, + tu_preorder_index=tu_preorder_index, + line_directive=_line_directive_for(parent_or_cursor), + declaration_line_directive=_line_directive_for(decl), + storage_class=cursor.storage_class, + is_top_level=parent is None, + ) - # Fallback: return full code - return self.code.text + def with_declaration(self, decl_symbol: "Symbol") -> "Symbol": + return replace( + self, + declaration=decl_symbol.code, + declaration_line_directive=decl_symbol.line_directive, + ) - @property - def declaration(self) -> CodeC | None: - return get_cursor_code(self.decl, pretty_print=True) if self.decl else None + def __getstate__(self) -> dict[str, object]: + state = dict(self.__dict__) + for field_name, value in state.items(): + if isinstance(value, CodeC): + state[field_name] = str(value) + return state - @property - def code(self) -> CodeC: - return get_cursor_code(self.parent or self.cursor, pretty_print=True) + def __setstate__(self, state: dict[str, object]): + annotations = {f.name: f.type for f in fields(type(self))} + for field_name, value in state.items(): + annotation = annotations.get(field_name) + if value is not None and (annotation is CodeC or CodeC in get_args(annotation)): + assert isinstance(value, str) + value = CodeC(value) + object.__setattr__(self, field_name, value) - @property - def is_definition(self) -> bool: - return self.cursor.is_definition() - @property - def is_variable(self) -> bool: - return self.cursor.kind == CursorKind.VAR_DECL +@dataclass +class TreeResult: + symbols: dict[str, Symbol] = field(default_factory=dict) + complete_graph: dict[str, list[str]] = field(default_factory=lambda: defaultdict(list)) + filename: str | None = None + arguments: list[str] | None = None + + +def _synthesize_llm_context_declaration(cursor: Cursor, fallback_code: CodeC) -> str: + # Synthesize forward declaration from cursor + if cursor.kind == CursorKind.FUNCTION_DECL: + result_type = cursor.result_type.spelling if cursor.result_type else "void" + params = ", ".join( + p.type.spelling + (" " + p.spelling if p.spelling else "") # type: ignore[reportOptionalMemberAccess] + for p in cursor.get_arguments() + ) + return f"{result_type} {cursor.spelling}({params});" + elif cursor.kind in ( + CursorKind.STRUCT_DECL, + CursorKind.UNION_DECL, + CursorKind.ENUM_DECL, + ): + kind_name = { + CursorKind.STRUCT_DECL: "struct", + CursorKind.UNION_DECL: "union", + CursorKind.ENUM_DECL: "enum", + }[cursor.kind] + return f"{kind_name} {cursor.spelling};" + elif cursor.kind == CursorKind.TYPEDEF_DECL: + underlying = cursor.underlying_typedef_type.spelling + return f"typedef {underlying} {cursor.spelling};" + elif cursor.kind == CursorKind.VAR_DECL: + return f"{cursor.type.spelling} {cursor.spelling};" + + # Fallback: return full code + return str(fallback_code) + + +def _line_directive_for(cursor: Cursor | None) -> CodeC | None: + if cursor is None: + return None + + location = cursor.location + if location.file is None or location.line == 0: + return None + + source_path = Path(str(location.file)).resolve().as_posix().replace('"', '\\"') + return CodeC(f'#line {location.line} "{source_path}"') + + +def _cursor_key(cursor: Cursor) -> tuple[str, str, str, str, int, int, int, int]: + location = cursor.location + source_path = Path(str(location.file)).resolve().as_posix() if location.file else "" + kind = str(cursor.kind) + return ( + cursor.get_usr(), + cursor.spelling, + kind, + source_path, + int(location.line), + int(location.column), + int(cursor.extent.start.offset), + int(cursor.extent.end.offset), + ) - @property - def is_function(self) -> bool: - return self.cursor.kind == CursorKind.FUNCTION_DECL - @property - def is_global(self) -> bool: - return self.cursor.linkage == LinkageKind.EXTERNAL +def _cursor_order_map(root: Cursor) -> dict[tuple[str, str, str, str, int, int, int, int], int]: + order: dict[tuple[str, str, str, str, int, int, int, int], int] = {} + for i, cursor in enumerate(root.walk_preorder()): + key = _cursor_key(cursor) + if key not in order: + order[key] = i + return order - @property - def is_system(self) -> bool: - return self.cursor.location.is_in_system_header - @property - def source_path(self) -> Path: - return Path(self.cursor.translation_unit.spelling).resolve() +def clang_get_presumed_location(cursor: Cursor | None) -> tuple[str, int, int] | None: + if cursor is None: + return None - def with_declaration(self, decl: Cursor) -> "Symbol": - return Symbol(self.name, self.cursor, self.parent, decl=decl) + filename = _CXString() + line = c_uint(0) + column = c_uint(0) + conf.lib.clang_getPresumedLocation( + cursor.location, byref(filename), byref(line), byref(column) + ) + path_text = _CXString.from_result(filename) + if not path_text or line.value == 0: + return None -@dataclass -class TreeResult: - symbols: dict[str, Symbol] = field(default_factory=dict) - complete_graph: dict[str, list[str]] = field( - default_factory=lambda: defaultdict(lambda: list()) - ) + path = Path(path_text).resolve().as_posix() + return path, int(line.value), int(column.value) def create_translation_unit(path_or_code: Path | CodeC) -> TranslationUnit: # Parse the code using clang if isinstance(path_or_code, CodeC): code = path_or_code - tu = TranslationUnit.from_source(FILENAME, unsaved_files=[(FILENAME, code.text)]) + tu = TranslationUnit.from_source(FILENAME, unsaved_files=[(FILENAME, str(code))]) else: tu = TranslationUnit.from_source(str(path_or_code.resolve())) if any(d.severity >= Diagnostic.Error for d in tu.diagnostics): @@ -130,21 +220,38 @@ def create_translation_unit(path_or_code: Path | CodeC) -> TranslationUnit: # Traverse the AST, extract symbols and resolve deep references def extract_info_c(tu: TranslationUnit) -> TreeResult: assert tu.cursor is not None - symbols = extract_symbol_info_c(tu.cursor) + tu_preorder_index_map = _cursor_order_map(tu.cursor) + symbols, reference_nodes = _extract_symbol_info_c( + tu.cursor, tu_preorder_index_map=tu_preorder_index_map + ) graph = { - # Prefer parent over cursor - name: extract_referenced_symbols(symbol.parent or symbol.cursor, symbols.keys()) + name: extract_referenced_symbols(reference_nodes[name], symbols.keys()) for name, symbol in symbols.items() } return TreeResult(symbols=symbols, complete_graph=graph) def extract_symbol_info_c(node: Cursor, parent: Cursor | None = None) -> dict[str, Symbol]: + tu_preorder_index_map = _cursor_order_map(node) + symbols, _ = _extract_symbol_info_c( + node, parent=parent, tu_preorder_index_map=tu_preorder_index_map + ) + return symbols + + +def _extract_symbol_info_c( + node: Cursor, + parent: Cursor | None = None, + tu_preorder_index_map: dict[tuple[str, str, str, str, int, int, int, int], int] + | None = None, +) -> tuple[dict[str, Symbol], dict[str, Cursor]]: symbols: dict[str, Symbol] = {} + reference_nodes: dict[str, Cursor] = {} + tu_preorder_index_map = tu_preorder_index_map or _cursor_order_map(node) # If enter new scope then exit early if node.kind == CursorKind.COMPOUND_STMT: - return symbols + return symbols, reference_nodes # Add declarative nodes to symbols usr = node.get_usr() @@ -158,36 +265,50 @@ def extract_symbol_info_c(node: Cursor, parent: Cursor | None = None) -> dict[st CursorKind.VAR_DECL, CursorKind.TYPEDEF_DECL, ): - symbols[usr] = Symbol(usr, node, parent=parent) + symbols[usr] = Symbol.from_cursor( + usr, + node, + parent=parent, + tu_preorder_index=tu_preorder_index_map.get(_cursor_key(node), -1), + ) + reference_nodes[usr] = parent or node # Recurse through children and merge them into symbols for child_node in node.get_children(): parent = node if parent is None and node.kind != CursorKind.TRANSLATION_UNIT else parent - child_symbols = extract_symbol_info_c(child_node, parent=parent) + child_symbols, child_refs = _extract_symbol_info_c( + child_node, + parent=parent, + tu_preorder_index_map=tu_preorder_index_map, + ) for child_name, child_symbol in child_symbols.items(): if child_name not in symbols: # Found a new symbol symbols[child_name] = child_symbol + reference_nodes[child_name] = child_refs[child_name] elif symbols[child_name].is_definition and child_symbol.is_definition: # Always keep current definition symbols[child_name] = child_symbol + reference_nodes[child_name] = child_refs[child_name] elif not symbols[child_name].is_definition and child_symbol.is_definition: # Previous symbol was a declaration so replace it with new definitional symbol - symbols[child_name] = child_symbol.with_declaration(symbols[child_name].cursor) + symbols[child_name] = child_symbol.with_declaration(symbols[child_name]) + reference_nodes[child_name] = child_refs[child_name] elif symbols[child_name].is_definition and not child_symbol.is_definition: if not symbols[child_name].is_system or not child_symbol.is_system: logger.debug(f"Ignoring declaration after definition of `{child_name}`") elif not symbols[child_name].is_definition and not child_symbol.is_definition: if ( - child_symbol.cursor.kind == CursorKind.VAR_DECL - and symbols[child_name].cursor.storage_class == StorageClass.EXTERN - and child_symbol.cursor.storage_class != StorageClass.EXTERN + child_symbol.kind == CursorKind.VAR_DECL + and symbols[child_name].storage_class == StorageClass.EXTERN + and child_symbol.storage_class != StorageClass.EXTERN ): # Prefer non-extern variable declaration (e.g. tentative definition) over extern one symbols[child_name] = child_symbol + reference_nodes[child_name] = child_refs[child_name] elif not symbols[child_name].is_system or not child_symbol.is_system: logger.debug(f"Ignoring re-declaration of `{child_name}`") - return symbols + return symbols, reference_nodes def extract_referenced_symbols(node: Cursor, global_symbols: Iterable[str]) -> list[str]: @@ -240,6 +361,8 @@ def get_cursor_prettyprinted(cursor: Cursor) -> CodeC: policy = PrintingPolicy.create(cursor) policy.set_property(PrintingPolicyProperty.IncludeTagDefinition, include_tag_definition) + # Emit C99 builtin spelling to avoid dependence on stdbool.h macro context. + policy.set_property(PrintingPolicyProperty.Bool, 0) return CodeC(cursor.pretty_printed(policy)) @@ -251,7 +374,7 @@ def get_cursor_code(cursor: Cursor, pretty_print: bool = False) -> CodeC: # Non-function definitions require statement terminations if cursor.kind != CursorKind.FUNCTION_DECL or not cursor.is_definition(): - code = CodeC(code.text.rstrip() + ";") + code = CodeC(str(code).rstrip() + ";") return code @@ -594,48 +717,6 @@ def _apply_edits(path: Path, edits: dict[tuple[int, int], bytes]): path.write_bytes(source) -def get_system_macro_undefs(includes: list[str], code: str) -> list[str]: - if not includes: - return [] - - # Parse the includes to enumerate system macros - include_text = "\n".join(includes) + "\n" - tu = TranslationUnit.from_source( - "undefs.c", - unsaved_files=[("undefs.c", include_text)], - options=TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD, - ) - - self_ref_macros: set[str] = set() - assert tu.cursor is not None - for cursor in tu.cursor.get_children(): - if cursor.kind != CursorKind.MACRO_DEFINITION: - continue - if not cursor.location.is_in_system_header: - continue - tokens = list(cursor.get_tokens()) - # Function-like macros have '(' immediately after the name token - if len(tokens) >= 2 and tokens[1].spelling == "(": - continue - # FIXME: This needs to be stronger and not detect #define stuff mystuff - # as self-referencing. - # We should ideally only check the replacement list tokens, - # but clang does not provide a way to get just those. - name = cursor.spelling - if any(tok.spelling == name for tok in tokens[1:]): - self_ref_macros.add(name) - - if not self_ref_macros: - return [] - - # Only #undef self-referencing macros whose name appears in the code - # FIXME: Would be nice if we had an AST list of already-expanded macros across all TUs - code_identifiers = set(re.findall(r"\b([A-Za-z_]\w*)\b", code)) - conflicting = self_ref_macros & code_identifiers - - return [f"#undef {name}" for name in sorted(conflicting)] - - def mangle(name: str) -> str: name = name.replace(" ", "_") name = name.replace(".", "_") diff --git a/src/ideas/ast_rust.py b/src/ideas/ast_rust.py index 81fa536..7467ffb 100644 --- a/src/ideas/ast_rust.py +++ b/src/ideas/ast_rust.py @@ -100,8 +100,8 @@ def get_macro_nodes(root: Node, placeholder: str) -> list[Node]: def validate_changes(code: CodeRust, template: CodeRust) -> OrderedDict[str, str]: - code_root = get_root(code.text) - template_root = get_root(template.text) + code_root = get_root(str(code)) + template_root = get_root(str(template)) nodes = get_nodes(code_root) template_nodes = get_nodes(template_root) @@ -187,10 +187,10 @@ def _rust_node_signature(node: Node, source: bytes) -> str | None: def get_signatures(code: CodeRust) -> CodeRust: - if not code.text.strip(): + if not str(code).strip(): return code - source = code.text.encode() + source = str(code).encode() root = get_root(source) parts: list[str] = [] diff --git a/src/ideas/convert_tests.py b/src/ideas/convert_tests.py index b1c78d8..6e2a6e8 100644 --- a/src/ideas/convert_tests.py +++ b/src/ideas/convert_tests.py @@ -195,7 +195,7 @@ def _main(cfg: ConvertConfig) -> None: rustfmt(output_file) # Add crate dependencies - crate = Crate(cargo_toml=cargo_toml, vcs=cfg.vcs) # type: ignore[reportArgumentType] + crate = Crate(cargo_toml, vcs=cfg.vcs) # type: ignore[reportArgumentType] if exec_tests: add_deps_for_exec(crate) if lib_tests: diff --git a/src/ideas/evaluate.py b/src/ideas/evaluate.py index 83718b9..234da2f 100644 --- a/src/ideas/evaluate.py +++ b/src/ideas/evaluate.py @@ -56,7 +56,7 @@ def list_tests(test_file: Path) -> list[str]: def _main(cfg: EvaluateConfig) -> None: # Resolve integration test file (error loudly if missing) - crate = Crate(cargo_toml=cfg.manifest, vcs="none") + crate = Crate(cfg.manifest, vcs="none") test_file = crate.cargo_toml.parent / "tests" / f"{cfg.test_cases}.rs" if not test_file.exists(): raise FileNotFoundError(f"Integration test file not found: {test_file}") diff --git a/src/ideas/init/build.py b/src/ideas/init/build.py index dbddd85..52283bd 100644 --- a/src/ideas/init/build.py +++ b/src/ideas/init/build.py @@ -10,13 +10,15 @@ import textwrap from pathlib import Path from dataclasses import dataclass +from concurrent.futures import ThreadPoolExecutor import hydra +from omegaconf import MISSING from hydra.core.config_store import ConfigStore from hydra.core.hydra_config import HydraConfig -from ideas.tools import Crate, LARGE_PROJECT -from ideas.tools import run_subprocess +from ideas.ast import Symbol +from ideas.tools import Crate, run_subprocess from ideas.ast_rust import CodeRust, mangle from ideas import create_translation_unit, extract_info_c from ideas.init.consolidate import get_symbols_and_dependencies @@ -26,6 +28,7 @@ @dataclass class BuildConfig: + cargo_toml: Path = MISSING vcs: str = "none" def __post_init__(self): @@ -40,7 +43,8 @@ def __post_init__(self): def write_build_script(crate: Crate) -> Path: c_src_path = crate.c_src_path.relative_to(crate.cargo_toml.parent) build_options = '.define("main", "_main")' if crate.is_bin else "" - build_rs_src = textwrap.dedent( + build_rs_path = crate.cargo_toml.parent / "build.rs" + build_rs_path.write_text( f""" fn main() {{ println!("cargo:rerun-if-changed={c_src_path}"); @@ -56,31 +60,32 @@ def write_build_script(crate: Crate) -> Path: }} """ ) - - build_rs_path = crate.cargo_toml.parent / "build.rs" - build_rs_path.write_text(build_rs_src) return build_rs_path -def write_main_binding(crate: Crate) -> str: - # Get binding for main (redefined as _main) - main_binding = get_linked_binding("_main", crate.c_src_path, "-Dmain=_main") +def write_main_binding(crate: Crate) -> CodeRust: + main_binding_path, main_function = _write_main_binding( + crate.c_src_path, crate.rust_src_path.parent / "binding" + ) + crate.vcs.add(main_binding_path) + return CodeRust(main_function) - main_binding_path = crate.rust_src_path.parent / "binding" / "main.rs" - main_binding_path.parent.mkdir(exist_ok=True) + +def _write_main_binding(c_src_path: Path, binding_dir: Path) -> tuple[Path, str]: + main_binding = _get_linked_binding("_main", c_src_path, "-Dmain=_main") + main_binding_path = binding_dir / "main.rs" + main_binding_path.parent.mkdir(exist_ok=True, parents=True) main_binding_path.write_text( "\n\n".join( [ "#![allow(unused_attributes)]", - main_binding.text, + str(main_binding), ] ) ) - crate.vcs.add(main_binding_path) - # Return appropriate main function instead of writing to binding.rs - if "fn _main()" in main_binding.text: - return textwrap.dedent( + if "fn _main()" in str(main_binding): + main_function = textwrap.dedent( """ pub fn main() { let ret = unsafe { binding::main::_main() }; @@ -89,7 +94,7 @@ def write_main_binding(crate: Crate) -> str: """ ) else: - return textwrap.dedent( + main_function = textwrap.dedent( """ pub fn main() { let mut args: Vec<_> = std::env::args().into_iter().map(|s| std::ffi::CString::new(s).unwrap().into_raw()).collect(); @@ -98,31 +103,36 @@ def write_main_binding(crate: Crate) -> str: } """ ) + return main_binding_path, main_function + +def _generate_binding(crate: Crate, symbol: Symbol) -> tuple[Path, str, str | None]: + c_src_path = crate.c_src_path + binding_dir = crate.rust_src_path.parent / "binding" + symbol_spelling = symbol.spelling + is_main = crate.is_bin and symbol_spelling == "main" -def write_symbol_binding(crate: Crate, symbol_name: str): - rust_spelling = mangle(symbol_name) - symbol_binding = get_linked_binding(rust_spelling, crate.c_src_path) + logger.info(f"Generating binding for symbol '{symbol_spelling}' ...") + if is_main: + main_binding_path, main_function = _write_main_binding(c_src_path, binding_dir) + return main_binding_path, "main", main_function - symbol_binding_path = crate.rust_src_path.parent / "binding" / f"{rust_spelling}.rs" - symbol_binding_path.parent.mkdir(exist_ok=True) + rust_spelling = mangle(symbol_spelling) + symbol_binding = _get_linked_binding(rust_spelling, c_src_path) + symbol_binding_path = binding_dir / f"{rust_spelling}.rs" + symbol_binding_path.parent.mkdir(exist_ok=True, parents=True) symbol_binding_path.write_text( "\n\n".join( [ "#![allow(unused_attributes)]", - symbol_binding.text, + str(symbol_binding), ] ) ) - crate.vcs.add(symbol_binding_path) - - binding_path = crate.rust_src_path.parent / "binding.rs" - with binding_path.open("a+") as f: - f.write(f"pub mod {rust_spelling};\n") - crate.vcs.add(binding_path) + return symbol_binding_path, rust_spelling, None -def get_linked_binding(function_name: str, c_src_path: Path, *bindgen_args: str) -> CodeRust: +def _get_linked_binding(function_name: str, c_src_path: Path, *bindgen_args: str) -> CodeRust: # Use bindgen to generate binding to C symbol bindgen = [ "bindgen", @@ -159,15 +169,8 @@ def get_linked_binding(function_name: str, c_src_path: Path, *bindgen_args: str) def _main(cfg: BuildConfig) -> None: output_dir = Path(HydraConfig.get().runtime.output_dir) - if LARGE_PROJECT: - logger.info("Hybrid build is disabled; skipping build.rs generation!") - return - # Fetch crate - crate = Crate( - cargo_toml=output_dir / "Cargo.toml", - vcs=cfg.vcs, # type: ignore[reportArgumentType] - ) + crate = Crate(cfg.cargo_toml, vcs=cfg.vcs) # type: ignore[reportArgumentType] # Get global symbol table tu = create_translation_unit(crate.c_src_path) @@ -189,32 +192,31 @@ def _main(cfg: BuildConfig) -> None: # Verify build with build.rs builds, feedback = crate.cargo_build() if not builds: - raise RuntimeError(f"Crate at {output_dir} does not build with build.rs!\n{feedback}") + raise RuntimeError(f"Crate does not build!\n{feedback}") # Generate a Rust binding for any global function since we need to force the Rust # linker to include that C function in the Rust artifact. # FIXME: If we ever test variables we should generate bindings for those here too! + bindings: list[tuple[Path, str, str | None]] = [] + if len(global_functions) <= 1: + bindings.append(_generate_binding(crate, global_functions[0])) + else: + with ThreadPoolExecutor() as pool: + futures = [ + pool.submit(_generate_binding, crate, symbol) for symbol in global_functions + ] + bindings = [future.result() for future in futures] + + # Write bindings to file and stage with VCS + binding_modules_by_file: dict[Path, str] = {} + for binding_file_path, rust_spelling, main_function in bindings: + if main_function is not None: + with crate.rust_src_path.open("a+") as rust_file: + rust_file.write(main_function) + binding_modules_by_file[binding_file_path] = f"pub mod {rust_spelling};\n" binding_path = crate.rust_src_path.parent / "binding.rs" - binding_path.write_text("") - main_function = "" - for symbol in global_functions: - if not (symbol.is_function and symbol.is_definition and symbol.is_global): - continue - if crate.is_bin and symbol.spelling == "main": - # main requires special handling because we must bind to it as _main and - # statically create a Rust main that calls it - main_function = write_main_binding(crate) - else: - write_symbol_binding(crate, symbol.spelling) - - # Write main function and binding to it - with crate.rust_src_path.open("a+") as f: - f.write(main_function) - if main_function: - with binding_path.open("a+") as f: - f.write("pub mod main;\n") - crate.vcs.add(crate.rust_src_path) - crate.vcs.add(binding_path) + binding_path.write_text("".join(binding_modules_by_file.values())) + crate.vcs.add(*binding_modules_by_file.keys(), crate.rust_src_path, binding_path) # Make the bindings module visible in the crate rust_src = crate.rust_src_path.read_text() @@ -230,7 +232,7 @@ def _main(cfg: BuildConfig) -> None: # Attempt a final build builds, feedback = crate.cargo_build() if not builds: - raise RuntimeError(f"Crate at {output_dir} does not build with build.rs!\n{feedback}") + raise RuntimeError(f"Crate does not build!\n{feedback}") msg = f"Generated build artifacts for `{crate.root_package['name']}`" logger.info(msg) crate.vcs.commit(msg) diff --git a/src/ideas/init/consolidate.py b/src/ideas/init/consolidate.py index b0cccd7..0d55998 100644 --- a/src/ideas/init/consolidate.py +++ b/src/ideas/init/consolidate.py @@ -4,41 +4,35 @@ # SPDX-License-Identifier: Apache-2.0 # -import sys import os +import sys import logging from pathlib import Path from functools import cmp_to_key from dataclasses import dataclass from graphlib import TopologicalSorter, CycleError +from concurrent.futures import ProcessPoolExecutor import hydra import networkx as nx from omegaconf import MISSING from hydra.core.config_store import ConfigStore from hydra.core.hydra_config import HydraConfig -from clang.cindex import CompilationDatabase, TranslationUnit, CursorKind, Cursor +from clang.cindex import CompilationDatabase, TranslationUnit, CursorKind from clang.cindex import TranslationUnitLoadError, Diagnostic, StorageClass -from ideas.ast import ( - extract_info_c, - TreeResult, - Symbol, - clang_rename_, - get_system_macro_undefs, - mangle, -) -from ideas.tools import Crate, check_c, LARGE_PROJECT +from ideas.ast import extract_info_c, TreeResult, Symbol, clang_rename_, mangle, CodeC +from ideas.tools import Crate, check_c logger = logging.getLogger("ideas.init.consolidate") @dataclass class ConsolidateConfig: - filename: Path = MISSING cargo_toml: Path = MISSING vcs: str = "none" + compile_commands: Path = MISSING source_priority: Path | None = None @@ -46,14 +40,16 @@ class ConsolidateConfig: cs.store(name="init.consolidate", node=ConsolidateConfig) -def init(compile_commands: Path, source_priority: list[Path]) -> str: +def init(compile_commands: Path, source_priority: list[Path]) -> CodeC: # Get symbol table and dependencies taking into account source priority asts = get_asts(compile_commands, source_priority) ast_order = create_ast_order(source_priority, asts) - symbols, dependencies = get_symbols_and_dependencies(asts, ast_order=ast_order) + symbols, dependencies = get_symbols_and_dependencies( + asts, ast_order=ast_order, filter_system_symbols=False + ) logger.info(f"Found {len(symbols)} symbols in {compile_commands}!") - # Consolidate C sources in lexicographical topological order + # Sort symbols in lexicographical topological order symbol_lexical_key = create_symbol_lexical_key_fn(symbols, ast_order) sorted_symbol_groups = list( nx.lexicographical_topological_sort( @@ -62,47 +58,67 @@ def init(compile_commands: Path, source_priority: list[Path]) -> str: ) ) - includes = get_includes(symbols) - feature_defines = get_feature_defines(compile_commands) - feature_undefs = [f"#undef {line.split()[1]}" for line in feature_defines] - sources = feature_defines + includes + feature_undefs + [""] + # Consolidate C sources keyed by raw snippet with optional line directive. + sources: dict[CodeC, str | None] = {} for group in sorted_symbol_groups: # Add forward declarations if more than one symbol in group if len(group) > 1: for name in group: - declaration = symbols[name].declaration - if declaration and declaration.text not in sources: - sources.append(declaration.text) + symbol = symbols[name] + declaration = symbol.declaration + if declaration is None: + continue + if declaration in sources: + continue + sources[declaration] = ( + str(symbol.declaration_line_directive) + if symbol.declaration_line_directive is not None + else None + ) - # Add symbol definitions + # Add symbol code for name in group: - definition = symbols[name].code.text - if definition not in sources: - sources.append(definition) - - # Prevent double-expansion of self-referencing macros from signal.h - header_len = len(feature_defines) + len(includes) + len(feature_undefs) - signal_includes = [inc for inc in includes if "signal.h" in inc] - code_text = "\n".join(sources[header_len:]) - undefs = get_system_macro_undefs(signal_includes, code_text) - if undefs: - sources = ( - feature_defines + includes + feature_undefs + undefs + [""] + sources[header_len:] - ) + symbol = symbols[name] + code = symbol.code + if code in sources: + continue + sources[code] = ( + str(symbol.line_directive) if symbol.line_directive is not None else None + ) - return "\n".join(sources) + return CodeC.join( + snippet if directive is None else CodeC(directive + str(snippet)) + for snippet, directive in sources.items() + ) def get_symbols_and_dependencies( asts: list[TreeResult], external_symbol_names: list[str] | None = None, ast_order: dict[Path, TreeResult] | None = None, + filter_system_symbols: bool = True, ) -> tuple[dict[str, Symbol], dict[tuple[str, ...], list[tuple[str, ...]]]]: - # Merge ASTs into non-system project dependencies - list_of_non_system_symbols = [ - {n: s for n, s in ast.symbols.items() if not s.is_system} for ast in asts - ] - project_symbols = merge_symbols(list_of_non_system_symbols, ast_order) + list_of_symbols: list[dict[str, Symbol]] = [ast.symbols for ast in asts] + if filter_system_symbols: + + def is_system_symbol(symbol: Symbol) -> bool: + if symbol.is_system: + logger.debug(f"Ignoring system symbol `{symbol.name}`") + return True + if symbol.presumed_path is None: + return False + if os.path.commonpath([symbol.presumed_path, symbol.tu_path]) == "/": + logger.debug(f"Ignoring system symbol {symbol.name}") + return True + return False + + list_of_symbols = [ + {name: symbol for name, symbol in symbols.items() if not is_system_symbol(symbol)} + for symbols in list_of_symbols + ] + + # Merge ASTs into project dependencies + project_symbols = merge_symbols(list_of_symbols, ast_order) project_dependencies = nx.compose_all( [ nx.from_dict_of_lists(ast.complete_graph, create_using=nx.DiGraph) # type: ignore @@ -135,12 +151,6 @@ def get_symbols_and_dependencies( scc_map = {n: tuple(sorted(C.nodes[n]["members"], key=symbol_lexical_key)) for n in C.nodes} dependencies = {scc_map[n]: [scc_map[s] for s in C.successors(n)] for n in C.nodes} - # Force pure type-declaration SCCs into one SCC unit without introducing cycles. - if not LARGE_PROJECT: - dependencies = _merge_pure_type_declaration_sccs( - C, scc_map, symbols, symbol_lexical_key - ) - # Make sure dependencies are topologically sortable try: list(TopologicalSorter(dependencies).static_order()) @@ -160,7 +170,7 @@ def create_ast_order( first_symbol = next(iter(tree.symbols.values()), None) if first_symbol is None: continue - path = first_symbol.source_path + path = first_symbol.tu_path ast_by_path[path] = tree ast_order: dict[Path, TreeResult] = {} @@ -193,13 +203,13 @@ def compare_symbol_lexical(a: str | tuple[str, ...], b: str | tuple[str, ...]) - a_symbol = symbols[a_name] b_symbol = symbols[b_name] - a_tu = a_symbol.source_path - b_tu = b_symbol.source_path + a_tu = a_symbol.tu_path + b_tu = b_symbol.tu_path - # If symbols are from the same translation unit, then we can directly - # compare their locations for lexical ordering. + # If symbols are from the same translation unit, compare their + # preorder traversal indices for lexical ordering. if a_tu == b_tu: - return _cmp_cursor_loc(a_symbol.cursor, b_symbol.cursor) + return _cmp_symbol_tu_order(a_symbol, b_symbol) if ast_order is None: raise RuntimeError( @@ -207,24 +217,24 @@ def compare_symbol_lexical(a: str | tuple[str, ...], b: str | tuple[str, ...]) - ) # If a's USR appears in b's TU with matching code, both symbols are - # present in b_tu and can be compared by their locations there + # present in b_tu and can be compared by TU preorder index there. b_ast = ast_order.get(b_tu) if ( b_ast is not None and a_name in b_ast.symbols - and b_ast.symbols[a_name].code.text == a_symbol.code.text + and b_ast.symbols[a_name].code == a_symbol.code ): - return _cmp_cursor_loc(b_ast.symbols[a_name].cursor, b_symbol.cursor) + return _cmp_symbol_tu_order(b_ast.symbols[a_name], b_symbol) # If b's USR appears in a's TU with matching code, both symbols are - # present in a_tu and can be compared by their locations there + # present in a_tu and can be compared by TU preorder index there. a_ast = ast_order.get(a_tu) if ( a_ast is not None and b_name in a_ast.symbols - and a_ast.symbols[b_name].code.text == b_symbol.code.text + and a_ast.symbols[b_name].code == b_symbol.code ): - return _cmp_cursor_loc(a_symbol.cursor, a_ast.symbols[b_name].cursor) + return _cmp_symbol_tu_order(a_symbol, a_ast.symbols[b_name]) # The symbol's USR is not shared across TUs, so fall back to ordering # by the position of each symbol's TU in ast_order (source priority) @@ -246,206 +256,49 @@ def compare_symbol_lexical(a: str | tuple[str, ...], b: str | tuple[str, ...]) - return cmp_to_key(compare_symbol_lexical) -def _cmp_cursor_loc(cursor_a: Cursor, cursor_b: Cursor) -> int: - loc_a = cursor_a.location - loc_b = cursor_b.location - if loc_a < loc_b: +def _cmp_symbol_tu_order(symbol_a: Symbol, symbol_b: Symbol) -> int: + if symbol_a.tu_path != symbol_b.tu_path: + raise ValueError( + "Cannot compare TU preorder indices for symbols from different translation units:" + f" {symbol_a.name} @ {symbol_a.tu_path} vs {symbol_b.name} @ {symbol_b.tu_path}" + ) + + order_a = symbol_a.tu_preorder_index + order_b = symbol_b.tu_preorder_index + if order_a < order_b: return -1 - if loc_b < loc_a: + if order_b < order_a: return 1 - if cursor_a.get_usr() != cursor_b.get_usr(): + if symbol_a.name != symbol_b.name: raise ValueError( f"Unable to order distinct symbols with identical lexical priority and location:" - f" {cursor_a.get_usr()} @ {loc_a} vs {cursor_b.get_usr()} @ {loc_b}" + f" {symbol_a.name} @ {order_a} vs {symbol_b.name} @ {order_b}" ) return 0 -def _merge_pure_type_declaration_sccs( - condensed: nx.DiGraph, - scc_map: dict[int, tuple[str, ...]], - symbols: dict[str, Symbol], - symbol_lexical_key_fn, -) -> dict[tuple[str, ...], list[tuple[str, ...]]]: - base_dependencies: dict[tuple[str, ...], list[tuple[str, ...]]] = { - scc_map[n]: sorted( - (scc_map[s] for s in condensed.successors(n)), key=symbol_lexical_key_fn - ) - for n in condensed.nodes - } - - # Only include SCCs where every member is a type declaration - type_scc_nodes = [ - n - for n, members in scc_map.items() - if members - and all( - symbols[name].kind - in ( - CursorKind.STRUCT_DECL, - CursorKind.UNION_DECL, - CursorKind.ENUM_DECL, - CursorKind.FIELD_DECL, - CursorKind.ENUM_CONSTANT_DECL, - CursorKind.TYPEDEF_DECL, - ) - for name in members - ) - ] - if len(type_scc_nodes) <= 1: - return base_dependencies - - # Merge all pure type declaration SCCs into one SCC unit and update dependencies - # accordingly without introducing cycles. Preserve the dependency order - # between the original type SCCs so by-value type definitions remain valid. - type_scc_set = set(type_scc_nodes) - ordered_type_members_by_scc = { - n: tuple(sorted(scc_map[n], key=symbol_lexical_key_fn)) for n in type_scc_nodes - } - ordered_type_scc_nodes = sorted( - type_scc_nodes, - key=lambda n: tuple( - symbol_lexical_key_fn(name) for name in ordered_type_members_by_scc[n] - ), - ) - type_scc_dependencies = { - n: tuple( - succ - for succ in ordered_type_scc_nodes - if succ in set(condensed.successors(n)) and succ in type_scc_set - ) - for n in ordered_type_scc_nodes - } - try: - merged_group = tuple( - name - for n in TopologicalSorter(type_scc_dependencies).static_order() - for name in ordered_type_members_by_scc[n] - ) - except CycleError: - merged_group = tuple( - name for n in ordered_type_scc_nodes for name in ordered_type_members_by_scc[n] - ) - - merged_dependencies: dict[tuple[str, ...], set[tuple[str, ...]]] = {} - merged_successors: set[tuple[str, ...]] = set() - - for n in condensed.nodes: - if n in type_scc_set: - for succ in condensed.successors(n): - if succ not in type_scc_set: - merged_successors.add(scc_map[succ]) - continue - - group = scc_map[n] - merged_dependencies.setdefault(group, set()) - for succ in condensed.successors(n): - if succ in type_scc_set: - merged_dependencies[group].add(merged_group) - else: - merged_dependencies[group].add(scc_map[succ]) - - merged_dependencies.setdefault(merged_group, set()) - merged_dependencies[merged_group].update(merged_successors) - - return { - group: sorted(successors, key=symbol_lexical_key_fn) - for group, successors in merged_dependencies.items() - } - - -def get_feature_defines(compile_commands: Path) -> list[str]: - db = CompilationDatabase.fromDirectory(compile_commands.parent) - cmds = db.getAllCompileCommands() - if cmds is None: - return [] - - defines: dict[str, str | None] = {} # name -> value (None if no value) - for cmd in cmds: - args = iter(cmd.arguments) - for arg in args: - # Glued "-Dstuff" - if arg.startswith("-D") and len(arg) > 2: - macro = arg[2:] - # Separate ["-D", "stuff"] - elif arg == "-D": - macro = next(args, None) - assert macro is not None, ( - f"Malformed compile command: -D without value in {cmd.filename}" - ) - else: - continue - - name, _, value = macro.partition("=") - # Implementation-reserved namespace (_[A-Z]...) per the C standard. - if not (len(name) >= 2 and name[0] == "_" and name[1].isupper()): - continue - - current_value = value if value else None - if name in defines and defines[name] != current_value: - logger.warning( - "Feature-test macro %s has conflicting values across TUs: " - "%s vs %s (keeping latter)", - name, - defines[name], - current_value, - ) - defines[name] = current_value - - return [f"#define {n} {v}" if v else f"#define {n}" for n, v in defines.items()] - - -def get_includes(symbols: dict[str, Symbol]) -> list[str]: - includes: list[str] = [] - for symbol in symbols.values(): - tu = symbol.cursor.translation_unit - for inclusion in tu.get_includes(): - # Source of the include should be in same path as TU while the include should NOT be - tu_path = str(Path(tu.spelling).resolve()) - inclusion_source_path = str(Path(inclusion.source.name).resolve()) - inclusion_include_path = str(Path(inclusion.include.name).resolve()) - # FIXME: Use is_in_system_header? Inclusion locations are always false though. - if (os.path.commonprefix((tu_path, inclusion_source_path)) != "/") and ( - os.path.commonprefix((tu_path, inclusion_include_path)) == "/" - ): - # Get include directive from source - with open(inclusion.location.file.name, "rb") as f: - f.seek(inclusion.location.offset) - include = f.readline().decode().strip() - include = f"#include {include}" - if include not in includes: - includes.append(include) - return includes - - def get_asts( compile_commands: Path, valid_paths: list[Path], rename_conflicting_symbols: bool = True ) -> list[TreeResult]: assert compile_commands.name == "compile_commands.json" db = CompilationDatabase.fromDirectory(compile_commands.parent) cmds = db.getAllCompileCommands() - assert cmds is not None - asts = [] - for i in range(len(cmds)): - cmd = cmds[i] - logger.info(f"Parsing TU {cmd.filename} ({i + 1}/{len(cmds)}) ...") - try: - tu = TranslationUnit.from_source(None, args=list(cmd.arguments)) - except TranslationUnitLoadError as e: - raise TranslationUnitLoadError( - f"Error parsing '{cmd.filename}' using args `{' '.join(cmd.arguments)}`\n{e}" - ) - if any(d.severity >= Diagnostic.Error for d in tu.diagnostics): - raise TranslationUnitLoadError( - "\n".join( - [d.format() for d in tu.diagnostics] - + [f"Error parsing '{cmd.filename}' using args `{' '.join(cmd.arguments)}`"] - ) - ) - assert tu.cursor is not None - if not valid_paths or Path(tu.cursor.spelling).resolve() in valid_paths: - ast = extract_info_c(tu) - asts.append(ast) + if cmds is None or len(cmds) == 0: + return [] + + valid_paths_set = {path.resolve() for path in valid_paths} + maybe_asts: list[TreeResult | None] + if len(cmds) <= 1: + maybe_asts = [_get_ast(cmds[0].filename, list(cmds[0].arguments), valid_paths_set)] + else: + with ProcessPoolExecutor() as pool: + futures = [ + pool.submit(_get_ast, cmd.filename, list(cmd.arguments), valid_paths_set) + for cmd in cmds + ] + maybe_asts = [future.result() for future in futures] + asts = [ast for ast in maybe_asts if ast is not None] + if rename_conflicting_symbols: original_sources = rename_conflicting_symbols_(asts) if original_sources: @@ -458,6 +311,29 @@ def get_asts( return asts +def _get_ast(filename: str, arguments: list[str], valid_paths: set[Path]) -> TreeResult | None: + logger.info(f"Parsing {filename} ...") + try: + tu = TranslationUnit.from_source(None, args=arguments) + if any(d.severity >= Diagnostic.Error for d in tu.diagnostics): + raise TranslationUnitLoadError("\n".join(d.format() for d in tu.diagnostics)) + except TranslationUnitLoadError as e: + raise TranslationUnitLoadError( + f"Failed to parse '{filename}' with compile arguments:\n" + f" {' '.join(arguments)}\n\n" + f"{e}" + ) + + assert tu.cursor is not None + source_path = Path(tu.cursor.spelling).resolve() + if valid_paths and source_path not in valid_paths: + return None + tree = extract_info_c(tu) + tree.filename = filename + tree.arguments = arguments + return tree + + def rename_conflicting_symbols_(asts: list[TreeResult]) -> dict[Path, bytes]: # Gather best representative symbol per spelling per AST into a single dict symbols_with_spelling: dict[str, list[Symbol]] = {} @@ -486,7 +362,7 @@ def rename_conflicting_symbols_(asts: list[TreeResult]) -> dict[Path, bytes]: # Find symbols with common spelling but different definitions across ASTs. # Group definitions by code to avoid O(n^2) pairwise comparison. - tu_renames: dict[TranslationUnit, dict[str, str]] = {} + tu_renames: dict[Path, dict[str, str]] = {} used_spellings = set(symbols_with_spelling.keys()) for spelling, symbols in symbols_with_spelling.items(): # Only definitions and variables can conflict @@ -495,9 +371,9 @@ def rename_conflicting_symbols_(asts: list[TreeResult]) -> dict[Path, bytes]: continue # Group definitions by their code text - identical code means no conflict - code_groups: dict[str, list[Symbol]] = {} + code_groups: dict[CodeC, list[Symbol]] = {} for sym in definitions: - code_groups.setdefault(sym.code.text, []).append(sym) + code_groups.setdefault(sym.code, []).append(sym) if len(code_groups) <= 1: continue @@ -517,16 +393,16 @@ def rename_conflicting_symbols_(asts: list[TreeResult]) -> dict[Path, bytes]: for sym in definitions: if sym.is_system: continue - if sym.is_global and sym.parent is None and sym.kind not in NON_LINKED_KINDS: + if sym.is_global and sym.is_top_level and sym.kind not in NON_LINKED_KINDS: continue - path = sym.source_path + path = sym.tu_path new_spelling = mangle(path.stem) + "_" + sym.spelling while new_spelling in used_spellings: path = path.parent new_spelling = mangle(path.stem) + "_" + new_spelling used_spellings.add(new_spelling) - tu_renames.setdefault(sym.cursor.translation_unit, {})[sym.name] = new_spelling + tu_renames.setdefault(sym.tu_path, {})[sym.name] = new_spelling if not tu_renames: return {} @@ -542,19 +418,27 @@ def rename_conflicting_symbols_(asts: list[TreeResult]) -> dict[Path, bytes]: existing_spellings.update(new_spellings) # Write renames to disk while keeping track of original source bytes - sources: dict[Path, bytes] = {} - try: - for tu, renames in tu_renames.items(): - # Reparse translation unit in case anything has changed on disk - tu.reparse() - clang_rename_(tu, renames, sources) + tu_args: dict[Path, list[str]] = {} + for ast in asts: + if ast.filename is None or ast.arguments is None: + continue + tu_args[Path(ast.filename).resolve()] = list(ast.arguments) + original_sources: dict[Path, bytes] = {} + try: + for tu_path, renames in tu_renames.items(): + args = tu_args.get(tu_path) + if args is not None: + tu = TranslationUnit.from_source(None, args=list(args)) + else: + tu = TranslationUnit.from_source(str(tu_path)) + clang_rename_(tu, renames, original_sources) except Exception: # Restore original source code if renaming fails before caller can reparse. - for path, source in sources.items(): + for path, source in original_sources.items(): path.write_bytes(source) raise - return sources + return original_sources def merge_symbols( @@ -575,32 +459,26 @@ def merge_symbols( if global_symbols[name].code == symbol.code: continue - global_source = global_symbols[name].source_path - symbol_source = symbol.source_path + global_source = global_symbols[name].tu_path + symbol_source = symbol.tu_path # If overwriting a symbol, then prefer one with a definition - if ( - global_symbols[name].cursor.is_definition() - and not symbol.cursor.is_definition() - ): + if global_symbols[name].is_definition and not symbol.is_definition: continue - elif ( - not global_symbols[name].cursor.is_definition() - and symbol.cursor.is_definition() - ): + elif not global_symbols[name].is_definition and symbol.is_definition: global_symbols[name] = symbol # Prefer non-extern variable declaration over extern one (e.g. tentative definition) elif ( - symbol.cursor.kind == CursorKind.VAR_DECL - and global_symbols[name].cursor.storage_class == StorageClass.EXTERN - and symbol.cursor.storage_class != StorageClass.EXTERN + symbol.kind == CursorKind.VAR_DECL + and global_symbols[name].storage_class == StorageClass.EXTERN + and symbol.storage_class != StorageClass.EXTERN ): global_symbols[name] = symbol # Never replace a non-extern variable with an extern one elif ( - global_symbols[name].cursor.kind == CursorKind.VAR_DECL - and global_symbols[name].cursor.storage_class != StorageClass.EXTERN - and symbol.cursor.storage_class == StorageClass.EXTERN + global_symbols[name].kind == CursorKind.VAR_DECL + and global_symbols[name].storage_class != StorageClass.EXTERN + and symbol.storage_class == StorageClass.EXTERN ): continue # Or prefer the symbol with source priority @@ -629,28 +507,24 @@ def merge_symbols( def _main(cfg: ConsolidateConfig): - if LARGE_PROJECT: - logger.info("LARGE_PROJECT mode enabled: consolidation is disabled!") - return - output_dir = Path(HydraConfig.get().runtime.output_dir) # Get crate information - crate = Crate(cargo_toml=cfg.cargo_toml, vcs=cfg.vcs) # type: ignore[reportArgumentType] + crate = Crate(cfg.cargo_toml, vcs=cfg.vcs) # type: ignore[reportArgumentType] source_priority: list[Path] = [] if cfg.source_priority: lines = cfg.source_priority.read_text().splitlines() source_priority = [Path(line.strip()).resolve() for line in lines if line.strip()] - output = init(cfg.filename, source_priority) + output = init(cfg.compile_commands, source_priority) # Only run preprocess, compile, and assemble steps on C code compiles, compile_errors = check_c(output, flags=["-c"]) # Write C code to disk crate.c_src_path.parent.mkdir(exist_ok=True, parents=True) - crate.c_src_path.write_text(output) + crate.c_src_path.write_text(str(output)) crate.vcs.add(crate.c_src_path) # Add hydra directory diff --git a/src/ideas/init/crate.py b/src/ideas/init/crate.py index 1bf3c3c..4b870a4 100644 --- a/src/ideas/init/crate.py +++ b/src/ideas/init/crate.py @@ -8,7 +8,7 @@ import logging import tomlkit from pathlib import Path -from dataclasses import dataclass +from dataclasses import dataclass, field import hydra from omegaconf import MISSING @@ -22,16 +22,21 @@ @dataclass class CrateConfig: - crate_type: str = MISSING + cargo_toml: Path = MISSING + template: str = MISSING vcs: str = "none" reexport_lib: bool = True + workspace_dependencies: list[str] = field(default_factory=list) def __post_init__(self): - if self.crate_type not in ["bin", "lib"]: - raise ValueError(f"Invalid crate type: {self.crate_type}!") + if self.template not in ["bin", "lib"]: + raise ValueError(f"Invalid crate template: {self.template}!") if self.vcs not in ["git", "none"]: raise ValueError(f"Invalid VCS: {self.vcs}!") + for dep in self.workspace_dependencies: + if not dep.strip(): + raise ValueError("workspace_dependencies entries must be non-empty crate names") cs = ConfigStore.instance() @@ -42,11 +47,7 @@ def _main(cfg: CrateConfig) -> None: output_dir = Path(HydraConfig.get().runtime.output_dir) # Initialize crate - crate = Crate( - cargo_toml=output_dir / "Cargo.toml", - type=cfg.crate_type, # type: ignore[reportArgumentType] - vcs=cfg.vcs, # type: ignore[reportArgumentType] - ) + crate = Crate(cfg.cargo_toml, template=cfg.template, vcs=cfg.vcs) # type: ignore[reportArgumentType] # Delete default cargo init code crate.rust_src_path.write_text("") @@ -61,15 +62,16 @@ def _main(cfg: CrateConfig) -> None: crate.cargo_add(dep="serde_json@1", section="dev") crate.cargo_add(dep="tempfile@3", section="dev") crate.cargo_add(dep="cc@1.2.53", section="build") + crate.add_workspace_dependencies(cfg.workspace_dependencies) - if cfg.crate_type == "bin": + if crate.is_bin: # Add static test dependencies crate.cargo_add(dep="assert_cmd@2.0.17", section="dev") crate.cargo_add(dep="predicates@3.1.3", section="dev") # Disable default tests cargo_toml = tomlkit.loads(crate.cargo_toml.read_text()) - if cfg.crate_type == "bin": + if crate.is_bin: bin, found = cargo_toml.get("bin", list()), False for target in bin: if target.get("name", None) == crate.root_package["name"]: @@ -78,14 +80,14 @@ def _main(cfg: CrateConfig) -> None: if not found: bin.append({"name": crate.root_package["name"], "test": False}) cargo_toml["bin"] = bin - if cfg.crate_type == "lib": + else: lib = cargo_toml.get("lib", dict()) lib.update({"test": False, "doctest": False}) cargo_toml["lib"] = lib crate.cargo_toml.write_text(tomlkit.dumps(cargo_toml)) # Export cdylib - if cfg.crate_type == "lib" and cfg.reexport_lib: + if not crate.is_bin and cfg.reexport_lib: cargo_toml = tomlkit.loads(crate.cargo_toml.read_text()) lib = cargo_toml.get("lib", dict()) lib.update({"crate-type": ["lib", "cdylib"]}) diff --git a/src/ideas/test_symbol.py b/src/ideas/test_symbol.py index ce4c01f..69d7a33 100644 --- a/src/ideas/test_symbol.py +++ b/src/ideas/test_symbol.py @@ -11,7 +11,7 @@ from ideas.tools import Crate from ideas.ast import Symbol -from ideas.init.build import write_main_binding +from ideas.init.build import write_main_binding, CodeRust logger = logging.getLogger("ideas.test_symbol") @@ -21,6 +21,7 @@ def __init__(self, crate: Crate, symbols: list[Symbol], tests: str): super().__init__() self.crate = crate self.tests = tests + self.main_function: CodeRust | None = None for symbol in symbols: if not (symbol.is_function and symbol.is_definition and symbol.is_global): @@ -28,7 +29,7 @@ def __init__(self, crate: Crate, symbols: list[Symbol], tests: str): if self.crate.is_bin and symbol.spelling == "main": # main requires special handling because we must bind to it as _main and # statically create a Rust main that calls it - self.main_function: str = write_main_binding(crate) + self.main_function = write_main_binding(crate) def test( self, tests: str, skip: list[str] | None = None @@ -56,9 +57,9 @@ def test( # Try building the crate to detect if we need to insert a main builds, feedback = self.crate.cargo_build(fix_E0601=False) - if "error[E0601]" in feedback and self.main_function: + if "error[E0601]" in feedback and self.main_function is not None: with self.crate.rust_src_path.open("a+") as f: - f.write(self.main_function) + f.write(str(self.main_function)) self.crate.vcs.add(wrapper_path, binding_path, self.crate.rust_src_path) @@ -70,7 +71,7 @@ def test( passes, jsonl, error, _ = self.crate.cargo_test( tests, skip=skip, test_harness="nextest run", message_format="libtest-json" ) - results = extract_test_results(jsonl) + results = _extract_test_results(jsonl) return passes, results, error def forward(self, symbol: Symbol, skip: list[str] | None = None) -> dspy.Prediction: @@ -117,7 +118,7 @@ def forward(self, symbol: Symbol, skip: list[str] | None = None) -> dspy.Predict return pred -def extract_test_results(output: str) -> dict[str, bool]: +def _extract_test_results(output: str) -> dict[str, bool]: test_results: dict[str, bool] = {} for line in output.splitlines(): diff --git a/src/ideas/tools.py b/src/ideas/tools.py index f35d151..e2aaf5e 100644 --- a/src/ideas/tools.py +++ b/src/ideas/tools.py @@ -17,6 +17,8 @@ from tempfile import TemporaryDirectory from pathlib import Path +from .ast import CodeC + TestCase = dict[str, None | str | int | float | list[int] | list[str] | list[float]] @@ -112,7 +114,7 @@ def __init__( cargo_toml: Path, vcs: Literal["none", "git"] = "none", ): - self.cargo_toml = cargo_toml + self.cargo_toml = cargo_toml.resolve() workspace_dir = self.cargo_toml.parent self.vcs = VCS(repo_dir=workspace_dir, vcs=vcs) @@ -132,18 +134,18 @@ def __init__( self, cargo_toml: Path, vcs: Literal["none", "git"] = "none", - type: Literal["bin", "lib"] | None = None, + template: Literal["bin", "lib"] | None = None, ): - self.cargo_toml = cargo_toml + self.cargo_toml = cargo_toml.resolve() crate_dir = self.cargo_toml.parent self.vcs = VCS(repo_dir=crate_dir, vcs=vcs) if not self.cargo_toml.exists(): - # Create a new crate with specified type, but without VCS - if not type: + # Create a new crate with specified template, but without VCS + if not template: raise ValueError( - f"Crate at {crate_dir} does not exist; type must be specified!" + f"Crate at {crate_dir} does not exist; template must be specified!" ) os.makedirs(crate_dir, exist_ok=True) success, output, error, _ = run_subprocess( @@ -151,7 +153,7 @@ def __init__( "cargo", "init", "--quiet", - f"--{type}", + f"--{template}", "--vcs=none", str(crate_dir), ] @@ -272,6 +274,51 @@ def cargo_feature(self, **features: list[str]) -> None: # Invalidate cached metadata self.invalidate_metadata() + def add_workspace_dependencies(self, names: list[str]) -> None: + if not names: + return + + workspace_root = self.workspace_root + workspace_toml_path = workspace_root / "Cargo.toml" + workspace_toml = tomlkit.loads(workspace_toml_path.read_text()) + workspace_table = workspace_toml.get("workspace", tomlkit.table()) + workspace_deps = workspace_table.get("dependencies", tomlkit.table()) + + workspace_member_ids = set(self.metadata.get("workspace_members", [])) + workspace_packages = { + pkg["name"]: pkg + for pkg in self.metadata.get("packages", []) + if pkg.get("id") in workspace_member_ids + } + + cargo_toml = tomlkit.loads(self.cargo_toml.read_text()) + dependencies = cargo_toml.get("dependencies", tomlkit.table()) + + for dep_name in names: + dep_name = dep_name.strip() + if not dep_name: + raise ValueError("workspace dependency names must be non-empty") + + pkg = workspace_packages.get(dep_name) + if pkg is None: + raise ValueError( + f"Workspace dependency `{dep_name}` was not found among workspace members" + ) + + manifest_path = Path(pkg["manifest_path"]) + dep_dir = manifest_path.parent + rel_path = dep_dir.relative_to(workspace_root) + workspace_deps[dep_name] = {"path": str(rel_path)} + dependencies[dep_name] = {"workspace": True} + + workspace_table["dependencies"] = workspace_deps + workspace_toml["workspace"] = workspace_table + workspace_toml_path.write_text(tomlkit.dumps(workspace_toml)) + + cargo_toml["dependencies"] = dependencies + self.cargo_toml.write_text(tomlkit.dumps(cargo_toml)) + self.invalidate_metadata() + def cargo_clean(self) -> None: cmd = [ "cargo", @@ -439,7 +486,7 @@ def run_subprocess( def check_c( - code: str, + code: CodeC, *, flags: list[str] | None = None, ) -> tuple[bool, str]: @@ -454,7 +501,7 @@ def check_c( cmd.append("-") cmd.extend(["-o", "/dev/null"]) - success, output, error, _ = run_subprocess(cmd, input=code) + success, output, error, _ = run_subprocess(cmd, input=str(code)) return success, output + error diff --git a/src/ideas/translate.py b/src/ideas/translate.py index d2638ef..1f856a0 100644 --- a/src/ideas/translate.py +++ b/src/ideas/translate.py @@ -18,7 +18,7 @@ from ideas import adapters, model, ModelConfig, GenerateConfig from ideas import SnippetTranslator, RecurrentTranslator, WrapperGenerator, SymbolTester from ideas import create_translation_unit, extract_info_c -from ideas.init.consolidate import get_symbols_and_dependencies, get_asts, create_ast_order +from ideas.init.consolidate import get_symbols_and_dependencies from .tools import Crate, LARGE_PROJECT logger = logging.getLogger("ideas.translate") @@ -33,8 +33,6 @@ class TranslateConfig: cargo_toml: Path = MISSING tests: str = MISSING - source_priority: Path | None = None - translator: str = "ChainOfThought" translator_max_iters: int = 5 wrapper_max_iters: int = 5 @@ -50,58 +48,41 @@ class TranslateConfig: def _main(cfg: TranslateConfig) -> None: output_dir = Path(HydraConfig.get().runtime.output_dir) logger.info(f"Saving results to {output_dir}") - crate = Crate(cargo_toml=cfg.cargo_toml.resolve(), vcs=cfg.vcs) # type: ignore[reportArgumentType] - - # Resolve source priority - source_priority: list[Path] = [] - if cfg.source_priority: - lines = cfg.source_priority.read_text().splitlines() - source_priority = [Path(line.strip()).resolve() for line in lines if line.strip()] + crate = Crate(cfg.cargo_toml, vcs=cfg.vcs) # type: ignore[reportArgumentType] # Save C source since it will be modified by the agent - if LARGE_PROJECT: - crate.c_src_path.write_text("") orig_c_src = crate.c_src_path.read_bytes() # Make sure Rust source is in known state (i.e., empty) crate.rust_src_path.write_text("") - if LARGE_PROJECT and (crate.cargo_toml.parent / "build.rs").exists(): - (crate.cargo_toml.parent / "build.rs").unlink() - crate.vcs.rm(crate.cargo_toml.parent / "build.rs", force=True) # Get global symbol table - if cfg.filename.suffix == ".c": - tu = create_translation_unit(cfg.filename) - asts = [extract_info_c(tu)] - ast_order = None - symbols, dependencies = get_symbols_and_dependencies( - asts, external_symbol_names=["c:@F@main"] if crate.is_bin else None - ) - else: - asts = get_asts(cfg.filename, source_priority) - ast_order = create_ast_order(source_priority, asts) - symbols, dependencies = get_symbols_and_dependencies( - asts, - external_symbol_names=["c:@F@main"] if crate.is_bin else None, - ast_order=ast_order, - ) + tu = create_translation_unit(cfg.filename) + asts = [extract_info_c(tu)] + symbols, dependencies = get_symbols_and_dependencies( + asts, external_symbol_names=["c:@F@main"] if crate.is_bin else None + ) # Create translation agent model.configure(cfg.model, cfg.generate) dspy.configure(adapter=adapters.ChatAdapter()) translator = getattr(dspy, cfg.translator) - snippet_translator = SnippetTranslator(translator, crate, cfg.translator_max_iters) - symbol_wrapper, symbol_tester = None, None + snippet_translator = SnippetTranslator(crate, translator, cfg.translator_max_iters) + symbol_wrapper = WrapperGenerator(crate, cfg.wrapper_max_iters) + symbol_tester = None if not LARGE_PROJECT: - symbol_wrapper = WrapperGenerator(crate, cfg.wrapper_max_iters) symbol_tester = SymbolTester(crate, symbols=list(symbols.values()), tests=cfg.tests) agent = RecurrentTranslator( crate, snippet_translator, symbol_wrapper, symbol_tester, cfg.max_iters ) # Run translation agent and write it to disk - pred = agent(symbols, dependencies, ast_order) - crate.rust_src_path.write_text(pred.translation.text) + try: + pred = agent(symbols, dependencies) + crate.rust_src_path.write_text(str(pred.translation)) + except Exception as e: + logger.exception(e) + pred = dspy.Prediction(success=False) usage = model.format_usage(pred) if pred.success: msg = f"Translated `{crate.root_package['name']}` to Rust: {usage}" @@ -114,7 +95,7 @@ def _main(cfg: TranslateConfig) -> None: logger.error(msg) # Clean up intermediate artifacts produced during translation - _cleanup(crate, symbols) + _cleanup(crate) # Commit translation if (output_subdir := HydraConfig.get().output_subdir) is not None: @@ -123,7 +104,7 @@ def _main(cfg: TranslateConfig) -> None: crate.vcs.commit(msg) -def _cleanup(crate: Crate, symbols: dict) -> None: +def _cleanup(crate: Crate) -> None: # Remove bindgen artifacts crate.vcs.rm( crate.rust_src_path.parent / "binding", diff --git a/src/ideas/translate_recurrent.py b/src/ideas/translate_recurrent.py index 0681e25..4302e54 100644 --- a/src/ideas/translate_recurrent.py +++ b/src/ideas/translate_recurrent.py @@ -27,7 +27,7 @@ def __init__( self, crate: Crate, symbol_translator: dspy.Module, - symbol_wrapper: dspy.Module | None = None, + symbol_wrapper: dspy.Module, symbol_tester: dspy.Module | None = None, max_iters: int = 1, ): @@ -112,12 +112,12 @@ def forward( break if g in immediate_to_be_translated: for name in g: - code = symbols[name].code.text + code = symbols[name].code char_count = len(str(code)) if LARGE_PROJECT and total_chars + char_count > MAX_DEPENDENT_CHARS: exceeded = True break - dependent_parts.append(symbols[name].code) + dependent_parts.append(code) total_chars += char_count dependent_code = CodeC.join(dependent_parts) @@ -284,15 +284,11 @@ def translate( # Write translation to crate translation = pred.translation with self.crate.rust_src_path.open("a") as f: - f.write(translation.text + "\n") + f.write(str(translation) + "\n") - # Generate wrapper, that may modify the translation, for each symbol - unsafe_translation = translation + # Generate wrapper for each symbol wrappers: dict[str, dspy.Prediction] = {} for symbol in symbols: - # If we don't have a wrapper function, then skip the symbol - if self.wrap_symbol is None: - continue # We can only hybrid build-test functions and variables if not (symbol.is_function and symbol.is_definition) and not symbol.is_variable: continue @@ -305,11 +301,10 @@ def translate( wrapper = self.wrap_symbol( symbol=symbol, reference_code=reference_code, - translation=unsafe_translation, + translation=pred.translation, support_code=support_code + snippet + dependent_code, prior_wrapper=prior_wrapper, ) - unsafe_translation = wrapper.translation # Save function wrappers for next retry and caching if symbol.is_function and symbol.is_definition and "wrapper" in wrapper: @@ -339,9 +334,8 @@ def translate( # Cache successful translation and wrappers if pred.success: self.translate_symbol.write_cache(pred) - if self.wrap_symbol is not None: - for wrapper in wrappers.values(): - self.wrap_symbol.write_cache(wrapper) + for wrapper in wrappers.values(): + self.wrap_symbol.write_cache(wrapper) # Return wrappers for next retry pred.wrappers = {name: wrapper.wrapper for name, wrapper in wrappers.items()} diff --git a/src/ideas/translate_snippet.py b/src/ideas/translate_snippet.py index 1e6afe0..dcf7653 100644 --- a/src/ideas/translate_snippet.py +++ b/src/ideas/translate_snippet.py @@ -7,6 +7,7 @@ import logging import sqlite3 from pathlib import Path +from textwrap import indent import dspy from dspy.utils.exceptions import AdapterParseError @@ -140,8 +141,8 @@ class SnippetTranslatorSignature(dspy.Signature): class SnippetTranslator(dspy.Module): def __init__( self, - translator: type[dspy.Module], crate: Crate, + translator: type[dspy.Module], max_iters: int = 5, ): super().__init__() @@ -151,8 +152,8 @@ def __init__( "\n\n".join([signature.instructions, _crate_dependencies]) ) - self._translate = translator(signature) self.crate = crate + self._translate = translator(signature) self.max_iters = max_iters self.cache = _init_cache(crate.workspace_root / "cache.db") @@ -165,19 +166,18 @@ def forward( dependent_code: CodeC, prior_translation: CodeRust | None = None, feedback: str = "", - translation: CodeRust | None = None, ) -> dspy.Prediction: logger.info(f"Translating snippet `{name}` ...") - # If the snippet is empty, use static translation - if not snippet.text: + # Use cache when no prior translation + if prior_translation is None and not str(snippet): + # Special case for empty snippets: fall back to a static comment translation = CodeRust(f"// Empty snippet `{name}`") - - # Use cache when no translation nor prior translation - if translation is None and prior_translation is None: + elif prior_translation is None: translation = _read_cache(self.cache, name, snippet) else: logger.info("Ignoring snippet cache...") + translation = None orig_rust_src = self.crate.rust_src_path.read_bytes() pred = dspy.Prediction() @@ -195,7 +195,7 @@ def forward( # This allows static translations that violate safety, which will be fixed by the LLM! try: pred = self.translate( - rust_src if not LARGE_PROJECT else reference_context, + reference_code if not LARGE_PROJECT else reference_context, snippet, dependent_code, prior_translation, @@ -221,7 +221,7 @@ def forward( # Append translation and check if it builds rust_src += translation - self.crate.rust_src_path.write_text(rust_src.text) + self.crate.rust_src_path.write_text(str(rust_src)) self.crate.vcs.add(self.crate.rust_src_path) # FIXME: Checking name for c:@F@main is brittle but we have no better way here. # The proper way to fix is to yield the translation back to the caller so it can @@ -240,14 +240,16 @@ def forward( if builds: msg = f"Translated snippet `{name}`: {usage}" logger.info(msg) - msg += f"\n\n# Reasoning\n{pred.reasoning}" if "reasoning" in pred else "" + if "reasoning" in pred: + msg += f"\n\n# Reasoning\n{indent(pred.reasoning, ' ')}" self.crate.vcs.commit(msg) break msg = f"Failed to translate snippet `{name}` ({i + 1}/{self.max_iters}): {usage}" logger.error(msg) - msg += f"\n\n# Reasoning\n{pred.reasoning}" if "reasoning" in pred else "" - msg += f"\n\n# Feedback\n{feedback}" if feedback else "" + if "reasoning" in pred: + msg += f"\n\n# Reasoning\n{indent(pred.reasoning, ' ')}" + msg += f"\n\n# Feedback\n{indent(feedback, ' ')}" if feedback else "" self.crate.vcs.commit(msg) self.crate.rust_src_path.write_bytes(orig_rust_src) pred.name = name @@ -262,7 +264,7 @@ def forward( def translate( self, - rust_src: CodeRust, + reference_code: CodeRust, snippet: CodeC, dependent_code: CodeC, prior_translation: CodeRust | None, @@ -278,7 +280,7 @@ def translate( else: if parent_usage_tracker is None: pred = self._translate( - reference_code=rust_src, + reference_code=reference_code, snippet=snippet, dependent_code=dependent_code, prior_translation=prior_translation or CodeRust(), @@ -287,7 +289,7 @@ def translate( else: with track_usage() as local_usage_tracker: pred = self._translate( - reference_code=rust_src, + reference_code=reference_code, snippet=snippet, dependent_code=dependent_code, prior_translation=prior_translation or CodeRust(), @@ -350,7 +352,7 @@ def _read_cache(cache: Path | None, name: str, snippet: CodeC) -> CodeRust | Non try: row = conn.execute( "SELECT translation FROM snippet_translations WHERE snippet=? AND success=1 ORDER BY id DESC LIMIT 1", - (snippet.text,), + (str(snippet),), ).fetchone() if row is None: row = conn.execute( @@ -389,12 +391,12 @@ def _write_cache( """, ( name, - snippet.text, - reference_code.text, - dependent_code.text, - prior_translation.text, + str(snippet), + str(reference_code), + str(dependent_code), + str(prior_translation), feedback, - translation.text, + str(translation), int(success), ), ) diff --git a/src/ideas/wrapper.py b/src/ideas/wrapper.py index 971847d..a2050a5 100644 --- a/src/ideas/wrapper.py +++ b/src/ideas/wrapper.py @@ -7,8 +7,8 @@ import re import sqlite3 import logging -import textwrap from pathlib import Path +from textwrap import indent from collections import OrderedDict import dspy @@ -191,15 +191,15 @@ def generate_unimplemented_wrapper(path: Path, symbol_name: str) -> CodeRust: # pub extern "C" fn match_( # threshold: f64, # ) -> ::std::os::raw::c_int { - # unimplemented!(); + # unimplemented!() # } unimplemented_wrapper = re.sub( r'unsafe extern "C" {\s*.*\s+pub fn (.*);\s+}', - rf'#[unsafe(export_name="{symbol_name}")]\npub extern "C" fn \1 {{\n unimplemented!();\n}}', - bindgen_wrapper.text, + rf'#[unsafe(export_name="{symbol_name}")]\npub extern "C" fn \1 {{\n unimplemented!()\n}}', + str(bindgen_wrapper), flags=re.DOTALL, ) - if unimplemented_wrapper == bindgen_wrapper.text: + if unimplemented_wrapper == str(bindgen_wrapper): raise ValueError( f"Failed to convert bindgen output to function for `{symbol_name}`!\nWrapper:\n{unimplemented_wrapper}" ) @@ -226,7 +226,7 @@ class WrapperGenerator(dspy.Module): def __init__( self, crate: Crate, - max_iters: int, + max_iters: int = 5, ) -> None: super().__init__() self.crate = crate @@ -243,7 +243,6 @@ def forward( reference_code: CodeRust, translation: CodeRust, prior_wrapper: CodeRust | None = None, - wrapper: CodeRust | None = None, support_code: CodeC | None = None, ) -> dspy.Prediction: if symbol.is_function and symbol.is_definition: @@ -252,12 +251,11 @@ def forward( reference_code, translation, prior_wrapper=prior_wrapper, - wrapper=wrapper, support_code=support_code, ) elif symbol.is_variable: self.wrap_variable_(symbol) - return dspy.Prediction(success=True, translation=translation) + return dspy.Prediction(success=True) else: raise NotImplementedError @@ -269,7 +267,7 @@ def wrap_variable_(self, symbol: Symbol): wrapper = bindgen(self.crate.c_src_path, symbol.spelling) symbol_wrapper_path = self.wrapper_path.parent / "wrapper" / f"{rust_spelling}.rs" symbol_wrapper_path.parent.mkdir(exist_ok=True, parents=True) - symbol_wrapper_path.write_text(wrapper.text) + symbol_wrapper_path.write_text(str(wrapper)) self.crate.vcs.add(symbol_wrapper_path) success, output = self._build(symbol) @@ -295,7 +293,6 @@ def wrap_function( reference_code: CodeRust, translation: CodeRust, prior_wrapper: CodeRust | None = None, - wrapper: CodeRust | None = None, support_code: CodeC | None = None, ) -> dspy.Prediction: # Don't bother wrapping main in binary crates @@ -304,7 +301,7 @@ def wrap_function( clang_make_extern_(self.crate.c_src_path, symbol.spelling) self.crate.vcs.add(self.crate.c_src_path) self.crate.vcs.commit(f"Made function `{symbol.name}` extern") - return dspy.Prediction(success=True, translation=translation) + return dspy.Prediction(success=True) logger.info(f"Generating wrapper for function `{symbol.name}` ...") @@ -315,16 +312,17 @@ def wrap_function( rust_spelling = mangle(symbol.spelling) symbol_wrapper_path = self.wrapper_path.parent / "wrapper" / f"{rust_spelling}.rs" symbol_wrapper_path.parent.mkdir(exist_ok=True, parents=True) - symbol_wrapper_path.write_text(unimplemented_wrapper.text) + symbol_wrapper_path.write_text(str(unimplemented_wrapper)) success, build_feedback = self._build(symbol) if not success: raise RuntimeError(f"The crate does not build!\n\n{build_feedback}") - # Use cache when no wrapper nor prior wrapper - if wrapper is None and prior_wrapper is None: + # Use cache when no prior wrapper + if prior_wrapper is None: wrapper = _read_cache(self.cache, symbol.spelling, unimplemented_wrapper) else: logger.info("Ignoring wrapper cache...") + wrapper = None # Generate dynamic signature and module for symbol signature_class = HybridSignature if not LARGE_PROJECT else Signature @@ -344,8 +342,7 @@ def wrap_function( # Try generating wrapper up to max_iter times msg = "" - success, build_feedback = False, "" - scope_feedback: OrderedDict[str, str] = OrderedDict() + success, build_feedback, scope_feedback = False, "", "" pred = dspy.Prediction() for i in range(max(self.max_iters, 1)): # Use the wrapper from the prior iteration as feedback for the next iteration @@ -360,7 +357,7 @@ def wrap_function( unimplemented_wrapper, prior_wrapper, build_feedback, - "\n\n".join(scope_feedback.values()), + scope_feedback, wrapper if i == 0 else None, ) except AdapterParseError: @@ -374,28 +371,26 @@ def wrap_function( continue # Reset scope feedback - scope_feedback.clear() - if "wrapper" not in pred or not isinstance(pred.wrapper, CodeRust): - scope_feedback["no_wrapper"] = ( - "No wrapper was generated. You must respect the template and instructions **exactly**!" - ) wrapper = unimplemented_wrapper + scope_feedback = "No wrapper was generated. You must respect the template and instructions **exactly**!" else: wrapper = pred.wrapper # Validate that changes are in scope - scope_feedback.update(validate_changes(wrapper, unimplemented_wrapper)) - + scope_feedback = "\n\n".join( + validate_changes(wrapper, unimplemented_wrapper).values() + ) # TODO: Check for a single crate function call in scope # Write wrapper to disk and check if we build with unsafe code since wrappers can use unsafe code - symbol_wrapper_path.write_text(wrapper.text) + symbol_wrapper_path.write_text(str(wrapper)) self.crate.vcs.add(symbol_wrapper_path) success, build_feedback = self._build(symbol) success = success and not build_feedback and not scope_feedback usage = format_usage(pred) + # Exit early if we build if success: # Permanently make function extern clang_make_extern_(self.crate.c_src_path, symbol.spelling) @@ -410,26 +405,26 @@ def wrap_function( msg = f"Wrapped function `{symbol.name}`: {usage}" logger.info(msg) if "reasoning" in pred: - msg += f"\n\n# Reasoning\n{pred.reasoning}" + msg += f"\n\n# Reasoning\n{indent(pred.reasoning, ' ')}" self.crate.vcs.commit(msg) break # Log and commit failure msg = f"Failed to wrap function `{symbol.name}` ({i + 1}/{self.max_iters}): {usage}" logger.error(msg) - msg += f"\n\n# Reasoning\n{pred.reasoning}" if "reasoning" in pred else "" - msg += f"\n\n# Build feedback\n{build_feedback}" - msg += f"\n\n# Scope Feedback\n{scope_feedback}" + if "reasoning" in pred: + msg += f"\n\n# Reasoning\n{indent(pred.reasoning, ' ')}" + msg += f"\n\n# Build Feedback\n{indent(build_feedback, ' ')}" + msg += f"\n\n# Scope Feedback\n{indent(scope_feedback, ' ')}" self.crate.vcs.commit(msg) pred.success = success pred.name = symbol.spelling - pred.translation = translation pred.wrapper = wrapper pred.bindgen_template = unimplemented_wrapper pred.prior_wrapper = prior_wrapper or CodeRust() pred.build_feedback = build_feedback - pred.scope_feedback = "\n\n".join(scope_feedback.values()) + pred.scope_feedback = scope_feedback if not success: # Feedback for translator pred.feedback = "It was difficult to generate a C-compatible FFI wrapper for the translation. Regenerate the translation with clear, explicit, wrapper-friendly Rust function boundaries and straightforward ownership, while keeping the translation fully memory-safe and free of unsafe constructs." @@ -496,16 +491,14 @@ def gather_wrappers(self, exclude_wrapper: str = "") -> CodeRust: continue modules[rust_spelling] = ( - f"pub mod {rust_spelling} {{\n" + textwrap.indent(wrapper_src, " ") + "\n}" + f"pub mod {rust_spelling} {{\n" + indent(wrapper_src, " ") + "\n}" ) if not modules: return CodeRust() return CodeRust( - "pub mod wrapper {\n" - + textwrap.indent("\n\n".join(modules.values()), " ") - + "\n}\n" + "pub mod wrapper {\n" + indent("\n\n".join(modules.values()), " ") + "\n}\n" ) def _build(self, symbol: Symbol) -> tuple[bool, str]: @@ -647,7 +640,7 @@ def _read_cache(cache: Path | None, name: str, bindgen_template: CodeRust) -> Co try: row = conn.execute( "SELECT wrapper FROM wrapper_translations WHERE bindgen_template=? AND success=1 ORDER BY id DESC LIMIT 1", - (bindgen_template.text,), + (str(bindgen_template),), ).fetchone() if row is None: row = conn.execute( @@ -685,11 +678,11 @@ def _write_cache( """, ( name, - bindgen_template.text, - prior_wrapper.text, + str(bindgen_template), + str(prior_wrapper), build_feedback, scope_feedback, - wrapper.text, + str(wrapper), int(success), ), ) diff --git a/test/test_cargo_test.py b/test/test_cargo_test.py index dd66c44..00a85e7 100644 --- a/test/test_cargo_test.py +++ b/test/test_cargo_test.py @@ -15,7 +15,7 @@ def tmp_crate(tmp_path: Path): """Create a minimal lib crate with passing and failing integration tests.""" crate_dir = tmp_path / "test_crate" - crate = tools.Crate(cargo_toml=crate_dir / "Cargo.toml", vcs="none", type="lib") + crate = tools.Crate(cargo_toml=crate_dir / "Cargo.toml", vcs="none", template="lib") (crate_dir / "src" / "lib.rs").write_text("pub fn add(a: i32, b: i32) -> i32 { a + b }\n") diff --git a/test/test_consolidate.py b/test/test_consolidate.py index 9febcc4..7118360 100644 --- a/test/test_consolidate.py +++ b/test/test_consolidate.py @@ -7,18 +7,12 @@ from pathlib import Path from textwrap import dedent -import networkx as nx -import pytest import json -from clang.cindex import TranslationUnit as TU - from ideas import ast from ideas.init.consolidate import ( create_ast_order, create_symbol_lexical_key_fn, - get_includes, - get_symbols_and_dependencies, init as consolidate_init, ) from ideas.tools import check_c @@ -55,8 +49,7 @@ def _ast_order_from_symbols( source_priority = source_priority or [] tu_representatives: dict[Path, ast.Symbol] = {} for symbol in symbols.values(): - tu_path = Path(symbol.cursor.translation_unit.spelling).resolve() - tu_representatives.setdefault(tu_path, symbol) + tu_representatives.setdefault(symbol.tu_path, symbol) fallback_asts = [ ast.TreeResult(symbols={symbol.name: symbol}) for symbol in tu_representatives.values() ] @@ -444,7 +437,7 @@ def test_macro_wrapped_declaration(tmp_path: Path): consolidated = consolidate_init(compile_commands, source_priority=[]) # The consolidated output must not contain the unexpanded macro - assert "LIB_EXPORT" not in consolidated, ( + assert "LIB_EXPORT" not in str(consolidated), ( f"Consolidated output contains unexpanded macro 'LIB_EXPORT':\n{consolidated}" ) @@ -623,9 +616,6 @@ def test_static_variable_tentative_defs_same_name_renamed(tmp_path: Path): ) -@pytest.mark.xfail( - reason="USR mismatch from -isystem; fixed at cmake level in ideas.cmake._normalize_isystem" -) def test_isystem_inline_function_dependency_not_lost(tmp_path: Path): """ When a header is included via -isystem in one TU but via -I in another, @@ -640,8 +630,6 @@ def test_isystem_inline_function_dependency_not_lost(tmp_path: Path): error: call to undeclared function 'my_alloc'; ISO C99 and later do not support implicit function declarations """ - from clang.cindex import TranslationUnit as TU - # alloc.h in util/ with a static inline function util_dir = tmp_path / "util" util_dir.mkdir() @@ -711,60 +699,28 @@ def test_isystem_inline_function_dependency_not_lost(tmp_path: Path): ) ) - # Parse caller.c with -isystem for util/ (ext target uses SYSTEM includes) - caller_tu = TU.from_source( - None, - args=["-c", str(caller_c), "-isystem", str(util_dir), f"-I{ext_dir}"], - ) - assert not any(d.severity >= 3 for d in caller_tu.diagnostics) - - # Parse user.c with regular -I for util/ - user_tu = TU.from_source(None, args=["-c", str(user_c), f"-I{util_dir}"]) - assert not any(d.severity >= 3 for d in user_tu.diagnostics) - - caller_tree = ast.extract_info_c(caller_tu) - user_tree = ast.extract_info_c(user_tu) - - # Verify the USR mismatch exists - caller_alloc_usr = next( - n for n, s in caller_tree.symbols.items() if s.spelling == "my_alloc" - ) - user_alloc_usr = next(n for n, s in user_tree.symbols.items() if s.spelling == "my_alloc") - assert caller_alloc_usr != user_alloc_usr, ( - "Expected USR mismatch between -isystem and -I includes" - ) - - # Put caller.c FIRST in ast_order so its symbols have higher priority - # in the lexicographic sort. This ensures my_alloc (from user.c, rank 1) - # sorts AFTER make_item (from caller.c, rank 0) when the dependency - # edge is missing. - asts = [caller_tree, user_tree] - ast_order = create_ast_order([caller_c, user_c], asts) - - symbols, dependencies = get_symbols_and_dependencies(asts, ast_order=ast_order) - - symbol_lexical_key = create_symbol_lexical_key_fn(symbols, ast_order) - sorted_symbol_groups = list( - nx.lexicographical_topological_sort( - nx.from_dict_of_lists(dependencies, create_using=nx.DiGraph).reverse(copy=False), # type: ignore[reportArgumentType] - key=symbol_lexical_key, + # Keep mixed include modes across TUs to exercise the USR normalization path. + compile_commands = tmp_path / "compile_commands.json" + compile_commands.write_text( + json.dumps( + [ + { + "directory": str(tmp_path), + "file": str(caller_c), + "command": f"cc -isystem {util_dir} -I{ext_dir} -c {caller_c}", + }, + { + "directory": str(tmp_path), + "file": str(user_c), + "command": f"cc -I{util_dir} -c {user_c}", + }, + ] ) ) - # Build consolidated output - sources: list[str] = get_includes(symbols) + [""] - for group in sorted_symbol_groups: - if len(group) > 1: - for name in group: - declaration = symbols[name].declaration - if declaration and declaration.text not in sources: - sources.append(declaration.text) - for name in group: - definition = symbols[name].code.text - if definition not in sources: - sources.append(definition) - - consolidated = "\n".join(sources) + consolidated = consolidate_init( + compile_commands, source_priority=[caller_c.resolve(), user_c.resolve()] + ) success, error = check_c(consolidated, flags=["-fsyntax-only", "-Wall"]) assert success, ( f"Consolidated code does not compile (isystem USR mismatch lost dependency):\n" @@ -812,50 +768,12 @@ def test_static_inline_in_scc_emitted_before_caller(tmp_path: Path): ) ) - caller_tu = TU.from_source(None, args=["-c", str(caller_c), f"-I{tmp_path}"]) - state_tu = TU.from_source(None, args=["-c", str(state_c), f"-I{tmp_path}"]) - assert not any(d.severity >= 3 for d in caller_tu.diagnostics) - assert not any(d.severity >= 3 for d in state_tu.diagnostics) - - caller_tree = ast.extract_info_c(caller_tu) - state_tree = ast.extract_info_c(state_tu) - - # state_tree FIRST in asts so merge_symbols picks helper from state.c - # (both have identical code; first encountered wins => state.c). - # ast_order = [caller.c, state.c]: rank 0, rank 1. - # Result: compute(rank 0) emitted before helper(rank 1) in SCC. - # helper has declaration=None (static inline), so no forward decl is emitted. - asts = [state_tree, caller_tree] - ast_order = create_ast_order([caller_c, state_c], asts) - - symbols, dependencies = get_symbols_and_dependencies(asts, ast_order=ast_order) - - # Verify cycle exists - scc_groups = [group for group in dependencies if len(group) > 1] - assert scc_groups, "Expected at least one multi-member SCC" - - # Build consolidated output (mirrors compose_all logic) - symbol_lexical_key = create_symbol_lexical_key_fn(symbols, ast_order) - sorted_symbol_groups = list( - nx.lexicographical_topological_sort( - nx.from_dict_of_lists(dependencies, create_using=nx.DiGraph).reverse(copy=False), # type: ignore[reportArgumentType] - key=symbol_lexical_key, - ) + compile_commands = _write_compile_commands( + tmp_path, [state_c, caller_c], extra_flags=f"-I{tmp_path}" + ) + consolidated = consolidate_init( + compile_commands, source_priority=[caller_c.resolve(), state_c.resolve()] ) - - sources: list[str] = get_includes(symbols) + [""] - for group in sorted_symbol_groups: - if len(group) > 1: - for name in group: - declaration = symbols[name].declaration - if declaration and declaration.text not in sources: - sources.append(declaration.text) - for name in group: - definition = symbols[name].code.text - if definition not in sources: - sources.append(definition) - - consolidated = "\n".join(sources) success, error = check_c(consolidated, flags=["-fsyntax-only", "-Wall"]) assert success, ( f"Consolidated code fails (static inline in SCC emitted after caller due to TU rank):\n" @@ -889,11 +807,12 @@ def test_system_macro_double_expansion(tmp_path: Path): ) -def test_gnu_source_preserved_in_consolidation(tmp_path: Path): +def test_cc_defines_preserved_in_consolidation(tmp_path: Path): main_c = tmp_path / "main.c" main_c.write_text( dedent( """\ + #include #include #include #include @@ -931,19 +850,20 @@ def test_gnu_source_preserved_in_consolidation(tmp_path: Path): ) ) - compile_commands = _write_compile_commands(tmp_path, [main_c], extra_flags="-D_GNU_SOURCE") + compile_commands = _write_compile_commands( + tmp_path, [main_c], extra_flags="-D_GNU_SOURCE -DPCRE2_CODE_UNIT_WIDTH=8" + ) consolidated = consolidate_init(compile_commands, source_priority=[]) - # All _GNU_SOURCE-gated symbols must appear in the consolidated output for sym in ("environ", "euidaccess", "pipe2", "qsort_r", "secure_getenv"): - assert sym in consolidated, ( + assert sym in str(consolidated), ( f"Consolidated output is missing '{sym}' usage:\n{consolidated}" ) success, error = check_c(consolidated, flags=["-fsyntax-only"]) assert success, ( - f"Consolidated code does not compile without -D_GNU_SOURCE " - f"(feature-test macro lost during consolidation):\n" + f"Consolidated code does not compile without -D_GNU_SOURCE and -DPCRE2_CODE_UNIT_WIDTH=8 " + f"(macros lost during consolidation):\n" f"{error}\n\nConsolidated output:\n{consolidated}" ) @@ -981,7 +901,7 @@ def test_posix_c_source_preserved_in_consolidation(tmp_path: Path): # All _POSIX_C_SOURCE-gated symbols must appear in the consolidated output for sym in ("clock_gettime", "strdup", "strtok_r"): - assert sym in consolidated, ( + assert sym in str(consolidated), ( f"Consolidated output is missing '{sym}' usage:\n{consolidated}" ) @@ -1045,3 +965,107 @@ def test_system_macro_undefs_preserve_benign_macros(tmp_path: Path): f"Consolidated code does not compile (benign macros broken):\n{error}\n\n" f"Consolidated output:\n{consolidated}" ) + + +def test_source_level_defines_required_by_system_headers_preserved(tmp_path: Path): + """ + Source-level #defines that affect system header behavior must be preserved + in consolidated output. Two sub-cases: + 1. PCRE2_CODE_UNIT_WIDTH: pcre2.h fires #error if missing. + 2. _GNU_SOURCE: changes strerror_r signature from int (POSIX) to char* (GNU). + """ + main_c = tmp_path / "main.c" + main_c.write_text( + dedent( + """\ + #define _GNU_SOURCE + #define PCRE2_CODE_UNIT_WIDTH 8 + #include + #include + + int use_pcre(const char *pat) { + (void)pat; + return 0; + } + + const char *get_err(int errnum) { + static char buf[256]; + const char *errstr = strerror_r(errnum, buf, sizeof(buf)); + return errstr; + } + """ + ) + ) + + compile_commands = _write_compile_commands(tmp_path, [main_c]) + consolidated = consolidate_init(compile_commands, source_priority=[]) + + success, error = check_c(consolidated, flags=["-fsyntax-only", "-Werror"]) + assert success, ( + f"Consolidated code does not compile (source-level defines dropped):\n{error}\n\n" + f"Consolidated output:\n{consolidated}" + ) + + +def test_header_defined_gnu_source_with_default_source_flag(tmp_path: Path): + """ + Compile commands have -D_DEFAULT_SOURCE, but _GNU_SOURCE is defined inside + a project header (config.h/first.h). The consolidator preserves + _DEFAULT_SOURCE (from -D flags) but loses _GNU_SOURCE (from the header). + Code using the GNU strerror_r (returns char*) then fails with: + error: incompatible integer to pointer conversion + + Uses two TUs to exercise the union: both include config.h which defines + _GNU_SOURCE, so the union should emit it exactly once. + """ + config_h = tmp_path / "config.h" + config_h.write_text( + dedent( + """\ + #define _GNU_SOURCE + """ + ) + ) + + main_c = tmp_path / "main.c" + main_c.write_text( + dedent( + """\ + #include "config.h" + #include + + const char *get_err(int errnum) { + static char buf[256]; + const char *errstr = strerror_r(errnum, buf, sizeof(buf)); + return errstr; + } + """ + ) + ) + + util_c = tmp_path / "util.c" + util_c.write_text( + dedent( + """\ + #include "config.h" + #include + + const char *get_err2(int errnum) { + static char buf[128]; + const char *errstr = strerror_r(errnum, buf, sizeof(buf)); + return errstr; + } + """ + ) + ) + + compile_commands = _write_compile_commands( + tmp_path, [main_c, util_c], extra_flags=f"-D_DEFAULT_SOURCE -I{tmp_path}" + ) + consolidated = consolidate_init(compile_commands, source_priority=[]) + + success, error = check_c(consolidated, flags=["-fsyntax-only", "-Werror"]) + assert success, ( + f"Consolidated code does not compile (header-defined _GNU_SOURCE lost):\n{error}\n\n" + f"Consolidated output:\n{consolidated}" + ) diff --git a/test/test_extract_code_from_tu.py b/test/test_extract_code_from_tu.py index cd03781..6baf583 100644 --- a/test/test_extract_code_from_tu.py +++ b/test/test_extract_code_from_tu.py @@ -5,6 +5,7 @@ # +import pickle import pytest from pathlib import Path from textwrap import dedent as d @@ -70,3 +71,28 @@ def test_newline(): ).strip() + "\n" ) + + +def test_symbol_is_picklable(): + tu = parse_c("int main(int argc, char **argv) { return 0; }") + symbol = ast.extract_info_c(tu).symbols["c:@F@main"] + + blob = pickle.dumps(symbol) + restored = pickle.loads(blob) + + assert restored.name == symbol.name + assert restored.spelling == symbol.spelling + assert restored.kind == symbol.kind + assert restored.tu_preorder_index == symbol.tu_preorder_index + assert restored.code == symbol.code + + +def test_tree_result_is_picklable(): + tu = parse_c("int main(int argc, char **argv) { return 0; }") + result = ast.extract_info_c(tu) + + blob = pickle.dumps(result) + restored = pickle.loads(blob) + + assert isinstance(restored, ast.TreeResult) + assert restored == result diff --git a/test/test_tools.py b/test/test_tools.py index 6022073..0e99a07 100644 --- a/test/test_tools.py +++ b/test/test_tools.py @@ -9,6 +9,7 @@ import pytest from ideas import tools +from ideas.ast import CodeC @pytest.fixture @@ -28,12 +29,12 @@ def rust_paths(fixtures_dir: Path) -> tuple[Path, Path]: def test_check_c(c_paths: tuple[Path, ...]): # Compilation should succeed - success1, out1 = tools.check_c(c_paths[0].read_text()) + success1, out1 = tools.check_c(CodeC(c_paths[0].read_text())) assert success1, out1 assert out1 == "" # Compilation should fail - success2, out2 = tools.check_c(c_paths[1].read_text()) + success2, out2 = tools.check_c(CodeC(c_paths[1].read_text())) assert not success2, out2 assert out2 != "" diff --git a/test/test_wrapper.py b/test/test_wrapper.py index e713f67..e777d4a 100644 --- a/test/test_wrapper.py +++ b/test/test_wrapper.py @@ -148,7 +148,7 @@ def test_bindgen_emits_expected_text_for_global_shapes( binding = wrapper_mod.bindgen(c_path, symbol) - assert binding.text.strip() == expected + assert str(binding).strip() == expected assert c_path.read_text() == source @@ -194,7 +194,7 @@ def test_bindgen_handles_dependent_declarations_for_target_global(tmp_path: Path dependent_binding = wrapper_mod.bindgen(c_path, "arr") assert c_path.read_text() == array_decl + dependent_decl - assert dependent_binding.text.strip() == baseline_binding.text.strip() + assert str(dependent_binding).strip() == str(baseline_binding).strip() def test_bindgen_handles_dependent_declarations_for_target_function(tmp_path: Path): @@ -210,4 +210,4 @@ def test_bindgen_handles_dependent_declarations_for_target_function(tmp_path: Pa dependent_binding = wrapper_mod.bindgen(c_path, "f") assert c_path.read_text() == dependent_source - assert dependent_binding.text.strip() == baseline_binding.text.strip() + assert str(dependent_binding).strip() == str(baseline_binding).strip() diff --git a/uv.lock b/uv.lock index b04941b..9a86732 100644 --- a/uv.lock +++ b/uv.lock @@ -1066,11 +1066,11 @@ wheels = [ [[package]] name = "json-repair" -version = "0.55.0" +version = "0.60.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/96/74/0f39677fa7c0127129c3f1a37c94d05c30a968ba3047200e54dea375b09a/json_repair-0.55.0.tar.gz", hash = "sha256:9fafb47d92582ef4bdd3520656bdb0fcb37b46cf6aa99c1926b7895abc0a3a4b", size = 38828, upload-time = "2026-01-01T20:29:02.684Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/a6/d69888cb4ffde30e80db1e6c32caaadd2f984a80067d5ea72c2cb3f61c3f/json_repair-0.60.1.tar.gz", hash = "sha256:841661cdd2df507c9a4e189097f38ca6bc372e06d4b4e36d72e590f68176c290", size = 49451, upload-time = "2026-06-03T17:28:44.451Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/00/d7d6b6b3257f9b1f997a558b6f7087b8af7c0a2f525f4fbd864c267a88ab/json_repair-0.55.0-py3-none-any.whl", hash = "sha256:bcf4880f5e6ad21a0f70ab034e3d1d398c2ae9698dc5717d7015afbac77b8ed7", size = 29570, upload-time = "2026-01-01T20:29:01.042Z" }, + { url = "https://files.pythonhosted.org/packages/32/1f/2a2b5eea8ef5762a86ad3f8fddddaaba2c0d76dd44e644b9158900868bec/json_repair-0.60.1-py3-none-any.whl", hash = "sha256:ba6ff974f2a8bef2f7768144a7f03f870a816443f03da27a49cdd0ec31a78049", size = 48045, upload-time = "2026-06-03T17:28:43.038Z" }, ] [[package]] @@ -1130,7 +1130,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.88.0" +version = "1.84.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -1146,9 +1146,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9b/8c/6cfce5d15554e076b8438a955436e9e6e2a2bd39c76539656c1c3861c369/litellm-1.88.0.tar.gz", hash = "sha256:4ff794493e40bd86c6f13e91dcb3e1aad697403fd46a96902196d93356ba48f4", size = 13886026, upload-time = "2026-06-06T23:25:17.93Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dd/e9/8941b7e72a187000561d932c0f2f2ed2b0fd080dfc33ba6e05961d45ca7d/litellm-1.84.0.tar.gz", hash = "sha256:b8ad0cbea11a5941b18d5af973017a340abd3d3ab41cb86e5401b970626d71a6", size = 15103206, upload-time = "2026-05-14T05:45:53.017Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/da/71/deb6637475253b83eb4577c058863eec4b4ddaef2d08dcd93981b1aa4209/litellm-1.88.0-py3-none-any.whl", hash = "sha256:abd3037e0bf5703f833f5565c87bdfd93578d3de46cbcb36dfa108c3ef58021c", size = 15276203, upload-time = "2026-06-06T23:25:07.257Z" }, + { url = "https://files.pythonhosted.org/packages/01/a6/77fa1bbf5e42eb596b06318b3f7e6af5d0f44028046d1d598c6a595d028f/litellm-1.84.0-py3-none-any.whl", hash = "sha256:2a58d6041e6aa27d1a28dc8d8828ab500fef1a00ef74ca65e60899035010c2f2", size = 16735062, upload-time = "2026-05-14T05:45:49.927Z" }, ] [[package]]