OpenDCAI · XZ0730 · Jul 18, 2026 · Jul 18, 2026 · Jul 18, 2026 · Jul 20, 2026
diff --git a/examples/config/starter.yaml b/examples/config/starter.yaml
@@ -149,7 +149,7 @@ default_states:
 
   trainer:
     train_framework: "llamafactory"
-    llamafactory_dir: "/home/lpc/repos/LLaMA-Factory/
+    llamafactory_dir: "/home/lpc/repos/LLaMA-Factory"
     llamafactory_env_path: "/home/lpc/miniconda3/envs/lmf/bin/"
     CUDA_VISIBLE_DEVICES: "0,1"
     swanlab_api_key: ""

diff --git a/examples/scripts/run_judger_standalone.py b/examples/scripts/run_judger_standalone.py
@@ -134,27 +134,13 @@ def _load_config(config_path: str) -> Dict[str, Any]:
 def _extract_state_from_starter_yaml(config: Dict[str, Any]) -> Dict[str, Any]:
     """从 starter.yaml 格式提取 state 字典。
 
-    starter.yaml 结构：
-        default_states:
-          task_id: "..."
-          output_dir: "..."
-          judger:
-            eval_model_path: "..."
-            ...
-        system:
-          CUDA_VISIBLE_DEVICES: "..."
+    仅提取 task_id 和 output_dir，不提取 judger 配置。
+    Judger 配置优先从 DB (taskmodel) 读取。
     """
     defaults = config.get("default_states", {})
-    system = config.get("system", {})
-
-    judger = dict(defaults.get("judger", {}))
-
-    # 从 system 补充 GPU 配置
-    if "cuda_visible_devices" not in judger and "CUDA_VISIBLE_DEVICES" in system:
-        judger["cuda_visible_devices"] = str(system["CUDA_VISIBLE_DEVICES"])
 
     return {
-        "judger": judger,
+        "judger": {},
         "task_id": defaults.get("task_id", ""),
         "output_dir": defaults.get("output_dir", "./outputs"),
     }

diff --git a/loopai/schema/states.py b/loopai/schema/states.py
@@ -997,14 +997,7 @@ class JudgerState(BaseModel):
         description="评估模型路径",
         json_schema_extra={"ui_type": "file_path", "ui_group": "评估模型"}
     )
-    eval_task_type: str = Field(
-        default="code",
-        title="评估任务类型",
-        description="评估任务类型, 支持代码生成(code), Text2sql(text2sql), 通用领域文本评估(general_text)",
-        json_schema_extra={"ui_type": "list", "ui_group": "评估模型",
-                           "allowed_values": ["code", "text2sql", "general_text"]}
-    )
-    # eval_base_url: str = Field(
+    #eval_base_url: str = Field(
     #    default=None,
     #    title="评估模型 Base URL",
     #    description="评估模型 Base URL，未设置或为空的时候，将会尝试通过本地开启vllm",
@@ -1028,13 +1021,7 @@ class JudgerState(BaseModel):
         description="评估模型 Top P",
         json_schema_extra={"ui_type": "slider", "max": 1, "ui_group": "评估模型"}
     )
-    eval_problem_path: str = Field(
-        default=None,
-        title="评估模型问题路径",
-        description="评估模型问题路径",
-        json_schema_extra={"ui_type": "file_path", "ui_group": "评估模型"}
-    )
-    # eval_format_type: str = Field(
+    #eval_format_type: str = Field(
     #    default=None,
     #    title="评估模型问题格式化类型",
     #    description="评估模型问题格式化类型，如果为空或None将不进入格式化节点，改格式化方式可以用户自由定义，目前支持\"human-eval\"和\"mbpp\"，格式化后的文件将存至output_dir定义的目录下",
@@ -1053,25 +1040,6 @@ class JudgerState(BaseModel):
         description="评估模型每个问题的样例生成数量",
         json_schema_extra={"ui_type": "number", "ui_group": "评估模型"}
     )
-    eval_text2sql_dir: str = Field(
-        default=None,
-        title="评估模型text2sql数据库目录",
-        description="评估模型text2sql数据库目录，仅text2sql任务下生效，并且数据文件中需要以字段db_id标注出相应的数据库文件夹至路径目录下",
-        json_schema_extra={"ui_type": "file_path", "ui_group": "评估模型"}
-    )
-    # 统一vllm配置删除
-    # eval_env_configs: str = Field(
-    #    default='{"NCCL_P2P_DISABLE": "1","NCCL_IB_DISABLE": "1","NCCL_DEBUG": "INFO","NCCL_SOCKET_IFNAME": "lo","NCCL_BLOCKING_WAIT": "1"}',
-    #    title="评估模型vllm启动环境参数",
-    #    description="评估模型vllm启动环境参数，需要完整字符串配置，为空则认为已启动vllm将会跳过启动vllm的过程",
-    #    json_schema_extra={"ui_type": "textarea", "language": "json", "ui_group": "评估模型"}
-    # )
-    # eval_vllm_port: int = Field(
-    #    default=8911,
-    #    title="vllm本地启动参数——port",
-    #    description="vllm本地启动参数——port，用于本地启动vllm服务的参数之一，当参数eval_base_url未设置或为空时生效",
-    #    json_schema_extra={"ui_type": "number", "ui_group": "评估模型"}
-    # )
     eval_vllm_tensor_parallel_size: int = Field(
         default=2,
         title="vllm本地启动参数——tensor_parallel_size",
@@ -1091,41 +1059,29 @@ class JudgerState(BaseModel):
     #    description="vllm本地启动参数——启动环境，用于本地启动vllm服务的参数之一，当参数eval_base_url未设置或为空时生效，为空时默认为当前环境启动。参数需要具体到python目录，格式应为<path>/miniconda3/envs/<env_name>/bin/python",
     #    json_schema_extra={"ui_type": "file_path", "ui_group": "评估模型"}
     # )
-    output_result_path: str = Field(
-        default="",
-        title="评测结果文件保存路径",
-        description="评测结果文件保存路径，该参数不支持用户自定义，运行后由程序根据任务ID等参数生成",
-        json_schema_extra={"ui_type": "file_path", "ui_group": "评估模型"}
-    )
-    output_case_path: str = Field(
-        default="",
-        title="评测样例集文件保存路径",
-        description="评测样例集文件保存路径，该参数不支持用户自定义，运行后由程序根据任务ID等参数生成",
-        json_schema_extra={"ui_type": "file_path", "ui_group": "评估模型"}
-    )
-    output_problem_path: str = Field(
-        default="",
-        title="评测格式化后问题集保存路径",
-        description="评测格式化后问题集，该参数不支持用户自定义，运行后由程序根据任务ID等参数生成，如未使用格式化模版该路径即为原始问题文件的路径",
-        json_schema_extra={"ui_type": "file_path", "ui_group": "评估模型"}
+    benchlist: List[Dict[str, Any]] = Field(
+        default_factory=list,
+        title="主任务评测集",
+        description="主任务评测集列表，每个元素包含 name、task_type、problem_path 等字段",
+        json_schema_extra={"ui_type": "textarea", "ui_group": "评估模型"}
     )
-    output_pred_path: str = Field(
-        default="",
-        title="评测预测结果保存路径",
-        description="通用文本评测结束后产生的预测文件路径",
-        json_schema_extra={"ui_type": "file_path", "ui_group": "评估模型"}
+    extra_benchlist: List[Dict[str, Any]] = Field(
+        default_factory=list,
+        title="附加任务评测集",
+        description="附加任务评测集列表，格式同 benchlist。失败不影响主任务",
+        json_schema_extra={"ui_type": "textarea", "ui_group": "评估模型"}
     )
-    bench: Dict[str, Any] = Field(
-        default_factory=dict,
-        title="Bench运行信息",
-        description="通用文本评测生成的 bench 信息，用于 Analyzer 后续指标计算",
-        json_schema_extra={"ui_type": "json_viewer", "ui_group": "评估模型"}
+    bench_result: List[Dict[str, Any]] = Field(
+        default_factory=list,
+        title="主任务评测结果",
+        description="主任务 bench 评测结果列表，供 Analyzer 读取",
+        json_schema_extra={"ui_type": "textarea", "ui_group": "评估模型"}
     )
-    bench_name: str = Field(
-        default="general_text_eval",
-        title="评测集名称",
-        description="通用文本评测使用的评测集名称",
-        json_schema_extra={"ui_type": "text", "ui_group": "评估模型"}
+    extra_bench_result: List[Dict[str, Any]] = Field(
+        default_factory=list,
+        title="附加任务评测结果",
+        description="附加任务 bench 评测结果列表，供 Analyzer 读取",
+        json_schema_extra={"ui_type": "textarea", "ui_group": "评估模型"}
     )
     cuda_visible_devices: str = Field(
         default="0",
@@ -1141,37 +1097,7 @@ class JudgerState(BaseModel):
     #    description="是否通过 API 调用模型",
     #    json_schema_extra={"ui_type": "toggle_switch", "ui_group": "评估模型"}
     # )
-    bench: List[Dict[str, Any]] = Field(
-        default="",
-        title="评测集名称",
-        description="通用文本评测使用的评测集相关信息",
-        json_schema_extra={"ui_type": "textarea", "ui_group": "评估模型"}
-    )
-    bench_dataflow_eval_type: str = Field(
-        default="",
-        title="通用文本评测类型",
-        description="通用文本 One-Eval DataFlow 评测类型，例如 key2_qa / key1_text_score",
-        json_schema_extra={"ui_type": "list", "ui_group": "评估模型",
-                           "allowed_values": ["key1_text_score", "key2_qa", "key2_q_ma", "key3_q_choices_a", "key3_q_choices_as", "key3_q_a_rejected"]}
-    )
-    key_mapping: Dict[str, Any] = Field(
-        default_factory=dict,
-        title="字段映射",
-        description="DataFlow 评测字段映射，如 input_question_key / input_target_key / input_pred_key",
-        json_schema_extra={"ui_type": "json_viewer", "ui_group": "评估模型"}
-    )
-    # skip_dataflow_eval: bool = Field(
-    #    default=False,
-    #    title="跳过 DataFlow 正式评测",
-    #    description="为 True 时仅准备 bench / records，不调用 DataFlowEvalTool.run_eval",
-    #    json_schema_extra={"ui_type": "toggle_switch", "ui_group": "评估模型"}
-    # )
-    # output_dir: str = Field(
-    #    default="",
-    #    title="通用文本输出路径",
-    #    description="通用文本任务结束后输出路径",
-    #    json_schema_ectra={"ui_type": "text", "ui_group": "评估模型"}
-    # )
+
 
 
 class AnalyzerState(BaseModel):

diff --git a/loopai/skills/Judger/__init__.py b/loopai/skills/Judger/__init__.py
@@ -64,20 +64,27 @@ def run(
 
     # 成功——标准 payload 输出到 stdout（Codex 消费）
     judger = result.get("judger", {})
-    bench = judger.get("bench") or {}
-    metrics = judger.get("metrics") or {}
-    if not metrics:
-        metrics = (bench.get("meta") or {}).get("eval_result") or {}
+    bench_list = judger.get("bench_result") or []
+    secondary_list = judger.get("extra_bench_result") or []
+
+    # 聚合 metrics：按 bench_name 索引
+    metrics = {}
+    for b in bench_list + secondary_list:
+        m = b.get("metrics") or {}
+        if m:
+            metrics[b.get("bench_name", "unknown")] = m
+        else:
+            meta = b.get("meta") or {}
+            m = meta.get("eval_result") or {}
+            if m:
+                metrics[b.get("bench_name", "unknown")] = m
+
     metrics_str = json.dumps(metrics, ensure_ascii=False) if metrics else ""
 
     emit_success(
         data={
-            "task_type": judger.get("eval_task_type"),
-            "output_result_path": judger.get("output_result_path", ""),
-            "output_case_path": judger.get("output_case_path", ""),
-            "output_problem_path": judger.get("output_problem_path", ""),
-            "output_pred_path": judger.get("output_pred_path", ""),
-            "bench": bench,
+            "bench_result": bench_list,
+            "extra_bench_result": secondary_list,
             "metrics": metrics_str,
         },
         stream_writer=writer,

diff --git a/loopai/skills/Judger/cli.py b/loopai/skills/Judger/cli.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+"""Judger CLI entry point — ``loopai-judger`` command."""
+
+from __future__ import annotations
+
+import argparse
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Run LoopAI Judger evaluation pipeline (standalone, no LangGraph)",
+    )
+    parser.add_argument(
+        "--resume", action="store_true", default=False,
+        help="Resume from last checkpoint",
+    )
+    parser.add_argument(
+        "--from-step", type=str, default=None,
+        help="Force start from a specific pipeline step",
+    )
+
+    args = parser.parse_args()
+
+    from loopai.skills.Judger import run
+    run(resume=args.resume, from_step=args.from_step)
+
+
+if __name__ == "__main__":
+    main()