Server config + observability refactor

.gitignore

@@ -146,6 +146,7 @@ images
 /custom/
 megatron_output/
 .qoder
+.kiro/
 
 # Pytorch
 *.pth

Dockerfile

@@ -37,7 +37,7 @@ RUN pip install flash-linear-attention -U --no-cache-dir
 RUN pip install numpy==2.2 --no-cache-dir
 
 # Install tinker, ray, and other deps
-RUN pip install --no-cache-dir tinker==0.16.1 "ray[serve]" transformers peft<=0.18 accelerate -U
+RUN pip install --no-cache-dir tinker==0.16.1 "ray[serve]" transformers peft<=0.18 accelerate redis opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp -U
 
 # Clone and install twinkle, checkout to latest v-tag
 RUN git clone https://github.com/modelscope/twinkle.git

cookbook/client/server/megatron/server_config.yaml

@@ -9,6 +9,17 @@ http_options:
   host: 0.0.0.0        # Listen on all network interfaces
   port: 9000            # Port number for the server
 
+# Persistence configuration for ServerState (sessions, models, futures, ...).
+# Top-level placement makes the launcher propagate this to every Ray worker
+# via env vars, so the configured backend is used regardless of which
+# deployment initializes the ServerState actor first.
+#   mode: memory | file | redis
+#   file_path: required for `file` mode
+#   redis_url / key_prefix: required for `redis` mode
+# persistence:
+#   mode: file
+#   file_path: /tmp/twinkle_state.json
+
 # Applications: each entry defines a service component deployed on the server
 applications:
 
@@ -84,7 +95,7 @@ applications:
     route_prefix: /api/v1/model/Qwen/Qwen3.6-27B
     import_path: model
     args:
-      use_megatron: true                          # Use Megatron-LM backend
+      backend: megatron                          # Use Megatron-LM backend
       model_id: "ms://Qwen/Qwen3.6-27B" # ModelScope model identifier
       max_length: 65536                           # model max length
       max_loras: 3                                # model max loras

cookbook/client/server/megatron/server_config_4b.yaml

@@ -38,7 +38,7 @@ applications:
     route_prefix: /api/v1/model/Qwen/Qwen3.5-4B
     import_path: model
     args:
-      use_megatron: true
+      backend: megatron
       model_id: "ms://Qwen/Qwen3.5-4B" # ModelScope model identifier
       max_length: 10240
       nproc_per_node: 2                            # Number of GPU processes per node

cookbook/client/server/transformer/server_config.yaml

@@ -9,6 +9,17 @@ http_options:
   host: 0.0.0.0        # Listen on all network interfaces
   port: 8000            # Port number for the server
 
+# Persistence configuration for ServerState (sessions, models, futures, ...).
+# Top-level placement makes the launcher propagate this to every Ray worker
+# via env vars, so the configured backend is used regardless of which
+# deployment initializes the ServerState actor first.
+#   mode: memory | file | redis
+#   file_path: required for `file` mode
+#   redis_url / key_prefix: required for `redis` mode
+persistence:
+  mode: file
+  file_path: /tmp/twinkle_state.json
+
 # Applications: each entry defines a service component deployed on the server
 applications:
 
@@ -38,7 +49,7 @@ applications:
     route_prefix: /api/v1/model/Qwen/Qwen3.5-4B
     import_path: model
     args:
-      use_megatron: false                          # Use HuggingFace Transformers backend
+      backend: transformers                        # Model backend: transformers | megatron
       model_id: "ms://Qwen/Qwen3.5-4B"    # ModelScope model identifier
       max_length: 10240
       nproc_per_node: 1                            # Number of GPU processes per node
@@ -64,43 +75,43 @@ applications:
           num_cpus: 0.1
           runtime_env:
             env_vars:
-              TWINKLE_TRUST_REMOTE_CODE: "0"
+              TWINKLE_TRUST_REMOTE_CODE: "1"
 
   # 3. Sampler Service - Runs inference / sampling using vLLM engine
   #    Used for generating text from the model (e.g., evaluating LoRA results).
-  - name: sampler-Qwen3.5-4B
-    route_prefix: /api/v1/sampler/Qwen/Qwen3.5-4B
-    import_path: sampler
-    args:
-      model_id: "ms://Qwen/Qwen3.5-4B"   # ModelScope model identifier
-      nproc_per_node: 2               # Number of GPU processes per node
-      sampler_type: vllm              # Inference engine: 'vllm' (fast) or 'torch' (TorchSampler)
-      engine_args:                    # vLLM engine-specific settings
-        max_model_len: 4096           # Maximum sequence length the engine supports
-        gpu_memory_utilization: 0.5   # Fraction of GPU memory to use (0.0-1.0)
-        enable_lora: true             # Allow loading LoRA adapters during inference
-        logprobs_mode: processed_logprobs # Logprobs mode for sampling results
-      device_group:                   # Logical device group for the sampler
-        name: sampler
-        ranks: 1                    # Number of GPUs to use
-        device_type: cuda
-      device_mesh:
-        device_type: cuda
-        dp_size: 1
-      queue_config:
-        rps_limit: 100                             # Max requests per second
-        tps_limit: 100000                           # Max tokens per second
-    deployments:
-      - name: SamplerManagement
-        autoscaling_config:
-          min_replicas: 1
-          max_replicas: 1
-          target_ongoing_requests: 16
-        ray_actor_options:
-          num_cpus: 0.1
-          runtime_env:
-            env_vars:
-              TWINKLE_TRUST_REMOTE_CODE: "0"
+  # - name: sampler-Qwen3.5-4B
+  #   route_prefix: /api/v1/sampler/Qwen/Qwen3.5-4B
+  #   import_path: sampler
+  #   args:
+  #     model_id: "ms://Qwen/Qwen3.5-4B"   # ModelScope model identifier
+  #     nproc_per_node: 2               # Number of GPU processes per node
+  #     sampler_type: vllm              # Inference engine: 'vllm' (fast) or 'torch' (TorchSampler)
+  #     engine_args:                    # vLLM engine-specific settings
+  #       max_model_len: 4096           # Maximum sequence length the engine supports
+  #       gpu_memory_utilization: 0.5   # Fraction of GPU memory to use (0.0-1.0)
+  #       enable_lora: true             # Allow loading LoRA adapters during inference
+  #       logprobs_mode: processed_logprobs # Logprobs mode for sampling results
+  #     device_group:                   # Logical device group for the sampler
+  #       name: sampler
+  #       ranks: 1                    # Number of GPUs to use
+  #       device_type: cuda
+  #     device_mesh:
+  #       device_type: cuda
+  #       dp_size: 1
+  #     queue_config:
+  #       rps_limit: 100                             # Max requests per second
+  #       tps_limit: 100000                           # Max tokens per second
+  #   deployments:
+  #     - name: SamplerManagement
+  #       autoscaling_config:
+  #         min_replicas: 1
+  #         max_replicas: 1
+  #         target_ongoing_requests: 16
+  #       ray_actor_options:
+  #         num_cpus: 0.1
+  #         runtime_env:
+  #           env_vars:
+  #             TWINKLE_TRUST_REMOTE_CODE: "1"
 
   # 4. Processor Service
   - name: processor

cookbook/client/twinkle/self_host/self_cognition.py

@@ -24,7 +24,7 @@
 base_model = 'Qwen/Qwen3.5-4B'
 base_url = 'http://localhost:8000'
 api_key = 'EMPTY_API_KEY'
-save_dir = '/model'
+save_dir = '/tmp/twinkle_sft_output'
 
 
 # Step 2: Initialize the Twinkle client to communicate with the remote server.
@@ -108,8 +108,10 @@ def train():
         start_step = progress['cur_step']
 
     # Step 7: Run the training loop
+    max_steps = 10  # Limit to 10 steps for quick verification
     logger.info(model.get_train_configs().model_dump())
 
+    global_step = 0
     for epoch in range(3):
         logger.info(f'Starting epoch {epoch}')
         for cur_step, batch in enumerate(dataloader, start=start_step + 1):
@@ -128,12 +130,22 @@ def train():
             # # Advance the learning rate scheduler by one step
             # model.lr_step()
 
+            global_step += 1
+
             # Log the loss every 2 steps (aligned with gradient accumulation)
             if cur_step % 2 == 0:
                 # Print metric
                 metric = model.calculate_metric(is_training=True)
                 logger.info(f'Current is step {cur_step} of {len(dataloader)}, metric: {metric.result}')
 
+            # Stop after max_steps
+            if global_step >= max_steps:
+                logger.info(f'Reached max_steps={max_steps}, stopping training.')
+                break
+
+        if global_step >= max_steps:
+            break
+
         # Step 8: Save the trained checkpoint
         twinkle_path = model.save(
             name=f'twinkle-epoch-{epoch}',

pyproject.toml

@@ -14,8 +14,12 @@ dependencies = [
   "safetensors",
   "peft>=0.11.0,<=0.19.0",
   "transformers",
+  "typer>=0.9.0",
 ]
 
+[project.scripts]
+twinkle-server = "twinkle.server.cli:main"
+
 [project.optional-dependencies]
 transformers = [
   "accelerate",
@@ -27,6 +31,9 @@ megatron = ["megatron-core>=0.12.0", "transformer-engine[pytorch]", "mcore_bridg
 vllm = ["vllm>=0.11"]
 ray = ["ray[serve]"]
 tinker = ["tinker==0.14.0"]
+test = ["hypothesis>=6.0"]
+telemetry = ["psutil>=5.9.0", "pynvml>=11.0.0"]
+redis = ["redis>=5.0"]
 docs = [
   "sphinx>=5.3.0,<6.0.0",
   "docutils>=0.16.0,<0.17.0",

src/twinkle/server/__main__.py

@@ -1,117 +1,22 @@
 # Copyright (c) ModelScope Contributors. All rights reserved.
-"""
-CLI entry point for Twinkle Server.
+"""CLI entry point for Twinkle Server.
+
+Thin shim — delegates to the typer-based :mod:`twinkle.server.cli` so the
+``python -m twinkle.server`` command and the ``twinkle-server`` console
+script share one implementation.
+
+Usage::
 
-Usage:
-    # From config file
-    python -m twinkle.server --config server_config.yaml
+    python -m twinkle.server launch --config server_config.yaml
+    python -m twinkle.server check-config --config server_config.yaml
+    python -m twinkle.server print-config --config server_config.yaml
+    python -m twinkle.server clear persistence --config server_config.yaml
 """
 from __future__ import annotations
 
-import argparse
-import os
 import sys
-from pathlib import Path
-
-from twinkle import get_logger
-
-logger = get_logger()
-
-
-def create_parser() -> argparse.ArgumentParser:
-    """Create the argument parser."""
-    parser = argparse.ArgumentParser(
-        prog='python -m twinkle.server',
-        description='Twinkle Server Launcher - Unified launcher supporting both Tinker and Twinkle clients',
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Examples:
-  # Start server from YAML config file
-  python -m twinkle.server --config server_config.yaml
-        """,
-    )
-
-    # Config file option
-    parser.add_argument(
-        '-c',
-        '--config',
-        type=str,
-        required=True,
-        metavar='PATH',
-        help='Path to YAML configuration file (required)',
-    )
-
-    # Ray options
-    parser.add_argument(
-        '--namespace',
-        type=str,
-        metavar='NS',
-        help="Ray namespace (default: 'twinkle_cluster')",
-    )
-
-    # Runtime options
-    parser.add_argument(
-        '--log-level',
-        type=str,
-        default='INFO',
-        choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
-        metavar='LEVEL',
-        help='Logging level (default: INFO)',
-    )
-
-    return parser
-
-
-def main(args: list[str] | None = None) -> int:
-    """
-    Main entry point for the CLI.
-
-    Args:
-        args: Command line arguments (uses sys.argv if None)
-
-    Returns:
-        Exit code (0 for success, non-zero for error)
-    """
-    parser = create_parser()
-    parsed_args = parser.parse_args(args)
-
-    try:
-        from twinkle.server.launcher import launch_server
-
-        # Apply log level so that all loggers (including those created later)
-        # pick up the user-specified level via the LOG_LEVEL env var that
-        # get_logger() already reads.
-        os.environ['LOG_LEVEL'] = parsed_args.log_level
-
-        config_path = Path(parsed_args.config)
-        if not config_path.exists():
-            logger.error(f'Config file not found: {config_path}')
-            return 1
-
-        launch_server(
-            config_path=config_path,
-            ray_namespace=parsed_args.namespace,
-        )
-
-        return 0
-
-    except KeyboardInterrupt:
-        logger.info('Server stopped by user')
-        return 0
-    except FileNotFoundError as e:
-        logger.error(f'File not found: {e}')
-        return 1
-    except ValueError as e:
-        logger.error(f'Configuration error: {e}')
-        return 1
-    except ImportError as e:
-        logger.error(f'Import error: {e}')
-        logger.error('Make sure all required dependencies are installed')
-        return 1
-    except Exception as e:
-        logger.exception(f'Unexpected error: {e}')
-        return 1
 
+from twinkle.server.cli import main
 
 if __name__ == '__main__':
     sys.exit(main())

src/twinkle/server/cli/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+"""Twinkle Server CLI (typer)."""
+from .app import app, main
+
+__all__ = ['app', 'main']