Add MOSS-TTS-Local model family

CMakeLists.txt

@@ -217,6 +217,16 @@ add_library(engine_runtime STATIC
     src/models/miotts/causal_lm.cpp
     src/models/miotts/session.cpp
     src/models/miotts/loader.cpp
+    src/models/moss_tts_local/assets.cpp
+    src/models/moss_tts_local/backbone.cpp
+    src/models/moss_tts_local/codec_decoder.cpp
+    src/models/moss_tts_local/codec_encoder.cpp
+    src/models/moss_tts_local/codec_quantizer.cpp
+    src/models/moss_tts_local/depth_transformer.cpp
+    src/models/moss_tts_local/generator.cpp
+    src/models/moss_tts_local/loader.cpp
+    src/models/moss_tts_local/processor.cpp
+    src/models/moss_tts_local/session.cpp
     src/models/voxcpm2/assets.cpp
     src/models/voxcpm2/audiovae.cpp
     src/models/voxcpm2/generator.cpp
@@ -488,6 +498,42 @@ if (ENGINE_ENABLE_OPENMP)
     target_link_libraries(vibevoice_finetune_overlay_check PRIVATE OpenMP::OpenMP_CXX)
 endif()
 
+add_executable(moss_tts_local_smoke
+    tests/moss_tts_local/moss_tts_local_smoke.cpp
+)
+
+target_link_libraries(moss_tts_local_smoke PRIVATE engine_runtime ggml)
+if (ENGINE_ENABLE_OPENMP)
+    target_link_libraries(moss_tts_local_smoke PRIVATE OpenMP::OpenMP_CXX)
+endif()
+
+add_executable(codec_dequant_parity
+    tests/moss_tts_local/codec_dequant_parity.cpp
+)
+
+target_link_libraries(codec_dequant_parity PRIVATE engine_runtime ggml)
+if (ENGINE_ENABLE_OPENMP)
+    target_link_libraries(codec_dequant_parity PRIVATE OpenMP::OpenMP_CXX)
+endif()
+
+add_executable(codec_decode_parity
+    tests/moss_tts_local/codec_decode_parity.cpp
+)
+
+target_link_libraries(codec_decode_parity PRIVATE engine_runtime ggml)
+if (ENGINE_ENABLE_OPENMP)
+    target_link_libraries(codec_decode_parity PRIVATE OpenMP::OpenMP_CXX)
+endif()
+
+add_executable(codec_encode_parity
+    tests/moss_tts_local/codec_encode_parity.cpp
+)
+
+target_link_libraries(codec_encode_parity PRIVATE engine_runtime ggml)
+if (ENGINE_ENABLE_OPENMP)
+    target_link_libraries(codec_encode_parity PRIVATE OpenMP::OpenMP_CXX)
+endif()
+
 if (ENGINE_BUILD_WARMBENCH)
     function(add_engine_warmbench target_name source_file)
         add_executable(${target_name}

app/workflow/execution.cpp

@@ -1,5 +1,6 @@
 #include "execution.h"
 
+#include <chrono>
 #include <stdexcept>
 
 namespace minitts::app {
@@ -53,18 +54,31 @@ AppBatchResult run_offline_batch(
     if (batch.requests.empty()) {
         throw std::runtime_error("offline batch requires at least one request");
     }
+    using Clock = std::chrono::steady_clock;
+    const auto to_ms = [](Clock::duration d) {
+        return std::chrono::duration<double, std::milli>(d).count();
+    };
+    const auto session_start = Clock::now();
+
+    const auto prepare_start = Clock::now();
     session.prepare(engine::runtime::build_preparation_request(batch.requests.front().request));
     AppBatchResult out;
+    out.prepare_ms = to_ms(Clock::now() - prepare_start);
     out.results.reserve(batch.requests.size());
     for (const auto & item : batch.requests) {
+        const auto run_start = Clock::now();
+        auto result = offline.run(item.request);
+        const double wall_ms = to_ms(Clock::now() - run_start);
         out.results.push_back(AppRequestResult{
             item.id,
-            offline.run(item.request),
+            std::move(result),
+            wall_ms,
         });
         if (on_result) {
             on_result(out.results.size() - 1, out.results.back());
         }
     }
+    out.session_wall_ms = to_ms(Clock::now() - session_start);
     if (audio_merge_mode == AudioMergeMode::Concat) {
         out.merged_audio = concat_audio_outputs(out.results, out.chapters);
     }

app/workflow/execution.h

@@ -19,6 +19,7 @@ struct AppRequest {
 struct AppRequestResult {
     std::string id;
     engine::runtime::TaskResult result;
+    double wall_ms = 0.0;
 };
 
 struct AudioChapter {
@@ -35,6 +36,8 @@ struct AppBatchResult {
     std::vector<AppRequestResult> results;
     std::optional<engine::runtime::AudioBuffer> merged_audio;
     std::vector<AudioChapter> chapters;
+    double prepare_ms = 0.0;
+    double session_wall_ms = 0.0;
 };
 
 enum class AudioMergeMode {

app/workflow/file_sink.cpp

@@ -318,6 +318,9 @@ void emit_batch_result(
     for (size_t i = 0; i < batch.results.size(); ++i) {
         emit_batch_item_result(i, batch.results[i], policy);
     }
+
+    std::cout << "[TIMING] session.prepare_ms " << batch.prepare_ms << "\n";
+    std::cout << "[TIMING] session.wall_ms " << batch.session_wall_ms << "\n";
 }
 
 void emit_batch_summary(
@@ -351,6 +354,7 @@ void emit_batch_item_result(
     const std::string request_id = safe_output_name(item.id);
     std::cout << "request_index=" << index << "\n";
     std::cout << "request_id=" << request_id << "\n";
+    std::cout << "[TIMING] request." << request_id << ".wall_ms " << item.wall_ms << "\n";
 
     std::optional<std::filesystem::path> audio_out;
     std::optional<std::filesystem::path> named_out_dir;

include/engine/models/moss_tts_local/assets.h

@@ -0,0 +1,83 @@
+#pragma once
+
+#include "engine/framework/assets/tensor_source.h"
+
+#include <cstdint>
+#include <filesystem>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace engine::models::moss_tts_local {
+
+struct MossBackboneConfig {
+    int64_t hidden_size = 0;
+    int64_t intermediate_size = 0;
+    int64_t num_hidden_layers = 0;
+    int64_t num_attention_heads = 0;
+    int64_t num_key_value_heads = 0;
+    int64_t head_dim = 0;
+    int64_t max_position_embeddings = 0;
+    int64_t vocab_size = 0;
+    float rms_norm_eps = 1.0e-6F;
+    float rope_theta = 1000000.0F;
+    bool tie_word_embeddings = true;
+};
+
+struct MossLocalTransformerConfig {
+    int64_t hidden_size = 0;
+    int64_t intermediate_size = 0;
+    int64_t num_layers = 0;
+    int64_t num_heads = 0;
+    int64_t max_positions = 0;
+    float layer_norm_eps = 1.0e-6F;
+    float rope_base = 1000000.0F;
+};
+
+struct MossTTSLocalConfig {
+    MossBackboneConfig backbone;
+    MossLocalTransformerConfig local;
+    int64_t num_codebooks = 0;
+    int64_t audio_vocab_size = 0;
+    std::vector<int64_t> audio_codebook_sizes;
+    int64_t audio_pad_token_id = 0;
+    int64_t pad_token_id = 0;
+    int64_t im_start_token_id = 0;
+    int64_t im_end_token_id = 0;
+    int64_t audio_start_token_id = 0;
+    int64_t audio_end_token_id = 0;
+    int64_t audio_user_slot_token_id = 0;
+    int64_t audio_assistant_slot_token_id = 0;
+    int64_t sampling_rate = 0;
+    std::string local_text_head_mode;
+    std::string audio_tokenizer_name_or_path;
+};
+
+struct MossTTSLocalAssetPaths {
+    std::filesystem::path model_root;
+    std::filesystem::path config_path;
+    std::filesystem::path model_weights_path;
+    std::optional<std::filesystem::path> tokenizer_json_path;
+    std::optional<std::filesystem::path> tokenizer_config_path;
+    std::optional<std::filesystem::path> tokenizer_vocab_path;
+    std::optional<std::filesystem::path> tokenizer_merges_path;
+    std::optional<std::filesystem::path> audio_tokenizer_root;
+};
+
+struct MossTTSLocalAssets {
+    MossTTSLocalAssetPaths paths;
+    MossTTSLocalConfig config;
+    std::shared_ptr<const assets::TensorSource> model_weights;
+};
+
+MossTTSLocalAssetPaths resolve_moss_tts_local_assets(const std::filesystem::path & model_path);
+MossTTSLocalConfig load_moss_tts_local_config(const std::filesystem::path & model_path);
+std::shared_ptr<const MossTTSLocalAssets> load_moss_tts_local_assets(const std::filesystem::path & model_path);
+
+// Locates the MOSS-Audio-Tokenizer-v2 codec weights: prefers a local audio_tokenizer/
+// directory next to the model, otherwise resolves the Hugging Face hub cache snapshot for
+// the repo id recorded in config (audio_tokenizer_name_or_path).
+std::filesystem::path resolve_moss_codec_dir(const MossTTSLocalAssets & assets);
+
+}  // namespace engine::models::moss_tts_local

include/engine/models/moss_tts_local/backbone.h

@@ -0,0 +1,57 @@
+#pragma once
+
+#include "engine/framework/assets/tensor_source.h"
+#include "engine/framework/core/execution_context.h"
+#include "engine/models/moss_tts_local/assets.h"
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+namespace engine::models::moss_tts_local {
+
+// Qwen3 backbone (transformer.*) runtime: loads the language-model weights and runs
+// a prefill forward that returns the final hidden states. Text tokens are embedded in
+// the graph; the summed audio-codebook contribution is supplied as a precomputed bias
+// so the depth transformer and the generator can share a single embedding table.
+class MossBackboneRuntime {
+public:
+    MossBackboneRuntime(
+        std::shared_ptr<const MossTTSLocalAssets> assets,
+        core::ExecutionContext & execution_context,
+        size_t graph_arena_bytes,
+        size_t weight_context_bytes,
+        assets::TensorStorageType weight_storage_type);
+    ~MossBackboneRuntime();
+
+    MossBackboneRuntime(const MossBackboneRuntime &) = delete;
+    MossBackboneRuntime & operator=(const MossBackboneRuntime &) = delete;
+
+    int64_t hidden_size() const noexcept;
+
+    // Incremental (KV-cached) generation. Call begin_generation once to size and allocate the
+    // per-layer cache, prefill the prompt in a single batched forward, then step one position
+    // at a time. This runs generation in O(T) work per step instead of the O(T^2) cost of
+    // re-forwarding the whole sequence every frame.
+    //
+    // begin_generation sizes the cache for max_positions and builds the reusable
+    // single-position step graph.
+    void begin_generation(int64_t max_positions) const;
+    // prefill forwards the whole prompt at once, writes every position's K/V into the cache,
+    // and returns the last position's hidden state ([hidden_size]) to seed the first frame.
+    // audio_bias is the summed audio-codebook embedding per position ([token_ids.size() *
+    // hidden_size] row-major), mirroring MossTTSLocalModel._build_inputs_embeds.
+    std::vector<float> prefill(const std::vector<int32_t> & token_ids, const std::vector<float> & audio_bias) const;
+    // step forwards one fused position (its text token embedded in-graph plus audio_bias_row),
+    // appends that position's K/V to the cache, and returns its hidden state ([hidden_size]).
+    std::vector<float> step(int32_t token_id, const std::vector<float> & audio_bias_row) const;
+    int64_t cached_positions() const noexcept;
+
+private:
+    void build_step_graph(int64_t cache_steps) const;
+
+    struct Impl;
+    std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace engine::models::moss_tts_local

include/engine/models/moss_tts_local/codec_decoder.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include "engine/framework/core/execution_context.h"
+
+#include <cstdint>
+#include <filesystem>
+#include <memory>
+#include <vector>
+
+namespace engine::models::moss_tts_local {
+
+// MOSS-Audio-Tokenizer-v2 decoder: turns generated RVQ codes into a 48 kHz
+// stereo waveform. The codec is "CNN-free" -- the decoder is a stack of causal
+// Transformer blocks (interleaved RoPE, LayerScale, GELU MLP) separated by
+// reshape-based patch upsamples, ending in a channel de-interleave that splits
+// the jointly-processed stream back into left/right. The RLFQ dequantizer
+// (codes -> latent) is provided by MossCodecDequantizer.
+class MossCodecDecoder {
+public:
+    MossCodecDecoder(
+        const std::filesystem::path & codec_dir,
+        core::ExecutionContext & execution_context,
+        int64_t num_quantizers,
+        size_t weight_context_bytes,
+        size_t graph_arena_bytes);
+    ~MossCodecDecoder();
+
+    MossCodecDecoder(const MossCodecDecoder &) = delete;
+    MossCodecDecoder & operator=(const MossCodecDecoder &) = delete;
+
+    int64_t sampling_rate() const noexcept;
+
+    // Decodes [num_quantizers][steps] codes into a stereo waveform returned as
+    // {left, right}, each with steps * 3840 samples at 48 kHz.
+    std::vector<std::vector<float>> decode(const std::vector<std::vector<int32_t>> & codes) const;
+
+private:
+    struct Impl;
+    std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace engine::models::moss_tts_local

include/engine/models/moss_tts_local/codec_encoder.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include "engine/framework/core/execution_context.h"
+
+#include <cstdint>
+#include <filesystem>
+#include <memory>
+#include <vector>
+
+namespace engine::models::moss_tts_local {
+
+// MOSS-Audio-Tokenizer-v2 encoder: turns a reference waveform into RLFQ codes
+// for zero-shot voice cloning. It is the structural mirror of MossCodecDecoder --
+// stereo is interleaved into one stream, patched down and run through a stack of
+// causal Transformer blocks (interleaved RoPE, LayerScale, GELU MLP), then the
+// RLFQ quantizer selects the nearest codes. Produces the same [num_quantizers,
+// frames] code matrix the generator consumes.
+class MossCodecEncoder {
+public:
+    MossCodecEncoder(
+        const std::filesystem::path & codec_dir,
+        core::ExecutionContext & execution_context,
+        int64_t num_quantizers,
+        size_t weight_context_bytes,
+        size_t graph_arena_bytes);
+    ~MossCodecEncoder();
+
+    MossCodecEncoder(const MossCodecEncoder &) = delete;
+    MossCodecEncoder & operator=(const MossCodecEncoder &) = delete;
+
+    // Encodes a waveform given as {left, right} channels (each with the same
+    // per-channel sample count, 48 kHz) into [num_quantizers][frames] codes.
+    std::vector<std::vector<int32_t>> encode(const std::vector<std::vector<float>> & channels) const;
+
+private:
+    struct Impl;
+    std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace engine::models::moss_tts_local