Add native Higgs Audio v3 TTS support

CMakeLists.txt

@@ -205,6 +205,13 @@ add_library(engine_runtime STATIC
     src/models/miotts/causal_lm.cpp
     src/models/miotts/session.cpp
     src/models/miotts/loader.cpp
+    src/models/higgs_tts/assets.cpp
+    src/models/higgs_tts/audio_codes.cpp
+    src/models/higgs_tts/codec.cpp
+    src/models/higgs_tts/generator.cpp
+    src/models/higgs_tts/tokenizer.cpp
+    src/models/higgs_tts/session.cpp
+    src/models/higgs_tts/loader.cpp
     src/models/voxcpm2/assets.cpp
     src/models/voxcpm2/audiovae.cpp
     src/models/voxcpm2/generator.cpp

docs/tts.md

@@ -268,12 +268,27 @@ Higgs Audio v3 TTS is a voice-clone TTS model. The current integration uses the
 audiocpp_cli --task tts --family higgs_tts --model models/higgs-audio-v3-tts-4b --backend cuda --text "Hello from Higgs Audio." --voice-ref assets/resources/b.wav --reference-text "Some call me nature. Others call me Mother Nature. I've been here for over 4.5 billion years. 22,500 times longer than you." --out out.wav
 ```
 
+Prepare WAV/MP3/M4A/FLAC references for `--voice-ref`; this normalizes them to mono 24 kHz WAV:
+
+```bash
+python3 tools/prepare_voice_ref.py sample-5.mp3 --output sample-5_ref_24k.wav --overwrite
+```
+
+If the reference transcript is unknown, install the native Qwen3 ASR package and transcribe the prepared reference first:
+
+```bash
+python3 tools/model_manager.py install qwen3_asr_0_6b --models-root models
+audiocpp_cli --task asr --family qwen3_asr --model models/Qwen3-ASR-0.6B --backend cuda --audio sample-5_ref_24k.wav --text ""
+```
+
+Copy the printed `text_output=...` value into Higgs `--reference-text`. Higgs does not run ASR inside the TTS session; ASR is an optional companion model.
+
 | Option | Values | Default | Meaning |
 |---|---|---:|---|
 | `--voice-ref` | WAV path | required | Reference speaker audio. |
 | `--reference-text` | text | empty string | Transcript for reference audio. |
 | `--text-chunk-size` | integer chars | `512` | Long-form chunk size. |
-| `--max-tokens` | integer | `1024` | Maximum generated AR tokens per chunk. |
+| `--max-tokens` | integer | `1024` | Maximum generated AR tokens per chunk. Raise this if speech stops before the text finishes. |
 | `--temperature` | float | `0.8` | AR sampling temperature. |
 | `--top-k` | integer | `30` | AR top-k sampling limit. |
 | `--top-p` | float | `0.8` | AR nucleus sampling limit. |

include/engine/models/higgs_tts/assets.h

@@ -0,0 +1,73 @@
+#pragma once
+
+#include <cstdint>
+#include <filesystem>
+#include <memory>
+#include <string>
+
+namespace engine::assets {
+class TensorSource;
+}
+
+namespace engine::models::higgs_tts {
+
+struct HiggsTextConfig {
+    std::string model_type;
+    int64_t vocab_size = 0;
+    int64_t hidden_size = 0;
+    int64_t intermediate_size = 0;
+    int64_t num_hidden_layers = 0;
+    int64_t num_attention_heads = 0;
+    int64_t num_key_value_heads = 0;
+    int64_t head_dim = 0;
+    int64_t max_position_embeddings = 0;
+    int64_t bos_token_id = 0;
+    int64_t eos_token_id = 0;
+    int64_t pad_token_id = 0;
+    float rms_norm_eps = 1.0e-6F;
+    float rope_theta = 1000000.0F;
+    bool tie_word_embeddings = true;
+};
+
+struct HiggsAudioEncoderConfig {
+    std::string model_type;
+    std::string encoder_type;
+    int64_t num_codebooks = 8;
+    int64_t vocab_size = 1026;
+    int64_t out_dim = 0;
+    int64_t max_chunk_size = 50;
+    int64_t mel_per_sample = 8;
+    bool tie_word_embeddings = true;
+    bool use_delay_pattern = true;
+};
+
+struct HiggsTTSConfig {
+    std::string model_type;
+    int64_t audio_token_id = -100;
+    int64_t ignore_index = -100;
+    HiggsTextConfig text;
+    HiggsAudioEncoderConfig audio_encoder;
+};
+
+struct HiggsTTSAssetPaths {
+    std::filesystem::path model_root;
+    std::filesystem::path small_assets_root;
+    std::filesystem::path config_path;
+    std::filesystem::path codec_config_path;
+    std::filesystem::path chat_template_path;
+    std::filesystem::path model_weights_path;
+    std::filesystem::path model_index_path;
+    std::filesystem::path tokenizer_json_path;
+    std::filesystem::path tokenizer_config_path;
+};
+
+struct HiggsTTSAssets {
+    HiggsTTSAssetPaths paths;
+    HiggsTTSConfig config;
+    std::shared_ptr<const assets::TensorSource> model_weights;
+};
+
+HiggsTTSAssetPaths resolve_higgs_tts_assets(const std::filesystem::path & model_path);
+std::shared_ptr<const HiggsTTSAssets> load_higgs_tts_assets(const std::filesystem::path & model_path);
+
+}  // namespace engine::models::higgs_tts

include/engine/models/higgs_tts/audio_codes.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <cstdint>
+#include <vector>
+
+namespace engine::models::higgs_tts {
+
+constexpr int32_t kHiggsAudioBocId = 1024;
+constexpr int32_t kHiggsAudioEocId = 1025;
+constexpr int32_t kHiggsAudioStopCode = -1;
+constexpr int32_t kHiggsAudioPlaceholderId = -100;
+constexpr int kHiggsAudioSampleRate = 24000;
+
+struct HiggsAudioCodeMatrix {
+    int64_t frames = 0;
+    int64_t codebooks = 0;
+    std::vector<int32_t> token_ids;
+};
+
+HiggsAudioCodeMatrix apply_delay_pattern(const HiggsAudioCodeMatrix & codes);
+HiggsAudioCodeMatrix reverse_delay_pattern(const HiggsAudioCodeMatrix & delayed_codes);
+
+}  // namespace engine::models::higgs_tts

include/engine/models/higgs_tts/codec.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "engine/framework/assets/tensor_source.h"
+#include "engine/framework/core/execution_context.h"
+#include "engine/framework/runtime/session.h"
+#include "engine/models/higgs_tts/assets.h"
+#include "engine/models/higgs_tts/audio_codes.h"
+
+#include <cstddef>
+#include <memory>
+
+namespace engine::models::higgs_tts {
+
+class HiggsAudioCodecDecoderRuntime {
+public:
+    struct Impl;
+
+    HiggsAudioCodecDecoderRuntime(
+        std::shared_ptr<const HiggsTTSAssets> assets,
+        core::ExecutionContext & execution,
+        size_t graph_arena_bytes,
+        size_t weight_context_bytes,
+        assets::TensorStorageType weight_storage_type);
+    ~HiggsAudioCodecDecoderRuntime();
+
+    HiggsAudioCodeMatrix encode_reference_audio(const runtime::AudioBuffer & audio);
+    runtime::AudioBuffer decode(const HiggsAudioCodeMatrix & raw_codes);
+
+private:
+    std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace engine::models::higgs_tts

include/engine/models/higgs_tts/generator.h

@@ -0,0 +1,51 @@
+#pragma once
+
+#include "engine/framework/assets/tensor_source.h"
+#include "engine/framework/core/execution_context.h"
+#include "engine/models/higgs_tts/assets.h"
+#include "engine/models/higgs_tts/audio_codes.h"
+#include "engine/models/higgs_tts/tokenizer.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+
+namespace engine::models::higgs_tts {
+
+struct HiggsTTSGenerationOptions {
+    int64_t max_tokens = 1024;
+    int top_k = 30;
+    float top_p = 0.8F;
+    float temperature = 0.8F;
+    uint32_t seed = 0;
+    bool do_sample = true;
+};
+
+struct HiggsTTSGeneratedCodes {
+    HiggsAudioCodeMatrix delayed_codes;
+    HiggsAudioCodeMatrix raw_codes;
+};
+
+class HiggsTTSGeneratorRuntime {
+public:
+    struct Impl;
+
+    HiggsTTSGeneratorRuntime(
+        std::shared_ptr<const HiggsTTSAssets> assets,
+        core::ExecutionContext & execution,
+        size_t prefill_graph_arena_bytes,
+        size_t decode_graph_arena_bytes,
+        size_t weight_context_bytes,
+        assets::TensorStorageType weight_storage_type);
+    ~HiggsTTSGeneratorRuntime();
+
+    HiggsTTSGeneratedCodes generate(
+        const HiggsTTSPrompt & prompt,
+        const HiggsTTSGenerationOptions & options,
+        const HiggsAudioCodeMatrix * reference_delayed_codes = nullptr);
+
+private:
+    std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace engine::models::higgs_tts

include/engine/models/higgs_tts/loader.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "engine/framework/runtime/model.h"
+#include "engine/models/higgs_tts/assets.h"
+
+#include <memory>
+
+namespace engine::models::higgs_tts {
+
+class HiggsTTSLoadedModel final : public runtime::ILoadedVoiceModel {
+public:
+    HiggsTTSLoadedModel(
+        runtime::ModelMetadata metadata,
+        runtime::CapabilitySet capabilities,
+        std::shared_ptr<const HiggsTTSAssets> assets);
+
+    const runtime::ModelMetadata & metadata() const noexcept override;
+    const runtime::CapabilitySet & capabilities() const noexcept override;
+    std::unique_ptr<runtime::IVoiceTaskSession> create_task_session(
+        const runtime::TaskSpec & task,
+        const runtime::SessionOptions & options) const override;
+
+private:
+    runtime::ModelMetadata metadata_;
+    runtime::CapabilitySet capabilities_;
+    std::shared_ptr<const HiggsTTSAssets> assets_;
+};
+
+std::unique_ptr<HiggsTTSLoadedModel> load_higgs_tts_model(const std::filesystem::path & model_path);
+std::shared_ptr<runtime::IVoiceModelLoader> make_higgs_tts_loader();
+
+}  // namespace engine::models::higgs_tts

include/engine/models/higgs_tts/session.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include "engine/framework/runtime/session.h"
+#include "engine/framework/runtime/session_base.h"
+#include "engine/models/higgs_tts/assets.h"
+#include "engine/models/higgs_tts/codec.h"
+#include "engine/models/higgs_tts/generator.h"
+#include "engine/models/higgs_tts/tokenizer.h"
+
+#include <memory>
+#include <string>
+
+namespace engine::models::higgs_tts {
+
+class HiggsTTSSession final
+    : public runtime::RuntimeSessionBase,
+      public runtime::IOfflineVoiceTaskSession {
+public:
+    HiggsTTSSession(
+        const runtime::TaskSpec & task,
+        const runtime::SessionOptions & options,
+        std::shared_ptr<const HiggsTTSAssets> assets);
+
+    std::string family() const override;
+    runtime::VoiceTaskKind task_kind() const override;
+    runtime::RunMode run_mode() const override;
+    void prepare(const runtime::SessionPreparationRequest & request) override;
+    runtime::TaskResult run(const runtime::TaskRequest & request) override;
+
+private:
+    runtime::TaskSpec task_;
+    std::shared_ptr<const HiggsTTSAssets> assets_;
+    HiggsTTSTokenizer tokenizer_;
+    std::unique_ptr<HiggsTTSGeneratorRuntime> generator_;
+    std::unique_ptr<HiggsAudioCodecDecoderRuntime> codec_decoder_;
+};
+
+}  // namespace engine::models::higgs_tts

include/engine/models/higgs_tts/tokenizer.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include "engine/models/higgs_tts/assets.h"
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace engine::models::higgs_tts {
+
+struct HiggsTTSPrompt {
+    std::string text;
+    std::string reference_text;
+    int64_t reference_audio_tokens = 0;
+    std::vector<int32_t> input_ids;
+};
+
+class HiggsTTSTokenizer {
+public:
+    explicit HiggsTTSTokenizer(std::shared_ptr<const HiggsTTSAssets> assets);
+
+    HiggsTTSPrompt build_prompt(
+        const std::string & text,
+        int64_t reference_audio_tokens = 0,
+        const std::string & reference_text = "") const;
+    std::vector<int32_t> encode_text(const std::string & text) const;
+
+    int32_t tts_token_id() const noexcept;
+    int32_t ref_audio_token_id() const noexcept;
+    int32_t ref_text_token_id() const noexcept;
+    int32_t text_token_id() const noexcept;
+    int32_t audio_token_id() const noexcept;
+
+private:
+    struct Impl;
+    std::shared_ptr<const Impl> impl_;
+};
+
+}  // namespace engine::models::higgs_tts

src/framework/runtime/registry.cpp

@@ -14,6 +14,7 @@
 // #include "engine/models/roformer/session.h"
 #include "engine/models/chatterbox/loader.h"
 #include "engine/models/citrinet_asr/session.h"
+#include "engine/models/higgs_tts/loader.h"
 #include "engine/models/marblenet_vad/session.h"
 #include "engine/models/miocodec/loader.h"
 #include "engine/models/miotts/loader.h"
@@ -211,8 +212,8 @@ ModelRegistry make_default_registry(const std::optional<std::filesystem::path> &
         // engine::models::roformer::make_mel_loader(),
         // engine::models::moss_tts::make_moss_tts_loader(),
         // engine::models::heartmula::make_heartmula_loader(),
-        // engine::models::higgs_tts::make_higgs_tts_loader(),
         // engine::models::parakeet_tdt::make_parakeet_tdt_loader(),
+        engine::models::higgs_tts::make_higgs_tts_loader(),
         engine::models::omnivoice::make_omnivoice_loader(),
         engine::models::miocodec::make_miocodec_loader(),
         engine::models::miotts::make_miotts_loader(),