[update] llm_asr suooported sensevoice, update llm_audio supported alsa cap & play, update static lib version

Author: LittleMouse

projects/llm_framework/SConstruct

@@ -5,7 +5,7 @@ import shutil
 os.environ['SDK_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'SDK'))
 os.environ['EXT_COMPONENTS_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'ext_components'))
 
-version = 'v0.1.2'
+version = 'v0.1.3'
 static_lib = 'static_lib'
 update = False
 

projects/llm_framework/main_asr/SConstruct

@@ -7,7 +7,7 @@ with open(env['PROJECT_TOOL_S']) as f:
 SRCS = Glob('src/*.c*')
 INCLUDE = [ADir('include'), ADir('.')]
 PRIVATE_INCLUDE = []
-REQUIREMENTS = ['pthread', 'utilities', 'eventpp', 'StackFlow', 'single_header_libs']
+REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow', 'single_header_libs']
 STATIC_LIB = []
 DYNAMIC_LIB = []
 DEFINITIONS = []
@@ -17,12 +17,22 @@ LINK_SEARCH_PATH = []
 STATIC_FILES = []
 
 LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./']
-DEFINITIONS += ['-std=c++17']
+DEFINITIONS += ['-std=c++17', '-fopenmp']
 LINK_SEARCH_PATH += [ADir('../static_lib')]
+REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys']
 
 INCLUDE += [ADir('../static_lib/include/sherpa'), ADir('../static_lib/include/sherpa/sherpa-ncnn')]
 LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/ncnn')]
-REQUIREMENTS += ['ncnn', 'sherpa-ncnn-core']
+LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/onnx')]
+REQUIREMENTS += ['ncnn', '', 'onnxruntime']
+LDFLAGS += [
+            '-l:libsherpa-onnx-core.a', '-l:libkaldi-native-fbank-core.a','-l:libkissfft-float.a',
+            '-l:libkaldi-decoder-core.a', '-l:libssentencepiece_core.a', '-l:libsherpa-onnx-fst.a',
+            '-l:libsherpa-onnx-kaldifst-core.a', '-l:libsherpa-onnx-fstfar.a',
+
+            '-l:libsherpa-ncnn-core.a', '-l:libsherpa-ncnn-fst.a', '-l:libsherpa-ncnn-kaldifst-core.a',
+            '-lgomp',
+            ]
 
 STATIC_FILES += Glob('mode_*.json')
 

projects/llm_framework/main_asr/mode_sense-voice-small-10s-ax650.json

@@ -0,0 +1,30 @@
+{
+    "mode": "sense-voice-small-10s-ax650",
+    "type": "asr",
+    "homepage": "https://huggingface.co/yunyu1258/qwen2.5-0.5b-ha",
+    "compile_flage": "pulsar2 build --input model-10-seconds.onnx --config config_sensevoice_main_u16.json --output_dir sensevoice-axmodel --output_name model-10-seconds.axmodel --target_hardware AX650 --compiler.check 0",
+    "pulsar_version": "5.0-patch1-fd447d0d",
+    "capabilities": [
+        "Chinese",
+        "English",
+        "Cantonese",
+        "Japanese",
+        "Korean"
+    ],
+    "input_type": [
+        "sys.pcm",
+        "sys.cap.0_0"
+    ],
+    "output_type": [
+        "asr.utf-8"
+    ],
+    "mode_param": {
+        "model_config.sense_voice.model": "model.axmodel",
+        "model_config.tokens": "tokens.txt",
+        "silero_vad.model": "silero_vad.onnx",
+        "model_config.provider": "axera",
+        "silence_timeout": 2000,
+        "awake_delay": 50
+    },
+    "mode_param_bak": {}
+}

projects/llm_framework/main_asr/src/main.cpp

@@ -5,7 +5,11 @@ with open(env['PROJECT_TOOL_S']) as f:
     exec(f.read())
 
 # SRCS = append_srcs_dir(ADir('src'))
-SRCS = Glob('src/*.c*')
+if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ:
+    SRCS = [AFile('src/sample_audio.c'), AFile('src/main.cpp')]
+else:
+    SRCS = [AFile('src/alsa_audio.c'), AFile('src/main.cpp')]
+
 INCLUDE = [ADir('include'), ADir('.')]
 PRIVATE_INCLUDE = []
 REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow', 'single_header_libs']

projects/llm_framework/main_audio/SConstruct

@@ -6,7 +6,11 @@
 #include <errno.h>
 #include <string.h>
 
-static int gcapLoopExit = 0;
+static struct pcm *g_play_pcm = NULL;
+static int gplayLoopExit      = 1;
+static int gcapLoopExit       = 1;
+AlsaConfig cap_config;
+AlsaConfig play_config;
 
 void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int channel, int rate, int bit,
                     AUDIOCallback callback)
@@ -23,7 +27,7 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch
 
     memset(&config, 0, sizeof(config));
     config.channels          = channel;
-    config.rate              = 48000;  // TODO: 部分USB MIC仅支持48k，暂时固定采集为48k
+    config.rate              = 48000;
     config.period_size       = 120;
     config.period_count      = 4;
     config.format            = PCM_FORMAT_S16_LE;
@@ -57,14 +61,16 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch
 
     SRC_STATE *src_state = NULL;
     float *in_float = NULL, *out_float = NULL;
+
     int in_frames  = pcm_get_buffer_size(pcm);
     int out_frames = (int)((float)in_frames * ((float)rate / 48000.0f) + 1);
-    int out_bytes  = out_frames * channel * sizeof(short);
+
+    int src_channels = 1;
 
     if (rate != 48000) {
-        src_state = src_new(SRC_SINC_FASTEST, channel, NULL);
-        in_float  = malloc(in_frames * channel * sizeof(float));
-        out_float = malloc(out_frames * channel * sizeof(float));
+        src_state = src_new(SRC_SINC_FASTEST, src_channels, NULL);
+        in_float  = malloc(in_frames * src_channels * sizeof(float));
+        out_float = malloc(out_frames * src_channels * sizeof(float));
         if (!src_state || !in_float || !out_float) {
             fprintf(stderr, "Unable to allocate resample buffers\n");
             free(buffer);
@@ -82,37 +88,62 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch
             fprintf(stderr, "Error capturing samples - %d (%s)\n", errno, strerror(errno));
             break;
         }
+
         frames_read = ret;
         total_frames_read += frames_read;
 
         if (rate == 48000) {
-            callback(buffer, frames_read * bytes_per_frame);
+            int in_channels = channel;  // 比如 4
+            int16_t *in     = (int16_t *)buffer;
+
+            int16_t *ch0 = malloc(frames_read * sizeof(int16_t));
+            if (!ch0) {
+                fprintf(stderr, "Unable to allocate ch0 buffer\n");
+                break;
+            }
+
+            for (unsigned int i = 0; i < frames_read; ++i) {
+                ch0[i] = in[i * in_channels + 0];
+            }
+
+            callback((const char *)ch0, frames_read * sizeof(int16_t));
+            free(ch0);
+
         } else {
+            int in_channels = channel;  // 比如 4
             short *in_short = (short *)buffer;
-            for (int i = 0; i < frames_read * channel; ++i) {
-                in_float[i] = in_short[i] / 32768.0f;
+
+            for (int i = 0; i < frames_read; ++i) {
+                short s     = in_short[i * in_channels + 0];
+                in_float[i] = s / 32768.0f;
             }
+
             SRC_DATA src_data;
             src_data.data_in       = in_float;
             src_data.input_frames  = frames_read;
             src_data.data_out      = out_float;
             src_data.output_frames = out_frames;
             src_data.src_ratio     = (double)rate / 48000.0;
             src_data.end_of_input  = 0;
-            int error              = src_process(src_state, &src_data);
+
+            int error = src_process(src_state, &src_data);
             if (error) {
                 fprintf(stderr, "SRC error: %s\n", src_strerror(error));
                 break;
             }
-            // float转short
-            short *out_short = malloc(src_data.output_frames_gen * channel * sizeof(short));
-            for (int i = 0; i < src_data.output_frames_gen * channel; ++i) {
+            int out_samples  = src_data.output_frames_gen;
+            short *out_short = malloc(out_samples * sizeof(short));
+            if (!out_short) {
+                fprintf(stderr, "Unable to allocate out_short buffer\n");
+                break;
+            }
+            for (int i = 0; i < out_samples; ++i) {
                 float sample = out_float[i];
                 if (sample > 1.0f) sample = 1.0f;
                 if (sample < -1.0f) sample = -1.0f;
                 out_short[i] = (short)(sample * 32767.0f);
             }
-            callback((const char *)out_short, src_data.output_frames_gen * channel * sizeof(short));
+            callback((const char *)out_short, out_samples * sizeof(short));
             free(out_short);
         }
     }
@@ -124,7 +155,6 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch
     }
     free(buffer);
     pcm_close(pcm);
-    printf("Total frames captured: %u\n", total_frames_read);
 }
 
 void alsa_close_cap()
@@ -135,4 +165,52 @@ void alsa_close_cap()
 int alsa_cap_status()
 {
     return gcapLoopExit;
+}
+
+void alsa_play(unsigned int card, unsigned int device, float Volume, int channel, int rate, int bit, const void *data,
+               int size)
+{
+    gplayLoopExit = 0;
+
+    struct pcm_config config;
+    memset(&config, 0, sizeof(config));
+    config.channels          = channel;
+    config.rate              = rate;
+    config.period_size       = 1024;
+    config.period_count      = 2;
+    config.format            = PCM_FORMAT_S16_LE;
+    config.silence_threshold = config.period_size * config.period_count;
+    config.stop_threshold    = config.period_size * config.period_count;
+    config.start_threshold   = config.period_size;
+
+    unsigned int pcm_open_flags = PCM_OUT;
+
+    struct pcm *pcm = pcm_open(card, device, pcm_open_flags, &config);
+    if (!pcm || !pcm_is_ready(pcm)) {
+        fprintf(stderr, "Unable to open PCM playback device (%s)\n", pcm ? pcm_get_error(pcm) : "invalid pcm");
+        if (pcm) {
+            pcm_close(pcm);
+        }
+        gplayLoopExit = 2;
+        return;
+    }
+
+    int frames         = pcm_bytes_to_frames(pcm, size);
+    int written_frames = pcm_writei(pcm, data, frames);
+    if (written_frames < 0) {
+        fprintf(stderr, "PCM playback error %s\n", pcm_get_error(pcm));
+    }
+    printf("Played %d frames\n", written_frames);
+    pcm_close(pcm);
+    gplayLoopExit = 2;
+}
+
+void alsa_close_play()
+{
+    gplayLoopExit = 1;
+}
+
+int alsa_play_status()
+{
+    return gplayLoopExit;
 }

projects/llm_framework/main_audio/src/alsa_audio.c

@@ -1,5 +1,18 @@
 #pragma once
 
+typedef struct AlsaConfig
+{
+    unsigned int card;
+    unsigned int device;
+    float volume;
+    int channel;
+    int rate;
+    int bit;
+} AlsaConfig;
+
+extern AlsaConfig cap_config;
+extern AlsaConfig play_config;
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -10,6 +23,13 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch
                     AUDIOCallback callback);
 void alsa_close_cap();
 
+void alsa_play(unsigned int card, unsigned int device, float Volume, int channel, int rate, int bit, const void *data,
+             int size);
+
+void alsa_close_play();
+int alsa_cap_status();
+int alsa_play_status();
+
 #ifdef __cplusplus
 }
 #endif