【C】回声消除-V3--音频技术-火山引擎

文档中心

导航

【C】回声消除-V3

最近更新时间：2023.07.07 11:16:16首次发布时间：2023.03.16 11:40:12

完整例子

#include "sami_core"

int num_channels = input_file->getNumChannel();
int sample_rate = input_file->getSampleRate();
int num_frames = input_file->getNumFrames();
auto in_samples = loadWholeAudioFile(file_src);
std::vector<uint8_t> modelBin = loadBinaryFromFile(modelPath);

// create handle
SAMICoreHandle handle = nullptr;
SAMICoreExecutorContextCreateParameter createParameter;
memset(&createParameter, 0, sizeof(SAMICoreExecutorContextCreateParameter));
createParameter.sampleRate = sample_rate;
createParameter.maxBlockSize = pre_define_block_size;
createParameter.numChannel = num_channels;
createParameter.modelBuffer = reinterpret_cast<char*>(modelBin.data());
createParameter.modelLen = modelBin.size();
createParameter.bussinessInfo = "aec_v3_demo";
createParameter.numAudioBuffer = 2;
createParameter.configInfo = R"( {"utility":"CommonAecUtility",
                                    "enable_stereo":true,
                                    "enable_pre_delay":true,
                                    "enable_pre_process":true,
                                    "enable_after_process":true,
                                    "after_process_param": {"dtd_threshold":0.2, 
                                                            "st_ramp":0.2,
                                                            "dt_ramp":0.2} } )";
int ret = SAMICoreCreateHandleByIdentify(&handle, SAMICoreIdentify_EngineExecutor_CE_AEC, &createParameter);
if(ret != SAMI_OK) {
    std::cerr << "create handler failed: " << ret;
    exit(-1);
}

SAMICoreAudioBuffer* in_audio_buffer = new SAMICoreAudioBuffer[2];
//存放mic数据
in_audio_buffer[0].numberChannels = num_channels;
in_audio_buffer[0].numberSamples = pre_define_block_size;
in_audio_buffer[0].data = new float*[num_channels];
in_audio_buffer[0].isInterleave = 0;

//存放ref数据
in_audio_buffer[1].numberChannels = num_channels;
in_audio_buffer[1].numberSamples = pre_define_block_size;
in_audio_buffer[1].data = new float*[num_channels];
in_audio_buffer[1].isInterleave = 0;

SAMICoreAudioBuffer out_audio_buffer;
out_audio_buffer.numberChannels = num_channels;
out_audio_buffer.numberSamples = pre_define_block_size;
out_audio_buffer.data = new float*[num_channels];
out_audio_buffer.isInterleave = isInterleave;

for(int c = 0; c < int(num_channels); ++c) {
    in_audio_buffer[0].data[c] = new float[pre_define_block_size];
    in_audio_buffer[0].data[c] = new float[pre_define_block_size];
    out_audio_buffer.data[c] = new float[pre_define_block_size];
}

SAMICoreBlock in_block;
memset(&in_block, 0, sizeof(SAMICoreBlock));
in_block.numberAudioData = 2;
in_block.dataType = SAMICoreDataType::SAMICoreDataType_AudioBuffer;
in_block.audioData = in_audio_buffer;

SAMICoreBlock out_block;
memset(&out_block, 0, sizeof(SAMICoreBlock));
out_block.numberAudioData = 1;
out_block.dataType = SAMICoreDataType::SAMICoreDataType_AudioBuffer;
out_block.audioData = &out_audio_buffer;

for(;hasAudioSamples();)
{
    copySamplesToInputBuffer(mic_data, in_audio_buffer[0]); //拷贝数据或者修改数据指针in_audio_buffer的指向
    copySamplesToInputBuffer(ref_data, in_audio_buffer[1]);
    in_audio_buffer[0].numberSamples = feed_size;
    in_audio_buffer[1].numberSamples = feed_size;
    out_audio_buffer.numberSamples = feed_size;
    ret = SAMICoreProcess(handle, &in_block, &out_block);
    if(ret != SAMI_OK && ret != SAMI_ENGINE_INPUT_NEED_MORE_DATA) {
        std::cerr << "process error: " << ret;
        exit(-1);
    }
    if(out_audio_buffer.numberSamples > 0) {
        // do something after process
        doSomethingAfterProcess(out_block);  //业务从out_block拷贝处理后的数据
    }
}

SAMICoreProperty flushProperty;
memset(&flushProperty, 0, sizeof(SAMICoreProperty));
flushProperty.type = SAMICoreDataType_AudioBuffer;
SAMICoreGetPropertyById(handle, SAMICorePropertyID_Common_Flush, &flushProperty);
if(flushProperty.dataLen > 0 && flushProperty.type == SAMICoreDataType_AudioBuffer  && flushProperty.data) {
    SAMICoreAudioBuffer* bufferArray = (SAMICoreAudioBuffer*)flushProperty.data;
    if(bufferArray[0].data && bufferArray[0].numberSamples > 0) {
        // do something after process
        doSomethingAfterProcess(out_block);  //业务从out_block拷贝处理后的数据
    }
}
SAMICoreDestroyProperty(&flushProperty);

SAMICoreProperty resetProperty;
memset(&resetProperty, 0, sizeof(SAMICoreProperty));
resetProperty.id = SAMICorePropertyID_Common_Reset;
resetProperty.type = SAMICoreDataType_Null;
SAMICoreSetProperty(handle, SAMICorePropertyID_Common_Reset, &resetProperty);

// release resources
for(int c = 0; c < int(num_channels); ++c) {
    delete[] in_audio_buffer[0].data[c];
    delete[] in_audio_buffer[1].data[c];
    delete[] out_audio_buffer.data[c];
}

delete[] in_audio_buffer.data[0];
delete[] in_audio_buffer.data[1];
delete[] in_audio_buffer;
delete[] out_audio_buffer.data;
SAMICoreDestroyHandle(handle);
handle = nullptr;

使用步骤

〇、从文件中读取模型

即读取整个模型文件到内存，实现方法自由发挥，例子中loadModelAsBinary仅供参考。算法模型详见回声消除介绍小节。

一、创建算法句柄

传入模型内存地址、模型大小、采样率和 maxBlockSize，通过 SAMICoreCreateHandleByIdentify 创建 handle。

SAMICoreExecutorContextCreateParameter参数介绍

参数	类型	说明
sampleRate	`int`	入参，指音频的采样率
maxBlockSize	`int`	入参，每次输入音频的最大的大小，算法需要根据此字段提前分配内存等，接下来每次送入process的大小不能超过该值
numChannel	`int`	入参，音频的通道数
modelBuffer	`const char*`	入参，模型的内容
modelLen	`int`	入参，模型的内容的长度
bussinessInfo	`const char*`	入参，表示调用的业务方信息
numAudioBuffer	`int`	入参，回声消除v3为2
configInfo	`const char*`	入参，见下文configInfo

configInfo

参数	类型	说明
utility	string	入参，固定设置为`CommonAecUtility`
enable_stereo	bool	入参，默认值：false；当处理两个通道的数据时候，enable_stereo为ture，两个通道单独处理；enable_stereo为false，处理第一个通道后，拷贝结果覆盖第二个通道，节省一半计算量；
enable_pre_delay	bool	入参，默认值：false；算法需要送入足够的数据才会输出结果，在实时场景需要等进等出，enable_pre_delay=true，会在一开始返回静音缓冲数据，减少接入难度，建议rtc场景默认开启
enable_pre_process	bool	入参，默认值：false； true时候打开防爆音前处理
enable_after_process	bool	入参，默认值：false； true时候打开单双讲检测后处理，在单讲场景提供更好的效果
after_process_param	json-map	入参，默认值：见下文单双讲检测后处理的配置参数

after_process_param参数介绍

参数	含义	范围
dtd_threshold	控制单双讲检测的阈值	0<=x<=1，默认取0.2
st_ramp	控制检测到单讲后gain的衰退速率	0<=x<=1，默认取0.2
dt_ramp	控制检测到双讲后gain的增益速率	0<=x<=1，默认取0.2

举例：
需要根据实际情况打开关闭功能

SAMICoreHandle handle = nullptr;
SAMICoreExecutorContextCreateParameter createParameter;
memset(&createParameter, 0, sizeof(SAMICoreExecutorContextCreateParameter));
createParameter.sampleRate = sample_rate;
createParameter.maxBlockSize = pre_define_block_size;
createParameter.numChannel = num_channels;
createParameter.modelBuffer = reinterpret_cast<char*>(modelBin.data());
createParameter.modelLen = modelBin.size();
createParameter.bussinessInfo = "aec_v3_demo";
createParameter.numAudioBuffer = 2;
createParameter.configInfo = R"( {"utility":"CommonAecUtility",
                                    "enable_stereo":true,
                                    "enable_pre_delay":true,
                                    "enable_pre_process":true,
                                    "enable_after_process":true,
                                    "after_process_param": {"dtd_threshold":0.2, 
                                                            "st_ramp":0.2,
                                                            "dt_ramp":0.2} } )";
int ret = SAMICoreCreateHandleByIdentify(&handle, SAMICoreIdentify_EngineExecutor_CE_AEC, &createParameter);
if(ret != SAMI_OK) {
    std::cerr << "create handler failed: " << ret;
    exit(-1);
}

有几种情况会导致创建失败：

模型数据不正确，例如模型数据损坏或者大小不对。
Block size 数据不正确。

二、创建 SAMICoreBlock 用于存放输入和输出

SAMICoreAudioBuffer，用于存放音频数据，它支持 Planar-Float 和 Interleaved-Float 类型数据。更多关于音频数据格式请参看名词解释 部分。SAMICoreBlock，用于存放需要处理的数据。
目前接口内部有ringBuffer，所以支持任意block_size, 当数据量不足的时候，不会有数据返回。
注意：输入SAMICoreAudioBuffer 需申请 2个，下标0存放mic音频数据，下标1存放ref音频数据

SAMICoreAudioBuffer* in_audio_buffer = new SAMICoreAudioBuffer[2];
//存放mic数据
in_audio_buffer[0].numberChannels = num_channels;
in_audio_buffer[0].numberSamples = pre_define_block_size;
in_audio_buffer[0].data = new float*[num_channels];
in_audio_buffer[0].isInterleave = 0;

//存放ref数据
in_audio_buffer[1].numberChannels = num_channels;
in_audio_buffer[1].numberSamples = pre_define_block_size;
in_audio_buffer[1].data = new float*[num_channels];
in_audio_buffer[1].isInterleave = 0;

SAMICoreAudioBuffer out_audio_buffer;
out_audio_buffer.numberChannels = num_channels;
out_audio_buffer.numberSamples = pre_define_block_size;
out_audio_buffer.data = new float*[num_channels];
out_audio_buffer.isInterleave = isInterleave;

for(int c = 0; c < int(num_channels); ++c) {
    in_audio_buffer[0].data[c] = new float[pre_define_block_size];
    in_audio_buffer[0].data[c] = new float[pre_define_block_size];
    out_audio_buffer.data[c] = new float[pre_define_block_size];
}

SAMICoreBlock in_block;
memset(&in_block, 0, sizeof(SAMICoreBlock));
in_block.numberAudioData = 2;
in_block.dataType = SAMICoreDataType::SAMICoreDataType_AudioBuffer;
in_block.audioData = in_audio_buffer;

SAMICoreBlock out_block;
memset(&out_block, 0, sizeof(SAMICoreBlock));
out_block.numberAudioData = 1;
out_block.dataType = SAMICoreDataType::SAMICoreDataType_AudioBuffer;
out_block.audioData = &out_audio_buffer;

三、处理音频

拷贝数据进行处理

将待处理的音频数据拷贝到 in_audio_buffer 中，经过 SAMICoreProcess 处理后，结果将拷贝至 output 中。示例中采用这种方法。

for(;hasAudioSamples();)
{
    copySamplesToInputBuffer(mic_data, in_audio_buffer[0]); //拷贝数据或者修改数据指针in_audio_buffer的指向
    copySamplesToInputBuffer(ref_data, in_audio_buffer[1]);
    in_audio_buffer[0].numberSamples = feed_size;
    in_audio_buffer[1].numberSamples = feed_size;
    out_audio_buffer.numberSamples = feed_size;
    ret = SAMICoreProcess(handle, &in_block, &out_block);
    if(ret != SAMI_OK && ret != SAMI_ENGINE_INPUT_NEED_MORE_DATA) {
        std::cerr << "process error: " << ret;
        exit(-1);
    }
    if(out_audio_buffer.numberSamples > 0) {
        // do something after process
        doSomethingAfterProcess(out_block);  //业务从out_block拷贝处理后的数据
    }
}

In-Place 进行处理

更新音频数据的指针，指向正确的内存即可，这样可以避免内存数据的拷贝。

for(;hasAudioSamples();)
{
    updateInputBuffer(mic_data, in_audio_buffer[0]);
    updateInputBuffer(ref_data, in_audio_buffer[1]);
    updateOutputBuffer(out_audio_buffer);
    
    int ret = SAMICoreProcess(handle, &in_block, &out_block);
    if(ret != SAMI_OK && ret != SAMI_ENGINE_INPUT_NEED_MORE_DATA) {
        std::cerr << "process error: " << ret;
        exit(-1);
    }
    if(out_audio_buffer.numberSamples > 0) {
        // do something after process
        doSomethingAfterProcess(out_block);  //业务从out_block拷贝处理后的数据
    }
}

有几种情况导致处理失败：

无效的 handle。handle 创建失败了，但仍然拿错误的 handle 进行 process
SAMICoreAudioBuffer 和 SAMICoreBlock 设置错误。

需要注意的是：当 block size 较小的情况下，前几帧会由于数据量不足无法处理而返回错误码SAMI_ENGINE_INPUT_NEED_MORE_DATA，但out_audio_buffer.numberSamples不一定为0

四、获取最后结果

由于算法存在缓存部分数据，因此当音频已经全部送入算法后，需要调用这个接口，将所有缓存的数据取出

SAMICoreProperty flushProperty;
memset(&flushProperty, 0, sizeof(SAMICoreProperty));
flushProperty.type = SAMICoreDataType_AudioBuffer;
SAMICoreGetPropertyById(handle, SAMICorePropertyID_Common_Flush, &flushProperty);
if(flushProperty.dataLen > 0 && flushProperty.type == SAMICoreDataType_AudioBuffer  && flushProperty.data) {
    SAMICoreAudioBuffer* bufferArray = (SAMICoreAudioBuffer*)flushProperty.data;
    if(bufferArray[0].data && bufferArray[0].numberSamples > 0) {
        // do something after process
        doSomethingAfterProcess(out_block);  //业务从out_block拷贝处理后的数据
    }
}
SAMICoreDestroyProperty(&flushProperty);

五、重置算法句柄

当处理完成后，后续有新数据需要处理，而又不想重复创建handle时，可以调用reset接口，之后就可以当做一个新实例环境来用

SAMICoreProperty resetProperty;
memset(&resetProperty, 0, sizeof(SAMICoreProperty));
resetProperty.id = SAMICorePropertyID_Common_Reset;
resetProperty.type = SAMICoreDataType_Null;
SAMICoreSetProperty(handle, SAMICorePropertyID_Common_Reset, &resetProperty);

六、释放资源

释放 handle

ret = SAMICoreDestroyHandle(handle);

此外，还要注意音频数据数据的内存释放（如果有）。例如：

for(int c = 0; c < int(num_channels); ++c) {
    delete[] in_audio_buffer[0].data[c];
    delete[] in_audio_buffer[1].data[c];
    delete[] out_audio_buffer.data[c];
}

delete[] in_audio_buffer.data[0];
delete[] in_audio_buffer.data[1];
delete[] in_audio_buffer;
delete[] out_audio_buffer.data;
SAMICoreDestroyHandle(handle);
handle = nullptr;