Downmix BT stream from stereo to mono

[M-Schiller]

Hi everyone,

I am currently working with the LyraT eval board and try to understand audio pipelines.
I want to receive an audio stream via Bluetooth, downmix stereo to mono, and sent it via I2S to one of the codec chips.
For this reason, I tried to expand the "play_bt_music_example" with the "downmix_pipeline_example" where I think it would be easiest to not reconfigure the audio codec but mix L and R onto both output channels and check that the output of a single speaker contains information of both input channels.

So, I tried the following code but to no avail.

Code: Select all

	downmix_cfg_t downmix_cfg = DEFAULT_DOWNMIX_CONFIG();
	downmix_cfg.downmix_info.source_num = 1;
	audio_element_handle_t downmixer = downmix_init(&downmix_cfg);

	esp_downmix_input_info_t source_info =
	{
		.samplerate = 48000,
		.channel = 2,
		.bits_num = 16,
		.gain = {0, 0},
		.transit_time = 500
	};
	
	source_info_init(downmixer, &source_info);
	downmix_set_input_rb_timeout(downmixer, 0, 0);
	downmix_set_input_rb(downmixer, audio_element_get_input_ringbuf(i2s_stream_writer), 0);

	audio_pipeline_register(pipeline, downmixer, "downmixer");

	const char *link_tag[3] = {"bt", "downmixer", "i2s"};
	audio_pipeline_link(pipeline, &link_tag[0], 3);

When I try to play a song, I always get the error
`A2DP_STREAM: No ineffecitive memory to allocate(190)`
but to me it seems like its basically the same usage of a downmixer as in the downmix example. Can the downmixer not be used to downmix a single (stereo) BT stream into a mono signal on I2S? Any help would be greatly appreciated.

[tempo.tian]

There is a hacky way, which use lowest resource.
After pipeline is linked, ringbuf is malloc and liked to each element.
Actually we can set output cb to overwrite the behavior, we reuse the malloc ringbuf but fill with stuff after processing in output callback.

Code: Select all

   // for example you have a mp3 decode and linked to i2s for output
    audio_pipeline_register(pipeline, mp3_decoder, "mp3");
    audio_pipeline_register(pipeline, i2s_stream_writer, "i2s");

    ESP_LOGI(TAG, "[2.4] Link it together [mp3_music_read_cb]-->mp3_decoder-->i2s_stream-->[codec_chip]");
    const char *link_tag[2] = {"mp3", "i2s"};
    audio_pipeline_link(pipeline, &link_tag[0], 2);
   // after pipeline is linked, we overwrite write function to use callback not ringbuf
    audio_element_set_write_cb(mp3_decoder, mp3_music_write_cb, i2s_stream_writer);
    //we can change the buffer output by mp3 decoder and process it, after process we resend it to i2s input
int mp3_music_write_cb(audio_element_handle_t el, char *buf, int len, TickType_t wait_time, void *ctx)
{
    ringbuf_handle_t fifo = audio_element_get_input_ringbuf((audio_element_handle_t)ctx);
    if (fifo && (len % 4) == 0) {
        int out_len = len / 2;
        int16_t* out = (int16_t*) malloc(out_len);
        if (out) {
            int16_t* left = (int16_t*) out;
            int16_t* in = (int16_t*) buf;
            int n = len / 4;
            while (n > 0) {
                *left = *in;
                left++;
                in += 2;
                n--;
            }
            out_len = rb_write(fifo, out, out_len, ctx);
            free(out);
        }
        return out_len * 2; 
    }
    return -1;
}

[tempo.tian]

To make all code runs ok you need change following code also:

Code: Select all

change code in audio_element.c
esp_err_t audio_element_set_ringbuf_done(audio_element_handle_t el)
{
    int ret = ESP_OK;
    if (NULL == el) {
        return ESP_FAIL;
    }
    if (el->out.output_rb) {
        ret |= rb_done_write(el->out.output_rb);
        for (int i = 0; i < el->multi_out.max_rb_num; ++i) {
            if (el->multi_out.rb[i]) {
                ret |= rb_done_write(el->multi_out.rb[i]);
            }
        }
        printf("%s write done\n", el->tag);
    }
    return ret;
}

change audio_element.h

struct audio_element {
    /* Functions/RingBuffers */
    el_io_func                  open;
    ctrl_func                   seek;
    process_func                process;
    el_io_func                  close;
    el_io_func                  destroy;
    io_type_t                   read_type;
    union {
        ringbuf_handle_t        input_rb;
        io_callback_t           read_cb;
    } in;
    io_type_t                   write_type;
    struct { // here use struct instead of union, so that rb and cb both kept
        ringbuf_handle_t        output_rb;
        io_callback_t           write_cb;
    } out;

[M-Schiller]

Thank you for your replies, tempo.tian. I still have a few questions regarding the implementation, though:

• Can you explain to me why the change in

  Code: Select all

  audio_element_set_ringbuf_done

  is necessary? It seems like you are disabling the check for the write_type here...
• When doing

  Code: Select all

      // after pipeline is linked, we overwrite write function to use callback not ringbuf
      audio_element_set_write_cb(mp3_decoder, mp3_music_write_cb, i2s_stream_writer);

  do I have to replace all parts of the pipeline to run on callbacks instead of ring buffers or is it possible to only use callbacks on some of the

  Code: Select all

  audio_element

in the pipeline?

EDIT: After trying your code and integrating into https://github.com/espressif/esp-adf/bl ... _example.c as

Code: Select all

int bt_stream_write_cb(audio_element_handle_t el, char *buf, int len, TickType_t wait_time, void *ctx);

...

	ESP_LOGI(TAG, "[4.3] Link it together [Bluetooth]-->bt_stream_reader-->i2s_stream_writer-->[codec_chip]");
	const char *link_tag[2] = {"bt", "i2s"};
	audio_pipeline_link(pipeline, &link_tag[0], 2);

	audio_element_set_write_cb(bt_stream_reader, bt_stream_write_cb, i2s_stream_writer); // HERE

...

int bt_stream_write_cb(audio_element_handle_t el, char *buf, int len, TickType_t wait_time, void *ctx)
{
	ringbuf_handle_t fifo = audio_element_get_input_ringbuf((audio_element_handle_t)ctx);

	if (NULL == fifo || (len % 4) != 0)
	{
		return AEL_IO_FAIL;
	}

	int out_len = len / 2;
	int16_t *out = (int16_t *)malloc(out_len);
	if (NULL == out)
	{
		return AEL_IO_FAIL;
	}

	int16_t *left = (int16_t *)out;
	int16_t *in = (int16_t *)buf;

	int n = len / 4;
	while (n > 0)
	{
		*left = *in;
		left++;
		in += 2;
		--n;
	}
	out_len = rb_write(fifo, out, out_len, wait_time);
	free(out);

	return out_len * 2;
}

the music is very high-pitched and stutters. Is it possible that the callback is not doing the right thing or executes too slow to process the stream?

[M-Schiller]

Thank you for your replies, tempo.tian. With your hints I got it to work the hacky way.

However, I'd prefer not to change code in the framework and use an `audio_element` to down-mix. Do you know which callback functions in `audio_element` are required to implement?

This is the code I have so far... parts that I expect to be unnecessary are omitted for brevity.

Code: Select all

mono_downmix.h/c:

#define DEFAULT_MONO_DOWNMIX_CFG()             \
	{                                            \
		.buf_sz = MONO_DOWNMIX_BUF_SIZE,           \
		.out_rb_sz = MONO_DOWNMIX_RINGBUFFER_SIZE, \
		.task_stack_sz = MONO_DOWNMIX_TASK_STACK,  \
		.task_core = MONO_DOWNMIX_TASK_CORE,       \
		.task_prio = MONO_DOWNMIX_TASK_PRIO,       \
		.stack_in_ext = false,                     \
	}

...

static esp_err_t _mono_downmix_process(audio_element_handle_t self, char *in, int len)
{
	ESP_LOGI(TAG, "%s", __func__);
	int diff = 0;
	if ((diff = len % 4) != 0)
	{
		ESP_LOGD(TAG, "Need to adapt buffer length %d to %d", len, len - diff);
	}

	int r_size = audio_element_input(self, in, len - diff);
	if (r_size <= 0)
	{
		ESP_LOGE(TAG, "ALARM! %d", r_size);
		return r_size;
	}

	if (r_size % 4 != 0)
	{
		ESP_LOGW(TAG, "Could not get full samples");
	}

	int new_len = r_size - (r_size % 4);
	int num_samples = new_len / 4;

	int_fast16_t *left = (int_fast16_t *)in;
	while (num_samples > 0)
	{
		int_fast16_t val = left[0] * 0.5 + left[1] * 0.5;
		left[0] = val;
		left[1] = val;

		left += 2;
		--num_samples;
	}

	return audio_element_output(self, in, new_len);
}

...

audio_element_handle_t mono_downmix_init(mono_downmix_cfg_t *config)
{
	ESP_LOGI(TAG, "%s", __func__);
	if (config == NULL)
	{
		ESP_LOGE(TAG, "config is NULL. (line %d)", __LINE__);
		return NULL;
	}

	audio_element_handle_t el;
	audio_element_cfg_t cfg = DEFAULT_AUDIO_ELEMENT_CONFIG();
	
	cfg.open = _mono_downmix_open;           // do nothing - return ESP_OK
	cfg.close = _mono_downmix_close;           // do nothing - return ESP_OK
	cfg.process = _mono_downmix_process; 
	cfg.destroy = _mono_downmix_destroy;  // do nothing - return ESP_OK

	cfg.task_stack = config->task_stack_sz;
	cfg.task_prio = config->task_prio;
	cfg.task_core = config->task_core;
	cfg.out_rb_size = config->out_rb_sz;
	cfg.buffer_len = config->buf_sz;
	cfg.stack_in_ext = config->stack_in_ext;

	cfg.tag = "downmix";

	el = audio_element_init(&cfg);
	AUDIO_MEM_CHECK(TAG, el, goto _mono_downmix_init_exit);
	return el;

_mono_downmix_init_exit:
	return NULL;
}

... 
app_main:
	const char *link_tag[2] = {"bt", "downmixer", "i2s"};
	audio_pipeline_link(pipeline, &link_tag[0], 3);

Now the program keeps crashing while trying to link the elements in the pipeline...

Code: Select all

I (1508) MONO_DOWNMIX: mono_downmix_init
I (1508) AUDIO_PIPELINE: link el->rb, el:0x3ffd4140, tag:bt, rb:0x3ffd75a8
I (1518) AUDIO_PIPELINE: link el->rb, el:0x3ffd6fc4, tag:downmixer, rb:0x3ffd6c50
Guru Meditation Error: Core  0 panic'ed (LoadProhibited). Exception was unhandled.

Core  0 register dump:
PC      : 0x400011f1  PS      : 0x00060330  A0      : 0x800d92a6  A1      : 0x3ffbc370
A2      : 0x00000062  A3      : 0xa5a5a5a5  A4      : 0x3f402ea0  A5      : 0x000005ee
A6      : 0x3f402e04  A7      : 0x3ffd6fc4  A8      : 0x00000000  A9      : 0xa5a5a5a5
A10     : 0x00000082  A11     : 0x3ff96354  A12     : 0x3ffd67b0  A13     : 0x3ffbc360
A14     : 0x3ffbc340  A15     : 0x0000000c  SAR     : 0x00000004  EXCCAUSE: 0x0000001c
EXCVADDR: 0xa5a5a5a5  LBEG    : 0x400014fd  LEND    : 0x4000150d  LCOUNT  : 0xfffffffd


Backtrace:0x400011ee:0x3ffbc3700x400d92a3:0x3ffbc390 0x400d988b:0x3ffbc3b0 0x400d70c8:0x3ffbc3d0 0x4017dfcd:0x3ffbc5e0 0x40093599:0x3ffbc600

Does anyone know what I could be missing here?

[expresspotato]

Hello,

Has anyone actually gotten downmix to work, to convert stereo to mono? While this is not the complete code, does anyone see something wrong with the downmix config?

All that ends up happening is BT packet overflow, so it would seem the downmix audio element is not reading the input properly.

Code: Select all

esp_downmix_input_info_t source_info ={
		.samplerate = 44100,
		.channel = 2,
		.bits_num = 16,
		.gain = {0, 0},
		.transit_time = 500
	};

	downmix_cfg_t downmix_cfg = DEFAULT_DOWNMIX_CONFIG();
	downmix_cfg.downmix_info.source_num = 1;
	downmix_cfg.downmix_info.output_type = ESP_DOWNMIX_OUTPUT_TYPE_ONE_CHANNEL;
	audio_element_handle_t downmix = downmix_init(&downmix_cfg);
	source_info_init(downmix, &source_info);

    i2s_stream_cfg_t i2s_cfg_writer = I2S_STREAM_CFG_DEFAULT();
	i2s_cfg_writer.i2s_config.channel_format = I2S_CHANNEL_FMT_ALL_LEFT;
    i2s_cfg_writer.type = AUDIO_STREAM_WRITER;
    i2s_cfg_writer.i2s_config.sample_rate = bt_service.a2dp_sample_rate;
    i2s_cfg_writer.use_alc = true;
    i2s_stream_writer = i2s_stream_init(&i2s_cfg_writer);

	audio_pipeline_register(pipeline_input, a2dp_input_stream, "a2dp_in_stream");
	audio_pipeline_register(pipeline_input, downmix, "downmix");
	audio_pipeline_register(pipeline_input, i2s_stream_writer, "i2s_stream_writer");

	const char *link_tag[3] = {"a2dp_in_stream", "downmix", "i2s_stream_writer"};
	audio_pipeline_link(pipeline_input, link_tag, 3);

	//i2s_alc_volume_set(i2s_stream_writer, 3); // Artificial Gain
    i2s_stream_set_clk(i2s_stream_writer, bt_service.a2dp_sample_rate, 16, 1);