How to stream audio from I2S Mic using Udp?

[Tao_spartan]

I've been struggling to send live audio from a huzzah32 with a ICS43434 over wifi using udp (not surprisingly as I'm a relative noob to esp32 and Arduino!). While audio can be obtained at the server end ( just at the moment capturing with netcat and playing with audacity) there is distortion and the audio appears to have been speeded up.
My code is as follows:

Code: Select all

#include <WiFi.h>
#include <WiFiUdp.h>
#include <stdio.h>
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "driver/i2s.h"
#include "esp_system.h"
#include <math.h>


#define SERIAL_BAUD 115200
/////////////////////////////////////////////////////////////////////
//Audio VAR
////////////////////////////////////////////////////////////////////

#define SAMPLE_RATE 16000
#define MTU 1440


////////////////////////////////////////////////////////////////////
//VAR
////////////////////////////////////////////////////////////////////
int16_t sample16 = 0;
int32_t sample = 0;
int16_t element = 0;
int sample_counter = 0;

const char * udpAddress = "192.168.200.59";
const int udpPort = 3333;
boolean transmit = false;
const char* ssid     = "myAp";
const char* password = "mypass";
QueueHandle_t queueSample;
int queueSize = SAMPLE_RATE * 2;

const i2s_port_t I2S_PORT = I2S_NUM_0;
//create UDP instance
WiFiUDP udp;

////////////////////////////////////////////////////////////////////
//task Setup
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
void setupSampleTask(){
xTaskCreate(
                    sampleAudio,          /* Task function. */
                    "SampleAudio",        /* String with name of task. */
                    10000,            /* Stack size in words. */
                    NULL,             /* Parameter passed as input of the task */
                    1,                /* Priority of the task. */
                    NULL);            /* Task handle. */
}

////////////////////////////////////////////////////////////////////
//I2S Setup
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
void  setupI2SMic() {
  Serial.println("Configuring I2S...");
  // The I2S config as per the example
  esp_err_t err;
 const  i2s_config_t i2s_config = {
    .mode = i2s_mode_t(I2S_MODE_MASTER | I2S_MODE_RX), // Receive, not transfer
    .sample_rate = SAMPLE_RATE,                         // 16KHz
    .bits_per_sample = I2S_BITS_PER_SAMPLE_32BIT, // could only get it to work with 32bits
    .channel_format = I2S_CHANNEL_FMT_ONLY_RIGHT, // although the SEL config should be left, it seems to transmit on right
    .communication_format = i2s_comm_format_t(I2S_COMM_FORMAT_I2S | I2S_COMM_FORMAT_I2S_MSB),
    .intr_alloc_flags = ESP_INTR_FLAG_LEVEL1,     // Interrupt level 1
    .dma_buf_count = 32,                           // number of buffers
    .dma_buf_len = 64,
    .use_apll = true
  };
  // The pin config as per the setup
  const i2s_pin_config_t pin_config = {
    .bck_io_num = 14,   // BCKL
    .ws_io_num = 15,    // LRCL
    .data_out_num = -1, // not used (only for speakers)
    .data_in_num = 32   // DOUT
  };
  err = i2s_driver_install(I2S_PORT, &i2s_config, 0, NULL);
  if (err != ESP_OK) {
    Serial.printf("Failed installing driver: %d\n", err);
    while (true);
  }
  err = i2s_set_pin(I2S_PORT, &pin_config);
  if (err != ESP_OK) {
    Serial.printf("Failed setting pin: %d\n", err);
    while (true);
  }
  
  Serial.println("I2S driver installed.");
};

////////////////////////////////////////////////////////////////////
//Setup
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
void setup() {

    Serial.begin(SERIAL_BAUD);
    Serial.println("I was built on " __DATE__ " at " __TIME__ "");
    Serial.print("Init Serial Communication:");
    Serial.println(SERIAL_BAUD);


    WiFi.begin(ssid, password);
    Serial.print("Connecting to ");
    Serial.println(ssid);
    while (WiFi.status() != WL_CONNECTED) {
        delay(500);
        Serial.print(".");
    }
    Serial.println("");
    Serial.println("WiFi connected with IP address: ");
    Serial.println(WiFi.localIP());

    queueSample = xQueueCreate( queueSize, sizeof( int16_t ) );
    
    if(queueSample == NULL){
      
         Serial.println("Error creating the queue");
    
    }
    
    setupI2SMic();
    
    setupSampleTask();

    Serial.println("Setup done!");
}

////////////////////////////////////////////////////////////////////
//Loop
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
void loop() {
              
        udp.beginPacket(udpAddress, udpPort);
        
        for(sample_counter=0; sample_counter<MTU/2; sample_counter++){
          
          xQueueReceive(queueSample, &element, portMAX_DELAY);
          udp.write((byte*)&element,2);
          
        }
        
        udp.endPacket();
      
}

////////////////////////////////////////////////////////////////////
//SampleAudio
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
void sampleAudio( void * parameter )
{
      
      while ( true ){
        
            i2s_pop_sample(I2S_PORT, (char *)&sample,10);
            sample>>=14;
            sample16 = (int16_t)sample;
            xQueueSend(queueSample, &sample16, 10);
            
      }
      vTaskDelete( NULL );
  
}

I think the issue may be the timing of the i2s_pop_sample but I'm only guessing, if there is anyone out there how could point me in the correct direction, that would be gratefully appreciated, if it can be done at all.

many thanks

Tao

[felipeduque]

Hi, spartan. I've faced similar issue, and solved it by reducing sampling rate to 8 kHz, and it works perfectly. I haven't tried to make it work at higher rates (it's possible, though, as per the datasheet), but from my experience with another I2S driver (for H3 processor), it's a matter of choosing the right system clock. Maybe there is some I2S master clock that only works correctly at 8 kHz rate. You'll have to dig deeper

[beckmx]

@tao_spartan

I also have the same setup, no surprise in there:

Code: Select all

#include <M5StickC.h>
#include <driver/i2s.h>
#include <WiFiUdp.h>
#include "WiFi.h"
#include "AsyncUDP.h"

#define PIN_CLK  0
#define PIN_DATA 34
#define READ_LEN (2 * 256)
#define GAIN_FACTOR 3
uint8_t BUFFER[READ_LEN] = {0};

uint16_t sound_wav[1024];
int bytesPacked = 0;
int16_t *adcBuffer = NULL;

//AsyncUDP udp;
WiFiUDP udp;

const char * ssid = "Unknown";
const char * password = "daredevilme";

void i2sInit()
{
   i2s_config_t i2s_config = {
    .mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_RX | I2S_MODE_PDM),
    .sample_rate =  44100,
    .bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT, // is fixed at 12bit, stereo, MSB
    .channel_format = I2S_CHANNEL_FMT_ALL_RIGHT,
    .communication_format = I2S_COMM_FORMAT_I2S,
    .intr_alloc_flags = ESP_INTR_FLAG_LEVEL1,
    .dma_buf_count = 2,
    .dma_buf_len = 128,
   };

   i2s_pin_config_t pin_config;
   pin_config.bck_io_num   = I2S_PIN_NO_CHANGE;
   pin_config.ws_io_num    = PIN_CLK;
   pin_config.data_out_num = I2S_PIN_NO_CHANGE;
   pin_config.data_in_num  = PIN_DATA;
  
   
   i2s_driver_install(I2S_NUM_0, &i2s_config, 0, NULL);
   i2s_set_pin(I2S_NUM_0, &pin_config);
   i2s_set_clk(I2S_NUM_0, 44100, I2S_BITS_PER_SAMPLE_16BIT, I2S_CHANNEL_MONO);
}



void mic_record_task (void* arg)
{   
  size_t bytesread;
  while(1){
    i2s_read(I2S_NUM_0,(char*) BUFFER, READ_LEN, &bytesread, (100 / portTICK_RATE_MS));
    adcBuffer = (int16_t *)BUFFER;
    showSignal();
    vTaskDelay(5 / portTICK_RATE_MS);
  }
}

void setup() {
  M5.begin();
  M5.Lcd.setRotation(3);
  M5.Lcd.fillScreen(WHITE);
  M5.Lcd.setTextColor(BLACK, WHITE);
  M5.Lcd.println("mic test");
  WiFi.mode(WIFI_STA);
  WiFi.begin(ssid, password);
  if (WiFi.waitForConnectResult() != WL_CONNECTED) {
      Serial.println("WiFi Failed");
      while(1) {
          delay(1000);
      }
  }
 
  i2sInit();
  xTaskCreate(mic_record_task, "mic_record_task", 2048, NULL, 1, NULL);
}


void showSignal(){
  int y;
  //Serial.println(sizeof(adcBuffer));
  
  for (int n = 0; n < 256; n++){
    sound_wav[bytesPacked] = adcBuffer[n];
    //Serial.println(adcBuffer[n]);
    bytesPacked++;
    //y = adcBuffer[n] * GAIN_FACTOR;
    /*Serial.print(n);
    Serial.print(":");*/
    
    //Serial.println(y);
    /*y = map(y, INT16_MIN, INT16_MAX, 10, 70);
    M5.Lcd.drawPixel(n, oldy[n],WHITE);
    M5.Lcd.drawPixel(n,y,BLACK);
    oldy[n] = y;*/
  }
  
  if(bytesPacked >= 1023){
    bytesPacked = 0;
    
    udp.beginPacket("x.x.x.x", 3005);
    for (int n2 = 0; n2 < 1023; n2++){
      udp.write(sound_wav[n2]);
      //Serial.println(sound_wav[n2]);
    }
    memset(sound_wav, 0, 1024);
    udp.endPacket();
    //vTaskDelay(15000 / portTICK_RATE_MS);
  }
}

void loop() {
  //printf("loop cycling\n");
  //vTaskDelay(10000 / portTICK_RATE_MS); // otherwise the main task wastes half of the cpu cycles
}

My audio is received by a python script:

Code: Select all

import pyaudio
import socket
import wave

UDP_IP = "x.x.x.x"
UDP_PORT = 3005
WAVE_OUTPUT_FILENAME = "output.wav"
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind((UDP_IP, UDP_PORT))

# p = pyaudio.PyAudio()

# stream = p.open(format=32, channels=1, rate=11111, input=True)

frames = []

try:
    while True:
        data, addr = sock.recvfrom(1024) # buffer de 1024 bytes
        print(addr)
        print(len(data))
        for i in range(0,len(data)):
            print(data[i])
            frames.append(data[i])
        
        # stream.write(data)
        
except KeyboardInterrupt:  
    print("Cerrando...")
    wf = wave.open(WAVE_OUTPUT_FILENAME, 'wb')
    wf.setnchannels(1)
    wf.setsampwidth(1)
    wf.setframerate(44100)
    wf.writeframes(b''.join(frames))
    wf.close()
    # stream.stop_stream()
    # stream.close()
    # p.terminate()

I am sampling at 16000, but I noticed that the UDP packet only writes 8 bit numbers and what we send is longer, my gut tells me it is cutting the number and losing precision, also the sample width in the python maybe is 2 instead of 1 byte,