Is there a way to improve POST request speed?

[elcrcp]

Hello guys and thanks in advance for any reply, I'm currently working on a project and using an ESP32-Wrover for that purpose. What I do is simply getting readings from a sensor and storing them to PSRAM, 2Byte x 1.200.000 readings per try.
Then I upload them to server ( I'm using a simple server script in my personal computer to save the data as txt). The problem is; Stored data is about 4MBytes on the server side and transferring this data takes about 10-11 minutes per try. Is there a way to improve upload time?
Here is my http post code;

Code: Select all

int postJsonData(char * ret_serverResponse, int16_t * xbuffer, int16_t * ybuffer, int16_t * zbuffer, const char * inp_hostname, const uint16_t inp_port, char * inp_endPoint,WiFiClient client)
{
  if(client.connect(inp_hostname,inp_port))
  {
    char headers[255]={0};
    sprintf(headers,"{\"Sensor_Type\":\"Sensor\",\"Sensor_Sensitivity\":%u,\"Measure_Periud_us\":%u,\"Measurement_Count\":%lu,",MEASUREMENT_SENSITIVITY,MEASURE_PERIOD_US,MEASURE_COUNT);
    char xheader[]="\"XARRAY\":[";
    char yheader[]="\"YARRAY\":[";
    char zheader[]="\"ZARRAY\":[";
    uint32_t datasize=0;
    for (uint32_t i=0;i<MEASURE_COUNT;i++)
    {
      datasize=datasize+String(xbuffer[i]).length()+String(ybuffer[i]).length()+String(zbuffer[i]).length();
    }
    // Sending "POST command to endpoint
    client.print("POST ");
    client.print(inp_endPoint);
    client.print(" HTTP/1.1\r\n");

    // Sending HTTP headers
    client.print("Host: ");
    client.print(inp_hostname);
    client.print("\r\n");
    client.print("Connection: close\r\n");
    client.print("Content-Length: "); 
    client.print(strlen(headers)+strlen(xheader)+strlen(yheader)+strlen(zheader)+datasize+(MEASURE_COUNT*3)+9);
//    
    client.print("\r\n");
    client.print("Content-Type: text/plain\r\n\r\n");
    // Sending headers and post together
    
    // Sending data
    client.print(headers);
    client.print(xheader);
    for(uint32_t i=0;i<MEASURE_COUNT;i++)
    {
      client.printf("%d,",xbuffer[i]);
    }
    client.printf("0],");
    client.print(yheader);
    for(uint32_t i=0;i<MEASURE_COUNT;i++)
    {
      client.printf("%d,",ybuffer[i]);
    }
    client.printf("0],");
    client.print(zheader);
    for(uint32_t i=0;i<MEASURE_COUNT;i++)
    {
      client.printf("%d,",zbuffer[i]);
    }
    client.printf("0]}");

    client.print("\r\n");
    unsigned long timeout = millis();

    // Get Client Response
    while (client.available() == 0){
      if (millis() - timeout > 3000){
        Serial.println(">>> Client Timeout !");
        client.stop();
        return -2;
      }
    }
    strcat(ret_serverResponse,(client.readStringUntil('\r')).c_str());
    client.stop();
    return parseHttpCode(ret_serverResponse);
  }
  else
  {
    Serial.println("Cannot connect to the host");
    return -1;
  }
}

[lbernstone]

Rearrange your data structure so you have all the common descriptor stuff at the top, and then use two arrays for all your elements. That way you only send the verbose stuff once, skip the sequencing, and can post the whole thing in one single transaction.
{
"Sensor type":"sensor",
"Sensitivity:"blah",
"Measure Period":"2us",
"Count":1200000,
"Sensor Data": {
"Sensor1":[23,42,92,4,.....],
"Sensor2":[...]
}
}

[elcrcp]

Rearrange your data structure so you have all the common descriptor stuff at the top, and then use two arrays for all your elements. That way you only send the verbose stuff once, skip the sequencing, and can post the whole thing in one single transaction.
{
"Sensor type":"sensor",
"Sensitivity:"blah",
"Measure Period":"2us",
"Count":1200000,
"Sensor Data": {
"Sensor1":[23,42,92,4,.....],
"Sensor2":[...]
}
}

Thank you for the advice lbernstone but I don't see how this change would reduce the transfer time from 10 mins to 7-8 mins. I haven't tried it yet but it seems to effect only for a few seconds at most. Can you eloborate if I'm mistaken?

[lbernstone]

I think most of your time there is spent pulling each individual element over SPI and then processing it into text. You might be able to improve that by making the transactions larger (packing multiple entries into each element or doing a more direct iteration on the pointer) in order to have less reads from psram and printf statements. Your best speed is going to be streaming the array directly to the server as a binary, and letting it do the processing. That would probably still take a couple minutes to move 4MB of data onto the wire.

[elcrcp]

Actually what I thought was "it gonna read all the data from spi anyway how it could change the time", But it did and drastically did.
I changed my code as to load more bulk data in each print command and data upload time reduced to 20 secs.
Here are test results;
Transferred Data Size ; 3278932 Bytes
1 Data per print command upload time ; more than 6 minutes
64 Data per print command upload time ; 19-21 seconds
I also tried to turn off-on the Nagle algorythm to see if it effects transfer time but there wasn't any observable result.
Thanks for the help.
But after the change I realised that I cannot calculate data size correctly. There wasn't any problem while I was sending data 1by1 but data size started to mismatch after starting to send bulk data. Actual data length is bigger then calculated data by 9500-10000 bytes. Any idea why?

Code: Select all

#define MEASURE_COUNT pow(2,18)
int postJsonData(char * ret_serverResponse, int16_t * xbuffer, int16_t * ybuffer, int16_t * zbuffer, const char * inp_hostname, const uint16_t inp_port, char * inp_endPoint,WiFiClient client)
{
  if(client.connect(inp_hostname,inp_port))
  {
    char headers[255]={0};
    uint32_t datasize=0;
    for (uint32_t i=0;i<MEASURE_COUNT;i++)
    {
      datasize=datasize+String(xbuffer[i]).length()+String(ybuffer[i]).length()+String(zbuffer[i]).length();
    }
    sprintf(headers,"{\"Sensor_Type\":\"Sensor\",\"Sensor_Sensitivity\":%u,\"Measure_Periud_us\":%u,\"Measurement_Count\":%lu,",MEASUREMENT_SENSITIVITY,MEASURE_PERIOD_US,MEASURE_COUNT);
    char xheader[]="\"XARRAY\":[";
    char yheader[]="\"YARRAY\":[";
    char zheader[]="\"ZARRAY\":[";
    Serial.println("Starting to data posting. ");
    // Sending "POST command to endpoint
    client.print("POST ");
    client.print(inp_endPoint);
    client.print(" HTTP/1.1\r\n");

    // Sending HTTP headers
    client.print("Host: ");
    client.print(inp_hostname);
    client.print("\r\n");
    client.print("Connection: close\r\n");
    client.print("Content-Length: "); 
    client.print(strlen(headers)+strlen(xheader)+strlen(yheader)+strlen(zheader)+datasize+(MEASURE_COUNT*3)+9);  
    client.print("\r\n");
    client.print("Content-Type: text/plain\r\n\r\n");
    // Sending headers and post together
    
    // Sending data
    client.print(headers);
    client.print(xheader);
    for(uint16_t i=0;i<(MEASURE_COUNT/64);i++)
    {
      client.printf("%d,%d,%d,..64times..,%d,",xbuffer[(64*i)+i+0],xbuffer[(64*i)+i+1],....,xbuffer[(64*i)+i+63]);
    }
    client.printf("0],");
    client.print(yheader);
    for(uint16_t i=0;i<(MEASURE_COUNT/64);i++)
    {
      client.printf("%d,%d,%d,..64times..,%d,",ybuffer[(64*i)+i+0],ybuffer[(64*i)+i+1],....,ybuffer[(64*i)+i+63]);
    }
    client.printf("0],");
    client.print(zheader);
    for(uint16_t i=0;i<(MEASURE_COUNT/64);i++)
    {
      client.printf("%d,%d,%d,..64times..,%d,",zbuffer[(64*i)+i+0],zbuffer[(64*i)+i+1],....,zbuffer[(64*i)+i+63]);
    }
    client.printf("0]}");
    Serial.println("Data post Finished.!");
    unsigned long timeout = millis();

    // Get Client Response
    while (client.available() == 0){
      if (millis() - timeout > 3000){
        Serial.println(">>> Client Timeout !");
        client.stop();
        return -2;
      }
    }
    strcat(ret_serverResponse,(client.readStringUntil('\r')).c_str());
    client.stop();
    return parseHttpCode(ret_serverResponse);
  }
  else
  {
    Serial.println("Cannot connect to the host");
    return -1;
  }
}