ESP32 Forum

ESP32 Official Forum
http://esp32.io/

Large Uploads Get Corrupted

http://esp32.io/viewtopic.php?f=14&t=9981

Page 1 of 1

Large Uploads Get Corrupted

Posted: Thu Apr 04, 2019 1:22 am
by sphillips
I have been working on a project that uploads data to the ESP32 http server. I have found that, above certain sizes, the mechanism does not properly buffer the incoming data - or at least that is what I think is going on.

When the CPU starts I can see the message: "I (306) wifi: Init static rx buffer size: 1600" on the console and I believe this has something to do with that. The size of the upload that gets corrupted happens at about twice that value ~ somewhere around 3200 bytes.

I have isolated the code into a sample file that loads a webserver, presents a page at the default url (just put the ip in the browser's address) and the page demonstrates the error.

The code preloads the data with '1's except at the i%160 point where it puts a '0'. When you get this from the ESP32 by clicking the "Get Data" button it paints this on the canvas in a grid 160 wide down the left hand side. Then when you press the "Put Data" button it uploads it to the ESP32 http server which stores it back to the original character array. However, it does not do so correctly as pressing the "Get Data" button once again will show.

I have gone through the post and get with wireshark and do not believe the webpage nor the browser has anything to do with this. I tested this with Chromium.
Code: [Select all] [Expand/Collapse]
  1. #include <sys/param.h>
  2. #include <esp_event_loop.h>
  3. #include <esp_log.h>
  4. #include <esp_http_server.h>
  5. #include <esp_wifi.h>
  6. #include "esp_system.h"
  7. #include "esp_spiffs.h"
  8. #include "nvs_flash.h"
  9.  
  10. #define TAG "Large_Upload_Test"
  11.  
  12. // Declare some functions so that we can order the code as we wish
  13. httpd_handle_t start_webserver(void);
  14. void stop_webserver(httpd_handle_t server);
  15.  
  16.  
  17. // This parameter determines the size of the test.   The data is downloaded to a webpage and uploaded to the ESP32 again.
  18. // Target for my project was 19200
  19. #define UPLOAD_DOWNLOAD_SIZE 19200
  20.  
  21. // This determines the buffer size for the upload.   Large sizes show bad corruption, even at 1600 or less it shows corruption
  22. #define UPLOAD_CACHE_SIZE 1600
  23.  
  24. // This determines the buffer size for the download.   It works at pretty much any reasonable size
  25. #define DOWNLOAD_CACHE_SIZE 1600
  26.  
  27. char detectionMask[UPLOAD_DOWNLOAD_SIZE];
  28.  
  29. esp_err_t event_handler(void *ctx, system_event_t *event) {
  30.   httpd_handle_t *server = (httpd_handle_t *)ctx;
  31.  
  32.   switch (event->event_id) {
  33.   case SYSTEM_EVENT_STA_START:
  34.     ESP_LOGD(TAG, "SYSTEM_EVENT_STA_START");
  35.     ESP_ERROR_CHECK(esp_wifi_connect());
  36.     break;
  37.   case SYSTEM_EVENT_STA_GOT_IP:
  38.     ESP_LOGD(TAG, "SYSTEM_EVENT_STA_GOT_IP");
  39.     ESP_LOGD(TAG, "Got IP: '%s'",
  40.              ip4addr_ntoa(&event->event_info.got_ip.ip_info.ip));
  41.  
  42.     /* Start the web server */
  43.     if (*server == NULL) {
  44.       *server = start_webserver();
  45.     }
  46.     break;
  47.   case SYSTEM_EVENT_STA_DISCONNECTED:
  48.     ESP_LOGD(TAG, "SYSTEM_EVENT_STA_DISCONNECTED");
  49.     ESP_ERROR_CHECK(esp_wifi_connect());
  50.  
  51.     /* Stop the web server */
  52.     if (*server) {
  53.       stop_webserver(*server);
  54.       *server = NULL;
  55.     }
  56.     break;
  57.   default:
  58.     break;
  59.   }
  60.   return ESP_OK;
  61. }
  62.  
  63. void initialise_wifi(void *arg) {
  64.   ESP_ERROR_CHECK(nvs_flash_init());
  65.   tcpip_adapter_init();
  66.   ESP_ERROR_CHECK(esp_event_loop_init(event_handler, arg));
  67.   wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();
  68.   ESP_ERROR_CHECK(esp_wifi_init(&cfg));
  69.   ESP_ERROR_CHECK(esp_wifi_set_storage(WIFI_STORAGE_RAM));
  70.   wifi_config_t wifi_config = {
  71.       .sta = {
  72.           .ssid = CONFIG_WIFI_SSID,
  73.           .password = CONFIG_WIFI_PASSWORD,
  74.       },
  75.   };
  76.   ESP_LOGD(TAG, "Setting WiFi configuration SSID %s...", wifi_config.sta.ssid);
  77.   ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_STA));
  78.   ESP_ERROR_CHECK(esp_wifi_set_config(ESP_IF_WIFI_STA, &wifi_config));
  79.   ESP_ERROR_CHECK(esp_wifi_start());
  80. }
  81.  
  82. void app_main() {
  83.   for (int i=0; i < sizeof(detectionMask); i++) {
  84.     if (i%160==0) detectionMask[i] = 0;
  85.     else detectionMask[i] = 1;
  86.   }
  87.  
  88.   static httpd_handle_t server = NULL;
  89.   initialise_wifi(&server);
  90. }
  91.  
  92. esp_err_t set_mask_handler(httpd_req_t *req) {
  93.     char contentBuffer[UPLOAD_CACHE_SIZE];
  94.     size_t recv_size = sizeof(contentBuffer);
  95.     size_t contentLen = req->content_len;
  96.  
  97.     // some sort of data error happens every 3200 bytes - turn on debug
  98.     esp_log_level_set("*", ESP_LOG_DEBUG);
  99.  
  100.  
  101.     ESP_LOGI(TAG, "Incoming transmission %d bytes", contentLen);
  102.  
  103.     size_t bytes_recvd = 0;
  104.     while (bytes_recvd < contentLen) {
  105.         if ((contentLen - bytes_recvd) < recv_size) recv_size = contentLen - bytes_recvd;
  106.         ESP_LOGI(TAG,"Recieving %d bytes in chunk, total bytes received: %d", recv_size, bytes_recvd);
  107.         int ret = httpd_req_recv(req, contentBuffer, recv_size);
  108.         if (ret <= ESP_OK) {  /* ESP_OK return value indicates connection closed */
  109.             /* Check if timeout occurred */
  110.             if (ret == HTTPD_SOCK_ERR_TIMEOUT) {
  111.                 /* In case of timeout one can choose to retry calling
  112.                 * httpd_req_recv(), but to keep it simple, here we
  113.                 * respond with an HTTP 408 (Request Timeout) error */
  114.                 httpd_resp_send_408(req);
  115.             }
  116.             /* In case of error, returning ESP_FAIL will
  117.             * ensure that the underlying socket is closed */
  118.             ESP_LOGE(TAG,"Something bad happened, bytes left: %d", contentLen - bytes_recvd);
  119.             return ESP_FAIL;
  120.         }
  121.  
  122.         // This is debug code to show the first few characters of the buffer.
  123.         // In the stock state the buffer size aligns with the encoded data so that each row should
  124.         // show the characters "011111111111"   ...while this works for the first few rows the
  125.         // data then skews about 12 places behind in index, and repeats a couple of other times
  126.         char printData[13];
  127.         printData[12] = '\0';
  128.         ESP_LOGI(TAG, "Writing bytes %d through %d", bytes_recvd, bytes_recvd+recv_size-1);
  129.         for (int i=0; i<recv_size; i++) {
  130.             if (i<12) printData[i] = contentBuffer[i];
  131.             detectionMask[bytes_recvd+i] = contentBuffer[i]=='1'?true:false;
  132.         }
  133.         ESP_LOGI(TAG, "12 Bytes: %s",printData);
  134.         // End debug code
  135.         bytes_recvd += recv_size;
  136.     }
  137.     /* Send a simple response */
  138.     const char resp[] = "Data uploaded successfully";
  139.     httpd_resp_send(req, resp, strlen(resp));
  140.     esp_log_level_set("*", ESP_LOG_INFO);
  141.     return ESP_OK;
  142. }
  143.  
  144. httpd_uri_t uri_upload_mask = {
  145.     .uri = "/bitmask",
  146.     .method = HTTP_POST,
  147.     .handler = set_mask_handler
  148. };
  149.  
  150. esp_err_t get_mask_handler(httpd_req_t *req) {
  151.     char contentBuffer[DOWNLOAD_CACHE_SIZE];
  152.     size_t send_size = sizeof(contentBuffer);
  153.  
  154.     ESP_LOGI(TAG, "Starting transmission of %d bytes", sizeof(detectionMask));
  155.     size_t bytes_sent = 0;
  156.     while (bytes_sent < sizeof(detectionMask)) {
  157.         if (sizeof(detectionMask) - bytes_sent < send_size) send_size = sizeof(detectionMask) - bytes_sent;
  158.  
  159.         ESP_LOGI(TAG, "Reading bytes %d through %d", bytes_sent, bytes_sent+send_size-1);
  160.         for (int i=0; i<send_size; i++) {
  161.             contentBuffer[i] = detectionMask[bytes_sent+i]?'1':'0';
  162.         }
  163.  
  164.         ESP_LOGI(TAG,"Sending %d bytes in chunk, total bytes sent: %d", send_size, bytes_sent);
  165.        
  166.         httpd_resp_send_chunk(req, (const char *)contentBuffer, send_size);
  167.  
  168.         // the return codes showed error when it actually worked here...
  169.  
  170.         // if (ret <= ESP_OK) {  /* ESP_OK return value indicates connection closed */
  171.         //     /* Check if timeout occurred */
  172.         //     if (ret == HTTPD_SOCK_ERR_TIMEOUT) {
  173.         //         /* In case of timeout one can choose to retry calling
  174.         //         * httpd_req_recv(), but to keep it simple, here we
  175.         //         * respond with an HTTP 408 (Request Timeout) error */
  176.         //         httpd_resp_send_408(req);
  177.         //     }
  178.         //     /* In case of error, returning ESP_FAIL will
  179.         //     * ensure that the underlying socket is closed */
  180.         //     ESP_LOGE(TAG,"Something bad happened, bytes left: %d", sizeof(imageDetectionMask) - bytes_sent);
  181.         //     return ESP_FAIL;
  182.         // }
  183.         bytes_sent += send_size;
  184.     }
  185.     // Send 0 size to terminate
  186.     httpd_resp_send_chunk(req, (const char *)contentBuffer, 0);
  187.     /* Send a simple response */
  188.     const char resp[] = "Data downloaded successfully";
  189.     httpd_resp_send(req, resp, strlen(resp));
  190.     return ESP_OK;
  191. }
  192.  
  193. httpd_uri_t uri_download_mask = {
  194.     .uri = "/bitmask",
  195.     .method = HTTP_GET,
  196.     .handler = get_mask_handler
  197. };
  198.  
  199. // This is a simple web page processed with gzip, opened in a hex editor and pasted here.  
  200. // I have wiresharked it for accuracy, it is not the browser doing this.   Tested in Chromium
  201. const char default_html_gz[] = {
  202. 0x1F, 0x8B, 0x08, 0x08, 0x20, 0x4F, 0xA5, 0x5C, 0x00, 0x03, 0x75, 0x70, 0x6C, 0x6F, 0x61, 0x64, 0x65, 0x72, 0x72, 0x6F, 0x72, 0x2E, 0x68, 0x74, 0x6D, 0x6C, 0x00, 0xD5, 0x57, 0xDD, 0x53, 0xDB, 0x46, 0x10, 0x7F, 0xCF, 0x5F, 0x71, 0xBD, 0x4E, 0x63, 0x51, 0x1C, 0x4B, 0xD0, 0x24, 0x93, 0x89, 0x2D, 0x3F, 0x40, 0x68, 0xC8, 0x0C, 0x14, 0x06, 0x9B, 0xB4, 0x9D, 0x34, 0xC3, 0x9C, 0xA5, 0x45, 0xBA, 0x20, 0xDD, 0xA9, 0xD2, 0xC9, 0xC6, 0xD3, 0xE1, 0x7F, 0xEF, 0x9E, 0x4E, 0x36, 0xA7, 0x0F, 0x08, 0xBC, 0xB5, 0xE2, 0x01, 0x69, 0x77, 0xEF, 0xB7, 0xDF, 0x7B, 0xEB, 0xC9, 0x0F, 0x1F, 0xCE, 0x0E, 0xE7, 0x7F, 0x9E, 0x1F, 0x91, 0x58, 0xA5, 0xC9, 0xF4, 0xC5, 0x44, 0xFF, 0x23, 0x09, 0x13, 0x91, 0x4F, 0x41, 0x50, 0x4D, 0x00, 0x16, 0x4E, 0x5F, 0x10, 0x7C, 0x26, 0x29, 0x28, 0x46, 0x82, 0x98, 0xE5, 0x05, 0x28, 0x9F, 0x5E, 0xCE, 0x7F, 0x7D, 0xF5, 0x8E, 0xD6, 0x2C, 0xC5, 0x55, 0x02, 0xD3, 0x13, 0x96, 0x47, 0x40, 0x2E, 0xB3, 0x44, 0xB2, 0x90, 0xCC, 0xA1, 0x50, 0x13, 0xD7, 0x30, 0x8C, 0x50, 0x11, 0xE4, 0x3C, 0x53, 0x44, 0xAD, 0x33, 0xF0, 0xA9, 0x82, 0x5B, 0xE5, 0x7E, 0x63, 0x4B, 0x66, 0xA8, 0x35, 0xD0, 0x92, 0xE5, 0xA4, 0x50, 0xE1, 0x41, 0x22, 0x83, 0x9B, 0xDF, 0x79, 0xA8, 0x62, 0xE2, 0x93, 0xD7, 0xE3, 0x0E, 0xEB, 0x18, 0x78, 0x14, 0xAB, 0x26, 0x8F, 0xA7, 0xD1, 0x2C, 0x60, 0x09, 0x20, 0xF5, 0x8D, 0xA1, 0x5E, 0x97, 0x22, 0x50, 0x5C, 0x0A, 0x92, 0x31, 0x2E, 0xD4, 0x29, 0x2B, 0x6E, 0xCE, 0xC4, 0x21, 0x13, 0xA8, 0xD2, 0xE1, 0x29, 0x8B, 0x20, 0x45, 0xCA, 0x0E, 0xF9, 0xA7, 0x92, 0xDD, 0xA0, 0x04, 0x15, 0x1F, 0x31, 0x42, 0x19, 0x94, 0x29, 0x08, 0x35, 0x8A, 0x40, 0x1D, 0x25, 0xA0, 0x5F, 0x0F, 0xD6, 0x9F, 0x42, 0x67, 0x90, 0xAE, 0x0D, 0xC6, 0x60, 0x67, 0xBC, 0x3D, 0xC9, 0xAF, 0x89, 0x63, 0x4E, 0x6A, 0xF1, 0x43, 0x29, 0xB4, 0x77, 0x36, 0xF4, 0x16, 0x5E, 0xDD, 0x22, 0x76, 0x47, 0xD4, 0x19, 0xEC, 0x87, 0x36, 0x5E, 0x65, 0xBD, 0xCC, 0x89, 0x53, 0x39, 0xE6, 0x7B, 0x63, 0xC2, 0x27, 0x5B, 0x93, 0x47, 0x09, 0x88, 0x48, 0xC5, 0x48, 0xDB, 0xDD, 0x6D, 0xEB, 0xD8, 0xE8, 0xC9, 0xE5, 0x0A, 0xF5, 0x9C, 0x32, 0x15, 0x8F, 0xAE, 0x13, 0x29, 0x73, 0x87, 0xBB, 0x7B, 0x6F, 0xBD, 0x96, 0x86, 0xAD, 0x51, 0x32, 0x41, 0x61, 0xFE, 0x13, 0x4A, 0x74, 0x05, 0xB4, 0x6B, 0x5B, 0xD5, 0x5F, 0xF8, 0x57, 0xE2, 0xFB, 0x64, 0xE0, 0x0D, 0xFA, 0x14, 0xEB, 0x07, 0x1D, 0x1C, 0x5D, 0xF3, 0x24, 0x99, 0xA9, 0x75, 0x95, 0x0A, 0x9A, 0x43, 0x48, 0xBB, 0xA8, 0xB6, 0xE8, 0x05, 0x04, 0xCA, 0xB1, 0x4C, 0x45, 0x73, 0x7E, 0xDE, 0x24, 0x73, 0x67, 0x68, 0x3B, 0x81, 0x5E, 0xD9, 0x9C, 0x46, 0xA1, 0x0C, 0x5B, 0xC5, 0xD1, 0xE3, 0xEB, 0x5D, 0x87, 0x02, 0x49, 0x01, 0x4F, 0x75, 0x64, 0x15, 0x73, 0x05, 0xFF, 0x4D, 0x57, 0xEE, 0xBF, 0xCC, 0xDB, 0x9D, 0xD5, 0x14, 0x75, 0xE6, 0xAE, 0x16, 0xDA, 0x07, 0xDA, 0x6A, 0x0C, 0xAC, 0xC0, 0xAD, 0x84, 0xD3, 0x69, 0x85, 0x47, 0xBA, 0x80, 0x6E, 0xBA, 0x80, 0x5A, 0xC6, 0x59, 0x05, 0x6E, 0xD7, 0x36, 0xDD, 0x0F, 0xB5, 0x54, 0x43, 0xAC, 0xCC, 0x75, 0xC9, 0x51, 0x77, 0xC1, 0x95, 0xD6, 0x4D, 0x9B, 0x20, 0xB7, 0xB1, 0x52, 0x19, 0xF2, 0x05, 0xAC, 0xC8, 0x1F, 0xA7, 0x27, 0xC7, 0xF8, 0x75, 0x01, 0x7F, 0x97, 0x38, 0x50, 0x1C, 0x4B, 0x5D, 0x25, 0x35, 0x92, 0x22, 0xC7, 0x09, 0xB5, 0x2E, 0x14, 0x53, 0x80, 0xB3, 0x49, 0x44, 0x3A, 0x59, 0x1B, 0x07, 0x9D, 0x76, 0x99, 0xEA, 0x6A, 0x56, 0x31, 0x2F, 0x46, 0xD5, 0xA1, 0x99, 0x3E, 0xA4, 0x0B, 0xFA, 0x35, 0x79, 0xF9, 0x92, 0x54, 0x74, 0x8D, 0x53, 0x16, 0x9A, 0xB6, 0xEF, 0x79, 0x7D, 0x45, 0xDE, 0x8C, 0x68, 0x8D, 0x55, 0x64, 0x52, 0x14, 0x30, 0x47, 0x6F, 0xBB, 0xA9, 0x0A, 0x90, 0x25, 0x13, 0x18, 0x25, 0x32, 0x72, 0x28, 0x16, 0x08, 0xF0, 0x25, 0x84, 0x84, 0x92, 0x5D, 0x1B, 0xAA, 0xEE, 0x69, 0x24, 0x52, 0xB2, 0x58, 0x2B, 0x68, 0xC4, 0x75, 0xF3, 0x3C, 0x32, 0xC6, 0xAE, 0x16, 0x0F, 0xF4, 0x76, 0x98, 0xB3, 0x15, 0x17, 0x91, 0x8E, 0x09, 0xC3, 0x62, 0x1F, 0x3F, 0x54, 0x37, 0x9D, 0xB0, 0x66, 0x20, 0x1C, 0xFA, 0xF1, 0x68, 0x4E, 0x87, 0x3A, 0x59, 0x43, 0xA2, 0xF2, 0x12, 0x3A, 0xC1, 0x2F, 0x40, 0x84, 0x9B, 0x94, 0xDC, 0xB5, 0x66, 0x6E, 0xF9, 0x48, 0x69, 0x3D, 0x9C, 0xFE, 0x46, 0xB4, 0x66, 0x08, 0xAF, 0x8D, 0x7F, 0x4E, 0xB0, 0x32, 0x59, 0xA8, 0xCF, 0x2C, 0x71, 0xE8, 0x06, 0x7B, 0xA8, 0xCD, 0xEF, 0x06, 0xAA, 0x65, 0x2E, 0xE4, 0xAC, 0x80, 0x87, 0x0D, 0xD6, 0x44, 0xD4, 0xAA, 0xC7, 0x64, 0xC7, 0x0E, 0xEB, 0x1A, 0xE8, 0xED, 0xB6, 0x4A, 0x4F, 0x6B, 0x98, 0xD7, 0x78, 0xF5, 0x0C, 0xB7, 0xCF, 0xED, 0xE2, 0x78, 0xDD, 0x1B, 0x58, 0xFE, 0x3C, 0x21, 0xE9, 0xED, 0xD0, 0xD7, 0x21, 0xE0, 0x02, 0x93, 0x70, 0x25, 0x58, 0x0A, 0x55, 0x0C, 0x96, 0x2C, 0xC1, 0x14, 0x5A, 0x8E, 0x35, 0x62, 0xFD, 0x49, 0x0B, 0x13, 0x2D, 0xFC, 0xDE, 0xC4, 0x7B, 0x7B, 0x58, 0xC7, 0xF9, 0x2F, 0x71, 0x79, 0x71, 0x62, 0x18, 0x3A, 0x77, 0x15, 0xE5, 0xB3, 0xC6, 0x33, 0xB4, 0x0A, 0x9A, 0xEC, 0xFC, 0x7F, 0xBA, 0xB8, 0x4A, 0x45, 0xE5, 0x71, 0xEF, 0x8C, 0x2B, 0x0E, 0xD6, 0x87, 0x09, 0x2B, 0x8A, 0xDF, 0xD0, 0x7D, 0x87, 0xE2, 0x88, 0xCE, 0x95, 0x1E, 0x65, 0x7D, 0x4D, 0x59, 0xA1, 0x7C, 0xF1, 0xBE, 0x8E, 0xB8, 0x10, 0x90, 0x1F, 0xCF, 0x4F, 0x4F, 0x9E, 0x36, 0x16, 0x70, 0x87, 0x9A, 0xF3, 0x14, 0x64, 0xA9, 0x9C, 0x7B, 0x3F, 0xFB, 0xAF, 0xA2, 0x5E, 0x15, 0xB4, 0xE7, 0x2A, 0xBA, 0x1B, 0xBE, 0xF1, 0xBC, 0xF6, 0x2D, 0xFF, 0xDD, 0x26, 0x3F, 0x3F, 0x9B, 0x7D, 0xAF, 0xCB, 0x55, 0x9D, 0xB8, 0x63, 0x8C, 0x37, 0xE4, 0x0E, 0xAD, 0x46, 0xBB, 0x50, 0xAF, 0xF4, 0x2A, 0x87, 0x47, 0x29, 0xCB, 0xB2, 0x84, 0x07, 0x4C, 0x3B, 0xE1, 0x7E, 0x2B, 0xA4, 0x18, 0x6F, 0xB6, 0x44, 0xB3, 0x24, 0xF6, 0xCF, 0x0D, 0x53, 0x8F, 0x76, 0x05, 0x4F, 0x5C, 0xB3, 0x0C, 0xE2, 0xDA, 0xE9, 0x9A, 0xBD, 0x73, 0xB2, 0x90, 0xE1, 0xBA, 0x5E, 0x1F, 0x43, 0xBE, 0x24, 0x81, 0xCE, 0x8A, 0x4F, 0xB1, 0x9D, 0x52, 0xFC, 0xAC, 0x97, 0xC6, 0x36, 0x77, 0x51, 0x2A, 0x85, 0x81, 0xB7, 0xB8, 0x95, 0x84, 0xC9, 0xB7, 0xD9, 0x3E, 0x8D, 0x08, 0x35, 0x85, 0xEB, 0xD3, 0x8F, 0xA0, 0xC8, 0x07, 0xA6, 0x18, 0x25, 0x52, 0x04, 0xE8, 0xC9, 0x8D, 0x4F, 0x9B, 0x57, 0xE3, 0x33, 0xB0, 0xCE, 0xCB, 0x0E, 0x56, 0x73, 0x16, 0x3E, 0x03, 0xEB, 0x30, 0x01, 0x96, 0x5B, 0x40, 0xED, 0x29, 0x65, 0x07, 0xC0, 0xC5, 0x08, 0xD4, 0x91, 0xB2, 0x5E, 0xAD, 0xB0, 0x64, 0x3C, 0x68, 0xC5, 0xAC, 0x5E, 0x76, 0x79, 0xE8, 0xDF, 0xDF, 0xE7, 0xB8, 0x7B, 0xE0, 0xC2, 0x83, 0x96, 0xC8, 0x1C, 0x53, 0xFD, 0x7E, 0x2F, 0xBB, 0x25, 0x38, 0x24, 0x78, 0x48, 0x7E, 0x0C, 0x7F, 0xD1, 0x7F, 0x63, 0x4A, 0x56, 0x7A, 0x49, 0xF1, 0xE9, 0x3B, 0xCF, 0xA3, 0x24, 0xAE, 0x36, 0x14, 0x9F, 0xBE, 0xC5, 0x8F, 0xE9, 0xC4, 0x35, 0x88, 0x8F, 0x9A, 0x71, 0xDF, 0x4D, 0xD3, 0x5A, 0x62, 0xE2, 0x9A, 0x2C, 0x63, 0xD2, 0xAB, 0x1F, 0x21, 0xFF, 0x02, 0x21, 0x16, 0x4F, 0xE6, 0x95, 0x0C, 0x00, 0x00
  203. };
  204.  
  205. esp_err_t defaultHtml(httpd_req_t *req) {
  206.   httpd_resp_set_hdr(req, "Content-Encoding","gzip");
  207.   httpd_resp_send(req, default_html_gz, sizeof(default_html_gz));  
  208.   return ESP_OK;
  209. }
  210.  
  211. httpd_uri_t uri_default = {
  212.     .uri = "/",
  213.     .method = HTTP_GET,
  214.     .handler = defaultHtml
  215. };
  216.  
  217. httpd_handle_t start_webserver(void) {
  218.   httpd_handle_t server = NULL;
  219.   httpd_config_t config = HTTPD_DEFAULT_CONFIG();
  220.   //config.stack_size = 32768;
  221.  
  222.   // Start the httpd server
  223.   ESP_LOGI(TAG, "Starting server on port: '%d'", config.server_port);
  224.   if (httpd_start(&server, &config) == ESP_OK) {
  225.     // Set URI handlers
  226.     ESP_LOGI(TAG, "Registering URI handlers");
  227.     httpd_register_uri_handler(server, &uri_default);
  228.     httpd_register_uri_handler(server, &uri_upload_mask);
  229.     httpd_register_uri_handler(server, &uri_download_mask);
  230.     return server;
  231.   }
  232.  
  233.   ESP_LOGE(TAG, "Error starting server!");
  234.   return NULL;
  235. }
  236.  
  237. void stop_webserver(httpd_handle_t server) {
  238.   // Stop the httpd server
  239.   httpd_stop(server);
  240. }
GeSHi © Codebox Plus Extension
I have put the code in DEBUG mode but got no relevant info, again this is to the best of my knowledge. Please help me verify that this is a bug - or point me to the mistake in my code if possible.

Thank you for any and all help

Re: Large Uploads Get Corrupted

Posted: Thu Apr 04, 2019 3:31 am
by ESP_Angus
Hi sphillips,

I don't know if this is the bug here, but the result of httpd_req_recv (when positive) is the number of bytes read, and may be less than the provided buffer size.

So this line:

Code: Select all

bytes_recvd += recv_size;
Should be:

Code: Select all

bytes_recvd += ret;
(Similarly, the debug code outputs ranges calculated with recv_size, but the actual received size may be different.)

Do you have an example of log output from this code when you issue the HTTP upload?

Re: Large Uploads Get Corrupted

Posted: Thu Apr 04, 2019 11:24 am
by sphillips
Brilliant! Thank you I had missed that detail and you are 100% correct. That is some interesting behavior by the internal receive buffers.

Before the change:
Code: [Select all] [Expand/Collapse]
  1. I (782986) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 0
  2. I (782996) Large_Upload_Test: Writing bytes 0 through 1599
  3. I (782996) Large_Upload_Test: 12 Bytes: 011111111111
  4. I (783006) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 1600
  5. I (783016) Large_Upload_Test: Writing bytes 1600 through 3199
  6. I (783016) Large_Upload_Test: 12 Bytes: 011111111111
  7. I (783026) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 3200
  8. I (783036) Large_Upload_Test: Writing bytes 3200 through 4799
  9. I (783046) Large_Upload_Test: 12 Bytes: 011111111111
  10. I (783046) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 4800
  11. I (783056) Large_Upload_Test: Writing bytes 4800 through 6399
  12. I (783066) Large_Upload_Test: 12 Bytes: 111111111111
  13. I (783066) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 6400
  14. I (783076) Large_Upload_Test: Writing bytes 6400 through 7999
  15. I (783086) Large_Upload_Test: 12 Bytes: 111111111111
  16. I (783086) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 8000
  17. I (783096) Large_Upload_Test: Writing bytes 8000 through 9599
  18. I (783106) Large_Upload_Test: 12 Bytes: 111111111111
  19. I (783106) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 9600
  20. I (783116) Large_Upload_Test: Writing bytes 9600 through 11199
  21. I (783126) Large_Upload_Test: 12 Bytes: 111111111111
  22. I (783126) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 11200
  23. I (783136) Large_Upload_Test: Writing bytes 11200 through 12799
  24. I (783146) Large_Upload_Test: 12 Bytes: 111111111111
  25. I (783146) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 12800
  26. I (783156) Large_Upload_Test: Writing bytes 12800 through 14399
  27. I (783166) Large_Upload_Test: 12 Bytes: 111111111111
  28. I (783176) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 14400
  29. I (783186) Large_Upload_Test: Writing bytes 14400 through 15999
  30. I (783186) Large_Upload_Test: 12 Bytes: 111111111111
  31. I (783196) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 16000
  32. I (783206) Large_Upload_Test: Writing bytes 16000 through 17599
  33. I (783206) Large_Upload_Test: 12 Bytes: 111111111111
  34. I (783216) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 17600
  35. I (783226) Large_Upload_Test: Writing bytes 17600 through 19199
  36. I (783226) Large_Upload_Test: 12 Bytes: 111111111111
GeSHi © Codebox Plus Extension
After the change:
Code: [Select all] [Expand/Collapse]
  1. I (13366) Large_Upload_Test: Incoming transmission 19200 bytes
  2. I (13366) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 0
  3. I (13366) Large_Upload_Test: Writing bytes 0 through 1599
  4. I (13376) Large_Upload_Test: 12 Bytes: 011111111111
  5. I (13376) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 1600
  6. I (13386) Large_Upload_Test: Writing bytes 1600 through 3199
  7. I (13396) Large_Upload_Test: 12 Bytes: 011111111111
  8. I (13396) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 3200
  9. I (13406) Large_Upload_Test: Writing bytes 3200 through 4799
  10. I (13416) Large_Upload_Test: 12 Bytes: 011111111111
  11. I (13416) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 4308
  12. I (13436) Large_Upload_Test: Writing bytes 4308 through 5907
  13. I (13436) Large_Upload_Test: 12 Bytes: 111111111111
  14. I (13446) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 5908
  15. I (13446) Large_Upload_Test: Writing bytes 5908 through 7507
  16. I (13456) Large_Upload_Test: 12 Bytes: 111111111111
  17. I (13466) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 7508
  18. I (13476) Large_Upload_Test: Writing bytes 7508 through 9107
  19. I (13476) Large_Upload_Test: 12 Bytes: 111111111111
  20. I (13486) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 8616
  21. I (13496) Large_Upload_Test: Writing bytes 8616 through 10215
  22. I (13496) Large_Upload_Test: 12 Bytes: 111111111111
  23. I (13506) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 10216
  24. I (13516) Large_Upload_Test: Writing bytes 10216 through 11815
  25. I (13516) Large_Upload_Test: 12 Bytes: 111111111111
  26. I (13526) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 11816
  27. I (13536) Large_Upload_Test: Writing bytes 11816 through 13415
  28. I (13536) Large_Upload_Test: 12 Bytes: 111111111111
  29. I (13546) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 12924
  30. I (13556) Large_Upload_Test: Writing bytes 12924 through 14523
  31. I (13556) Large_Upload_Test: 12 Bytes: 111111111111
  32. I (13566) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 14524
  33. I (13576) Large_Upload_Test: Writing bytes 14524 through 16123
  34. I (13576) Large_Upload_Test: 12 Bytes: 111111111111
  35. I (13586) Large_Upload_Test: Recieving 1600 bytes in chunk, total bytes received: 16124
  36. I (13596) Large_Upload_Test: Writing bytes 16124 through 17723
  37. I (13606) Large_Upload_Test: 12 Bytes: 111111111111
  38. I (13606) Large_Upload_Test: Recieving 1476 bytes in chunk, total bytes received: 17724
  39. I (13616) Large_Upload_Test: Writing bytes 17724 through 19199
  40. I (13626) Large_Upload_Test: 12 Bytes: 111111111111
GeSHi © Codebox Plus Extension
Since the returned data size skews the placement from the i%160==0 boundaries the test to see the first few characters of the buffer becomes meaningless - however the download from the webpage and subsequent upload does not alter the data. The data can now be round-tripped endlessly.

Thank you, and sorry for my ignorance on what was being returned from the recv function...

Re: Large Uploads Get Corrupted

Posted: Fri Apr 05, 2019 8:16 pm
by sphillips
Hey I posted a thank you but never saw it appear in the thread. That was exactly the problem. Thank you
All times are UTC
Page 1 of 1
Powered by phpBB® Forum Software © phpBB Limited
https://www.phpbb.com/