Issues with ESP-IDF v5.2 using async handlers

[agatrost1620]

I am trying to set up the webserver to be able to receive a file from the web page and then be able to query the progress from the web page to the webserver. Before I was using the async feature addition in 5.2 I had been able to upload the file from v 5.0. I had been reading that I had to upgrade to 5.2 to be able to do as I was needing. I have updated esp-idf to 5.2 and corrected the errors that were there and tested it working before attempting to use the async feature.

Now Here is the webserver implementation that I have:

Initializing the webserver:

Code: [Select all] [Expand/Collapse]

1. esp_err_t WEBSERVER_Start(void)
2. {
3.     ESP_LOGI(TAG, "Starting Webserver");
4.     esp_err_t retValue = ESP_FAIL;
5.     httpd_handle_t server = NULL;
6.     httpd_config_t config = HTTPD_DEFAULT_CONFIG();
7.     httpd_uri_t http_uri[] =
8.     {
9.         {
10.             .uri = "/api/progress",
11.             .method = HTTP_GET,
12.             .handler = FirmwareProgressGetHandler,
13.             .user_ctx = NULL,
14.         },
15.         {
16.             .uri = "/api/upload",
17.             .method = HTTP_POST,
18.             .handler = FirmwareUploadPostHandler,
19.             .user_ctx = NULL,
20.  
21.         },
22.             .
23.             .
24.             .
25.         /*
26.          * Handler for serving the webpage files
27.          */
28.         {
29.             .uri = "/*",
30.             .method = HTTP_GET,
31.             .handler = WebServer,
32.             .user_ctx = "/www_0",//TODO: Set base path based on OTA_0 or OTA_1
33.         }
34.     };
35.    
36.     mAsyncQueue = xQueueCreate(10, sizeof(async_req_t));
37.  
38.     if (NULL != mAsyncQueue)
39.     {
40.         retValue = (pdTRUE == xTaskCreate( AsyncRequestHandler, "Async Request", 4096, NULL, configMAX_PRIORITIES - 1, NULL)) ?
41.                    ESP_OK : ESP_FAIL;
42.     }
43.  
44.     if (ESP_OK == retValue)
45.     {
46.         config.uri_match_fn = httpd_uri_match_wildcard;
47.         config.stack_size = 10 * 1024;
48.         httpd_start(&server, &config);
49.         for (int i = 0; i < (sizeof(http_uri) / sizeof(http_uri[0])); ++i)
50.         {
51.             httpd_register_uri_handler(server, &http_uri[i]);
52.         }
53.  
54.         httpd_register_err_handler(server, HTTPD_404_NOT_FOUND, Error404handler);
55.     }
56.  
57.     return retValue;
58. }

GeSHi © Codebox Plus Extension

Worker thread that receives the queue:

Code: [Select all] [Expand/Collapse]

1. static void AsyncRequestHandler(void* param)
2. {
3.     async_req_t asyncReq;
4.  
5.     while (1)
6.     {
7.         BaseType_t recvCheck = xQueueReceive(mAsyncQueue, &asyncReq, portMAX_DELAY);
8.         if (recvCheck != pdTRUE)
9.         {
10.             ESP_LOGE(TAG, "EVENT Receive error");
11.         }
12.         else
13.         {
14.             ESP_LOGI(TAG, "Received Async Request");
15.             asyncReq.handler(asyncReq.req);
16.             httpd_req_async_handler_complete(asyncReq.req);
17.         }
18.     }
19. }

GeSHi © Codebox Plus Extension

Update handler called from webserver:

Code: [Select all] [Expand/Collapse]

1. static esp_err_t FirmwareUploadPostHandler(httpd_req_t* req)
2. {
3.     esp_err_t retValue = ESP_FAIL;
4.     httpd_req_t* asyncReq = NULL;
5.     retValue = httpd_req_async_handler_begin(req, &asyncReq);
6.  
7.     async_req_t async =
8.     {
9.         .handler = FirmwareUpload,
10.         .req = asyncReq,
11.     };
12.     ESP_LOGI(TAG, "FirmwareUpload Post Handler");
13.     if (xQueueSend(mAsyncQueue, &async, pdMS_TO_TICKS(100)) == false)
14.     {
15.         ESP_LOGE(TAG, "worker queue is full");
16.         httpd_req_async_handler_complete(asyncReq); // cleanup
17.         return ESP_FAIL;
18.     }
19.  
20.     return retValue;
21. }

GeSHi © Codebox Plus Extension

The code that I am having issues with receiveing the data:

Code: [Select all] [Expand/Collapse]

1. static esp_err_t FirmwareUpload(httpd_req_t* req)
2. {
3.     ESP_LOGI(TAG, "Firmware Upload");
4.  
5.     esp_err_t retValue = ESP_FAIL;
6.     int bytesRecv = 0;
7.     int bytesReqRecv = -1;
8.     int reqRecReturn = 0;
9.     int totalBytesRemaining = req->content_len;
10.     int timeoutRetry = 3;
11.     httpd_err_code_t errorCode = HTTPD_500_INTERNAL_SERVER_ERROR;
12.     char buffer[BUFFER_SIZE] = {0};
13.     char* bufferPtr = buffer;
14.  
15.     //while (totalBytesRemaining > 0) ***Commented out to see print out of TEST to get see what first read is ***
16.     {
17.         bytesRecv = 0;
18.         bytesReqRecv = (bytesReqRecv != 0) ? (3 * sizeof(uint32_t) +  16) :
19.                     (totalBytesRemaining < BUFFER_SIZE) ?  totalBytesRemaining : BUFFER_SIZE;
20.         bufferPtr = buffer;
21.         while (bytesRecv != bytesReqRecv)
22.         {
23.             reqRecReturn = httpd_req_recv(req, bufferPtr, bytesReqRecv - bytesRecv);
24.             if ((HTTPD_SOCK_ERR_TIMEOUT == reqRecReturn) && (0 < timeoutRetry))
25.             {
26.                 --timeoutRetry;
27.                 ESP_LOGW(TAG, "Recv TIMEOUT retrying %d more time(s)", timeoutRetry);
28.             }
29.             else if ((HTTPD_SOCK_ERR_TIMEOUT == reqRecReturn) && (0 >= timeoutRetry))
30.             {
31.                 ESP_LOGE(TAG, "Recv TIMEOUT");
32.                 break;
33.             }
34.             else if (HTTPD_SOCK_ERR_INVALID == reqRecReturn)
35.             {
36.                 ESP_LOGE(TAG, "Invalid sock");
37.                 break;
38.             }
39.             else if (HTTPD_SOCK_ERR_FAIL == reqRecReturn)
40.             {
41.                 ESP_LOGE(TAG, "sock failed");
42.                 break;
43.             }
44.             else if (0 == reqRecReturn)
45.             {
46.                 ESP_LOGW(TAG, "Received 0 bytes");
47.             }
48.             else if (0 < reqRecReturn)
49.             {
50.                 timeoutRetry = 3;
51.                 bytesRecv += reqRecReturn;
52.                 bufferPtr += reqRecReturn;
53.                 ESP_LOGV(TAG, "Received: %4d of %4d", bytesRecv, bytesReqRecv);
54.             }
55.             else
56.             {
57.                 ESP_LOGW(TAG, "Unknown Received %d", reqRecReturn);
58.             }
59.         }
60.  
61.         printf("TEST - %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X\n",
62.                buffer[0], buffer[1], buffer[2], buffer[3], buffer[4], buffer[5], buffer[6], buffer[7], buffer[8], buffer[9]);
63.         if (bytesRecv != bytesReqRecv)
64.         {
65.             retValue = ESP_FAIL;
66.             ESP_LOGE(TAG, "Failed to recv expected data - %d/%d", bytesRecv, bytesReqRecv);
67.             totalBytesRemaining = -1;
68.         }
69.         else
70.         {
71.             *** Operate on received data ***
72.             totalBytesRemaining = (ESP_OK == retValue) ? totalBytesRemaining - bytesRecv : -1;
73.             bytesReqRecv = 0;
74.         }
75.     }//END commented out loop to get one iteration
76.  
77.  
78.     return retValue;
79.     #endif
80. }

GeSHi © Codebox Plus Extension

My issue is when I printf out the buffer I am operating on it is not at the beginning of the data being transferred.
I have seen it close to 0x1080, 0x500(ish) and it is not consistent that I have seen on how far in it goes on the same transfer of data.
Any help would be appreciated.
Please let me know if there is any other information needed to help resolve my issue.

Thank you
it

[thethinker]

I am trying to set up the webserver to be able to receive a file from the web page and then be able to query the progress from the web page to the webserver. Before I was using the async feature addition in 5.2 I had been able to upload the file from v 5.0. I had been reading that I had to upgrade to 5.2 to be able to do as I was needing. I have updated esp-idf to 5.2 and corrected the errors that were there and tested it working before attempting to use the async feature.

Now Here is the webserver implementation that I have:

Initializing the webserver:

Code: [Select all] [Expand/Collapse]

1. esp_err_t WEBSERVER_Start(void)
2. {
3.     ESP_LOGI(TAG, "Starting Webserver");
4.     esp_err_t retValue = ESP_FAIL;
5.     httpd_handle_t server = NULL;
6.     httpd_config_t config = HTTPD_DEFAULT_CONFIG();
7.     httpd_uri_t http_uri[] =
8.     {
9.         {
10.             .uri = "/api/progress",
11.             .method = HTTP_GET,
12.             .handler = FirmwareProgressGetHandler,
13.             .user_ctx = NULL,
14.         },
15.         {
16.             .uri = "/api/upload",
17.             .method = HTTP_POST,
18.             .handler = FirmwareUploadPostHandler,
19.             .user_ctx = NULL,
20.  
21.         },
22.             .
23.             .
24.             .
25.         /*
26.          * Handler for serving the webpage files
27.          */
28.         {
29.             .uri = "/*",
30.             .method = HTTP_GET,
31.             .handler = WebServer,
32.             .user_ctx = "/www_0",//TODO: Set base path based on OTA_0 or OTA_1
33.         }
34.     };
35.    
36.     mAsyncQueue = xQueueCreate(10, sizeof(async_req_t));
37.  
38.     if (NULL != mAsyncQueue)
39.     {
40.         retValue = (pdTRUE == xTaskCreate( AsyncRequestHandler, "Async Request", 4096, NULL, configMAX_PRIORITIES - 1, NULL)) ?
41.                    ESP_OK : ESP_FAIL;
42.     }
43.  
44.     if (ESP_OK == retValue)
45.     {
46.         config.uri_match_fn = httpd_uri_match_wildcard;
47.         config.stack_size = 10 * 1024;
48.         httpd_start(&server, &config);
49.         for (int i = 0; i < (sizeof(http_uri) / sizeof(http_uri[0])); ++i)
50.         {
51.             httpd_register_uri_handler(server, &http_uri[i]);
52.         }
53.  
54.         httpd_register_err_handler(server, HTTPD_404_NOT_FOUND, Error404handler);
55.     }
56.  
57.     return retValue;
58. }

GeSHi © Codebox Plus Extension

Worker thread that receives the queue:

Code: [Select all] [Expand/Collapse]

1. static void AsyncRequestHandler(void* param)
2. {
3.     async_req_t asyncReq;
4.  
5.     while (1)
6.     {
7.         BaseType_t recvCheck = xQueueReceive(mAsyncQueue, &asyncReq, portMAX_DELAY);
8.         if (recvCheck != pdTRUE)
9.         {
10.             ESP_LOGE(TAG, "EVENT Receive error");
11.         }
12.         else
13.         {
14.             ESP_LOGI(TAG, "Received Async Request");
15.             asyncReq.handler(asyncReq.req);
16.             httpd_req_async_handler_complete(asyncReq.req);
17.         }
18.     }
19. }

GeSHi © Codebox Plus Extension

Update handler called from webserver:

Code: [Select all] [Expand/Collapse]

1. static esp_err_t FirmwareUploadPostHandler(httpd_req_t* req)
2. {
3.     esp_err_t retValue = ESP_FAIL;
4.     httpd_req_t* asyncReq = NULL;
5.     retValue = httpd_req_async_handler_begin(req, &asyncReq);
6.  
7.     async_req_t async =
8.     {
9.         .handler = FirmwareUpload,
10.         .req = asyncReq,
11.     };
12.     ESP_LOGI(TAG, "FirmwareUpload Post Handler");
13.     if (xQueueSend(mAsyncQueue, &async, pdMS_TO_TICKS(100)) == false)
14.     {
15.         ESP_LOGE(TAG, "worker queue is full");
16.         httpd_req_async_handler_complete(asyncReq); // cleanup
17.         return ESP_FAIL;
18.     }
19.  
20.     return retValue;
21. }

GeSHi © Codebox Plus Extension

The code that I am having issues with receiveing the data:

Code: [Select all] [Expand/Collapse]

1. static esp_err_t FirmwareUpload(httpd_req_t* req)
2. {
3.     ESP_LOGI(TAG, "Firmware Upload");
4.  
5.     esp_err_t retValue = ESP_FAIL;
6.     int bytesRecv = 0;
7.     int bytesReqRecv = -1;
8.     int reqRecReturn = 0;
9.     int totalBytesRemaining = req->content_len;
10.     int timeoutRetry = 3;
11.     httpd_err_code_t errorCode = HTTPD_500_INTERNAL_SERVER_ERROR;
12.     char buffer[BUFFER_SIZE] = {0};
13.     char* bufferPtr = buffer;
14.  
15.     //while (totalBytesRemaining > 0) ***Commented out to see print out of TEST to get see what first read is ***
16.     {
17.         bytesRecv = 0;
18.         bytesReqRecv = (bytesReqRecv != 0) ? (3 * sizeof(uint32_t) +  16) :
19.                     (totalBytesRemaining < BUFFER_SIZE) ?  totalBytesRemaining : BUFFER_SIZE;
20.         bufferPtr = buffer;
21.         while (bytesRecv != bytesReqRecv)
22.         {
23.             reqRecReturn = httpd_req_recv(req, bufferPtr, bytesReqRecv - bytesRecv);
24.             if ((HTTPD_SOCK_ERR_TIMEOUT == reqRecReturn) && (0 < timeoutRetry))
25.             {
26.                 --timeoutRetry;
27.                 ESP_LOGW(TAG, "Recv TIMEOUT retrying %d more time(s)", timeoutRetry);
28.             }
29.             else if ((HTTPD_SOCK_ERR_TIMEOUT == reqRecReturn) && (0 >= timeoutRetry))
30.             {
31.                 ESP_LOGE(TAG, "Recv TIMEOUT");
32.                 break;
33.             }
34.             else if (HTTPD_SOCK_ERR_INVALID == reqRecReturn)
35.             {
36.                 ESP_LOGE(TAG, "Invalid sock");
37.                 break;
38.             }
39.             else if (HTTPD_SOCK_ERR_FAIL == reqRecReturn)
40.             {
41.                 ESP_LOGE(TAG, "sock failed");
42.                 break;
43.             }
44.             else if (0 == reqRecReturn)
45.             {
46.                 ESP_LOGW(TAG, "Received 0 bytes");
47.             }
48.             else if (0 < reqRecReturn)
49.             {
50.                 timeoutRetry = 3;
51.                 bytesRecv += reqRecReturn;
52.                 bufferPtr += reqRecReturn;
53.                 ESP_LOGV(TAG, "Received: %4d of %4d", bytesRecv, bytesReqRecv);
54.             }
55.             else
56.             {
57.                 ESP_LOGW(TAG, "Unknown Received %d", reqRecReturn);
58.             }
59.         }
60.  
61.         printf("TEST - %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X\n",
62.                buffer[0], buffer[1], buffer[2], buffer[3], buffer[4], buffer[5], buffer[6], buffer[7], buffer[8], buffer[9]);
63.         if (bytesRecv != bytesReqRecv)
64.         {
65.             retValue = ESP_FAIL;
66.             ESP_LOGE(TAG, "Failed to recv expected data - %d/%d", bytesRecv, bytesReqRecv);
67.             totalBytesRemaining = -1;
68.         }
69.         else
70.         {
71.             *** Operate on received data ***
72.             totalBytesRemaining = (ESP_OK == retValue) ? totalBytesRemaining - bytesRecv : -1;
73.             bytesReqRecv = 0;
74.         }
75.     }//END commented out loop to get one iteration
76.  
77.  
78.     return retValue;
79.     #endif
80. }

GeSHi © Codebox Plus Extension

My issue is when I printf out the buffer I am operating on it is not at the beginning of the data being transferred.
I have seen it close to 0x1080, 0x500(ish) and it is not consistent that I have seen on how far in it goes on the same transfer of data.
Any help would be appreciated.
Please let me know if there is any other information needed to help resolve my issue.

Thank you
it

Hello @agatrost1620,
I have the same exact issue at the moment. Did you figrue this out?