esp_http_client, returning wrong length

[mikemoy]

In the application example for "ESP HTTP Client" here:
https://docs.espressif.com/projects/esp ... ttp-client

It does not seem to work properly for returning the length received. The event_handle shows the length and data received correctly, but in if (err == ESP_OK) it does not. Here is the output when run and my code example below that.

Code: Select all

I (463) HTTP_CLIENT: Setting WiFi configuration SSID WhosLooking...
I (533) phy: phy_version: 4008, 544f89f, Jan 24 2019, 14:54:06, 0, 0
I (533) wifi: mode : sta (24:0a:c4:04:5c:f4)
I (1263) wifi: n:5 0, o:1 0, ap:255 255, sta:5 0, prof:1
I (2243) wifi: state: init -> auth (b0)
I (2253) wifi: state: auth -> assoc (0)
I (2273) wifi: state: assoc -> run (10)
I (2403) wifi: connected with WhosLooking, channel 5
I (2403) wifi: pm start, type: 1

I (3393) event: sta ip: 192.168.0.3, mask: 255.255.255.0, gw: 192.168.0.1
I (3393) HTTP_CLIENT: Requesting From: http://api.thingspeak.com/apps/thinghttp/send_request?api_key=AWCB96AR5EDPI79B

I (4273) HTTP_CLIENT: HTTP_EVENT_ON_DATA, len=40
Not Chunked Data:<strong class="time">5:43:18 AM</strong>
I (4273) HTTP_CLIENT: HTTP reader Status = 200, content_length = -1
Content Length=-1
I (4283) HTTP_CLIENT: No Content To Read

My source code.

Code: Select all

#include <string.h>
#include <stdlib.h>
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
//#define LOG_LOCAL_LEVEL ESP_LOG_VERBOSE		// Use for ESP_LOGD
#include "esp_log.h"
#include "esp_system.h"
#include "nvs_flash.h"
#include "esp_wifi.h"
#include "esp_event_loop.h"
#include "freertos/event_groups.h"
#include "esp_http_client.h"
 
#define STATION_SSID		"WhosLooking"
#define STATION_PASSPHRASE  "12345678"

 
static const char *TAG = "HTTP_CLIENT";
static EventGroupHandle_t wifi_event_group;
 
const int CONNECTED_BIT = BIT0;
 
/*
 *
 */
 static esp_err_t event_handler(void *ctx, system_event_t *event)
{
	switch (event->event_id) {
	case SYSTEM_EVENT_STA_START:
		esp_wifi_connect();
		break;
	case SYSTEM_EVENT_STA_GOT_IP:
		xEventGroupSetBits(wifi_event_group, CONNECTED_BIT);
		break;
	case SYSTEM_EVENT_STA_DISCONNECTED:
		/* This is a workaround as ESP32 WiFi libs don't currently
		   auto-reassociate. */
		esp_wifi_connect();
		xEventGroupClearBits(wifi_event_group, CONNECTED_BIT);
		break;
	default:
		break;
	}
	return ESP_OK;
}
 /*
  *
  */
 esp_err_t _http_event_handler(esp_http_client_event_t *evt)
{
	switch (evt->event_id) {
	case HTTP_EVENT_ERROR:
		ESP_LOGD(TAG, "HTTP_EVENT_ERROR");
		break;
	case HTTP_EVENT_ON_CONNECTED:
		ESP_LOGD(TAG, "HTTP_EVENT_ON_CONNECTED");
		break;
	case HTTP_EVENT_HEADER_SENT:
		ESP_LOGD(TAG, "HTTP_EVENT_HEADER_SENT");
		break;
	case HTTP_EVENT_ON_HEADER:
		ESP_LOGD(TAG, "HTTP_EVENT_ON_HEADER, key=%s, value=%s", evt->header_key, evt->header_value);
		break;
	case HTTP_EVENT_ON_DATA:
		ESP_LOGI(TAG, "HTTP_EVENT_ON_DATA, len=%d", evt->data_len);
		if (!esp_http_client_is_chunked_response(evt->client)) 
		{
			printf("Chunked Data:%.*s\r\n", evt->data_len, (char*)evt->data);
		}
		else
		{
			printf("Not Chunked Data:%.*s\r\n", evt->data_len, (char*)evt->data);
		}
 
		break;
	case HTTP_EVENT_ON_FINISH:
		ESP_LOGD(TAG, "HTTP_EVENT_ON_FINISH");
		break;
	case HTTP_EVENT_DISCONNECTED:
		ESP_LOGD(TAG, "HTTP_EVENT_DISCONNECTED");
		break;
	}
	return ESP_OK;
}
/*
 *
 */
int RequestSiteData(char *website)
{
	char buffer[128] = { 0 };
	int read_len = 0;
	
	esp_http_client_config_t config = {
		.url = website,
		.event_handler = _http_event_handler,
	};
	esp_http_client_handle_t client = esp_http_client_init(&config);

	ESP_LOGI(TAG, "Requesting From: %s\r\n", website);
	
	esp_err_t err = esp_http_client_perform(client);
	
	if (err == ESP_OK) 
	{
		ESP_LOGI(TAG, "HTTP reader Status = %d, content_length = %d", esp_http_client_get_status_code(client), esp_http_client_get_content_length(client));
	
		int content_length =  esp_http_client_fetch_headers(client);
                printf("Content Length=%d\r\n", content_length);
                
		if (content_length > 0)
		{
			read_len = esp_http_client_read(client, buffer, content_length);
			if (read_len <= 0) 
			{
				ESP_LOGI(TAG, "Error read data");
			}
			else
			{
				buffer[read_len] = 0;
				printf("RX[%d]:%s\r\n", read_len, buffer);
			}
		}
		else
		{
			ESP_LOGI(TAG, "No Content To Read");
		}
	}
	
	esp_http_client_cleanup(client);
	
	return read_len;
}
/*
 *
 */
void wifi_initialise(void)
{
	tcpip_adapter_init();
	wifi_event_group = xEventGroupCreate();
	ESP_ERROR_CHECK(esp_event_loop_init(event_handler, NULL));
	wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();
	ESP_ERROR_CHECK(esp_wifi_init(&cfg));
	ESP_ERROR_CHECK(esp_wifi_set_storage(WIFI_STORAGE_RAM));
	wifi_config_t wifi_config = {
		.sta = {
		.ssid = STATION_SSID,
		.password = STATION_PASSPHRASE,
	},
	};
	ESP_LOGI(TAG, "Setting WiFi configuration SSID %s...", wifi_config.sta.ssid);
	ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_STA));
	ESP_ERROR_CHECK(esp_wifi_set_config(ESP_IF_WIFI_STA, &wifi_config));
	ESP_ERROR_CHECK(esp_wifi_start());
 
}
/*
 *
 */
 void app_main()
{
	esp_err_t ret = nvs_flash_init();
	
	if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
		ESP_ERROR_CHECK(nvs_flash_erase());
		ret = nvs_flash_init();
	}
	ESP_ERROR_CHECK(ret);
	
	wifi_initialise();

	// Wait here until Wifi is connected
	xEventGroupWaitBits(wifi_event_group, CONNECTED_BIT, false, true, portMAX_DELAY);
	
	RequestSiteData("http://api.thingspeak.com/apps/thinghttp/send_request?api_key=AWCB96AR5EDPI79B");
	
	
}

[ESP_Sprite]

The reason is that esp_http_client_get_content_length() returns whatever the server set in the Content-Length header. This requires the webserver knows the content length before sending the page. As your page seems to be dynamically generated by a script on the server side, the webserver likely can't know the length of the response in advance and as such doesn't send a valid Content-Length header, causing esp_http_client_get_content_length() to return -1.

[mikemoy]

But when esp_err_t err = esp_http_client_perform(client); is fired, and then shortly thereafter "_http_event_handle" fires it does show the correct length. Upon "esp_http_client_perform" return is falls into if (err == ESP_OK) which esp_http_client_get_content_length(client)); is called and returns a wrong value.

[ESP_Sprite]

...I don't know how to answer that. The behaviour as I described is from the documentation; the fact that you actually do get a valid response at some point could be seen as a bug. (That is, if your webserver indeed does not send out a Content-Length field.) Maybe the parser is nice enough to fill in the length at the end, after it received all the data.

[boarchuz]

I (3374) INFO: HTTP_EVENT_ON_DATA, len=40

If you're referring to this, that value is the number of bytes received and ready for your app to process from that chunk, not the value of the content length header (and not necessarily the total length). It happens to be equivalent to the total length in this case because it all arrives in one chunk anyway, but suppose it came in 3 lots instead:

I (3374) INFO: HTTP_EVENT_ON_DATA, len=16
I (3384) INFO: HTTP_EVENT_ON_DATA, len=16
I (3394) INFO: HTTP_EVENT_ON_DATA, len=8

There might be a 'cumulative length' member of the client struct, otherwise you could track it yourself with each data event.

[mikemoy]

Thank guys for your time in this. I am failing to understand why
esp_http_client_get_content_length(client));
is returning -1.
When clearly a event was fired showing that i did indeed receive 40 bytes.
I (3374) INFO: HTTP_EVENT_ON_DATA, len=40

Whether is was chunked or not, wouldn't it see logical when
esp_err_t err = esp_http_client_perform(client);
has returned that a call to
esp_http_client_get_content_length(client));
should return the total data length received ?

[chegewara]

Get http response content length (from header Content-Length) the valid value if this function invoke after esp_http_client_perform

https://docs.espressif.com/projects/esp ... t_handle_t

It can be used in 2 cases(if Content-Length is set):
1. to allocate buffer which can hold all retrieved data
2. to compare with amount of data you read from website

[mikemoy]

This is why it does not make any sense. It was not a chunked response. it was 40 bytes, all received at once.
Anyone can pop this in their browser and see for themselves.
http://api.thingspeak.com/apps/thinghtt ... AR5EDPI79B


after calling esp_http_client_perform(client);
you can see that the event_handler fired and showed the data received as not chunked and was 40 bytes.
Later, when "esp_http_client_get_content_length(client)" is called it returns -1. Which is wrong.
Also, even more later when "int content_length = esp_http_client_fetch_headers(client);" is called its also is returning -1, which is wrong.

I need a way to find out how much data was received after calling esp_http_client_perform, so that I can call "esp_http_client_read" to actually get the data.

[boarchuz]

Use any http traffic monitor and you'll see that it is:

Code: Select all

Transfer-Encoding: chunked

(with no "Content-Length" header)

esp_http_client_get_content_length(client) should be returning -1 for chunked. No problem there.

esp_http_client_fetch_headers(client) should be returning 0 for chunked...

Code: Select all

after calling esp_http_client_perform, so that I can call "esp_http_client_read" to actually get the data

You can't use esp_http_client like this.

There are two ways: in the basic way where you set it all up then let it off the leash with esp_http_client_perform(); or as a 'stream' where you are responsible for handling the flow of a http request/response while the esp_http_client will take care of the boring stuff under the hood.
These two uses are not compatible. You shouldn't be using esp_http_client_fetch_headers or esp_http_client_read with esp_http_client_perform.
Have another look at the "Overview" and "HTTP Stream" here: https://docs.espressif.com/projects/esp ... lient.html

[mikemoy]

Have another look at the "Overview" and "HTTP Stream" here: https://docs.espressif.com/projects/esp ... lient.html

I started out using that same example, only changing the .url
It does the same thing. Shows receiving 40 bytes. yet when its all done, this line here:
"ESP_LOGI(TAG, "Status = %d, content_length = %d""
shows "Status = 200, content_length = -1"
But the content length is not -1, its 40.

All i am trying to do is connect to that weblink, and get the data it responds back with.