[SOLVED] HTTP GET, not able to read the server data

[filo_gr]

Hello community,

I'm using two ESP32.

• ESP32-C3 behaves as a webserver and access point. If I make an HTTP get request (using a web browser or TestMace) I can read the html page content.

• ESP32 behaves as a client and it needs to connect itself to the ESP32-C3. I connects correctly to the ESP32-C3 and it makes a good HTTP get request (it returns status 200).

The problem is that I obtain Status = 200, content_length = -1. In other words, I'm not able to read the content of the html page.
It is sure I'm making something wrong. However the example "esp_http_client" works well!

Piece of code on the server side:

Code: Select all

esp_err_t
file_server_init (const char * p_base_path)
{
    esp_err_t ret = ESP_FAIL;
    static file_server_data_t * p_server_data = NULL;
    httpd_handle_t server = NULL;
    // Per configurare correttamente la struttura di base, va sempre eseguito
    // HTTPD_DEFAULT_CONFIG.
    //
    httpd_config_t config = HTTPD_DEFAULT_CONFIG();
    // Si deve fornire il percorso valido, il webserver supporta solo il
    // percorso '/spiffs'.
    //
    if ((NULL == p_base_path) || (strcmp(p_base_path, "/spiffs") != 0))
    {
        ESP_LOGE(g_p_tag,
                 "File server presently supports only '/spiffs' as base path");
        ret = ESP_ERR_INVALID_ARG;
    }
    // Verifica che il file server non sia già stato inizializzato.
    //
    else if (p_server_data != NULL)
    {
        ESP_LOGE(g_p_tag, "File server already started");
        ret = ESP_ERR_INVALID_STATE;
    }
    else
    {
        // Alloca memoria per la struttura del file server.
        //
        p_server_data = calloc(1, sizeof(file_server_data_t));
        if (NULL == p_server_data)
        {
            ESP_LOGE(g_p_tag, "Failed to allocate memory for server data");
            ret = ESP_ERR_NO_MEM;
        }
        else
        {
            strlcpy(p_server_data->base_path, p_base_path,
                    sizeof(p_server_data->base_path));
            // Aumento il numero massimo di handler URI registrabili.
            //
            config.max_uri_handlers = 20;

            /* Use the URI wildcard matching function in order to
            * allow the same handler to respond to multiple different
            * target URIs which match the wildcard scheme */
            config.uri_match_fn = httpd_uri_match_wildcard;

            ESP_LOGI(g_p_tag, "Starting HTTP Server on port: '%d'",
                     config.server_port);
            // Fa partire il server web creando una istanza HTTP e allocando
            // memoria e risorse per esso in base alla configurazione
            // specificata.
            //
            if (httpd_start(&server, &config) != ESP_OK)
            {
                ESP_LOGE(g_p_tag, "Failed to start file server!");
                ret = ESP_FAIL;
            }
            // Impostazione dell'apertura della pagina principale, attraverso la
            // definizione dell'URI, del metodo URI, della funzione legata a tale URI.
            //
            else
            {
                httpd_uri_t file_partition = {
                .uri       = "/index",
                .method    = HTTP_GET,
                .handler   = new_page_handler,
                .user_ctx  = p_server_data
                };
                ESP_ERROR_CHECK(httpd_register_uri_handler(server, &file_partition));
                ret = ESP_OK;
            }
        }
    }

    return ret;
}   /* file_server_init() */

static esp_err_t
new_page_handler (httpd_req_t * p_req)
{
    int data_read = 0;
    char buffer[100];
    esp_err_t ret = ESP_FAIL;
    ESP_LOGI(g_p_tag, "example.html");
    // Apertura di example.txt dalla SPIFFS.
    //
    FILE * h_file_handler = fopen("/spiffs/example.html", "r");
    if (NULL == h_file_handler)
    {
        ESP_LOGE(g_p_tag, "Failed to open example.html");
        ret = ESP_FAIL;
    }
    else
    {
        // Riempio buf di zeri (quindi 64 zeri).
        //
        memset(buffer, 0, sizeof(buffer));
        httpd_resp_sendstr_chunk(p_req, "<!DOCTYPE html>");

        do
        {
            // Metto gli oggetti nel buffer.
            data_read = fread(buffer, 1, sizeof(buffer), h_file_handler);
            httpd_resp_sendstr_chunk(p_req, buffer);
            // Display the read contents from the file
            ESP_LOGI(g_p_tag, "Read from example.html: %s", buffer);
        } while (100 == data_read);

        httpd_resp_sendstr_chunk(p_req, NULL);
        // Chiude il file example.html.
        //
        fclose(h_file_handler);
        ret = ESP_OK;
    }
    
    return ret;
}   /* new_page_handler() */

Piece of code on the client side

Code: Select all

void
https_client_init (void)
{
    // Configurazione di default per il client.
    //
    esp_http_client_config_t config = {
        .host = "192.168.1.1",
        .path = "/index",
        .query = "esp",
        .event_handler = client_event_handler,
        .user_data = g_local_response_buffer,
        .disable_auto_redirect = true,
        .transport_type = HTTP_TRANSPORT_OVER_TCP,

    };

    g_http_client = esp_http_client_init(&config);

    return;
}   /* https_client_init() */

esp_err_t
https_client_connection (const char * data_in)
{
    int data_len = 0;
    esp_err_t ret = ESP_OK;

    (void) esp_http_client_set_header(g_http_client, HTTP_HEADER_NAME,
                                      HTTP_HEADER_VALUE);
    (void) esp_http_client_set_url(g_http_client, data_in);
    esp_http_client_set_method(g_http_client, HTTP_METHOD_GET);
    ret = esp_http_client_perform(g_http_client);
    // Azione da compiere in base alla risposta della chiamata per l'esecuzione
    // delle azioni del client.
    //
    if (ret == ESP_OK)
    {
        ESP_LOGI(g_p_tag, "Status = %d, content_length = %d",
                 esp_http_client_get_status_code(g_http_client),
                 esp_http_client_get_content_length(g_http_client));
        if (true == esp_http_client_is_chunked_response(g_http_client))
        {
            (void) esp_http_client_get_chunk_length(g_http_client, &data_len);
            if (data_len != 0)
            {
                ESP_LOGI(g_p_tag, "Dati chunk presenti");
            }
            else
            {
                ESP_LOGI(g_p_tag, "Dati chunk nulli");
            }
        }
        ESP_LOG_BUFFER_HEX(g_p_tag, g_local_response_buffer,
                           strlen(g_local_response_buffer));
    }
    else
    {
        ESP_LOGE(g_p_tag, "Error perform http request %s",
                 esp_err_to_name(ret));
    }

    return ret;
}   /* https_client_connection() */

[ESP_Sprite]

That's all expected. Content_length=-1 indicates it's a chunked transfer, in other words, the server doesn't know the length of the data to be sent yet and you should just receive until it's done.

[filo_gr]

That's all expected. Content_length=-1 indicates it's a chunked transfer, in other words, the server doesn't know the length of the data to be sent yet and you should just receive until it's done.

Thanks for letting me know I'm on the right way.
So, if the server doesn't know the total size of data, it sends it as chunked.
On the client size I have content_length = -1 because the content is chunked.
Hence

Code: Select all

true == esp_http_client_is_chunked_response(g_http_client)

is true.

But how should I read the content now? Maybe through esp_http_client_read()?

[ESP_Mahavir]

You can use `HTTP_EVENT_ON_DATA` from event handler registered in `esp_http_client_config_t`. Here is reference for this https://github.com/espressif/esp-idf/bl ... mple.c#L63

[filo_gr]

You can use `HTTP_EVENT_ON_DATA` from event handler registered in `esp_http_client_config_t`. Here is reference for this https://github.com/espressif/esp-idf/bl ... mple.c#L63

Ok, I created the handler, as in the example, and I modified the code adding an else condition (related to chunk data):

Code: Select all

        case HTTP_EVENT_ON_DATA:
            ESP_LOGI(g_p_tag, "HTTP_EVENT_ON_DATA, len=%d", evt->data_len);
            /*
             *  Check for chunked encoding is added as the URL for chunked
             *  encoding used in this example returns binary data.
             *  However, event handler can also be used in case chunked encoding
             *  is used.
             */
            // Verifica se il dato in risposta è "chunked".
            //
            if (!esp_http_client_is_chunked_response(evt->client))
            {
                // If user_data buffer is configured, copy the response into the
                // buffer
                //
                if (evt->user_data)
                {
                    memcpy(evt->user_data + output_len, evt->data,
                           evt->data_len);
                }
                else
                {
                    if (p_output_buffer == NULL)
                    {
                        p_output_buffer = (char *) malloc(
                                esp_http_client_get_content_length(
                                    evt->client));
                        output_len = 0;
                        if (p_output_buffer == NULL)
                        {
                            ESP_LOGE(g_p_tag, "Failed to allocate memory");
                            ret = ESP_FAIL;
                        }
                    }
                    if (ESP_OK == ret)
                    {
                        memcpy(p_output_buffer + output_len, evt->data,
                               evt->data_len);
                    }
                }
                if (ESP_OK == ret)
                {
                    output_len += evt->data_len;
                }
            }
            else
            {
                ESP_LOGI(g_p_tag, "Dati: %s", (char *) evt->data);   // HERE!
            }
        break;

Now, I can read chunked data. Next step I think it will the storage of the received data in order to process it. But at the moment it is another topic.

[CodingArco]

You can use `HTTP_EVENT_ON_DATA` from event handler registered in `esp_http_client_config_t`. Here is reference for this https://github.com/espressif/esp-idf/bl ... mple.c#L63

Ok, I created the handler, as in the example, and I modified the code adding an else condition (related to chunk data):

Code: Select all

        case HTTP_EVENT_ON_DATA:
            ESP_LOGI(g_p_tag, "HTTP_EVENT_ON_DATA, len=%d", evt->data_len);
            /*
             *  Check for chunked encoding is added as the URL for chunked
             *  encoding used in this example returns binary data.
             *  However, event handler can also be used in case chunked encoding
             *  is used.
             */
            // Verifica se il dato in risposta è "chunked".
            //
            if (!esp_http_client_is_chunked_response(evt->client))
            {
                // If user_data buffer is configured, copy the response into the
                // buffer
                //
                if (evt->user_data)
                {
                    memcpy(evt->user_data + output_len, evt->data,
                           evt->data_len);
                }
                else
                {
                    if (p_output_buffer == NULL)
                    {
                        p_output_buffer = (char *) malloc(
                                esp_http_client_get_content_length(
                                    evt->client));
                        output_len = 0;
                        if (p_output_buffer == NULL)
                        {
                            ESP_LOGE(g_p_tag, "Failed to allocate memory");
                            ret = ESP_FAIL;
                        }
                    }
                    if (ESP_OK == ret)
                    {
                        memcpy(p_output_buffer + output_len, evt->data,
                               evt->data_len);
                    }
                }
                if (ESP_OK == ret)
                {
                    output_len += evt->data_len;
                }
            }
            else
            {
                ESP_LOGI(g_p_tag, "Dati: %s", (char *) evt->data);   // HERE!
            }
        break;

Now, I can read chunked data. Next step I think it will the storage of the received data in order to process it. But at the moment it is another topic.

Thanks! This helped a lot. I also added the length of the data to be printed to prevent a buffer overrun:

Code: Select all

ESP_LOGI(TAG, "Data: %.*s", evt->data_len, (char *)evt->data);