HTTP Client uploading file via POST

[dawime]

I am trying to use the ESP32S3 to read a file from the SD card, and send it to a remote webserver via HTTP POST. It does not have to be POST, but seemed like the easiest thing to do.

I tested the PHP script by uploading a file via a html form, and it seems to work ok.
Here is the PHP script:

Code: Select all

<?php
if ($_SERVER["REQUEST_METHOD"] == "POST") {
    // Get the uploaded file information
    $file = $_FILES["filename"];
    $fileName = $file["name"];
    $fileTmpName = $file["tmp_name"];
    $fileError = $file["error"];

    if ($fileError === UPLOAD_ERR_OK) {
        // Get the uploader's IP address
        $uploaderIP = $_SERVER["REMOTE_ADDR"];

        // Generate a new filename with the uploader's IP and current date/time
        $newFileName = $uploaderIP . "_" . date("YmdHis") . "_" . $fileName;
        error_log($newFileName);
        // Specify the directory to save the file
        $uploadDirectory = "/kunden/homepages/8/d963708782/htdocs/Data/sounds/";
         // Move the uploaded file to the destination directory with the new filename
        if (move_uploaded_file($fileTmpName, $uploadDirectory . $newFileName)) {
            echo "File uploaded successfully!";
                error_log("uploaded file");
        } else {
            echo "Error uploading file.";
                error_log("Error uploading file");
        }
    } else {
        echo "Error: " . $fileError;
        }
}
?>

Now, on the ESP32S3 side, I connect to the wifi, and call the http_task to execute the file transfer. in this case its a simple text file with 49 bytes of length:

Code: Select all

esp_err_t _http_event_handler(esp_http_client_event_t *evt) 
{

    switch(evt->event_id) 
    {
        case HTTP_EVENT_ERROR:
            ESP_LOGE(TAG, "HTTP_EVENT_ERROR");
            break;
        case HTTP_EVENT_ON_CONNECTED:
            ESP_LOGI(TAG, "HTTP_EVENT_ON_CONNECTED");
            break;
        case HTTP_EVENT_HEADER_SENT:
            ESP_LOGI(TAG, "HTTP_EVENT_HEADER_SENT");
            break;
        case HTTP_EVENT_ON_HEADER:
            ESP_LOGI(TAG, "HTTP_EVENT_ON_HEADER, key=%s, value=%s", evt->header_key, evt->header_value);
            break;
        case HTTP_EVENT_ON_DATA:
            ESP_LOGI(TAG, "HTTP_EVENT_ON_DATA, len=%d", evt->data_len);
            break;
        case HTTP_EVENT_ON_FINISH:
            ESP_LOGI(TAG, "HTTP_EVENT_ON_FINISH");
            break;
        case HTTP_EVENT_DISCONNECTED:
            ESP_LOGI(TAG, "HTTP_EVENT_DISCONNECTED");
            break;
        case HTTP_EVENT_REDIRECT:
            ESP_LOGI(TAG, "HTTP_EVENT_REDIRECT");
            break;
    }
    return ESP_OK;
}

void http_task(void *args)
{
 // Set up HTTP client
    esp_err_t err;
    esp_http_client_config_t config = {
        .url = SERVER_URL,
        .method = HTTP_METHOD_POST,
        .event_handler = _http_event_handler,
        .keep_alive_enable = true,
    };
    esp_http_client_handle_t client = esp_http_client_init(&config);


    //esp_http_client_set_header(client, "Transfer-Encoding", "chunked");

    // Open file for reading
    char fname[32];
    char header_fname[32];
    char header_size[16];
    FILE *file;
    strcpy(fname,"/sdcard/FILE.TXT");
    strcpy(header_fname,"FILE.TXT");
    file= fopen(fname, "r");
    if(file==NULL)
    {
         ESP_LOGE(TAG,"error opening file");
    }
    else
        {

        }

    // Get file size
    fseek(file, 0, SEEK_END);
    long file_size = ftell(file);
    fseek(file, 0, SEEK_SET);
    ESP_LOGI(TAG, "File size: %ld bytes", file_size);

    // POST
    esp_http_client_set_method(client, HTTP_METHOD_POST);
    esp_http_client_set_header(client, "Content-Type", "application/octet-stream");
    itoa(file_size,header_size,10);
    ESP_LOGI(TAG,"file size for header is %s",header_size);
    esp_http_client_set_header(client, "Content-Length", header_size);
    esp_http_client_set_header(client, "filename", header_fname);
   // char content_disposition_header[64];
   // esp_http_client_set_header(client, "Content-Disposition", content_disposition_header);



    err = esp_http_client_open(client, file_size);
    if (err != ESP_OK) {
        ESP_LOGE(TAG, "Failed to open HTTP connection: %s", esp_err_to_name(err));
        esp_http_client_cleanup(client);
        return;
    } 

/*
        err = esp_http_client_perform(client);
        if (err == ESP_OK) {
            ESP_LOGI(TAG, "HTTP POST Status = %d, content_length = %lld",
                    esp_http_client_get_status_code(client),
                    esp_http_client_get_content_length(client));
        } else {
            ESP_LOGE(TAG, "HTTP POST request failed: %s", esp_err_to_name(err));
        }

*/    

    // Send file in chunks
    size_t total_sent = 0;
    char data[CHUNK_SIZE];

    #if 1
    while (total_sent < file_size) 
    {
        size_t to_send = CHUNK_SIZE;
        if (total_sent + to_send > file_size) {
            to_send = file_size - total_sent;
        }
        fread(data, 1, to_send, file);
        ESP_LOGI(TAG,"Read %d bytes, sending now",to_send);

        int write_len = esp_http_client_write(client, &data[0], file_size);
        if (write_len <= 0) {
            ESP_LOGE(TAG, "Error occurred during file write: %s", esp_err_to_name(write_len));
            esp_http_client_cleanup(client);
            return;
        }
        
        err = esp_http_client_perform(client);
        if (err != ESP_OK) {
            ESP_LOGE(TAG, "HTTP POST request failed: %s", esp_err_to_name(err));
        }
        
        total_sent += to_send;
    }    
#endif

        // Clean up
    fclose(file);
    esp_http_client_cleanup(client);
    ESP_LOGI(TAG, "File upload complete");
    while(1)
    {
            vTaskDelay(100);
    }
    vTaskDelete(NULL);

}

The log output of the code is:
I (2936) HTTP: File size: 49 bytes
I (2936) HTTP: file size for header is 49
I (3056) HTTP: HTTP_EVENT_ON_CONNECTED
I (3066) HTTP: HTTP_EVENT_HEADER_SENT
I (3066) HTTP: Read 49 bytes, sending now
I (3206) HTTP: HTTP_EVENT_ON_HEADER, key=Content-Type, value=text/html; charset=UTF-8
I (3206) HTTP: HTTP_EVENT_ON_HEADER, key=Transfer-Encoding, value=chunked
I (3206) HTTP: HTTP_EVENT_ON_HEADER, key=Connection, value=keep-alive
I (3216) HTTP: HTTP_EVENT_ON_HEADER, key=Keep-Alive, value=timeout=15
I (3226) HTTP: HTTP_EVENT_ON_HEADER, key=Date, value=Sun, 04 Jun 2023 22:03:05 GMT
I (3236) HTTP: HTTP_EVENT_ON_HEADER, key=Server, value=Apache
I (3236) HTTP: HTTP_EVENT_ON_DATA, len=8
I (3246) HTTP: HTTP_EVENT_ON_FINISH
I (3246) HTTP: HTTP_EVENT_DISCONNECTED
I (3256) HTTP: File upload complete

However, the php script never sees the filename , and therefore does not create the file, erroring out

Any idea of what am I missing?

[Craige Hales]

here's a snip of some php not used in the last 7 years but worked with a raspberry to do about the same thing. Might point you somewhere...as I recall, returning messages from the server to the pi as html was a good debugging technique. You could send the returned html page to the serial console. Can't find the pi code right now; it was probably python that generated the post with curl.

Code: Select all

if ( $_POST['name']!="camera5" )
{
  echo "!1";//FILES"; var_dump($_FILES); echo "POST"; var_dump($_POST); echo "GET"; var_dump($_GET);
}
else
{
  $source = "/var/www/5/" .
  // ip2long($_SERVER["REMOTE_ADDR"])
  $_POST['name']
   . "/";
  $path = $source . $today["year"] . "-" . 
   substr("0" . $today["mon"],   -2, 2)  . "-" . 
   substr("0" . $today["mday"],   -2, 2)
   . "/";
  $fname =
   $today["year"] . "-" . 
   substr("0" . $today["mon"],   -2, 2) . "-" .
   substr("0" . $today["mday"],   -2, 2) . "-" .
   substr("0" . $today["hours"],   -2, 2) . "-" .
   substr("0" . $today["minutes"], -2, 2) . "-" .
   substr("0" . $today["seconds"], -2, 2) /*. substr($usec,2,3)*/ . ".jpg";
  if ( empty($_FILES["file"] ) )
  {
    echo "!2";//nothing in $ FILES " . var_dump($_FILES) . $path . $fname;
  }
  else if ( $_FILES["file"]["error"] != 0 )
  {
    echo "!3";//error " . $_FILES["file"]["error"];
  }
  else
  {
    $filename = basename( $_FILES["file"]["name"] );
    $extension = substr( $filename, strrpos( $filename, "." ) + 1 );
    if ( ( $extension == "jpg" ) && ( $_FILES["file"]["type"] == "image/jpeg" ) && ( $_FILES["file"]["size"] < 5000000 ) )
    {
      createPath( $path,0 );

[MicroController]

Notice that

Code: Select all

$file = $_FILES["filename"];

will yield a file from a form input field named "filename" as you'd get from HTML <input name="filename" type="file" />.
That is explained e.g. here aswell as a possible solution/alternative to form-based upload: "xmlHttpRequest.setRequestHeader('Content-Disposition', 'attachment; filename="' + fileName + '"')".

[dawime]

Tried some new things, but still no success...
I attempted to change the headers as described for upload:

Code: Select all

 snprintf(contentDispositionHeader, sizeof(contentDispositionHeader), "attachment; filename=\"%s\"", header_fname);
    ESP_LOGI(TAG,"%s",contentDispositionHeader);
    esp_http_client_set_header(client, "Content-Disposition", contentDispositionHeader);
    esp_http_client_set_header(client, "Content-Type", "application/octet-stream");
    esp_http_client_set_header(client, "Content-Length", header_size);

ESP code complets ok
I (2957) HTTP: attachment; filename="FILE.TXT"
I (3097) HTTP: HTTP_EVENT_ON_CONNECTED
I (3107) HTTP: HTTP_EVENT_HEADER_SENT
I (3317) HTTP: HTTP_EVENT_ON_HEADER, key=Content-Type, value=text/html; charset=UTF-8
I (3317) HTTP: HTTP_EVENT_ON_HEADER, key=Transfer-Encoding, value=chunked
I (3327) HTTP: HTTP_EVENT_ON_HEADER, key=Connection, value=keep-alive
I (3337) HTTP: HTTP_EVENT_ON_HEADER, key=Keep-Alive, value=timeout=15
I (3337) HTTP: HTTP_EVENT_ON_HEADER, key=Date, value=Tue, 06 Jun 2023 02:44:02 GMT
I (3347) HTTP: HTTP_EVENT_ON_HEADER, key=Server, value=Apache
I (3357) HTTP: HTTP_EVENT_ON_DATA, len=8
I (3357) HTTP: HTTP_EVENT_ON_FINISH
I (3367) HTTP: HTTP_EVENT_DISCONNECTED
I (3367) HTTP: File upload complete

I modified the PHP script to give me more info:

Code: Select all

<?php
if ($_SERVER["REQUEST_METHOD"] == "POST") {
    // Log file path
    $logFilePath = "/kunden/homepages/8/d963708782/htdocs/Data/error.log";

    // Get all headers and parameters
    $headers = getallheaders();
    $parameters = $_POST;

    // Get the uploaded file information
    $file = $_FILES["filename"];
    file_put_contents($logFilePath, "file is ", FILE_APPEND |LOCK_EX);
    file_put_contents($logFilePath, $file, FILE_APPEND |LOCK_EX);
    file_put_contents($logFilePath, "\n basename is \n", FILE_APPEND |LOCK_EX);
    file_put_contents($logFilePath, basename( $_FILES["file"]["name"] ));

    $fileName = $file["name"];
    $fileTmpName = $file["tmp_name"];
    $fileError = $file["error"];

    file_put_contents($logFilePath, "\nfileName is ", FILE_APPEND |LOCK_EX);
    file_put_contents($logFilePath, $fileName, FILE_APPEND |LOCK_EX);
    file_put_contents($logFilePath, "\n", FILE_APPEND |LOCK_EX);
   
    
    // Log headers and parameters to the log file
    file_put_contents($logFilePath, "Headers:\n", FILE_APPEND | LOCK_EX);
    file_put_contents($logFilePath, print_r($headers, true), FILE_APPEND | LOCK_EX);
    file_put_contents($logFilePath, "\n\nParameters:\n", FILE_APPEND | LOCK_EX);
    file_put_contents($logFilePath, print_r($parameters, true), FILE_APPEND | LOCK_EX);

   if ($fileError === UPLOAD_ERR_OK) {
        // Get the uploader's IP address
        $uploaderIP = $_SERVER["REMOTE_ADDR"];

        // Generate a new filename with the uploader's IP and current date/time
        $newFileName = $uploaderIP . "_" . date("YmdHis") . "_" . $fileName;

        // Specify the directory to save the file
        $uploadDirectory = "/kunden/homepages/8/d963708782/htdocs/Data/sounds/";

        // Move the uploaded file to the destination directory with the new filename
        if (move_uploaded_file($fileTmpName, $uploadDirectory . $newFileName)) {
            file_put_contents($logFilePath, "\n\nFile uploaded successfully!\n", FILE_APPEND | LOCK_EX);
            echo "File uploaded successfully!";
        } else {
            file_put_contents($logFilePath, "\n\nError uploading file.\n", FILE_APPEND | LOCK_EX);
            echo "Error uploading file.";
        }
    } else {
        file_put_contents($logFilePath, "\n\nError: " . $fileError, FILE_APPEND | LOCK_EX);
        echo "Error: " . $fileError;
    }
}
?>

I still get no file, and the output of the log file shows the right headers, but maybe the issues is the PHP script now?
(uiserver):u112369002:~/Data$ more error.log

fileName is
Headers:
Array
(
[Host] => data.wateriq.ai
[Content-Length] => 49
[User-Agent] => ESP32 HTTP Client/1.0
[Content-Disposition] => attachment; filename="FILE.TXT"
[Content-Type] => application/octet-stream
)


Parameters:
Array
(
)


Error:
(uiserver):u112369002:~/Data$

Looking at this it looks like the ""--boundary" is not included that I can see (unless its built up implicitly). that could be the reason this fails ,as when I look at arduino code the headers are built up manually via strings then sent over. Is there an IDF function for this or I am really needing to build the headers via strings to send them?

Thanks

[Craige Hales]

Found the upload part, it did use curl in a shell script...Sorry, don't have anything else. I kind of remember needing to read server logs to figure out some of the answers.

Code: Select all

  
raspistill -q 15 -n -rot 180 -w 2592 -h 1944 -t 5000 -o /run/shm/temp.jpg > /run/shm/cmra.log
curl -k -i -F "name=camera5" -F "file=@/run/shm/temp.jpg" https://example.com/xxx.php > /run/shm/curl.log

-k -- insecure self-signed certificate on my server "example.com"
-i -- http response headers in output, probably for debugging
-F -- form - used twice - tiny part of curl doc:

This enables uploading of binary files etc. To force the 'content' part to be a file,
prefix the file name with an @ sign. To just get the content part from a file, prefix
the file name with the symbol <. The difference between @ and < is then that @ makes a
file get attached in the post as a file upload, while the < makes a text field and just
get the contents for that text field from a file.

edit: so yes, probably the boundary thing, https://everything.curl.dev/http/multipart
Curl was taking care of that for me.