HTTPS Multipart File Transfer to Server

[gunar.kroeger]

Hi again!

I am required to upload files from SD card to a server with a specified url and header using multipart request
I have already successfully used esp_http_client example and was able to use POST commands to send data to the specified url through HTTPS.
But I couldn't find an example on how to send files with multipart protocol.

Could someone guide me to the information needed to perform this task? links, libraries or examples would be appreciated!
Reading from SD card is not the problem. It's the sending of the chunks that I don't understand.

Thanks (:

[gunar.kroeger]

So, from what I understand so far, Multipart request is just a regular POST message.

I have to
1. esp_http_client_init
2. esp_http_client_set_header for each header needed in the multipart request protocol
3. esp_http_client_open
4. esp_http_client_write every chunk of the file I wish to send
5. esp_http_client_fetch_headers of the response
6. esp_http_client_read to read the response

from what I understand, I can't use a simple esp_http_client_perform because the files are too large to be loaded into memory
Is all this correct? I will test my theory as soon as I can

thanks

[gunar.kroeger]

This works! I have been able to send small files to the server.
I came across another problem for larger files but I think I should open another thread for it

[jfernandes]

Hello @gunar.kroeger,

By any chance could you post your code that you used to send files to server? Im in the same boat at the moment. I have got basic small byte files (128 bytes ish) going up no problem using just the wireless client, but I would like to send some larger sensor log files across. How big were your files that you got successfully sent?

Thank you

[dmitrij999]

Hello everyone!

I'm also trying to send large file from LittleFS to server using esp_http_client. Server responds the 400 error code, and "No file part". Server code works, tested with Postman

The code for sending using esp_http_client:

Code: Select all

esp_err_t upload_file_to_server(string fname, string basename, int file_size) {
    ESP_LOGI(TAG, "Sending %s with size %d", basename.c_str(), file_size);
    esp_err_t err = ESP_OK;

    int status_code = 0;
    int content_length = 0;
	int total_read_len = 0, read_len = 0;
    int total_len_to_send = 0;

    char tmp_buf[256];
    char BODY[512];
    char END[128];
    memset(BODY, 0, sizeof(BODY));
    memset(END, 0, sizeof(END));
    memset(tmp_buf, 0, sizeof(tmp_buf));

    FILE * f = fopen(fname.c_str(), "rb");
    if (!f) {
        ESP_LOGE(TAG, "Cannot open file to upload!");
        return ESP_FAIL;
    }

    uint8_t * send_buffer = (uint8_t*)malloc(MAX_HTTP_OUTPUT_BUFFER*sizeof(uint8_t));
    if (!send_buffer) {
        ESP_LOGE(TAG, "Cannot allocate mem for temp buffer!");
        err = ESP_ERR_NO_MEM;
        goto file_upload_buffer_allocate_failed;
    }
    bzero(send_buffer, MAX_HTTP_OUTPUT_BUFFER);

    esp_http_client_set_method(wav_sender_client, HTTP_METHOD_POST);

    sprintf(tmp_buf, "multipart/form-data; boundary=%s", WAV_FILE_BOUNDARY);
    esp_http_client_set_header(wav_sender_client, "Content-Type", tmp_buf);
    memset(tmp_buf, 0, sizeof(tmp_buf));
    sprintf(tmp_buf, "%d", file_size);
    esp_http_client_set_header(wav_sender_client, "Content-Length", tmp_buf);
    // memset(tmp_buf, 0, sizeof(tmp_buf));
    // sprintf(tmp_buf, "attachment; name=\"file\"; filename=\"%s\"", basename.c_str());
    // esp_http_client_set_header(wav_sender_client, "Content-Disposition", tmp_buf);
    memset(tmp_buf, 0, sizeof(tmp_buf));
    esp_http_client_set_header(wav_sender_client, "Accept", "*/*");

    
    sprintf(tmp_buf, "--%s\r\n", WAV_FILE_BOUNDARY);
    strcpy(BODY, tmp_buf);
    memset(tmp_buf, 0, sizeof(tmp_buf));
    sprintf(tmp_buf, "Content-Disposition: form-data; name=\"file\"; filename=\"%s\"\r\n", basename.c_str());
    strcat(BODY, tmp_buf);
    memset(tmp_buf, 0, sizeof(tmp_buf));
    sprintf(tmp_buf, "Content-Type: application/octet-stream\r\n\r\n");
    strcat(BODY, tmp_buf);

    memset(tmp_buf, 0, sizeof(tmp_buf));
    sprintf(tmp_buf, "\r\n%s--\r\n\r\n", WAV_FILE_BOUNDARY);
    strcpy(END, tmp_buf);

    memset(tmp_buf, 0, sizeof(tmp_buf));
    total_len_to_send = file_size+strlen(BODY)+strlen(END);
    sprintf(tmp_buf, "%d", total_len_to_send);
    esp_http_client_set_header(wav_sender_client, "Content-Length", tmp_buf);

    printf("%s", BODY);
    printf("%s", END);

    if ((err = esp_http_client_open(wav_sender_client, total_len_to_send)) != ESP_OK) {
		ESP_LOGE(TAG, "Failed to open HTTP connection: %s", esp_err_to_name(err));
		goto file_upload_connection_failed;
	}
    ESP_LOGI(TAG, "Connection opened");
    // vTaskDelay(pdMS_TO_TICKS(100));

    if (esp_http_client_write(wav_sender_client, BODY, sizeof(BODY)) == ESP_FAIL) {
        ESP_LOGE(TAG, "Cannot send BODY preamble");
        goto file_upload_connection_end;
    }

    while (fread(&send_buffer[0], sizeof(uint8_t), MAX_HTTP_OUTPUT_BUFFER, f) > 0) {
		int snt = esp_http_client_write(wav_sender_client, (char*)send_buffer, MAX_HTTP_OUTPUT_BUFFER);
		bzero(send_buffer, MAX_HTTP_OUTPUT_BUFFER);
        if (snt == ESP_FAIL) {
            ESP_LOGE(TAG, "Cannot send part");
            goto file_upload_connection_end;
        }
	}

    if (esp_http_client_write(wav_sender_client, END, sizeof(END)) == ESP_FAIL) {
        ESP_LOGE(TAG, "Cannot send END preamble");
        goto file_upload_connection_end;
    }

    content_length =  esp_http_client_fetch_headers(wav_sender_client);
	total_read_len = 0;
    if (total_read_len < content_length && content_length <= MAX_HTTP_OUTPUT_BUFFER) {
		read_len = esp_http_client_read(wav_sender_client, (char*)send_buffer, content_length);
		if (read_len <= 0) {
			ESP_LOGE(TAG, "Error read data");
            err = ESP_FAIL;
		}
		send_buffer[read_len] = 0;
		ESP_LOGI(TAG, "read_len = %d", read_len);
	}

    status_code = esp_http_client_get_status_code(wav_sender_client);

	ESP_LOGI(TAG, "HTTP Status = %d, content_length = %d",
	                    status_code,
	                    esp_http_client_get_content_length(wav_sender_client));

    if (status_code >= 400) {
        ESP_LOGE(TAG, "Error sending file!");
        err = ESP_FAIL;
    }

file_upload_connection_end:
    esp_http_client_close(wav_sender_client);
    ESP_LOGI(TAG, "Connection closed");
file_upload_connection_failed:    
    esp_http_client_delete_header(wav_sender_client, "Content-Type");
    esp_http_client_delete_header(wav_sender_client, "Content-Length");
    esp_http_client_delete_header(wav_sender_client, "Content-Disposition");
    esp_http_client_delete_header(wav_sender_client, "Accept");
    free(send_buffer);
file_upload_buffer_allocate_failed:
    fclose(f);
    return err;
}

The server code:

Code: Select all

@app.route('/upload-record', methods=['POST'])
def upload_record():
    # check if the post request has the file part
    if 'file' not in request.files:
        print("No file part")
        return make_response('No file part', 400)
    file = request.files['file']
    # if user does not select file, browser also
    # submit an empty part without filename
    if file.filename == '':
        print("No selected file")
        return make_response('No selected file', 400)
    if file and allowed_file(file.filename):
        filename = secure_filename(file.filename)
        file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
        return "Record uploaded!"
    else:
        return make_response('Wrong file or its format', 400)