Super slow sdcard on Wrover kit

[__vnv__]

Hello everyone,

trying to download mp3 file from the net (URL) and saving it to SDCARD seems to be almost impossible.
I have successfully downloaded small file few KB and saved it to SDCARD but trying to do it with 500k or 1MB file it seems that it takes forever (and in general it does).

I have tried to write a test with for look to write to file ~1MB but for some reason it fails.
This is the code I used (it is rewritten version of example, merged, SDCARD and http request).

Can anyone hint what could be the issue?

Code: Select all

/* ESP HTTP Client Example

   This example code is in the Public Domain (or CC0 licensed, at your option.)

   Unless required by applicable law or agreed to in writing, this
   software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
   CONDITIONS OF ANY KIND, either express or implied.
*/

#define LOG_LOCAL_LEVEL ESP_LOG_ERROR 

#include <string.h>
#include <stdlib.h>
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_log.h"
#include "esp_system.h"
#include "nvs_flash.h"
#include "app_wifi.h"

#include "esp_http_client.h"


/* SD CARD */
#include "esp_vfs_fat.h"
#include "driver/sdmmc_host.h"
#include "driver/sdspi_host.h"
#include "sdmmc_cmd.h"
/* SD CARD */


#ifdef USE_SPI_MODE
// Pin mapping when using SPI mode.
// With this mapping, SD card can be used both in SPI and 1-line SD mode.
// Note that a pull-up on CS line is required in SD mode.
#define PIN_NUM_MISO 2
#define PIN_NUM_MOSI 15
#define PIN_NUM_CLK  14
#define PIN_NUM_CS   13
#endif //USE_SPI_MODE



//#define MAX_HTTP_RECV_BUFFER 512
#define MAX_HTTP_RECV_BUFFER 1024
static const char *TAG = "HTTP_CLIENT";

// ------------------ GLOBAL VARS -----------------------------

FILE *fp=NULL;
// ------------------ GLOBAL VARS -----------------------------


/* Root cert for howsmyssl.com, taken from howsmyssl_com_root_cert.pem

   The PEM file was extracted from the output of this command:
   openssl s_client -showcerts -connect www.howsmyssl.com:443 </dev/null

   The CA root cert is the last cert given in the chain of certs.

   To embed it in the app binary, the PEM file is named
   in the component.mk COMPONENT_EMBED_TXTFILES variable.
*/
extern const char howsmyssl_com_root_cert_pem_start[] asm("_binary_howsmyssl_com_root_cert_pem_start");
extern const char howsmyssl_com_root_cert_pem_end[]   asm("_binary_howsmyssl_com_root_cert_pem_end");

esp_err_t _http_event_handler(esp_http_client_event_t *evt)
{
    switch(evt->event_id) {
        case HTTP_EVENT_ERROR:
            ESP_LOGE (TAG, "HTTP_EVENT_ERROR");
            break;
        case HTTP_EVENT_ON_CONNECTED:
            ESP_LOGE (TAG, "HTTP_EVENT_ON_CONNECTED");
            break;
        case HTTP_EVENT_HEADER_SENT:
            ESP_LOGE (TAG, "HTTP_EVENT_HEADER_SENT");
            break;
        case HTTP_EVENT_ON_HEADER:
            ESP_LOGE (TAG, "HTTP_EVENT_ON_HEADER, key=%s, value=%s", evt->header_key, evt->header_value);
            
            break;
        case HTTP_EVENT_ON_DATA:
            ESP_LOGE (TAG, "HTTP_EVENT_ON_DATA, len=%d", evt->data_len);
            if (!esp_http_client_is_chunked_response(evt->client)) {
                // Write out data
                // printf("%.*s", evt->data_len, (char*)evt->data);

                if(fp == NULL){
                    fp = fopen("/sdcard/muzika.mp3","wb");
                }

                if(fp != NULL){
                    fwrite(evt->data,1,evt->data_len,fp);
                    ESP_LOGE (TAG, "---------- HTTP_EVENT_ON_DATA - WRITING TO SD CARD ----------");
                }

            }

            break;
        case HTTP_EVENT_ON_FINISH:
            ESP_LOGE (TAG, "HTTP_EVENT_ON_FINISH");

            fclose(fp);
            fp = NULL;

            break;
        case HTTP_EVENT_DISCONNECTED:
            ESP_LOGE (TAG, "HTTP_EVENT_DISCONNECTED");
            break;
    }
    return ESP_OK;
}


static void http_download_chunk()
{
    esp_http_client_config_t config = {
        //.url = "http://httpbin.org/stream-bytes/8912",
        .url = "http://www.theoctopusproject.com/mp3/whatthey.mp3",
        .event_handler = _http_event_handler,
    };
    esp_http_client_handle_t client = esp_http_client_init(&config);
    esp_err_t err = esp_http_client_perform(client);

    if (err == ESP_OK) {
        ESP_LOGI(TAG, "HTTP chunk encoding Status = %d, content_length = %d",
                esp_http_client_get_status_code(client),
                esp_http_client_get_content_length(client));
    } else {
        ESP_LOGE(TAG, "Error perform http request %s", esp_err_to_name(err));
    }
    esp_http_client_cleanup(client);
}



static void http_test_task(void *pvParameters)
{
    app_wifi_wait_connected();
    ESP_LOGI(TAG, "Connected to AP, begin http example");

    http_download_chunk();

    ESP_LOGI(TAG, "Finish http example");
    vTaskDelete(NULL);
}

void app_main()
{
    esp_err_t ret = nvs_flash_init();
    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
      ESP_ERROR_CHECK(nvs_flash_erase());
      ret = nvs_flash_init();
    }
    ESP_ERROR_CHECK(ret);
    // -- app_wifi_initialise();

    /* SDCARD SETUP */
    #ifndef USE_SPI_MODE
    ESP_LOGI(TAG, "Using SDMMC peripheral");
    sdmmc_host_t host = SDMMC_HOST_DEFAULT();

    // This initializes the slot without card detect (CD) and write protect (WP) signals.
    // Modify slot_config.gpio_cd and slot_config.gpio_wp if your board has these signals.
    sdmmc_slot_config_t slot_config = SDMMC_SLOT_CONFIG_DEFAULT();

    // To use 1-line SD mode, uncomment the following line:
    // slot_config.width = 1;

    // GPIOs 15, 2, 4, 12, 13 should have external 10k pull-ups.
    // Internal pull-ups are not sufficient. However, enabling internal pull-ups
    // does make a difference some boards, so we do that here.
    gpio_set_pull_mode(15, GPIO_PULLUP_ONLY);   // CMD, needed in 4- and 1- line modes
    gpio_set_pull_mode(2, GPIO_PULLUP_ONLY);    // D0, needed in 4- and 1-line modes
    gpio_set_pull_mode(4, GPIO_PULLUP_ONLY);    // D1, needed in 4-line mode only
    gpio_set_pull_mode(12, GPIO_PULLUP_ONLY);   // D2, needed in 4-line mode only
    gpio_set_pull_mode(13, GPIO_PULLUP_ONLY);   // D3, needed in 4- and 1-line modes

#else
    ESP_LOGI(TAG, "Using SPI peripheral");

    sdmmc_host_t host = SDSPI_HOST_DEFAULT();
    sdspi_slot_config_t slot_config = SDSPI_SLOT_CONFIG_DEFAULT();
    slot_config.gpio_miso = PIN_NUM_MISO;
    slot_config.gpio_mosi = PIN_NUM_MOSI;
    slot_config.gpio_sck  = PIN_NUM_CLK;
    slot_config.gpio_cs   = PIN_NUM_CS;
    // This initializes the slot without card detect (CD) and write protect (WP) signals.
    // Modify slot_config.gpio_cd and slot_config.gpio_wp if your board has these signals.
#endif //USE_SPI_MODE

    // Options for mounting the filesystem.
    // If format_if_mount_failed is set to true, SD card will be partitioned and
    // formatted in case when mounting fails.
    esp_vfs_fat_sdmmc_mount_config_t mount_config = {
        .format_if_mount_failed = false,
        .max_files = 5,
        .allocation_unit_size = 16 * 1024
    };

    // Use settings defined above to initialize SD card and mount FAT filesystem.
    // Note: esp_vfs_fat_sdmmc_mount is an all-in-one convenience function.
    // Please check its source code and implement error recovery when developing
    // production applications.
    sdmmc_card_t* card;
    ret = esp_vfs_fat_sdmmc_mount("/sdcard", &host, &slot_config, &mount_config, &card);

    if (ret != ESP_OK) {
        if (ret == ESP_FAIL) {
            ESP_LOGE(TAG, "Failed to mount filesystem. "
                "If you want the card to be formatted, set format_if_mount_failed = true.");
        } else {
            ESP_LOGE(TAG, "Failed to initialize the card (%s). "
                "Make sure SD card lines have pull-up resistors in place.", esp_err_to_name(ret));
        }
        return;
    }

    // Card has been initialized, print its properties
    sdmmc_card_print_info(stdout, card);


    // Use POSIX and C standard library functions to work with files.
    // First create a file.
    ESP_LOGE(TAG, "Opening file");
    FILE* f = fopen("/sdcard/hello.txt", "w");
    if (f == NULL) {
        ESP_LOGE(TAG, "Failed to open file for writing");
        return;
    }
    
    
    // ----------------------------------------------------------------------------
    // This is used to test SDCARD but it fails
    int j=0;
    for(int i = 0 ; i < 1048576  ; i++,j++){
    //fprintf(f, "Hello %s!\n", card->cid.name);
        fprintf(f, "X");
        if(j > 1024){
            vTaskDelay(3);
            j = 0;
        }
    }
    fclose(f);
    ESP_LOGE(TAG, "File written");
    // ----------------------------------------------------------------------------


     // Uncomment this to procede with download and save to SDCARD
    //-- xTaskCreate(&http_test_task, "http_test_task", 8192, NULL, 5, NULL);
    //xTaskCreate(&http_test_task, "http_test_task", 16384, NULL, 5, NULL);
}

[ESP_igrr]

Could you please describe the failure in any detail, for example by posting the console output?

With regards to write speed: please keep in mind that FILEs are buffered by default, and BUFSIZE is 128 bytes. This means that in your attached code, for every 128 fprintf calls, there will be 1 write operation to FAT. Given that FAT works with 512 byte sectors, writing just 128 bytes is inefficient, and will result in a read-modify-write operation. SD card writes are also far more efficient when a large buffer is written in a single operation (due to the way wear levelling process of an SD card works).

To increase the write speed, allocate a large (16kB or 32kB) buffer. Fill it with incoming data. Then write to the file, bypassing stdio buffering (using write(fileno(f), buffer, size);, instead of fprintf). On WROVER-KIT you can also enable high-speed mode of SD interface. See https://docs.espressif.com/projects/esp ... -frequency for the code snippet. Overall you should be able to get ~ 3MB/sec write speed, after these changes.

[__vnv__]

Thank you for helping out , will try out buffering method and report progress.

[lucaot]

Thank you for helping out , will try out buffering method and report progress.

Hello, did you have any success on increasing the speed?
I am experiencing a similar issue and the writing speed is chopped at 420kBps more or less.

I am using a 4BIT SDcard SDMMC connection at 20MHz clock.
Increasing the SD driver clock to 40MHz doesn't have any effect, therefore it looks like it's not a SD card communication issue but rather an implementation inefficiency in the fwrite function.

I haven't found the place where to optimize it though.

[dizcza]

To increase the write speed, allocate a large (16kB or 32kB) buffer. Fill it with incoming data. Then write to the file, bypassing stdio buffering (using write(fileno(f), buffer, size);, instead of fprintf).

I'm trying to find the fastest way to dump a JPEG image from ESP32-CAM to my SD card mounted at "/sd". I have trouble with the "open" command:

Code: Select all

    int fd = open("/sd/a.bin", O_WRONLY | O_CREAT);
    if (fd == -1) {
        ESP_LOGW(TAG, "%s (%d)", strerror(errno), errno);
        return;
    }

I'm getting "No such file or directory" (errno 2). If I use the "fopen" command,

Code: Select all

static uint8_t framebuffer_static[50 * 1024];
FILE *f = fopen("/sd/a.bin", "w");

then the following two scenarios yield the same write duration:

Code: Select all

fwrite(framebuffer_static, sizeof(uint8_t), sizeof(framebuffer_static), f);
fclose(f);

Code: Select all

write(fileno(f), framebuffer_static, sizeof(framebuffer_static));
close(fileno(f));

Does it mean I can no further speed the IO operations?