SD lib file IO errors

[dizcza]

Hardware:

Board: TTGO T8 V1.7 and M5Core2-AWS
Core Installation version: arduino-esp32 v2.0.1
IDE name: Arduino IDE
Flash Frequency: 240Mhz
PSRAM enabled: Enabled for M5Core2 and no such option for TTGO
Upload Speed: 115200
Computer OS: Ubuntu 20.04

Description:

There is a number of issues already addressed to the SD card library (#524 is the most cited one), but the majority if not all are related to mounting errors. I want to extend the horizon to which SD lib fails to function. This issue addresses file IO errors related to SD card lib. In short, writing to a file with the SD lib surely fails, sooner or later. I've been preparing the material for the issue for two days, and I hope it won't be ranked as a "please help dunno what to do" one.

I have two different SD cards and two boards: TTGO T8 V1.7 and M5Core2. They all perform equally (in the same fashion, either both fail or none), and I'll focus on the TTGO T8 just to pick one.

Until recently, I've been running the applications in ESP-IDF and had no SD-card-related errors. I was happy till I hit Arduino.

Sketch

I've come up with two minimal Arduino programs. One uses the SD card lib and constantly fails (in a flash, a minute, or a few hours; it often crashes immediately, in the first run after SD is successfully initialized), and the other does not (I've been running it for 20 hours in Arduino + months in ESP-IDF).

Arduino sketch using the arduino-esp32 SD lib. Always fails.

#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_log.h"
#include "SD.h"

#define SDCARD_SPI_GPIO_MISO 2
#define SDCARD_SPI_GPIO_MOSI 15
#define SDCARD_SPI_GPIO_CLK  14
#define SDCARD_SPI_GPIO_CS   13

bool stopped = false;


static bool testSDcardIOFailed() {
  static byte buff[512];
  size_t bytes_written;
  File file = SD.open("/file1.bin", FILE_WRITE);
  bool sdFailed = false;
  
  for (int i = 0; i < 100; i++) {
    bytes_written = file.write(buff, sizeof(buff));
    if (bytes_written != sizeof(buff)) {
      ESP_LOGE("sd", "bytes_written %zu, expected %zu; file.size() = %zu bytes", bytes_written, sizeof(buff), file.size());
      Serial.println("SD card IO error write");
      sdFailed = true;
      break;
    }
    file.flush();
    if (file.size() != sizeof(buff) * (i + 1)) {
      ESP_LOGE("sd", "i=%d file.size() = %zu bytes", i, file.size());
      Serial.println("SD card IO error flush");
      sdFailed = true;
      break;
    }
  }
  
  file.close();
  return sdFailed;
}


void setup() {
  Serial.begin(115200);
  delay(1000);
  ESP_LOGI("main", "Freertos tick rate: %d", configTICK_RATE_HZ);

  SPI.begin(SDCARD_SPI_GPIO_CLK,
            SDCARD_SPI_GPIO_MISO,
            SDCARD_SPI_GPIO_MOSI,
            SDCARD_SPI_GPIO_CS); //SCK, MISO, MOSI, SS (CS)
  while (!SD.begin(SDCARD_SPI_GPIO_CS, SPI)) delay(100);

  sdcard_type_t cardType = SD.cardType();
  String cardTypeStr = "UNKNOWN";
  switch (cardType) {
      case CARD_MMC:
          cardTypeStr = "MMC";
          break;
      case CARD_SD:
          cardTypeStr = "SDSC";
          break;
      case CARD_SDHC:
          cardTypeStr = "SDHC";
          break;
  }

  ESP_LOGI("main", "SD card type: %s", cardTypeStr);
  ESP_LOGI("main", "SD card size: %lluMB", SD.totalBytes() >> 20);
  ESP_LOGI("main", "SD card free: %lluMB", (SD.totalBytes() - SD.usedBytes()) >> 20);

  Serial.println("SD started");
}


void loop() {
  if (stopped) return;
  
  if (testSDcardIOFailed()) {
    Serial.println("SD IO Error");
    stopped = true;
  }
  delay(10);
}

Output

[  1010][I][sketch_dec09a.ino:59] setup(): Freertos tick rate: 1000
[  1021][I][sketch_dec09a.ino:81] setup(): SD card type: SDHC
[  1021][I][sketch_dec09a.ino:82] setup(): SD card size: 14912MB
[  1021][I][sketch_dec09a.ino:83] setup(): SD card free: 12853MB
SD started
[1542764][W][sd_diskio.cpp:104] sdWait(): Wait Failed
[1542764][E][sd_diskio.cpp:126] sdSelectCard(): Select Failed
[1542764][E][sd_heavy_task.ino:40] testSDcardIOFailed(): bytes_written 0, expected 512; file.size() = 49152 bytes
SD card IO error write

Arduino sketch with a self-written SD SPI lib. Never failed.

#include <stdio.h>
#include <string.h>

#include "esp_log.h"
#include "esp_err.h"
#include "esp_vfs_fat.h"
#include "sdmmc_cmd.h"
#include "driver/sdmmc_host.h"
#include "driver/sdmmc_types.h"
#include "vfs_fat_internal.h"

#define SDCARD_SPI_GPIO_MISO 2
#define SDCARD_SPI_GPIO_MOSI 15
#define SDCARD_SPI_GPIO_CLK  14
#define SDCARD_SPI_GPIO_CS   13

#define SDCARD_ALLOCATION_UNIT_SIZE (16 * 1024)


static sdmmc_card_t *m_card = NULL;
const char *sdcard_mount_point = "/sd";
static const char *TAG = "sdcard";
bool stopped = false;


esp_err_t sdcard_init() {
  // Options for mounting the filesystem.
  // If format_if_mount_failed is set to true, SD card will be partitioned and
  // formatted in case when mounting fails.
  esp_vfs_fat_sdmmc_mount_config_t mount_config = {
    .format_if_mount_failed = false,
    .max_files = 5,
    .allocation_unit_size = SDCARD_ALLOCATION_UNIT_SIZE
  };

  // Use settings defined above to initialize SD card and mount FAT filesystem.
  // Note: esp_vfs_fat_sdmmc/sdspi_mount is all-in-one convenience functions.
  // Please check its source code and implement error recovery when developing
  // production applications.

    sdmmc_host_t host = SDSPI_HOST_DEFAULT();

    spi_bus_config_t bus_cfg = {
        .mosi_io_num = SDCARD_SPI_GPIO_MOSI,
        .miso_io_num = SDCARD_SPI_GPIO_MISO,
        .sclk_io_num = SDCARD_SPI_GPIO_CLK,
        .quadwp_io_num = -1,
        .quadhd_io_num = -1,
        .max_transfer_sz = 4092,
    };

    ESP_ERROR_CHECK(spi_bus_initialize((spi_host_device_t) host.slot, &bus_cfg, SPI_DMA_CH1));

    // This initializes the slot without card detect (CD) and write protect (WP) signals.
    // Modify slot_config.gpio_cd and slot_config.gpio_wp if your board has these signals.
    sdspi_device_config_t slot_config = SDSPI_DEVICE_CONFIG_DEFAULT();
    slot_config.gpio_cs = (gpio_num_t) SDCARD_SPI_GPIO_CS;
    slot_config.host_id = (spi_host_device_t) host.slot;

    esp_err_t err = esp_vfs_fat_sdspi_mount(sdcard_mount_point, &host, &slot_config,
        &mount_config, &m_card);

  if (err == ESP_OK) {
    ESP_LOGI(TAG, "SD card mounted at %s", sdcard_mount_point);
    sdmmc_card_print_info(stdout, m_card);
  } else {
    ESP_LOGW(TAG, "esp_vfs_fat_sdmmc_mount failed (%s)",
        esp_err_to_name(err));
  }

  return err;
}


static bool testSDcardIOFailed() {
  static uint8_t buff[512];
  size_t bytes_written;
  FILE *file = fopen("/sd/file1.bin", "w");
  bool sdFailed = false;

  for (int i = 0; i < 100; i++) {
    bytes_written = fwrite(buff, sizeof(uint8_t), sizeof(buff), file);
    if (bytes_written != sizeof(buff)) {
      ESP_LOGE(TAG, "bytes_written %zu, expected %zu", bytes_written, sizeof(buff));
      ESP_LOGE(TAG, "SD card IO error write");
      sdFailed = true;
      break;
    }
    fflush(file);
    if (ftell(file) != sizeof(buff) * (i + 1)) {
      ESP_LOGE(TAG, "SD card IO error flush");
      sdFailed = true;
      break;
    }
  }

  fclose(file);
  return sdFailed;
}


void setup() {
  Serial.begin(115200);
  delay(1000);

  ESP_LOGI("main", "Freertos tick rate: %d", configTICK_RATE_HZ);
  ESP_ERROR_CHECK(sdcard_init());
  ESP_LOGI("main", "STARTED");
}


void loop() {
  if (stopped) return;
  
  if (testSDcardIOFailed()) {
    stopped = true;
  }
  delay(10);
}

Output

[  1010][I][sketch_dec09b.ino:120] setup(): Freertos tick rate: 1000
[  1072][I][sketch_dec09b.ino:78] sdcard_init(): SD card mounted at /sd
Name: SD16G
Type: SDHC/SDXC
Speed: 20 MHz
Size: 14916MB
[  1073][I][sketch_dec09b.ino:122] setup(): STARTED

Internet gurus suggest connecting an internal pull-up resistor to the MISO line pinMode(SDCARD_SPI_GPIO_MISO, INPUT_PULLUP), but this doesn't help either. There is no way to connect an external pullup resistor to the dedicated SD card slot of the M5Core2 and TTGO T8 boards I'm using. It won't help though - the problem exists in the arduino-esp library - I do not have any problems if I switch the project to ESP-IDF (in fact, the code for the second Arduino example is borrowed from an ESP-IDF project I'm using). I've been running ESP-IDF for months (!) with a similar heavy payload to an SD card, and I had no SD-card-related issues, using the same internal SD card slots.

At least now you have things to compare. And I mean, the second example is just your orthodox code from ESP-IDF... I've tried to look into Arduino SD lib, but I recoiled from its code complexity.

[dizcza]

I've tested a script that combines both approaches: I took SD lib card initialization but operated directly with the sdtio.h functions.

Arduino sketch with SD lib initialization and stdio.h file operations

#include <stdio.h>
#include <string.h>

#include "esp_log.h"
#include "esp_err.h"
#include "SD.h"

#define SDCARD_SPI_GPIO_MISO 2
#define SDCARD_SPI_GPIO_MOSI 15
#define SDCARD_SPI_GPIO_CLK  14
#define SDCARD_SPI_GPIO_CS   13

#define GPIO_LED_PIN 21


static const char *TAG = "sdcard";

bool stopped = false;

static bool testSDcardIOFailed() {
  static uint8_t buff[512];
  size_t bytes_written;
  FILE *file = fopen("/sd/file1.bin", "w");
  bool sdFailed = false;

  for (int i = 0; i < 100; i++) {
    bytes_written = fwrite(buff, sizeof(uint8_t), sizeof(buff), file);
    if (bytes_written != sizeof(buff)) {
      ESP_LOGE(TAG, "bytes_written %zu, expected %zu", bytes_written, sizeof(buff));
      ESP_LOGE(TAG, "SD card IO error write");
      sdFailed = true;
      break;
    }
    fflush(file);
    if (ftell(file) != sizeof(buff) * (i + 1)) {
      ESP_LOGE(TAG, "SD card IO error flush");
      sdFailed = true;
      break;
    }
  }

  fclose(file);
  return sdFailed;
}


void setup() {
  Serial.begin(115200);
  delay(1000);

  ESP_LOGI("main", "Freertos tick rate: %d", configTICK_RATE_HZ);
  SPI.begin(SDCARD_SPI_GPIO_CLK,
            SDCARD_SPI_GPIO_MISO,
            SDCARD_SPI_GPIO_MOSI,
            SDCARD_SPI_GPIO_CS); //SCK, MISO, MOSI, SS (CS)
  while (!SD.begin(SDCARD_SPI_GPIO_CS, SPI)) delay(100);

  ESP_LOGI("main", "STARTED");
  pinMode(GPIO_LED_PIN, OUTPUT);
}


void loop() {
  if (stopped) return;
  
  if (testSDcardIOFailed()) {
    digitalWrite(GPIO_LED_PIN, HIGH);
    stopped = true;
  }
  delay(10);
}

It has failed with the same error as in the first sketch. Although it lasted longer.

[Vigeant]

I have a similar error. I had to revert my esp32 libs to 1.06 for my SD card to mount again.

[GeoffroyPdd]

We are having similar issues.

Hardware:
Board: ESP 32 Devkit
Core Installation version: see below
IDE name: Arduino IDE
Flash Frequency: 240Mhz
PSRAM enabled: No
Upload Speed: 115200
Computer OS: Windows 10

Sketch
Tests are done using the SD example sketch given with the corresponding Arduino-ESP32 version.

Description
Arduino-ESP32 : V1.0.6
No mounting issues, no writing issues.

Arduino-ESP32 : V2.0.1
Mounting issue exists. Once it appears, it only goes away by flashing back to 1.0.6.

Arduino-ESP32 : V2.0.2
Mounting issue exists. Once it appears, it only goes away by flashing back to 1.0.6.
Additionnal issue is that the I/O operations of the SD Card are much slower (~10 times compared to 2.0.1 and 1.0.6).