From aa0b7aef08eeaf51eccbb4081ab41f28cabc645f Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Thu, 18 Jul 2019 15:43:21 -0700 Subject: [PATCH 1/5] Add time to filesystems, new API calls Support the ESP32 File::getLastWrite() call and setting the time on all filesystems automatically (assuming the system clock has been set properly and time(NULL) returns the proper time!). Adds Dir::fileTime() to get the time of a file being listed, similar to Dir::fileName() and Dir::fileSize(). Adds ::setTimeCallback(time_t (*cb)()) to File, Dir, and FS, allowing users to override the default timestamp on a per-file, directory, or filesystem basis. By default, a simple callback returning time(nullptr) is implemented. LittleFS uses the 't' attribute and should be backwards compatible. SD/SDFS work and include wrappers for obsolete SdFat timestamp callbacks using the MSDOS time. SPIFFS has been updated to support either the existing on-flash format without any timestamps, or a new on-flash format with a flag area and a timestamp metadata element added. SPIFFS will try and mount any existing filesystem as non-timestamp, and continue to write them in the old format. New filesystems may be formatted to include timestamps (default) or in the old format, configurable via the SPIFFS.setConfig call. The SPIFFS work took as starting point all of work of @luc-github in #3730 and extended it into a backwards compatible mode by doing much surgery on the SPIFFS library itself and the on-flash format. Includes an updated SD/listfiles and SPIFFS_time example. --- cores/esp8266/FS.cpp | 50 +++- cores/esp8266/FS.h | 28 ++- cores/esp8266/FSImpl.h | 35 +++ cores/esp8266/spiffs/spiffs.h | 12 +- cores/esp8266/spiffs/spiffs_check.cpp | 6 +- cores/esp8266/spiffs/spiffs_config.h | 16 ++ cores/esp8266/spiffs/spiffs_gc.cpp | 2 +- cores/esp8266/spiffs/spiffs_hydrogen.cpp | 28 +-- cores/esp8266/spiffs/spiffs_nucleus.cpp | 69 +++--- cores/esp8266/spiffs/spiffs_nucleus.h | 6 +- cores/esp8266/spiffs_api.cpp | 11 + cores/esp8266/spiffs_api.h | 147 ++++++++++-- doc/filesystem.rst | 116 +++++++++- libraries/LittleFS/src/LittleFS.h | 39 ++++ libraries/SD/examples/listfiles/listfiles.ino | 14 +- libraries/SD/src/SD.cpp | 2 + libraries/SD/src/SD.h | 50 ++++ libraries/SDFS/src/SDFS.h | 33 +++ .../examples/SPIFFS_Time/SPIFFS_Time.ino | 214 ++++++++++++++++++ 19 files changed, 784 insertions(+), 94 deletions(-) create mode 100644 libraries/esp8266/examples/SPIFFS_Time/SPIFFS_Time.ino diff --git a/cores/esp8266/FS.cpp b/cores/esp8266/FS.cpp index 7006a8afaf..6e2abf209f 100644 --- a/cores/esp8266/FS.cpp +++ b/cores/esp8266/FS.cpp @@ -180,6 +180,19 @@ String File::readString() return ret; } +time_t File::getLastWrite() { + if (!_p) + return 0; + + return _p->getLastWrite(); +} + +void File::setTimeCallback(time_t (*cb)(void)) { + if (!_p) + return; + _p->setTimeCallback(cb); +} + File Dir::openFile(const char* mode) { if (!_impl) { return File(); @@ -192,7 +205,9 @@ File Dir::openFile(const char* mode) { return File(); } - return File(_impl->openFile(om, am), _baseFS); + auto f = File(_impl->openFile(om, am), _baseFS); + f.setTimeCallback(timeCallback); + return f; } String Dir::fileName() { @@ -203,6 +218,12 @@ String Dir::fileName() { return _impl->fileName(); } +time_t Dir::fileTime() { + if (!_impl) + return 0; + return _impl->fileTime(); +} + size_t Dir::fileSize() { if (!_impl) { return 0; @@ -241,6 +262,20 @@ bool Dir::rewind() { return _impl->rewind(); } +time_t Dir::getLastWrite() { + if (!_impl) + return 0; + + return _impl->getLastWrite(); +} + +void Dir::setTimeCallback(time_t (*cb)(void)) { + if (!_impl) + return; + _impl->setTimeCallback(cb); +} + + bool FS::setConfig(const FSConfig &cfg) { if (!_impl) { return false; @@ -308,7 +343,9 @@ File FS::open(const char* path, const char* mode) { DEBUGV("FS::open: invalid mode `%s`\r\n", mode); return File(); } - return File(_impl->open(path, om, am), this); + auto f = File(_impl->open(path, om, am), this); + f.setTimeCallback(timeCallback); + return f; } bool FS::exists(const char* path) { @@ -327,7 +364,9 @@ Dir FS::openDir(const char* path) { return Dir(); } DirImplPtr p = _impl->openDir(path); - return Dir(p, this); + auto d = Dir(p, this); + d.setTimeCallback(timeCallback); + return d; } Dir FS::openDir(const String& path) { @@ -378,6 +417,11 @@ bool FS::rename(const String& pathFrom, const String& pathTo) { return rename(pathFrom.c_str(), pathTo.c_str()); } +void FS::setTimeCallback(time_t (*cb)(void)) { + if (!_impl) + return; + _impl->setTimeCallback(cb); +} static bool sflags(const char* mode, OpenMode& om, AccessMode& am) { diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index 2d33f48084..018e8fab18 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -110,12 +110,16 @@ class File : public Stream String readString() override; + time_t getLastWrite(); + void setTimeCallback(time_t (*cb)(void)); + protected: FileImplPtr _p; // Arduino SD class emulation std::shared_ptr _fakeDir; FS *_baseFS; + time_t (*timeCallback)(void) = nullptr; }; class Dir { @@ -126,15 +130,21 @@ class Dir { String fileName(); size_t fileSize(); + time_t fileTime(); bool isFile() const; bool isDirectory() const; bool next(); bool rewind(); + time_t getLastWrite(); + void setTimeCallback(time_t (*cb)(void)); + protected: DirImplPtr _impl; FS *_baseFS; + time_t (*timeCallback)(void) = nullptr; + }; // Backwards compatible, <4GB filesystem usage @@ -179,17 +189,26 @@ class FSConfig class SPIFFSConfig : public FSConfig { public: - SPIFFSConfig(bool autoFormat = true) { + SPIFFSConfig(bool autoFormat = true, bool enableTime = true) { _type = SPIFFSConfig::fsid::FSId; _autoFormat = autoFormat; + _enableTime = enableTime; } enum fsid { FSId = 0x53504946 }; + + SPIFFSConfig setEnableTime(bool val = true) { + _enableTime = val; + return *this; + } + + // Inherit _type and _autoFormat + bool _enableTime; }; class FS { public: - FS(FSImplPtr impl) : _impl(impl) { } + FS(FSImplPtr impl) : _impl(impl) { timeCallback = _defaultTimeCB; } bool setConfig(const FSConfig &cfg); @@ -223,10 +242,14 @@ class FS bool gc(); + void setTimeCallback(time_t (*cb)(void)); + friend class ::SDClass; // More of a frenemy, but SD needs internal implementation to get private FAT bits protected: FSImplPtr _impl; FSImplPtr getImpl() { return _impl; } + time_t (*timeCallback)(void); + static time_t _defaultTimeCB(void) { return time(NULL); } }; } // namespace fs @@ -241,6 +264,7 @@ using fs::SeekCur; using fs::SeekEnd; using fs::FSInfo; using fs::FSConfig; +using fs::SPIFFSConfig; #endif //FS_NO_GLOBALS #if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_SPIFFS) diff --git a/cores/esp8266/FSImpl.h b/cores/esp8266/FSImpl.h index 6caea9ca2f..4ffc8af77e 100644 --- a/cores/esp8266/FSImpl.h +++ b/cores/esp8266/FSImpl.h @@ -41,6 +41,19 @@ class FileImpl { virtual const char* fullName() const = 0; virtual bool isFile() const = 0; virtual bool isDirectory() const = 0; + + // Filesystems *may* support a timestamp per-file, so allow the user to override with + // their own callback for *this specific* file (as opposed to the FSImpl call of the + // same name. The default implementation simply returns time(&null) + virtual void setTimeCallback(time_t (*cb)(void)) { timeCallback = cb; } + + // Return the last written time for a file. Undefined when called on a writable file + // as the FS is allowed to return either the time of the last write() operation or the + // time present in the filesystem metadata (often the last time the file was closed) + virtual time_t getLastWrite() { return 0; } // Default is to not support timestamps + +protected: + time_t (*timeCallback)(void) = nullptr; }; enum OpenMode { @@ -62,10 +75,24 @@ class DirImpl { virtual FileImplPtr openFile(OpenMode openMode, AccessMode accessMode) = 0; virtual const char* fileName() = 0; virtual size_t fileSize() = 0; + virtual time_t fileTime() { return 0; } // By default, FS doesn't report file times virtual bool isFile() const = 0; virtual bool isDirectory() const = 0; virtual bool next() = 0; virtual bool rewind() = 0; + + // Filesystems *may* support a timestamp per-file, so allow the user to override with + // their own callback for *this specific* file (as opposed to the FSImpl call of the + // same name. The default implementation simply returns time(&null) + virtual void setTimeCallback(time_t (*cb)(void)) { timeCallback = cb; } + + // Return the last written time for a file. Undefined when called on a writable file + // as the FS is allowed to return either the time of the last write() operation or the + // time present in the filesystem metadata (often the last time the file was closed) + virtual time_t getLastWrite() { return 0; } // Default is to not support timestamps + +protected: + time_t (*timeCallback)(void) = nullptr; }; class FSImpl { @@ -85,6 +112,14 @@ class FSImpl { virtual bool mkdir(const char* path) = 0; virtual bool rmdir(const char* path) = 0; virtual bool gc() { return true; } // May not be implemented in all file systems. + + // Filesystems *may* support a timestamp per-file, so allow the user to override with + // their own callback for all files on this FS. The default implementation simply + // returns the present time as reported by time(&null) + virtual void setTimeCallback(time_t (*cb)(void)) { timeCallback = cb; } + +protected: + time_t (*timeCallback)(void) = nullptr; }; } // namespace fs diff --git a/cores/esp8266/spiffs/spiffs.h b/cores/esp8266/spiffs/spiffs.h index e659bf2f37..746ad11963 100644 --- a/cores/esp8266/spiffs/spiffs.h +++ b/cores/esp8266/spiffs/spiffs.h @@ -300,9 +300,7 @@ typedef struct { spiffs_obj_type type; spiffs_page_ix pix; u8_t name[SPIFFS_OBJ_NAME_LEN]; -#if SPIFFS_OBJ_META_LEN - u8_t meta[SPIFFS_OBJ_META_LEN]; -#endif + u8_t meta[4]; // Set to the largest supported metadata len } spiffs_stat; struct spiffs_dirent { @@ -311,9 +309,7 @@ struct spiffs_dirent { spiffs_obj_type type; u32_t size; spiffs_page_ix pix; -#if SPIFFS_OBJ_META_LEN - u8_t meta[SPIFFS_OBJ_META_LEN]; -#endif + u8_t meta[4]; // Set to the largest supported metadata len }; typedef struct { @@ -534,7 +530,7 @@ s32_t SPIFFS_close(spiffs *fs, spiffs_file fh); */ s32_t SPIFFS_rename(spiffs *fs, const char *old, const char *newPath); -#if SPIFFS_OBJ_META_LEN + /** * Updates file's metadata * @param fs the file system struct @@ -550,7 +546,7 @@ s32_t SPIFFS_update_meta(spiffs *fs, const char *name, const void *meta); * @param meta new metadata. must be SPIFFS_OBJ_META_LEN bytes long. */ s32_t SPIFFS_fupdate_meta(spiffs *fs, spiffs_file fh, const void *meta); -#endif + /** * Returns last error of last file operation. diff --git a/cores/esp8266/spiffs/spiffs_check.cpp b/cores/esp8266/spiffs/spiffs_check.cpp index 258efb30c7..a23dc1bcb5 100644 --- a/cores/esp8266/spiffs/spiffs_check.cpp +++ b/cores/esp8266/spiffs/spiffs_check.cpp @@ -64,7 +64,7 @@ static s32_t spiffs_object_get_data_page_index_reference( u32_t addr = SPIFFS_PAGE_TO_PADDR(fs, *objix_pix); if (objix_spix == 0) { // get referenced page from object index header - addr += sizeof(spiffs_page_object_ix_header) + data_spix * sizeof(spiffs_page_ix); + addr += sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN + data_spix * sizeof(spiffs_page_ix); } else { // get referenced page from object index addr += sizeof(spiffs_page_object_ix) + SPIFFS_OBJ_IX_ENTRY(fs, data_spix) * sizeof(spiffs_page_ix); @@ -136,7 +136,7 @@ static s32_t spiffs_rewrite_index(spiffs *fs, spiffs_obj_id obj_id, spiffs_span_ // rewrite in mem if (objix_spix == 0) { - ((spiffs_page_ix*)((u8_t *)fs->lu_work + sizeof(spiffs_page_object_ix_header)))[data_spix] = new_data_pix; + ((spiffs_page_ix*)((u8_t *)fs->lu_work + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN))[data_spix] = new_data_pix; } else { ((spiffs_page_ix*)((u8_t *)fs->lu_work + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)] = new_data_pix; } @@ -574,7 +574,7 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) { // object header page index entries = SPIFFS_OBJ_HDR_IX_LEN(fs); data_spix_offset = 0; - object_page_index = (spiffs_page_ix *)((u8_t *)fs->lu_work + sizeof(spiffs_page_object_ix_header)); + object_page_index = (spiffs_page_ix *)((u8_t *)fs->lu_work + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN); } else { // object page index entries = SPIFFS_OBJ_IX_LEN(fs); diff --git a/cores/esp8266/spiffs/spiffs_config.h b/cores/esp8266/spiffs/spiffs_config.h index c3343fe2fe..df59a71f6a 100644 --- a/cores/esp8266/spiffs/spiffs_config.h +++ b/cores/esp8266/spiffs/spiffs_config.h @@ -25,6 +25,22 @@ typedef uint16_t u16_t; typedef int8_t s8_t; typedef uint8_t u8_t; + +// Enable supporting both timestamped and non-timestamped FS images by making +// the meta-len a global variable here and by changing the logic of SPIFFS.begin. +extern int __SPIFFS_obj_meta_len; +#define SPIFFS_OBJ_META_LEN (__SPIFFS_obj_meta_len) +#define SPIFFS_MAX_META (4) // Maximum metadata size allowed at runtime +// Because SPIFFS reads binary images of blocks, we need to be sure to actually ensure there is space to +// keep the metadata (since 0 bytes are allocated in the struct itself). Use an anonymous union to ensure +// that there is at least MAX_META bytes after the end of the struct. +#define alloc_spiffs_page_object_ix_header(name) \ + union { \ + spiffs_page_object_ix_header name; \ + char xxx[sizeof(spiffs_page_object_ix_header) + SPIFFS_MAX_META]; \ + }; + + #ifndef SEEK_SET #define SEEK_SET 0 /* set file offset to offset */ #endif diff --git a/cores/esp8266/spiffs/spiffs_gc.cpp b/cores/esp8266/spiffs/spiffs_gc.cpp index fe556d1fc7..0afa257423 100644 --- a/cores/esp8266/spiffs/spiffs_gc.cpp +++ b/cores/esp8266/spiffs/spiffs_gc.cpp @@ -467,7 +467,7 @@ s32_t spiffs_gc_clean(spiffs *fs, spiffs_block_ix bix) { // update memory representation of object index page with new data page if (gc.cur_objix_spix == 0) { // update object index header page - ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[p_hdr.span_ix] = new_data_pix; + ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN))[p_hdr.span_ix] = new_data_pix; SPIFFS_GC_DBG("gc_clean: MOVE_DATA wrote page " _SPIPRIpg" to objix_hdr entry " _SPIPRIsp" in mem\n", new_data_pix, (spiffs_span_ix)SPIFFS_OBJ_IX_ENTRY(fs, p_hdr.span_ix)); } else { // update object index page diff --git a/cores/esp8266/spiffs/spiffs_hydrogen.cpp b/cores/esp8266/spiffs/spiffs_hydrogen.cpp index fffc5a83a8..9310474dff 100644 --- a/cores/esp8266/spiffs/spiffs_hydrogen.cpp +++ b/cores/esp8266/spiffs/spiffs_hydrogen.cpp @@ -729,10 +729,10 @@ s32_t SPIFFS_fremove(spiffs *fs, spiffs_file fh) { static s32_t spiffs_stat_pix(spiffs *fs, spiffs_page_ix pix, spiffs_file fh, spiffs_stat *s) { (void)fh; - spiffs_page_object_ix_header objix_hdr; + alloc_spiffs_page_object_ix_header(objix_hdr); spiffs_obj_id obj_id; s32_t res =_spiffs_rd(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_READ, fh, - SPIFFS_PAGE_TO_PADDR(fs, pix), sizeof(spiffs_page_object_ix_header), (u8_t *)&objix_hdr); + SPIFFS_PAGE_TO_PADDR(fs, pix), sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN, (u8_t *)&objix_hdr); SPIFFS_API_CHECK_RES(fs, res); u32_t obj_id_addr = SPIFFS_BLOCK_TO_PADDR(fs, SPIFFS_BLOCK_FOR_PAGE(fs , pix)) + @@ -746,9 +746,8 @@ static s32_t spiffs_stat_pix(spiffs *fs, spiffs_page_ix pix, spiffs_file fh, spi s->size = objix_hdr.size == SPIFFS_UNDEFINED_LEN ? 0 : objix_hdr.size; s->pix = pix; strncpy((char *)s->name, (char *)objix_hdr.name, SPIFFS_OBJ_NAME_LEN); -#if SPIFFS_OBJ_META_LEN - _SPIFFS_MEMCPY(s->meta, objix_hdr.meta, SPIFFS_OBJ_META_LEN); -#endif + if (SPIFFS_OBJ_META_LEN) + _SPIFFS_MEMCPY(s->meta, objix_hdr.meta, SPIFFS_OBJ_META_LEN); return res; } @@ -928,7 +927,7 @@ s32_t SPIFFS_rename(spiffs *fs, const char *old_path, const char *new_path) { #endif // SPIFFS_READ_ONLY } -#if SPIFFS_OBJ_META_LEN + s32_t SPIFFS_update_meta(spiffs *fs, const char *name, const void *meta) { #if SPIFFS_READ_ONLY (void)fs; (void)name; (void)meta; @@ -953,8 +952,7 @@ s32_t SPIFFS_update_meta(spiffs *fs, const char *name, const void *meta) { } SPIFFS_API_CHECK_RES_UNLOCK(fs, res); - res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, fd->objix_hdr_pix, 0, 0, meta, - 0, &pix_dummy); + res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, fd->objix_hdr_pix, 0, 0, (const u8_t*)meta, 0, &pix_dummy); spiffs_fd_return(fs, fd->file_nbr); @@ -988,8 +986,7 @@ s32_t SPIFFS_fupdate_meta(spiffs *fs, spiffs_file fh, const void *meta) { SPIFFS_API_CHECK_RES_UNLOCK(fs, res); } - res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, fd->objix_hdr_pix, 0, 0, meta, - 0, &pix_dummy); + res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, fd->objix_hdr_pix, 0, 0, (const u8_t*)meta, 0, &pix_dummy); SPIFFS_API_CHECK_RES_UNLOCK(fs, res); @@ -998,7 +995,7 @@ s32_t SPIFFS_fupdate_meta(spiffs *fs, spiffs_file fh, const void *meta) { return res; #endif // SPIFFS_READ_ONLY } -#endif // SPIFFS_OBJ_META_LEN + spiffs_DIR *SPIFFS_opendir(spiffs *fs, const char *name, spiffs_DIR *d) { SPIFFS_API_DBG("%s\n", __func__); @@ -1029,7 +1026,7 @@ static s32_t spiffs_read_dir_v( void *user_var_p) { (void)user_const_p; s32_t res; - spiffs_page_object_ix_header objix_hdr; + alloc_spiffs_page_object_ix_header(objix_hdr); if (obj_id == SPIFFS_OBJ_ID_FREE || obj_id == SPIFFS_OBJ_ID_DELETED || (obj_id & SPIFFS_OBJ_ID_IX_FLAG) == 0) { return SPIFFS_VIS_COUNTINUE; @@ -1037,7 +1034,7 @@ static s32_t spiffs_read_dir_v( spiffs_page_ix pix = SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, ix_entry); res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ, - 0, SPIFFS_PAGE_TO_PADDR(fs, pix), sizeof(spiffs_page_object_ix_header), (u8_t *)&objix_hdr); + 0, SPIFFS_PAGE_TO_PADDR(fs, pix), sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN, (u8_t *)&objix_hdr); if (res != SPIFFS_OK) return res; if ((obj_id & SPIFFS_OBJ_ID_IX_FLAG) && objix_hdr.p_hdr.span_ix == 0 && @@ -1049,9 +1046,8 @@ static s32_t spiffs_read_dir_v( e->type = objix_hdr.type; e->size = objix_hdr.size == SPIFFS_UNDEFINED_LEN ? 0 : objix_hdr.size; e->pix = pix; -#if SPIFFS_OBJ_META_LEN - _SPIFFS_MEMCPY(e->meta, objix_hdr.meta, SPIFFS_OBJ_META_LEN); -#endif + if (SPIFFS_OBJ_META_LEN) + _SPIFFS_MEMCPY(e->meta, objix_hdr.meta, SPIFFS_OBJ_META_LEN); return SPIFFS_OK; } return SPIFFS_VIS_COUNTINUE; diff --git a/cores/esp8266/spiffs/spiffs_nucleus.cpp b/cores/esp8266/spiffs/spiffs_nucleus.cpp index 35238f351b..afaae9ac15 100644 --- a/cores/esp8266/spiffs/spiffs_nucleus.cpp +++ b/cores/esp8266/spiffs/spiffs_nucleus.cpp @@ -319,6 +319,7 @@ s32_t spiffs_probe( #endif // SPIFFS_USE_MAGIC && SPIFFS_USE_MAGIC_LENGTH && SPIFFS_SINGLETON==0 + static s32_t spiffs_obj_lu_scan_v( spiffs *fs, spiffs_obj_id obj_id, @@ -635,7 +636,7 @@ static void spiffs_update_ix_map(spiffs *fs, spiffs_page_ix objix_data_pix; if (objix_spix == 0) { // get data page from object index header page - objix_data_pix = ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix_header)))[map_spix]; + objix_data_pix = ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN))[map_spix]; } else { // get data page from object index page objix_data_pix = ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, map_spix)]; @@ -915,7 +916,7 @@ s32_t spiffs_object_create( spiffs_page_ix *objix_hdr_pix) { s32_t res = SPIFFS_OK; spiffs_block_ix bix; - spiffs_page_object_ix_header oix_hdr; + alloc_spiffs_page_object_ix_header(oix_hdr); int entry; res = spiffs_gc_check(fs, SPIFFS_DATA_PAGE_SIZE(fs)); @@ -942,19 +943,19 @@ s32_t spiffs_object_create( oix_hdr.type = type; oix_hdr.size = SPIFFS_UNDEFINED_LEN; // keep ones so we can update later without wasting this page strncpy((char*)oix_hdr.name, (const char*)name, SPIFFS_OBJ_NAME_LEN); -#if SPIFFS_OBJ_META_LEN - if (meta) { - _SPIFFS_MEMCPY(oix_hdr.meta, meta, SPIFFS_OBJ_META_LEN); + if (SPIFFS_OBJ_META_LEN) { + if (meta) { + _SPIFFS_MEMCPY(oix_hdr.meta, meta, SPIFFS_OBJ_META_LEN); + } else { + memset(oix_hdr.meta, 0xff, SPIFFS_OBJ_META_LEN); + } } else { - memset(oix_hdr.meta, 0xff, SPIFFS_OBJ_META_LEN); + (void) meta; } -#else - (void) meta; -#endif // update page res = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_DA | SPIFFS_OP_C_UPDT, - 0, SPIFFS_OBJ_LOOKUP_ENTRY_TO_PADDR(fs, bix, entry), sizeof(spiffs_page_object_ix_header), (u8_t*)&oix_hdr); + 0, SPIFFS_OBJ_LOOKUP_ENTRY_TO_PADDR(fs, bix, entry), sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN, (u8_t*)&oix_hdr); SPIFFS_CHECK_RES(res); spiffs_cb_object_event(fs, (spiffs_page_object_ix *)&oix_hdr, @@ -1006,13 +1007,13 @@ s32_t spiffs_object_update_index_hdr( if (name) { strncpy((char*)objix_hdr->name, (const char*)name, SPIFFS_OBJ_NAME_LEN); } -#if SPIFFS_OBJ_META_LEN - if (meta) { - _SPIFFS_MEMCPY(objix_hdr->meta, meta, SPIFFS_OBJ_META_LEN); + if (SPIFFS_OBJ_META_LEN) { + if (meta) { + _SPIFFS_MEMCPY(objix_hdr->meta, meta, SPIFFS_OBJ_META_LEN); + } + } else { + (void) meta; } -#else - (void) meta; -#endif if (size) { objix_hdr->size = size; } @@ -1174,11 +1175,11 @@ s32_t spiffs_object_open_by_page( spiffs_mode mode) { (void)mode; s32_t res = SPIFFS_OK; - spiffs_page_object_ix_header oix_hdr; + alloc_spiffs_page_object_ix_header(oix_hdr); spiffs_obj_id obj_id; res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_IX | SPIFFS_OP_C_READ, - fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, pix), sizeof(spiffs_page_object_ix_header), (u8_t *)&oix_hdr); + fd->file_nbr, SPIFFS_PAGE_TO_PADDR(fs, pix), sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN, (u8_t *)&oix_hdr); SPIFFS_CHECK_RES(res); spiffs_block_ix bix = SPIFFS_BLOCK_FOR_PAGE(fs, pix); @@ -1354,7 +1355,7 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { // append to existing page, fill out free data in existing page if (cur_objix_spix == 0) { // get data page from object index header page - data_page = ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[data_spix]; + data_page = ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN))[data_spix]; } else { // get data page from object index page data_page = ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)]; @@ -1374,7 +1375,7 @@ s32_t spiffs_object_append(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { // update memory representation of object index page with new data page if (cur_objix_spix == 0) { // update object index header page - ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[data_spix] = data_page; + ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN))[data_spix] = data_page; SPIFFS_DBG("append: " _SPIPRIid " wrote page " _SPIPRIpg " to objix_hdr entry " _SPIPRIsp " in mem\n", fd->obj_id , data_page, data_spix); objix_hdr->size = offset+written; @@ -1543,7 +1544,7 @@ s32_t spiffs_object_modify(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { spiffs_page_ix orig_data_pix; if (cur_objix_spix == 0) { // get data page from object index header page - orig_data_pix = ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[data_spix]; + orig_data_pix = ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN))[data_spix]; } else { // get data page from object index page orig_data_pix = ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)]; @@ -1606,7 +1607,7 @@ s32_t spiffs_object_modify(spiffs_fd *fd, u32_t offset, u8_t *data, u32_t len) { // update memory representation of object index page with new data page if (cur_objix_spix == 0) { // update object index header page - ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[data_spix] = data_pix; + ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN))[data_spix] = data_pix; SPIFFS_DBG("modify: wrote page " _SPIPRIpg " to objix_hdr entry " _SPIPRIsp " in mem\n", data_pix, data_spix); } else { // update object index page @@ -1664,14 +1665,14 @@ static s32_t spiffs_object_find_object_index_header_by_name_v( void *user_var_p) { (void)user_var_p; s32_t res; - spiffs_page_object_ix_header objix_hdr; + alloc_spiffs_page_object_ix_header(objix_hdr); spiffs_page_ix pix = SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, ix_entry); if (obj_id == SPIFFS_OBJ_ID_FREE || obj_id == SPIFFS_OBJ_ID_DELETED || (obj_id & SPIFFS_OBJ_ID_IX_FLAG) == 0) { return SPIFFS_VIS_COUNTINUE; } res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ, - 0, SPIFFS_PAGE_TO_PADDR(fs, pix), sizeof(spiffs_page_object_ix_header), (u8_t *)&objix_hdr); + 0, SPIFFS_PAGE_TO_PADDR(fs, pix), sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN, (u8_t *)&objix_hdr); SPIFFS_CHECK_RES(res); if (objix_hdr.p_hdr.span_ix == 0 && (objix_hdr.p_hdr.flags & (SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_FINAL | SPIFFS_PH_FLAG_IXDELE)) == @@ -1814,8 +1815,8 @@ s32_t spiffs_object_truncate( if (cur_objix_spix == 0) { // get data page from object index header page - data_pix = ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[data_spix]; - ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[data_spix] = SPIFFS_OBJ_ID_FREE; + data_pix = ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN))[data_spix]; + ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN))[data_spix] = SPIFFS_OBJ_ID_FREE; } else { // get data page from object index page data_pix = ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)]; @@ -1887,7 +1888,7 @@ s32_t spiffs_object_truncate( // update memory representation of object index page with new data page if (cur_objix_spix == 0) { // update object index header page - ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[data_spix] = new_data_pix; + ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN))[data_spix] = new_data_pix; SPIFFS_DBG("truncate: wrote page " _SPIPRIpg " to objix_hdr entry " _SPIPRIsp " in mem\n", new_data_pix, (spiffs_span_ix)SPIFFS_OBJ_IX_ENTRY(fs, data_spix)); } else { // update object index page @@ -1920,8 +1921,8 @@ s32_t spiffs_object_truncate( } else { // make uninitialized object SPIFFS_DBG("truncate: reset objix_hdr page " _SPIPRIpg "\n", objix_pix); - memset(fs->work + sizeof(spiffs_page_object_ix_header), 0xff, - SPIFFS_CFG_LOG_PAGE_SZ(fs) - sizeof(spiffs_page_object_ix_header)); + memset(fs->work + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN, 0xff, + SPIFFS_CFG_LOG_PAGE_SZ(fs) - sizeof(spiffs_page_object_ix_header)-SPIFFS_OBJ_META_LEN); res = spiffs_object_update_index_hdr(fs, fd, fd->obj_id, objix_pix, fs->work, 0, 0, SPIFFS_UNDEFINED_LEN, &new_objix_hdr_pix); SPIFFS_CHECK_RES(res); @@ -2014,7 +2015,7 @@ s32_t spiffs_object_read( if (cur_objix_spix == 0) { // get data page from object index header page - data_pix = ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[data_spix]; + data_pix = ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN))[data_spix]; } else { // get data page from object index page data_pix = ((spiffs_page_ix*)((u8_t *)objix + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)]; @@ -2070,9 +2071,9 @@ static s32_t spiffs_obj_lu_find_free_obj_id_bitmap_v(spiffs *fs, spiffs_obj_id i if (conflicting_name && (id & SPIFFS_OBJ_ID_IX_FLAG)) { spiffs_page_ix pix = SPIFFS_OBJ_LOOKUP_ENTRY_TO_PIX(fs, bix, ix_entry); int res; - spiffs_page_object_ix_header objix_hdr; + alloc_spiffs_page_object_ix_header(objix_hdr); res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ, - 0, SPIFFS_PAGE_TO_PADDR(fs, pix), sizeof(spiffs_page_object_ix_header), (u8_t *)&objix_hdr); + 0, SPIFFS_PAGE_TO_PADDR(fs, pix), sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN, (u8_t *)&objix_hdr); SPIFFS_CHECK_RES(res); if (objix_hdr.p_hdr.span_ix == 0 && (objix_hdr.p_hdr.flags & (SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_FINAL | SPIFFS_PH_FLAG_IXDELE)) == @@ -2099,10 +2100,10 @@ static s32_t spiffs_obj_lu_find_free_obj_id_compact_v(spiffs *fs, spiffs_obj_id if (id != SPIFFS_OBJ_ID_FREE && id != SPIFFS_OBJ_ID_DELETED && (id & SPIFFS_OBJ_ID_IX_FLAG)) { s32_t res; const spiffs_free_obj_id_state *state = (const spiffs_free_obj_id_state*)user_const_p; - spiffs_page_object_ix_header objix_hdr; + alloc_spiffs_page_object_ix_header(objix_hdr); res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU2 | SPIFFS_OP_C_READ, - 0, SPIFFS_OBJ_LOOKUP_ENTRY_TO_PADDR(fs, bix, ix_entry), sizeof(spiffs_page_object_ix_header), (u8_t*)&objix_hdr); + 0, SPIFFS_OBJ_LOOKUP_ENTRY_TO_PADDR(fs, bix, ix_entry), sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN, (u8_t*)&objix_hdr); if (res == SPIFFS_OK && objix_hdr.p_hdr.span_ix == 0 && ((objix_hdr.p_hdr.flags & (SPIFFS_PH_FLAG_INDEX | SPIFFS_PH_FLAG_FINAL | SPIFFS_PH_FLAG_DELET)) == (SPIFFS_PH_FLAG_DELET))) { diff --git a/cores/esp8266/spiffs/spiffs_nucleus.h b/cores/esp8266/spiffs/spiffs_nucleus.h index cc195c7eca..f39a5def50 100644 --- a/cores/esp8266/spiffs/spiffs_nucleus.h +++ b/cores/esp8266/spiffs/spiffs_nucleus.h @@ -248,7 +248,7 @@ extern "C" { // entries in an object header page index #define SPIFFS_OBJ_HDR_IX_LEN(fs) \ - ((SPIFFS_CFG_LOG_PAGE_SZ(fs) - sizeof(spiffs_page_object_ix_header))/sizeof(spiffs_page_ix)) + ((SPIFFS_CFG_LOG_PAGE_SZ(fs) - sizeof(spiffs_page_object_ix_header)-SPIFFS_OBJ_META_LEN)/sizeof(spiffs_page_ix)) // entries in an object page index #define SPIFFS_OBJ_IX_LEN(fs) \ ((SPIFFS_CFG_LOG_PAGE_SZ(fs) - sizeof(spiffs_page_object_ix))/sizeof(spiffs_page_ix)) @@ -509,10 +509,8 @@ typedef struct SPIFFS_PACKED spiffs_obj_type type; // name of object u8_t name[SPIFFS_OBJ_NAME_LEN]; -#if SPIFFS_OBJ_META_LEN // metadata. not interpreted by SPIFFS in any way. - u8_t meta[SPIFFS_OBJ_META_LEN]; -#endif + u8_t meta[]; // Leave array size undefined, since it is variable at runtime } spiffs_page_object_ix_header; // object index page header diff --git a/cores/esp8266/spiffs_api.cpp b/cores/esp8266/spiffs_api.cpp index 833dd3edb0..32deedc208 100644 --- a/cores/esp8266/spiffs_api.cpp +++ b/cores/esp8266/spiffs_api.cpp @@ -25,8 +25,11 @@ using namespace fs; +int __SPIFFS_obj_meta_len = 0; + namespace spiffs_impl { + FileImplPtr SPIFFSImpl::open(const char* path, OpenMode openMode, AccessMode accessMode) { if (!isSpiffsFilenameValid(path)) { @@ -51,6 +54,14 @@ FileImplPtr SPIFFSImpl::open(const char* path, OpenMode openMode, AccessMode acc fd, path, openMode, accessMode, _fs.err_code); return FileImplPtr(); } + if (!(mode & SPIFFS_O_RDONLY) && __SPIFFS_obj_meta_len==sizeof(4)) { + time_t t = time(NULL); + struct tm tmr; + localtime_r(&t, &tmr); + time_t meta = mktime(&tmr); + DEBUGV("SPIFFSImpl::open updating file write time to %ld\r\n", meta); + SPIFFS_fupdate_meta(&_fs, fd, &meta); + } return std::make_shared(this, fd); } diff --git a/cores/esp8266/spiffs_api.h b/cores/esp8266/spiffs_api.h index 0491508f4d..40aeea4904 100644 --- a/cores/esp8266/spiffs_api.h +++ b/cores/esp8266/spiffs_api.h @@ -39,6 +39,7 @@ extern "C" { using namespace fs; +extern int __SPIFFS_obj_meta_len; namespace spiffs_impl { int getSpiffsMode(OpenMode openMode, AccessMode accessMode); @@ -143,6 +144,7 @@ class SPIFFSImpl : public FSImpl bool setConfig(const FSConfig &cfg) override { if ((cfg._type != SPIFFSConfig::fsid::FSId) || (SPIFFS_mounted(&_fs) != 0)) { + DEBUGV("SPIFFSImpl::setConfig error. Either wrong object used or SPIFFS mounted\n"); return false; } _cfg = *static_cast(&cfg); @@ -167,9 +169,7 @@ class SPIFFSImpl : public FSImpl return true; } if (_cfg._autoFormat) { - auto rc = SPIFFS_format(&_fs); - if (rc != SPIFFS_OK) { - DEBUGV("SPIFFS_format: rc=%d, err=%d\r\n", rc, _fs.err_code); + if (!_formatOldOrNew()) { return false; } return _tryMount(); @@ -201,9 +201,8 @@ class SPIFFSImpl : public FSImpl if (_tryMount()) { SPIFFS_unmount(&_fs); } - auto rc = SPIFFS_format(&_fs); - if (rc != SPIFFS_OK) { - DEBUGV("SPIFFS_format: rc=%d, err=%d\r\n", rc, _fs.err_code); + + if (!_formatOldOrNew()) { return false; } @@ -228,7 +227,7 @@ class SPIFFSImpl : public FSImpl return &_fs; } - bool _tryMount() + spiffs_config _setupSpiffsConfig(bool metadata) { spiffs_config config; memset(&config, 0, sizeof(config)); @@ -236,13 +235,12 @@ class SPIFFSImpl : public FSImpl config.hal_read_f = &spiffs_hal_read; config.hal_write_f = &spiffs_hal_write; config.hal_erase_f = &spiffs_hal_erase; - config.phys_size = _size; - config.phys_addr = _start; + config.phys_size = metadata ? _size - _blockSize : _size; + config.phys_addr = metadata ? _start + _blockSize :_start; config.phys_erase_block = FLASH_SECTOR_SIZE; config.log_block_size = _blockSize; config.log_page_size = _pageSize; - if (((uint32_t) std::numeric_limits::max()) < (_size / _blockSize)) { DEBUGV("spiffs_block_ix type too small"); abort(); @@ -263,6 +261,49 @@ class SPIFFSImpl : public FSImpl abort(); } + return config; + } + + bool _formatOldOrNew() { + // Make sure we format with requested metadata len + if (_cfg._enableTime) { + if (!_mark_metadata_fs(_start, _blockSize, _pageSize)) { + DEBUGV("SPIFFSImpl::_formatOldOrNew error, can't mark as metadata filesystem\n"); + return false; + } + } + // Make sure we format with requested metadata len + __SPIFFS_obj_meta_len = _cfg._enableTime ? 4 : 0; + DEBUGV("SPIFFSImpl::_formatOldOrNew formatting with metadata==%d\r\n", __SPIFFS_obj_meta_len); + spiffs_config config = _setupSpiffsConfig(_cfg._enableTime); + + // We need to try a mount on SPIFFS, even though it will probably fail, to make the opaque + // SPIFFS _fs struct take the new sizes specified in the configuration. + size_t fdsBufSize = SPIFFS_buffer_bytes_for_filedescs(&_fs, _maxOpenFds); + size_t cacheBufSize = SPIFFS_buffer_bytes_for_cache(&_fs, _maxOpenFds); + auto err = SPIFFS_mount(&_fs, &config, _workBuf.get(), + _fdsBuf.get(), fdsBufSize, _cacheBuf.get(), cacheBufSize, + &SPIFFSImpl::_check_cb); + if (err == SPIFFS_OK) { + SPIFFS_unmount(&_fs); + } + auto rc = SPIFFS_format(&_fs); + if (rc != SPIFFS_OK) { + DEBUGV("SPIFFS_format: rc=%d, err=%d\r\n", rc, _fs.err_code); + return false; + } + if (_cfg._enableTime && !_is_metadata_fs(_start, _blockSize, _pageSize)) { + DEBUGV("SPIFFSImpl::_formatOldOrNew lost metadata flag somehow\n"); + return false; + } + return true; + } + + + bool _tryMount() + { + spiffs_config config = _setupSpiffsConfig(false); + // hack: even though fs is not initialized at this point, // SPIFFS_buffer_bytes_for_cache uses only fs->config.log_page_size // suggestion: change SPIFFS_buffer_bytes_for_cache to take @@ -282,18 +323,76 @@ class SPIFFSImpl : public FSImpl _cacheBuf.reset(new uint8_t[cacheBufSize]); } - DEBUGV("SPIFFSImpl: mounting fs @%x, size=%x, block=%x, page=%x\r\n", - _start, _size, _blockSize, _pageSize); - - auto err = SPIFFS_mount(&_fs, &config, _workBuf.get(), + // First, can we mount w/o metadata (preserve backwards) + int err; + if ( !_is_metadata_fs(_start, _blockSize, _pageSize) ) { + __SPIFFS_obj_meta_len = 0; + DEBUGV("SPIFFSImpl: trying fs @%x, size=%x, block=%x, page=%x, metadata=%d\r\n", + _start, _size, _blockSize, _pageSize, __SPIFFS_obj_meta_len); + err = SPIFFS_mount(&_fs, &config, _workBuf.get(), + _fdsBuf.get(), fdsBufSize, _cacheBuf.get(), cacheBufSize, + &SPIFFSImpl::_check_cb); + } else { + // Flag matched, it's a metadata FS + __SPIFFS_obj_meta_len = 4; + config = _setupSpiffsConfig(true); + DEBUGV("SPIFFSImpl: doesn't look like old metadata==0, so trying fs @%x, size=%x, block=%x, page=%x, metadata=%d\r\n", + _start, _size, _blockSize, _pageSize, __SPIFFS_obj_meta_len); + err = SPIFFS_mount(&_fs, &config, _workBuf.get(), _fdsBuf.get(), fdsBufSize, _cacheBuf.get(), cacheBufSize, &SPIFFSImpl::_check_cb); + } DEBUGV("SPIFFSImpl: mount rc=%d\r\n", err); return err == SPIFFS_OK; } + // The value that's repeated through the entire first blockSize bytes of a timestamp FS + enum { __SPIFFS_MD_FS__ = 0xc0de0004 }; + + // Check the first blockSize bytes, and if they match the flag it's a timestamp FS + static bool _is_metadata_fs(uint32_t _start, uint32_t _blockSize, uint32_t _pageSize) + { + char buff[_pageSize]; + for (uint32_t page = 0; page < _blockSize / _pageSize; page++) { + uint32_t addr = _start + page * _pageSize; + auto ret = spiffs_hal_read(addr, _pageSize, (uint8_t*)buff); + if (ret != SPIFFS_OK) { + return false; + } + for (uint32_t *ptr=reinterpret_cast(buff), cnt=0; cnt < sizeof(buff)/sizeof(uint32_t); cnt++, ptr++) { + if (*ptr != __SPIFFS_MD_FS__) { + return false; + } + } + } + DEBUGV("SPIFFSImpl: metadata FS\n"); + return true; + } + + // Fills the first block with the timestamp flag + static bool _mark_metadata_fs(uint32_t _start, uint32_t _blockSize, uint32_t _pageSize) + { + char buff[_pageSize]; + for (uint32_t *ptr=reinterpret_cast(buff), cnt=0; cnt < sizeof(buff)/sizeof(uint32_t); cnt++, ptr++) { + *ptr = __SPIFFS_MD_FS__; + } + auto ret = spiffs_hal_erase(_start, _blockSize); + if (ret != SPIFFS_OK) { + return false; + } + for (uint32_t page = 0; page < _blockSize / _pageSize; page++) { + uint32_t addr = _start + page * _pageSize; + ret = spiffs_hal_write(addr, _pageSize, (uint8_t *)buff); + if (ret != SPIFFS_OK) { + return false; + } + } + DEBUGV("SPIFFSImpl: set metadata FS flag\n"); + return true; + } + static void _check_cb(spiffs_check_type type, spiffs_check_report report, uint32_t arg1, uint32_t arg2) { @@ -470,6 +569,18 @@ class SPIFFSFileImpl : public FileImpl return name(); // No dirs, they're the same on SPIFFS } + time_t getLastWrite() override + { + CHECKFD(); + time_t t = 0; + if (__SPIFFS_obj_meta_len) { + _getStat() ; + memcpy(&t, _stat.meta, sizeof(time_t)); + } + return t; + } + + protected: void _getStat() const { @@ -539,6 +650,14 @@ class SPIFFSDirImpl : public DirImpl return _dirent.size; } + time_t fileTime() override { + if (!_valid) { + return 0; + } + // TODO - When SPIFFS supports time metadata, return it + return 0; + } + bool isFile() const override { // No such thing as directories on SPIFFS diff --git a/doc/filesystem.rst b/doc/filesystem.rst index b7d8b421ef..426d95d076 100644 --- a/doc/filesystem.rst +++ b/doc/filesystem.rst @@ -92,6 +92,27 @@ and ``SPIFFS.open()`` to ``LittleFS.open()`` with the rest of the code remaining untouched. +SDFS and SD +----------- +FAT filesystems are supported on the ESP8266 using the old Arduino wrapper +"SD.h" which wraps the "SDFS.h" filesystem transparently. + +Any commands discussed below pertaining to SPIFFS or LittleFS are +applicable to SD/SDFS. + +For legacy applications, the classic SD filesystem may continue to be used, +but for new applications, directly accessing the SDFS filesystem is +recommended as it may expose additional functionality that the old Arduino +SD filesystem didn't have. + +Note that in earlier releases of the core, using SD and SPIFFS in the same +sketch was complicated and required the use of ``NO_FS_GLOBALS``. The +current design makes SD, SDFS, SPIFFS, and LittleFS fully source compatible +and so please remove any ``NO_FS_GLOBALS`` definitions in your projects +when updgrading core versions. + + + SPIFFS file system limitations ------------------------------ @@ -160,6 +181,31 @@ in the specific subdirectory. This mimics the POSIX behavior for directory traversal most C programmers are used to. +SPIFFS on-flash versions +------------------------ + +There are two on-flash formats for SPIFFS. The original one has no +metadata support (SPIFFS_OBJ_META_LEN==0), while the current one supports +a timestamp (SPIFFS_OBJ_META_LEN==4). + +Because the on-flash SPIFFS layout changes, depending on whether or not +metadata is present, the two formats are incompatible. + +To ensure backwards compatibility, a unique layout on-flash is defined for +the new version, with the first page of flash set to 0xc0de0004. + +When SPIFFS is attempted to be mounted the core first checks the first +page of the flash filesystem. If it is not 0xc0de0004 filled, then +it immediately attempts to mount using the non-timestamped version. +Only if the first page is all 0xc0de0004 will it attempt to mount with +timestamps. + +When formatting a new filesystem, by default it builds a timestamp +filesystem, but this can be overridden using +``SPIFFS.setConfig(SPIFFSConfig().setEnableTime(true));`` before any +``SPIFFS.begin()`` or ``SPIFFS.format()``. + + Uploading files to file system ------------------------------ @@ -196,8 +242,8 @@ use esptool.py. - To upload a LittleFS filesystem use Tools > ESP8266 LittleFS Data Upload -File system object (SPIFFS/LittleFS) ------------------------------------- +File system object (SPIFFS/LittleFS/SD/SDFS) +-------------------------------------------- setConfig ~~~~~~~~~ @@ -377,7 +423,7 @@ info or LittleFS.info(fs_info); Fills `FSInfo structure <#filesystem-information-structure>`__ with -information about the file system. Returns ``true`` is successful, +information about the file system. Returns ``true`` if successful, ``false`` otherwise. Filesystem information structure @@ -402,6 +448,44 @@ block size - ``pageSize`` — filesystem logical page size - ``maxOpenFiles`` ``maxPathLength`` — max file name length (including one byte for zero termination) +info64 +~~~~~~ + +.. code:: cpp + + FSInfo64 fsinfo; + SD.info(fsinfo); + or LittleFS(fsinfo); + +Performs the same operation as ``info`` but allows for reporting greater than +4GB for filesystem size/used/etc. Should be used with the SD and SDFS +filesystems since most SD cards today are greater than 4GB in size. + +setTimeCallback(time_t (*cb)(void)) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: cpp + + time_t myTimeCallback() { + return 1455451200; // UNIX timestamp + } + void setup () { + LittleFS.setTimeCallback(myTimeCallback); + ... + // Any files will now be made with Pris' incept date + } + + +The SD, SDFS, and LittleFS filesystems support a file timestamp, updated when the file is +opened for writing. By default, the ESP8266 will use the internal time returned from +``time(NULL)``. If your app sets the system time using NTP before file operations, then +you should not need to use this function. However, if you need to set a specific time +for a file, or the system clock isn't correct and you need to read the time from an external +RTC or use a fixed time, this call allows you do to so. + +In general use, with a functioning ``time()`` call, user applications should not need +to use this function. + Directory object (Dir) ---------------------- @@ -441,6 +525,12 @@ fileSize Returns the size of the current file pointed to by the internal iterator. +fileTime +~~~~~~~~ + +Returns the time_t write time of the current file pointed +to by the internal iterator. + isFile ~~~~~~ @@ -464,6 +554,13 @@ rewind Resets the internal pointer to the start of the directory. +setTimeCallback(time_t (*cb)(void)) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sets the time callback for any files accessed from this Dir object via openNextFile. +Note that the SD and SDFS filesystems only support a filesystem-wide callback and +calls to ``Dir::setTimeCallback`` may produce unexpected behavior. + File object ----------- @@ -535,6 +632,12 @@ fullName Returns the full path file name as a ``const char*``. +getLastWrite +~~~~~~~~~~~~ + +Returns the file last write time, and only valid for files opened in read-only +mode. If a file is opened for writing, the returned time may be indeterminate. + isFile ~~~~~~ @@ -589,3 +692,10 @@ rewindDirectory (compatibiity method, not recommended for new code) Resets the ``openNextFile`` pointer to the top of the directory. Only valid when ``File.isDirectory() == true``. + +setTimeCallback(time_t (*cb)(void)) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sets the time callback for this specific file. Note that the SD and +SDFS filesystems only support a filesystem-wide callback and calls to +``Dir::setTimeCallback`` may produce unexpected behavior. diff --git a/libraries/LittleFS/src/LittleFS.h b/libraries/LittleFS/src/LittleFS.h index 467255ca31..e0875b9ad7 100644 --- a/libraries/LittleFS/src/LittleFS.h +++ b/libraries/LittleFS/src/LittleFS.h @@ -422,7 +422,25 @@ class LittleFSFileImpl : public FileImpl lfs_file_close(_fs->getFS(), _getFD()); _opened = false; DEBUGV("lfs_file_close: fd=%p\n", _getFD()); + if (timeCallback) { + // Add metadata with last write time + time_t now = timeCallback(); + int rc = lfs_setattr(_fs->getFS(), _name.get(), 't', (const void *)&now, sizeof(now)); + if (rc < 0) { + DEBUGV("Unable to set time on '%s' to %d\n", _name.get(), now); + } + } + } + } + + time_t getLastWrite() override { + time_t ftime = 0; + if (_opened && _fd) { + int rc = lfs_getattr(_fs->getFS(), _name.get(), 't', (void *)&ftime, sizeof(ftime)); + if (rc != sizeof(ftime)) + ftime = 0; // Error, so clear read value } + return ftime; } const char* name() const override { @@ -520,6 +538,27 @@ class LittleFSDirImpl : public DirImpl return _dirent.size; } + time_t fileTime() override { + if (!_valid) { + return 0; + } + int nameLen = 3; // Slashes, terminator + nameLen += _dirPath.get() ? strlen(_dirPath.get()) : 0; + nameLen += strlen(_dirent.name); + char *tmpName = (char*)malloc(nameLen); + if (!tmpName) { + return 0; + } + snprintf(tmpName, nameLen, "%s%s%s", _dirPath.get() ? _dirPath.get() : "", _dirPath.get()&&_dirPath.get()[0]?"/":"", _dirent.name); + time_t ftime = 0; + int rc = lfs_getattr(_fs->getFS(), tmpName, 't', (void *)&ftime, sizeof(ftime)); + if (rc != sizeof(ftime)) + ftime = 0; // Error, so clear read value + free(tmpName); + return ftime; + } + + bool isFile() const override { return _valid && (_dirent.type == LFS_TYPE_REG); } diff --git a/libraries/SD/examples/listfiles/listfiles.ino b/libraries/SD/examples/listfiles/listfiles.ino index 2b5eaf1ab8..fd35d0c338 100644 --- a/libraries/SD/examples/listfiles/listfiles.ino +++ b/libraries/SD/examples/listfiles/listfiles.ino @@ -28,14 +28,14 @@ File root; void setup() { // Open serial communications and wait for port to open: - Serial.begin(9600); + Serial.begin(115200); while (!Serial) { ; // wait for serial port to connect. Needed for Leonardo only } Serial.print("Initializing SD card..."); - if (!SD.begin(4)) { + if (!SD.begin(SS)) { Serial.println("initialization failed!"); return; } @@ -70,11 +70,13 @@ void printDirectory(File dir, int numTabs) { } else { // files have sizes, directories do not Serial.print("\t\t"); - Serial.println(entry.size(), DEC); + Serial.print(entry.size(), DEC); + Serial.print("\t\t"); + time_t ft = entry.getLastWrite(); + struct tm *tm = localtime(&ft); + // US format. Feel free to convert to your own locale... + Serial.printf("%02d-%02d-%02d %02d:%02d:%02d\n", tm->tm_mon + 1, tm->tm_mday, tm->tm_year % 100, tm->tm_hour, tm->tm_min, tm->tm_sec); } entry.close(); } } - - - diff --git a/libraries/SD/src/SD.cpp b/libraries/SD/src/SD.cpp index 6b40b0e494..a3f0431dd8 100644 --- a/libraries/SD/src/SD.cpp +++ b/libraries/SD/src/SD.cpp @@ -3,3 +3,5 @@ #if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_SD) SDClass SD; #endif + +void (*__SD__userDateTimeCB)(uint16_t*, uint16_t*) = nullptr; diff --git a/libraries/SD/src/SD.h b/libraries/SD/src/SD.h index bb1c696a58..d15d8f85b4 100644 --- a/libraries/SD/src/SD.h +++ b/libraries/SD/src/SD.h @@ -29,6 +29,7 @@ #undef FILE_WRITE #define FILE_WRITE (sdfat::O_READ | sdfat::O_WRITE | sdfat::O_CREAT | sdfat::O_APPEND) + class SDClass { public: boolean begin(uint8_t csPin, SPISettings cfg = SPI_HALF_SPEED) { @@ -137,6 +138,17 @@ class SDClass { return ((uint64_t)clusterSize() * (uint64_t)totalClusters()); } + void setTimeCallback(time_t (*cb)(void)) { + SDFS.setTimeCallback(cb); + } + + // Wrapper to allow obsolete datetimecallback use, silently convert to time_t in wrappertimecb + void dateTimeCallback(void (*cb)(uint16_t*, uint16_t*)) { + extern void (*__SD__userDateTimeCB)(uint16_t*, uint16_t*); + __SD__userDateTimeCB = cb; + SDFS.setTimeCallback(wrapperTimeCB); + } + private: const char *getMode(uint8_t mode) { bool read = (mode & sdfat::O_READ) ? true : false; @@ -150,8 +162,46 @@ class SDClass { else { return "r"; } } + static time_t wrapperTimeCB(void) { + extern void (*__SD__userDateTimeCB)(uint16_t*, uint16_t*); + if (__SD__userDateTimeCB) { + uint16_t d, t; + __SD__userDateTimeCB(&d, &t); + return sdfs::SDFSImpl::FatToTimeT(d, t); + } + return time(nullptr); + } + }; + +// Expose FatStructs.h helpers for MSDOS date/time for use with dateTimeCallback +static inline uint16_t FAT_DATE(uint16_t year, uint8_t month, uint8_t day) { + return (year - 1980) << 9 | month << 5 | day; +} +static inline uint16_t FAT_YEAR(uint16_t fatDate) { + return 1980 + (fatDate >> 9); +} +static inline uint8_t FAT_MONTH(uint16_t fatDate) { + return (fatDate >> 5) & 0XF; +} +static inline uint8_t FAT_DAY(uint16_t fatDate) { + return fatDate & 0X1F; +} +static inline uint16_t FAT_TIME(uint8_t hour, uint8_t minute, uint8_t second) { + return hour << 11 | minute << 5 | second >> 1; +} +static inline uint8_t FAT_HOUR(uint16_t fatTime) { + return fatTime >> 11; +} +static inline uint8_t FAT_MINUTE(uint16_t fatTime) { + return (fatTime >> 5) & 0X3F; +} +static inline uint8_t FAT_SECOND(uint16_t fatTime) { + return 2*(fatTime & 0X1F); +} + + #if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_SD) extern SDClass SD; #endif diff --git a/libraries/SDFS/src/SDFS.h b/libraries/SDFS/src/SDFS.h index db55fe547a..9443642ee9 100644 --- a/libraries/SDFS/src/SDFS.h +++ b/libraries/SDFS/src/SDFS.h @@ -203,6 +203,20 @@ class SDFSImpl : public FSImpl return (clusterSize() * totalClusters()); } + // Helper function, takes FAT and makes standard time_t + static time_t FatToTimeT(uint16_t d, uint16_t t) { + struct tm tiempo; + memset(&tiempo, 0, sizeof(tiempo)); + tiempo.tm_sec = (((int)t) << 1) & 0x3e; + tiempo.tm_min = (((int)t) >> 5) & 0x3f; + tiempo.tm_hour = (((int)t) >> 11) & 0x1f; + tiempo.tm_mday = (int)(d & 0x1f); + tiempo.tm_mon = ((int)(d >> 5) & 0x0f) - 1; + tiempo.tm_year = ((int)(d >> 9) & 0x7f) + 80; + tiempo.tm_isdst = -1; + return mktime(&tiempo); + } + protected: friend class SDFileImpl; friend class SDFSDirImpl; @@ -212,6 +226,7 @@ class SDFSImpl : public FSImpl return &_fs; } + static uint8_t _getFlags(OpenMode openMode, AccessMode accessMode) { uint8_t mode = 0; if (openMode & OM_CREATE) { @@ -350,6 +365,17 @@ class SDFSFileImpl : public FileImpl return _opened ? _fd->isDirectory() : false; } + time_t getLastWrite() override { + time_t ftime = 0; + if (_opened && _fd) { + sdfat::dir_t tmp; + if (_fd.get()->dirEntry(&tmp)) { + ftime = SDFSImpl::FatToTimeT(tmp.lastWriteDate, tmp.lastWriteTime); + } + } + return ftime; + } + protected: SDFSImpl* _fs; @@ -425,6 +451,12 @@ class SDFSDirImpl : public DirImpl _size = file.fileSize(); _isFile = file.isFile(); _isDirectory = file.isDirectory(); + sdfat::dir_t tmp; + if (file.dirEntry(&tmp)) { + _time = SDFSImpl::FatToTimeT(tmp.lastWriteDate, tmp.lastWriteTime); + } else { + _time = 0; + } file.getName(_lfn, sizeof(_lfn)); file.close(); } else { @@ -447,6 +479,7 @@ class SDFSDirImpl : public DirImpl std::shared_ptr _dir; bool _valid; char _lfn[64]; + time_t _time; std::shared_ptr _dirPath; uint32_t _size; bool _isFile; diff --git a/libraries/esp8266/examples/SPIFFS_Time/SPIFFS_Time.ino b/libraries/esp8266/examples/SPIFFS_Time/SPIFFS_Time.ino new file mode 100644 index 0000000000..ed093ae10c --- /dev/null +++ b/libraries/esp8266/examples/SPIFFS_Time/SPIFFS_Time.ino @@ -0,0 +1,214 @@ +#include +#include +#include + +#ifndef STASSID +#define STASSID "your-ssid" +#define STAPSK "your-password" +#endif + +const char *ssid = STASSID; +const char *pass = STAPSK; + +long timezone = 2; +byte daysavetime = 1; + + +bool getLocalTime(struct tm * info, uint32_t ms) { + uint32_t count = ms / 10; + time_t now; + + time(&now); + localtime_r(&now, info); + + if (info->tm_year > (2016 - 1900)) { + return true; + } + + while (count--) { + delay(10); + time(&now); + localtime_r(&now, info); + if (info->tm_year > (2016 - 1900)) { + return true; + } + } + return false; +} + + +void listDir(const char * dirname) { + Serial.printf("Listing directory: %s\n", dirname); + + Dir root = SPIFFS.openDir(dirname); + + while (root.next()) { + File file = root.openFile("r"); + Serial.print(" FILE: "); + Serial.print(root.fileName()); + Serial.print(" SIZE: "); + Serial.print(file.size()); + time_t t = file.getLastWrite(); + struct tm * tmstruct = localtime(&t); + file.close(); + Serial.printf(" LAST WRITE: %d-%02d-%02d %02d:%02d:%02d\n", (tmstruct->tm_year) + 1900, (tmstruct->tm_mon) + 1, tmstruct->tm_mday, tmstruct->tm_hour, tmstruct->tm_min, tmstruct->tm_sec); + } +} + + +void readFile(const char * path) { + Serial.printf("Reading file: %s\n", path); + + File file = SPIFFS.open(path, "r"); + if (!file) { + Serial.println("Failed to open file for reading"); + return; + } + + Serial.print("Read from file: "); + while (file.available()) { + Serial.write(file.read()); + } + file.close(); +} + +void writeFile(const char * path, const char * message) { + Serial.printf("Writing file: %s\n", path); + + File file = SPIFFS.open(path, "w"); + if (!file) { + Serial.println("Failed to open file for writing"); + return; + } + if (file.print(message)) { + Serial.println("File written"); + } else { + Serial.println("Write failed"); + } + file.close(); +} + +void appendFile(const char * path, const char * message) { + Serial.printf("Appending to file: %s\n", path); + + File file = SPIFFS.open(path, "a"); + if (!file) { + Serial.println("Failed to open file for appending"); + return; + } + if (file.print(message)) { + Serial.println("Message appended"); + } else { + Serial.println("Append failed"); + } + file.close(); +} + +void renameFile(const char * path1, const char * path2) { + Serial.printf("Renaming file %s to %s\n", path1, path2); + if (SPIFFS.rename(path1, path2)) { + Serial.println("File renamed"); + } else { + Serial.println("Rename failed"); + } +} + +void deleteFile(const char * path) { + Serial.printf("Deleting file: %s\n", path); + if (SPIFFS.remove(path)) { + Serial.println("File deleted"); + } else { + Serial.println("Delete failed"); + } +} + +void setup() { + Serial.begin(115200); + // We start by connecting to a WiFi network + Serial.println(); + Serial.println(); + Serial.print("Connecting to "); + Serial.println(ssid); + + WiFi.begin(ssid, pass); + + while (WiFi.status() != WL_CONNECTED) { + delay(500); + Serial.print("."); + } + Serial.println("WiFi connected"); + Serial.println("IP address: "); + Serial.println(WiFi.localIP()); + Serial.println("Contacting Time Server"); + configTime(3600 * timezone, daysavetime * 3600, "time.nist.gov", "0.pool.ntp.org", "1.pool.ntp.org"); + struct tm tmstruct ; + delay(2000); + tmstruct.tm_year = 0; + getLocalTime(&tmstruct, 5000); + Serial.printf("\nNow is : %d-%02d-%02d %02d:%02d:%02d\n", (tmstruct.tm_year) + 1900, (tmstruct.tm_mon) + 1, tmstruct.tm_mday, tmstruct.tm_hour, tmstruct.tm_min, tmstruct.tm_sec); + Serial.println(""); + + Serial.printf("First, format an old-style filesystem without metadata\n"); + SPIFFS.setConfig(SPIFFSConfig(false, false)); + SPIFFS.format(); + SPIFFS.end(); + Serial.printf("Now mount it\n"); + if (!SPIFFS.begin()) { + Serial.println("SPIFFS mount failed"); + return; + } + + listDir("/"); + deleteFile("/hello.txt"); + writeFile("/hello.txt", "Hello "); + appendFile("/hello.txt", "World!\n"); + listDir("/"); + + Serial.printf("The timestamp should be invalid above since the filesystem doesn't have metadata\n"); + + Serial.printf("Now unmount and remount and perform the same operation.\n"); + Serial.printf("Timestamp should be invalid, but data should be good.\n"); + SPIFFS.end(); + Serial.printf("Now mount it\n"); + if (!SPIFFS.begin()) { + Serial.println("SPIFFS mount failed"); + return; + } + readFile("/hello.txt"); + listDir("/"); + + Serial.printf("Now format a new SPIFFS FS with timestamp support\n"); + + SPIFFS.end(); + SPIFFS.setConfig(SPIFFSConfig().setEnableTime(true)); + SPIFFS.format(); + SPIFFS.end(); + Serial.printf("Now mount it\n"); + if (!SPIFFS.begin()) { + Serial.println("SPIFFS mount failed"); + return; + } + + listDir("/"); + deleteFile("/hello.txt"); + writeFile("/hello.txt", "Hello "); + appendFile("/hello.txt", "World!\n"); + listDir("/"); + + Serial.printf("The timestamp should be valid above\n"); + + Serial.printf("Now unmount and remount and perform the same operation.\n"); + Serial.printf("Timestamp should be valid, data should be good.\n"); + SPIFFS.end(); + Serial.printf("Now mount it\n"); + if (!SPIFFS.begin()) { + Serial.println("SPIFFS mount failed"); + return; + } + readFile("/hello.txt"); + listDir("/"); + + +} + +void loop() { } From 2487e2212cb024d5af2d7addc179d69ff13dd714 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Fri, 19 Jul 2019 21:06:45 -0700 Subject: [PATCH 2/5] Remove global, make metadata length a member of FS --- cores/esp8266/spiffs/spiffs.h | 6 ++++-- cores/esp8266/spiffs/spiffs_config.h | 3 +-- cores/esp8266/spiffs_api.cpp | 4 +--- cores/esp8266/spiffs_api.h | 15 +++++++-------- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/cores/esp8266/spiffs/spiffs.h b/cores/esp8266/spiffs/spiffs.h index 746ad11963..61f3190928 100644 --- a/cores/esp8266/spiffs/spiffs.h +++ b/cores/esp8266/spiffs/spiffs.h @@ -291,6 +291,8 @@ typedef struct spiffs_t { void *user_data; // config magic u32_t config_magic; + // active metadata length + u32_t obj_meta_len; } spiffs; /* spiffs file status struct */ @@ -300,7 +302,7 @@ typedef struct { spiffs_obj_type type; spiffs_page_ix pix; u8_t name[SPIFFS_OBJ_NAME_LEN]; - u8_t meta[4]; // Set to the largest supported metadata len + u8_t meta[SPIFFS_MAX_META]; // Set to the largest supported metadata len } spiffs_stat; struct spiffs_dirent { @@ -309,7 +311,7 @@ struct spiffs_dirent { spiffs_obj_type type; u32_t size; spiffs_page_ix pix; - u8_t meta[4]; // Set to the largest supported metadata len + u8_t meta[SPIFFS_MAX_META]; // Set to the largest supported metadata len }; typedef struct { diff --git a/cores/esp8266/spiffs/spiffs_config.h b/cores/esp8266/spiffs/spiffs_config.h index df59a71f6a..2c0cc455bc 100644 --- a/cores/esp8266/spiffs/spiffs_config.h +++ b/cores/esp8266/spiffs/spiffs_config.h @@ -28,8 +28,7 @@ typedef uint8_t u8_t; // Enable supporting both timestamped and non-timestamped FS images by making // the meta-len a global variable here and by changing the logic of SPIFFS.begin. -extern int __SPIFFS_obj_meta_len; -#define SPIFFS_OBJ_META_LEN (__SPIFFS_obj_meta_len) +#define SPIFFS_OBJ_META_LEN (fs->obj_meta_len) #define SPIFFS_MAX_META (4) // Maximum metadata size allowed at runtime // Because SPIFFS reads binary images of blocks, we need to be sure to actually ensure there is space to // keep the metadata (since 0 bytes are allocated in the struct itself). Use an anonymous union to ensure diff --git a/cores/esp8266/spiffs_api.cpp b/cores/esp8266/spiffs_api.cpp index 32deedc208..c45883adc1 100644 --- a/cores/esp8266/spiffs_api.cpp +++ b/cores/esp8266/spiffs_api.cpp @@ -25,8 +25,6 @@ using namespace fs; -int __SPIFFS_obj_meta_len = 0; - namespace spiffs_impl { @@ -54,7 +52,7 @@ FileImplPtr SPIFFSImpl::open(const char* path, OpenMode openMode, AccessMode acc fd, path, openMode, accessMode, _fs.err_code); return FileImplPtr(); } - if (!(mode & SPIFFS_O_RDONLY) && __SPIFFS_obj_meta_len==sizeof(4)) { + if (!(mode & SPIFFS_O_RDONLY) && _fs.obj_meta_len) { time_t t = time(NULL); struct tm tmr; localtime_r(&t, &tmr); diff --git a/cores/esp8266/spiffs_api.h b/cores/esp8266/spiffs_api.h index 40aeea4904..9e108cf3f4 100644 --- a/cores/esp8266/spiffs_api.h +++ b/cores/esp8266/spiffs_api.h @@ -39,7 +39,6 @@ extern "C" { using namespace fs; -extern int __SPIFFS_obj_meta_len; namespace spiffs_impl { int getSpiffsMode(OpenMode openMode, AccessMode accessMode); @@ -273,8 +272,8 @@ class SPIFFSImpl : public FSImpl } } // Make sure we format with requested metadata len - __SPIFFS_obj_meta_len = _cfg._enableTime ? 4 : 0; - DEBUGV("SPIFFSImpl::_formatOldOrNew formatting with metadata==%d\r\n", __SPIFFS_obj_meta_len); + _fs.obj_meta_len = _cfg._enableTime ? 4 : 0; + DEBUGV("SPIFFSImpl::_formatOldOrNew formatting with metadata==%d\r\n", _fs.obj_meta_len); spiffs_config config = _setupSpiffsConfig(_cfg._enableTime); // We need to try a mount on SPIFFS, even though it will probably fail, to make the opaque @@ -326,18 +325,18 @@ class SPIFFSImpl : public FSImpl // First, can we mount w/o metadata (preserve backwards) int err; if ( !_is_metadata_fs(_start, _blockSize, _pageSize) ) { - __SPIFFS_obj_meta_len = 0; + _fs.obj_meta_len = 0; DEBUGV("SPIFFSImpl: trying fs @%x, size=%x, block=%x, page=%x, metadata=%d\r\n", - _start, _size, _blockSize, _pageSize, __SPIFFS_obj_meta_len); + _start, _size, _blockSize, _pageSize, _fs.obj_meta_len); err = SPIFFS_mount(&_fs, &config, _workBuf.get(), _fdsBuf.get(), fdsBufSize, _cacheBuf.get(), cacheBufSize, &SPIFFSImpl::_check_cb); } else { // Flag matched, it's a metadata FS - __SPIFFS_obj_meta_len = 4; + _fs.obj_meta_len = 4; config = _setupSpiffsConfig(true); DEBUGV("SPIFFSImpl: doesn't look like old metadata==0, so trying fs @%x, size=%x, block=%x, page=%x, metadata=%d\r\n", - _start, _size, _blockSize, _pageSize, __SPIFFS_obj_meta_len); + _start, _size, _blockSize, _pageSize, _fs.obj_meta_len); err = SPIFFS_mount(&_fs, &config, _workBuf.get(), _fdsBuf.get(), fdsBufSize, _cacheBuf.get(), cacheBufSize, &SPIFFSImpl::_check_cb); @@ -573,7 +572,7 @@ class SPIFFSFileImpl : public FileImpl { CHECKFD(); time_t t = 0; - if (__SPIFFS_obj_meta_len) { + if (_fs->getFs()->obj_meta_len) { _getStat() ; memcpy(&t, _stat.meta, sizeof(time_t)); } From a1231e7959c74107739a287d0a51a8682a67c4e9 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sat, 20 Jul 2019 08:16:54 -0700 Subject: [PATCH 3/5] Move the meta-len to the config struct for SPIFFS It seems more appropriate to use the config struct tio hold the meta-len and not the spiffs fs object itself. Cleans up the higher layewr code a bit, too. --- cores/esp8266/spiffs/spiffs.h | 5 ++--- cores/esp8266/spiffs/spiffs_config.h | 2 +- cores/esp8266/spiffs_api.cpp | 2 +- cores/esp8266/spiffs_api.h | 12 +++++------- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/cores/esp8266/spiffs/spiffs.h b/cores/esp8266/spiffs/spiffs.h index 61f3190928..e2e61f854e 100644 --- a/cores/esp8266/spiffs/spiffs.h +++ b/cores/esp8266/spiffs/spiffs.h @@ -220,12 +220,13 @@ typedef struct { // logical size of a page, must be at least // log_block_size / 8 u32_t log_page_size; - #endif #if SPIFFS_FILEHDL_OFFSET // an integer offset added to each file handle u16_t fh_ix_offset; #endif + // active metadata length + u32_t obj_meta_len; } spiffs_config; typedef struct spiffs_t { @@ -291,8 +292,6 @@ typedef struct spiffs_t { void *user_data; // config magic u32_t config_magic; - // active metadata length - u32_t obj_meta_len; } spiffs; /* spiffs file status struct */ diff --git a/cores/esp8266/spiffs/spiffs_config.h b/cores/esp8266/spiffs/spiffs_config.h index 2c0cc455bc..41c035d9a9 100644 --- a/cores/esp8266/spiffs/spiffs_config.h +++ b/cores/esp8266/spiffs/spiffs_config.h @@ -28,7 +28,7 @@ typedef uint8_t u8_t; // Enable supporting both timestamped and non-timestamped FS images by making // the meta-len a global variable here and by changing the logic of SPIFFS.begin. -#define SPIFFS_OBJ_META_LEN (fs->obj_meta_len) +#define SPIFFS_OBJ_META_LEN (fs->cfg.obj_meta_len) #define SPIFFS_MAX_META (4) // Maximum metadata size allowed at runtime // Because SPIFFS reads binary images of blocks, we need to be sure to actually ensure there is space to // keep the metadata (since 0 bytes are allocated in the struct itself). Use an anonymous union to ensure diff --git a/cores/esp8266/spiffs_api.cpp b/cores/esp8266/spiffs_api.cpp index c45883adc1..8fb7cb158a 100644 --- a/cores/esp8266/spiffs_api.cpp +++ b/cores/esp8266/spiffs_api.cpp @@ -52,7 +52,7 @@ FileImplPtr SPIFFSImpl::open(const char* path, OpenMode openMode, AccessMode acc fd, path, openMode, accessMode, _fs.err_code); return FileImplPtr(); } - if (!(mode & SPIFFS_O_RDONLY) && _fs.obj_meta_len) { + if (!(mode & SPIFFS_O_RDONLY) && _fs.cfg.obj_meta_len) { time_t t = time(NULL); struct tm tmr; localtime_r(&t, &tmr); diff --git a/cores/esp8266/spiffs_api.h b/cores/esp8266/spiffs_api.h index 9e108cf3f4..35e4a07250 100644 --- a/cores/esp8266/spiffs_api.h +++ b/cores/esp8266/spiffs_api.h @@ -239,6 +239,7 @@ class SPIFFSImpl : public FSImpl config.phys_erase_block = FLASH_SECTOR_SIZE; config.log_block_size = _blockSize; config.log_page_size = _pageSize; + config.obj_meta_len = metadata ? 4 : 0; if (((uint32_t) std::numeric_limits::max()) < (_size / _blockSize)) { DEBUGV("spiffs_block_ix type too small"); @@ -272,9 +273,8 @@ class SPIFFSImpl : public FSImpl } } // Make sure we format with requested metadata len - _fs.obj_meta_len = _cfg._enableTime ? 4 : 0; - DEBUGV("SPIFFSImpl::_formatOldOrNew formatting with metadata==%d\r\n", _fs.obj_meta_len); spiffs_config config = _setupSpiffsConfig(_cfg._enableTime); + DEBUGV("SPIFFSImpl::_formatOldOrNew formatting with metadata==%d\r\n", config.obj_meta_len); // We need to try a mount on SPIFFS, even though it will probably fail, to make the opaque // SPIFFS _fs struct take the new sizes specified in the configuration. @@ -325,18 +325,16 @@ class SPIFFSImpl : public FSImpl // First, can we mount w/o metadata (preserve backwards) int err; if ( !_is_metadata_fs(_start, _blockSize, _pageSize) ) { - _fs.obj_meta_len = 0; DEBUGV("SPIFFSImpl: trying fs @%x, size=%x, block=%x, page=%x, metadata=%d\r\n", - _start, _size, _blockSize, _pageSize, _fs.obj_meta_len); + _start, _size, _blockSize, _pageSize, _fs.cfg.obj_meta_len); err = SPIFFS_mount(&_fs, &config, _workBuf.get(), _fdsBuf.get(), fdsBufSize, _cacheBuf.get(), cacheBufSize, &SPIFFSImpl::_check_cb); } else { // Flag matched, it's a metadata FS - _fs.obj_meta_len = 4; config = _setupSpiffsConfig(true); DEBUGV("SPIFFSImpl: doesn't look like old metadata==0, so trying fs @%x, size=%x, block=%x, page=%x, metadata=%d\r\n", - _start, _size, _blockSize, _pageSize, _fs.obj_meta_len); + _start, _size, _blockSize, _pageSize, _fs.cfg.obj_meta_len); err = SPIFFS_mount(&_fs, &config, _workBuf.get(), _fdsBuf.get(), fdsBufSize, _cacheBuf.get(), cacheBufSize, &SPIFFSImpl::_check_cb); @@ -572,7 +570,7 @@ class SPIFFSFileImpl : public FileImpl { CHECKFD(); time_t t = 0; - if (_fs->getFs()->obj_meta_len) { + if (_fs->getFs()->cfg.obj_meta_len) { _getStat() ; memcpy(&t, _stat.meta, sizeof(time_t)); } From 7dccf0eb1f18c00ae0e5ae19bf25981506eecb53 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sat, 20 Jul 2019 09:40:15 -0700 Subject: [PATCH 4/5] Update to latest SDFat with #ifdef configurations Allow user overrides of SDFat configuration settings in platform.txt by bracketing the config.h file defines with ifndefs. --- libraries/ESP8266SdFat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266SdFat b/libraries/ESP8266SdFat index 05fd2c86ad..f84b1a5aee 160000 --- a/libraries/ESP8266SdFat +++ b/libraries/ESP8266SdFat @@ -1 +1 @@ -Subproject commit 05fd2c86ad6493e99821cbd06a66fc05c5325ba8 +Subproject commit f84b1a5aeecdeb80e05b4e7b1e58c64c62183088 From 3ae2f63c09474ec0b7e5770af681835d3e087114 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Fri, 27 Dec 2019 07:53:49 -0800 Subject: [PATCH 5/5] Update FS.h --- cores/esp8266/FS.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index 08921e490d..c737516cfa 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -190,7 +190,7 @@ class SPIFFSConfig : public FSConfig public: static constexpr uint32_t FSId = 0x53504946; SPIFFSConfig(bool autoFormat = true, bool enableTime = true) { - _type = SPIFFSConfig::fsid::FSId; + _type = SPIFFSConfig::FSId; _autoFormat = autoFormat; _enableTime = enableTime; }