Add time to filesystems, new API calls

Support the ESP32 File::getLastWrite() call and setting the time on
all filesystems automatically (assuming the system clock has
been set properly and time(NULL) returns the proper time!).

Adds Dir::fileTime() to get the time of a file being listed, similar to
Dir::fileName() and Dir::fileSize().

Adds ::setTimeCallback(time_t (*cb)()) to File, Dir, and FS, allowing
users to override the default timestamp on a per-file, directory, or
filesystem basis.  By default, a simple callback returning time(nullptr)
is implemented.

LittleFS uses the 't' attribute and should be backwards compatible.

SD/SDFS work and include wrappers for obsolete SdFat timestamp callbacks
using the MSDOS time.

SPIFFS has been updated to support either the existing on-flash format
without any timestamps, or a new on-flash format with a flag area and a
timestamp metadata element added.

SPIFFS will try and mount any existing filesystem as non-timestamp, and
continue to write them in the old format.

New filesystems may be formatted to include timestamps (default) or in
the old format, configurable via the SPIFFS.setConfig call.

The SPIFFS work took as starting point all of work of @luc-github in #3730
and extended it into a backwards compatible mode by doing much surgery
on the SPIFFS library itself and the on-flash format.

Includes an updated SD/listfiles and SPIFFS_time example.

Author: earlephilhower

cores/esp8266/FS.cpp

@@ -180,6 +180,19 @@ String File::readString()
     return ret;
 }
 
+time_t File::getLastWrite() {
+    if (!_p)
+        return 0;
+
+    return _p->getLastWrite();
+}
+
+void File::setTimeCallback(time_t (*cb)(void)) {
+    if (!_p)
+        return;
+    _p->setTimeCallback(cb);
+}
+
 File Dir::openFile(const char* mode) {
     if (!_impl) {
         return File();
@@ -192,7 +205,9 @@ File Dir::openFile(const char* mode) {
         return File();
     }
 
-    return File(_impl->openFile(om, am), _baseFS);
+    auto f = File(_impl->openFile(om, am), _baseFS);
+    f.setTimeCallback(timeCallback);
+    return f;
 }
 
 String Dir::fileName() {
@@ -203,6 +218,12 @@ String Dir::fileName() {
     return _impl->fileName();
 }
 
+time_t Dir::fileTime() {
+    if (!_impl)
+        return 0;
+    return _impl->fileTime();
+}
+
 size_t Dir::fileSize() {
     if (!_impl) {
         return 0;
@@ -241,6 +262,20 @@ bool Dir::rewind() {
     return _impl->rewind();
 }
 
+time_t Dir::getLastWrite() {
+    if (!_impl)
+        return 0;
+
+    return _impl->getLastWrite();
+}
+
+void Dir::setTimeCallback(time_t (*cb)(void)) {
+    if (!_impl)
+        return;
+    _impl->setTimeCallback(cb);
+}
+
+
 bool FS::setConfig(const FSConfig &cfg) {
     if (!_impl) {
         return false;
@@ -308,7 +343,9 @@ File FS::open(const char* path, const char* mode) {
         DEBUGV("FS::open: invalid mode `%s`\r\n", mode);
         return File();
     }
-    return File(_impl->open(path, om, am), this);
+    auto f = File(_impl->open(path, om, am), this);
+    f.setTimeCallback(timeCallback);
+    return f;
 }
 
 bool FS::exists(const char* path) {
@@ -327,7 +364,9 @@ Dir FS::openDir(const char* path) {
         return Dir();
     }
     DirImplPtr p = _impl->openDir(path);
-    return Dir(p, this);
+    auto d = Dir(p, this);
+    d.setTimeCallback(timeCallback);
+    return d;
 }
 
 Dir FS::openDir(const String& path) {
@@ -378,6 +417,11 @@ bool FS::rename(const String& pathFrom, const String& pathTo) {
     return rename(pathFrom.c_str(), pathTo.c_str());
 }
 
+void FS::setTimeCallback(time_t (*cb)(void)) {
+    if (!_impl)
+        return;
+    _impl->setTimeCallback(cb);
+}
 
 
 static bool sflags(const char* mode, OpenMode& om, AccessMode& am) {

cores/esp8266/FS.h

@@ -110,12 +110,16 @@ class File : public Stream
 
     String readString() override;
 
+    time_t getLastWrite();
+    void setTimeCallback(time_t (*cb)(void));
+
 protected:
     FileImplPtr _p;
 
     // Arduino SD class emulation
     std::shared_ptr<Dir> _fakeDir;
     FS                  *_baseFS;
+    time_t (*timeCallback)(void) = nullptr;
 };
 
 class Dir {
@@ -126,15 +130,21 @@ class Dir {
 
     String fileName();
     size_t fileSize();
+    time_t fileTime();
     bool isFile() const;
     bool isDirectory() const;
 
     bool next();
     bool rewind();
 
+    time_t getLastWrite();
+    void setTimeCallback(time_t (*cb)(void));
+
 protected:
     DirImplPtr _impl;
     FS       *_baseFS;
+    time_t (*timeCallback)(void) = nullptr;
+
 };
 
 // Backwards compatible, <4GB filesystem usage
@@ -179,17 +189,26 @@ class FSConfig
 class SPIFFSConfig : public FSConfig
 {
 public:
-    SPIFFSConfig(bool autoFormat = true) {
+    SPIFFSConfig(bool autoFormat = true, bool enableTime = true) {
         _type = SPIFFSConfig::fsid::FSId;
 	_autoFormat = autoFormat;
+	_enableTime = enableTime;
     }
     enum fsid { FSId = 0x53504946 };
+
+    SPIFFSConfig setEnableTime(bool val = true) {
+        _enableTime = val;
+        return *this;
+    }
+
+    // Inherit _type and _autoFormat
+    bool _enableTime;
 };
 
 class FS
 {
 public:
-    FS(FSImplPtr impl) : _impl(impl) { }
+    FS(FSImplPtr impl) : _impl(impl) { timeCallback = _defaultTimeCB; }
 
     bool setConfig(const FSConfig &cfg);
 
@@ -223,10 +242,14 @@ class FS
 
     bool gc();
 
+    void setTimeCallback(time_t (*cb)(void));
+
     friend class ::SDClass; // More of a frenemy, but SD needs internal implementation to get private FAT bits
 protected:
     FSImplPtr _impl;
     FSImplPtr getImpl() { return _impl; }
+    time_t (*timeCallback)(void);
+    static time_t _defaultTimeCB(void) { return time(NULL); }
 };
 
 } // namespace fs
@@ -241,6 +264,7 @@ using fs::SeekCur;
 using fs::SeekEnd;
 using fs::FSInfo;
 using fs::FSConfig;
+using fs::SPIFFSConfig;
 #endif //FS_NO_GLOBALS
 
 #if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_SPIFFS)

cores/esp8266/FSImpl.h

@@ -41,6 +41,19 @@ class FileImpl {
     virtual const char* fullName() const = 0;
     virtual bool isFile() const = 0;
     virtual bool isDirectory() const = 0;
+
+    // Filesystems *may* support a timestamp per-file, so allow the user to override with
+    // their own callback for *this specific* file (as opposed to the FSImpl call of the
+    // same name.  The default implementation simply returns time(&null)
+    virtual void setTimeCallback(time_t (*cb)(void)) { timeCallback = cb; }
+
+    // Return the last written time for a file.  Undefined when called on a writable file
+    // as the FS is allowed to return either the time of the last write() operation or the
+    // time present in the filesystem metadata (often the last time the file was closed)
+    virtual time_t getLastWrite() { return 0; } // Default is to not support timestamps
+
+protected:
+    time_t (*timeCallback)(void) = nullptr;
 };
 
 enum OpenMode {
@@ -62,10 +75,24 @@ class DirImpl {
     virtual FileImplPtr openFile(OpenMode openMode, AccessMode accessMode) = 0;
     virtual const char* fileName() = 0;
     virtual size_t fileSize() = 0;
+    virtual time_t fileTime() { return 0; } // By default, FS doesn't report file times
     virtual bool isFile() const = 0;
     virtual bool isDirectory() const = 0;
     virtual bool next() = 0;
     virtual bool rewind() = 0;
+
+    // Filesystems *may* support a timestamp per-file, so allow the user to override with
+    // their own callback for *this specific* file (as opposed to the FSImpl call of the
+    // same name.  The default implementation simply returns time(&null)
+    virtual void setTimeCallback(time_t (*cb)(void)) { timeCallback = cb; }
+
+    // Return the last written time for a file.  Undefined when called on a writable file
+    // as the FS is allowed to return either the time of the last write() operation or the
+    // time present in the filesystem metadata (often the last time the file was closed)
+    virtual time_t getLastWrite() { return 0; } // Default is to not support timestamps
+
+protected:
+    time_t (*timeCallback)(void) = nullptr;
 };
 
 class FSImpl {
@@ -85,6 +112,14 @@ class FSImpl {
     virtual bool mkdir(const char* path) = 0;
     virtual bool rmdir(const char* path) = 0;
     virtual bool gc() { return true; } // May not be implemented in all file systems.
+
+    // Filesystems *may* support a timestamp per-file, so allow the user to override with
+    // their own callback for all files on this FS.  The default implementation simply
+    // returns the present time as reported by time(&null)
+    virtual void setTimeCallback(time_t (*cb)(void)) { timeCallback = cb; }
+
+protected:
+    time_t (*timeCallback)(void) = nullptr;
 };
 
 } // namespace fs

cores/esp8266/spiffs/spiffs.h

@@ -300,9 +300,7 @@ typedef struct {
   spiffs_obj_type type;
   spiffs_page_ix pix;
   u8_t name[SPIFFS_OBJ_NAME_LEN];
-#if SPIFFS_OBJ_META_LEN
-  u8_t meta[SPIFFS_OBJ_META_LEN];
-#endif
+  u8_t meta[4]; // Set to the largest supported metadata len
 } spiffs_stat;
 
 struct spiffs_dirent {
@@ -311,9 +309,7 @@ struct spiffs_dirent {
   spiffs_obj_type type;
   u32_t size;
   spiffs_page_ix pix;
-#if SPIFFS_OBJ_META_LEN
-  u8_t meta[SPIFFS_OBJ_META_LEN];
-#endif
+  u8_t meta[4]; // Set to the largest supported metadata len
 };
 
 typedef struct {
@@ -534,7 +530,7 @@ s32_t SPIFFS_close(spiffs *fs, spiffs_file fh);
  */
 s32_t SPIFFS_rename(spiffs *fs, const char *old, const char *newPath);
 
-#if SPIFFS_OBJ_META_LEN
+
 /**
  * Updates file's metadata
  * @param fs            the file system struct
@@ -550,7 +546,7 @@ s32_t SPIFFS_update_meta(spiffs *fs, const char *name, const void *meta);
  * @param meta          new metadata. must be SPIFFS_OBJ_META_LEN bytes long.
  */
 s32_t SPIFFS_fupdate_meta(spiffs *fs, spiffs_file fh, const void *meta);
-#endif
+
 
 /**
  * Returns last error of last file operation.

cores/esp8266/spiffs/spiffs_check.cpp

@@ -64,7 +64,7 @@ static s32_t spiffs_object_get_data_page_index_reference(
   u32_t addr = SPIFFS_PAGE_TO_PADDR(fs, *objix_pix);
   if (objix_spix == 0) {
     // get referenced page from object index header
-    addr += sizeof(spiffs_page_object_ix_header) + data_spix * sizeof(spiffs_page_ix);
+    addr += sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN + data_spix * sizeof(spiffs_page_ix);
   } else {
     // get referenced page from object index
     addr += sizeof(spiffs_page_object_ix) + SPIFFS_OBJ_IX_ENTRY(fs, data_spix) * sizeof(spiffs_page_ix);
@@ -136,7 +136,7 @@ static s32_t spiffs_rewrite_index(spiffs *fs, spiffs_obj_id obj_id, spiffs_span_
 
   // rewrite in mem
   if (objix_spix == 0) {
-    ((spiffs_page_ix*)((u8_t *)fs->lu_work + sizeof(spiffs_page_object_ix_header)))[data_spix] = new_data_pix;
+    ((spiffs_page_ix*)((u8_t *)fs->lu_work + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN))[data_spix] = new_data_pix;
   } else {
     ((spiffs_page_ix*)((u8_t *)fs->lu_work + sizeof(spiffs_page_object_ix)))[SPIFFS_OBJ_IX_ENTRY(fs, data_spix)] = new_data_pix;
   }
@@ -574,7 +574,7 @@ static s32_t spiffs_page_consistency_check_i(spiffs *fs) {
             // object header page index
             entries = SPIFFS_OBJ_HDR_IX_LEN(fs);
             data_spix_offset = 0;
-            object_page_index = (spiffs_page_ix *)((u8_t *)fs->lu_work + sizeof(spiffs_page_object_ix_header));
+            object_page_index = (spiffs_page_ix *)((u8_t *)fs->lu_work + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN);
           } else {
             // object page index
             entries = SPIFFS_OBJ_IX_LEN(fs);

cores/esp8266/spiffs/spiffs_config.h

@@ -25,6 +25,22 @@ typedef uint16_t u16_t;
 typedef int8_t s8_t;
 typedef uint8_t u8_t;
 
+
+// Enable supporting both timestamped and non-timestamped FS images by making
+// the meta-len a global variable here and by changing the logic of SPIFFS.begin.
+extern int __SPIFFS_obj_meta_len;
+#define SPIFFS_OBJ_META_LEN (__SPIFFS_obj_meta_len)
+#define SPIFFS_MAX_META (4) // Maximum metadata size allowed at runtime
+// Because SPIFFS reads binary images of blocks, we need to be sure to actually ensure there is space to
+// keep the metadata (since 0 bytes are allocated in the struct itself).  Use an anonymous union to ensure
+// that there is at least MAX_META bytes after the end of the struct.
+#define alloc_spiffs_page_object_ix_header(name) \
+  union { \
+    spiffs_page_object_ix_header name; \
+    char xxx[sizeof(spiffs_page_object_ix_header) + SPIFFS_MAX_META]; \
+  };
+
+
 #ifndef SEEK_SET
 #define	SEEK_SET	0	/* set file offset to offset */
 #endif

cores/esp8266/spiffs/spiffs_gc.cpp

@@ -467,7 +467,7 @@ s32_t spiffs_gc_clean(spiffs *fs, spiffs_block_ix bix) {
               // update memory representation of object index page with new data page
               if (gc.cur_objix_spix == 0) {
                 // update object index header page
-                ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)))[p_hdr.span_ix] = new_data_pix;
+                ((spiffs_page_ix*)((u8_t *)objix_hdr + sizeof(spiffs_page_object_ix_header)+SPIFFS_OBJ_META_LEN))[p_hdr.span_ix] = new_data_pix;
                 SPIFFS_GC_DBG("gc_clean: MOVE_DATA wrote page " _SPIPRIpg" to objix_hdr entry " _SPIPRIsp" in mem\n", new_data_pix, (spiffs_span_ix)SPIFFS_OBJ_IX_ENTRY(fs, p_hdr.span_ix));
               } else {
                 // update object index page