Merge branch 'feature/nvs_version_check_v3.1' into 'release/v3.1'

nvs_flash: Version compatibility check for nvs storage (backport 3.1)

See merge request idf/esp-idf!2958

Author: jack0c

components/driver/test/test_adc2.c

@@ -56,8 +56,8 @@ TEST_CASE("adc2 work with wifi","[adc]")
     //init wifi
     printf("nvs init\n");
     esp_err_t r = nvs_flash_init();
-    if (r == ESP_ERR_NVS_NO_FREE_PAGES) {
-        printf("no free pages, erase..\n");
+    if (r == ESP_ERR_NVS_NO_FREE_PAGES || r == ESP_ERR_NVS_NEW_VERSION_FOUND) {
+        printf("no free pages or nvs version mismatch, erase..\n");
         TEST_ESP_OK(nvs_flash_erase());
         r = nvs_flash_init();
     } 

components/esp32/esp_err_to_name.c

@@ -157,6 +157,11 @@ static const esp_err_msg_t esp_err_msg_table[] = {
 #   ifdef      ESP_ERR_NVS_PART_NOT_FOUND
     ERR_TBL_IT(ESP_ERR_NVS_PART_NOT_FOUND),                 /*  4367 0x110f Partition with specified name is not found
                                                                             in the partition table */
+#   endif
+#   ifdef      ESP_ERR_NVS_NEW_VERSION_FOUND
+    ERR_TBL_IT(ESP_ERR_NVS_NEW_VERSION_FOUND),              /*  4368 0x1110 NVS partition contains data in new format
+                                                                            and cannot be recognized by this version of
+                                                                            code */
 #   endif
     // components/ulp/include/esp32/ulp.h
 #   ifdef      ESP_ERR_ULP_BASE

components/esp32/test/test_esp32.c

@@ -83,8 +83,8 @@ TEST_CASE("wifi stop and deinit","[wifi]")
     //init nvs
     ESP_LOGI(TAG, EMPH_STR("nvs_flash_init"));
     esp_err_t r = nvs_flash_init();
-    if (r == ESP_ERR_NVS_NO_FREE_PAGES) {
-        ESP_LOGI(TAG, EMPH_STR("no free pages, erase.."));
+    if (r == ESP_ERR_NVS_NO_FREE_PAGES || r == ESP_ERR_NVS_NEW_VERSION_FOUND) {
+        ESP_LOGI(TAG, EMPH_STR("no free pages or nvs version mismatch, erase.."));
         TEST_ESP_OK(nvs_flash_erase());
         r = nvs_flash_init();
     } 

components/nvs_flash/include/nvs.h

@@ -44,6 +44,7 @@ typedef uint32_t nvs_handle;
 #define ESP_ERR_NVS_NO_FREE_PAGES       (ESP_ERR_NVS_BASE + 0x0d)  /*!< NVS partition doesn't contain any empty pages. This may happen if NVS partition was truncated. Erase the whole partition and call nvs_flash_init again. */
 #define ESP_ERR_NVS_VALUE_TOO_LONG      (ESP_ERR_NVS_BASE + 0x0e)  /*!< String or blob length is longer than supported by the implementation */
 #define ESP_ERR_NVS_PART_NOT_FOUND      (ESP_ERR_NVS_BASE + 0x0f)  /*!< Partition with specified name is not found in the partition table */
+#define ESP_ERR_NVS_NEW_VERSION_FOUND   (ESP_ERR_NVS_BASE + 0x10)  /*!< NVS partition contains data in new format and cannot be recognized by this version of code */
 
 #define NVS_DEFAULT_PART_NAME           "nvs"   /*!< Default partition name of the NVS partition in the partition table */
 /**

components/nvs_flash/src/nvs_page.cpp

@@ -64,6 +64,11 @@ esp_err_t Page::load(uint32_t sectorNumber)
     } else {
         mState = header.mState;
         mSeqNumber = header.mSeqNumber;
+        if(header.mVersion < NVS_VERSION) {
+            return ESP_ERR_NVS_NEW_VERSION_FOUND;
+        } else {
+            mVersion = header.mVersion;
+        }
     }
 
     switch (mState) {
@@ -628,6 +633,7 @@ esp_err_t Page::initialize()
     Header header;
     header.mState = mState;
     header.mSeqNumber = mSeqNumber;
+    header.mVersion = mVersion;
     header.mCrc32 = header.calculateCrc32();
 
     auto rc = spi_flash_write(mBaseAddress, &header, sizeof(header));
@@ -799,6 +805,15 @@ esp_err_t Page::setSeqNumber(uint32_t seqNumber)
     return ESP_OK;
 }
 
+esp_err_t Page::setVersion(uint8_t ver)
+{
+    if (mState != PageState::UNINITIALIZED) {
+        return ESP_ERR_NVS_INVALID_STATE;
+    }
+    mVersion = ver;
+    return ESP_OK;
+}
+
 esp_err_t Page::erase()
 {
     auto sector = mBaseAddress / SPI_FLASH_SEC_SIZE;

components/nvs_flash/src/nvs_page.hpp

@@ -51,6 +51,8 @@ class Page : public intrusive_list_node<Page>
     static const uint8_t NS_INDEX = 0;
     static const uint8_t NS_ANY = 255;
 
+    static const uint8_t NVS_VERSION = 0xff; // Decrement to upgrade
+
     enum class PageState : uint32_t {
         // All bits set, default state after flash erase. Page has not been initialized yet.
         UNINITIALIZED = 0xffffffff,
@@ -83,6 +85,8 @@ class Page : public intrusive_list_node<Page>
     esp_err_t getSeqNumber(uint32_t& seqNumber) const;
 
     esp_err_t setSeqNumber(uint32_t seqNumber);
+ 
+    esp_err_t setVersion(uint8_t version);
 
     esp_err_t writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize);
 
@@ -142,12 +146,13 @@ class Page : public intrusive_list_node<Page>
     public:
         Header()
         {
-            std::fill_n(mReserved, sizeof(mReserved)/sizeof(mReserved[0]), UINT32_MAX);
+            std::fill_n(mReserved, sizeof(mReserved)/sizeof(mReserved[0]), UINT8_MAX);
         }
 
         PageState mState;       // page state
         uint32_t mSeqNumber;    // sequence number of this page
-        uint32_t mReserved[5];  // unused, must be 0xffffffff
+        uint8_t mVersion;       // nvs format version
+        uint8_t mReserved[19];  // unused, must be 0xff
         uint32_t mCrc32;        // crc of everything except mState
 
         uint32_t calculateCrc32();
@@ -198,6 +203,7 @@ class Page : public intrusive_list_node<Page>
     uint32_t mBaseAddress = 0;
     PageState mState = PageState::INVALID;
     uint32_t mSeqNumber = UINT32_MAX;
+    uint8_t mVersion = NVS_VERSION;
     typedef CompressedEnumTable<EntryState, 2, ENTRY_COUNT> TEntryTable;
     TEntryTable mEntryTable;
     size_t mNextFreeEntry = INVALID_ENTRY;

components/nvs_flash/test/test_nvs.c

@@ -16,7 +16,7 @@ TEST_CASE("various nvs tests", "[nvs]")
 {
     nvs_handle handle_1;
     esp_err_t err = nvs_flash_init();
-    if (err == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (err == ESP_ERR_NVS_NO_FREE_PAGES || err == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_LOGW(TAG, "nvs_flash_init failed (0x%x), erasing partition and retrying", err);
         const esp_partition_t* nvs_partition = esp_partition_find_first(
                 ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS, NULL);
@@ -83,7 +83,7 @@ TEST_CASE("calculate used and free space", "[nvs]")
     TEST_ASSERT_TRUE(h_count_entries == 0);
 
     esp_err_t err = nvs_flash_init();
-    if (err == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (err == ESP_ERR_NVS_NO_FREE_PAGES || err == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_LOGW(TAG, "nvs_flash_init failed (0x%x), erasing partition and retrying", err);
         const esp_partition_t* nvs_partition = esp_partition_find_first(
                 ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS, NULL);

examples/bluetooth/a2dp_gatts_coex/main/main.c

@@ -636,7 +636,7 @@ void app_main()
 
     // Initialize NVS.
     ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }
@@ -703,4 +703,4 @@ void app_main()
     bt_app_work_dispatch(bt_av_hdl_stack_evt, BT_APP_EVT_STACK_UP, NULL, 0, NULL);
     //gatt server init
     ble_gatts_init();
-}
+}

examples/bluetooth/a2dp_sink/main/main.c

@@ -46,7 +46,7 @@ void app_main()
 {
     /* Initialize NVS — it is used to store PHY calibration data */
     esp_err_t ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/a2dp_source/main/main.c

@@ -100,7 +100,7 @@ void app_main()
 {
     // Initialize NVS.
     esp_err_t ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/ble_adv/main/app_bt.c

@@ -214,7 +214,7 @@ void app_main()
 {
     /* Initialize NVS — it is used to store PHY calibration data */
     esp_err_t ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/ble_hid_device_demo/main/ble_hidd_demo_main.c

@@ -265,7 +265,7 @@ void app_main()
 
     // Initialize NVS.
     ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/ble_spp_server/main/ble_spp_server_demo.c

@@ -655,7 +655,7 @@ void app_main()
 
     // Initialize NVS
     ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/ble_throughput/throughput_client/main/example_ble_client_throughput.c

@@ -519,7 +519,7 @@ void app_main()
 {
     // Initialize NVS.
     esp_err_t ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/ble_throughput/throughput_server/main/example_ble_server_throughput.c

@@ -641,7 +641,7 @@ void app_main()
 
     // Initialize NVS.
     ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/blufi/main/blufi_example_main.c

@@ -381,7 +381,7 @@ void app_main()
 
     // Initialize NVS
     ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/bt_discovery/main/bt_discovery.c

@@ -273,7 +273,7 @@ void app_main()
 {
     /* Initialize NVS — it is used to store PHY calibration data */
     esp_err_t ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/bt_spp_acceptor/main/example_spp_acceptor_demo.c

@@ -108,7 +108,7 @@ static void esp_spp_cb(esp_spp_cb_event_t event, esp_spp_cb_param_t *param)
 void app_main()
 {
     esp_err_t ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/bt_spp_initiator/main/example_spp_initiator_demo.c

@@ -202,7 +202,7 @@ void app_main()
     }
 
     esp_err_t ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/bt_spp_vfs_acceptor/main/example_spp_vfs_acceptor_demo.c

@@ -108,7 +108,7 @@ static void esp_spp_stack_cb(esp_spp_cb_event_t event, esp_spp_cb_param_t *param
 void app_main()
 {
     esp_err_t ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/bt_spp_vfs_initiator/main/example_spp_vfs_initiator_demo.c

@@ -187,7 +187,7 @@ void app_main()
     }
 
     esp_err_t ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/controller_hci_uart/main/controller_hci_uart_demo.c

@@ -37,7 +37,7 @@ void app_main()
 
     /* Initialize NVS — it is used to store PHY calibration data */
     ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/gatt_client/main/gattc_demo.c

@@ -414,7 +414,7 @@ void app_main()
 {
     // Initialize NVS.
     esp_err_t ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/gatt_client/tutorial/Gatt_Client_Example_Walkthrough.md

@@ -41,7 +41,7 @@ void app_main()
 {
     // Initialize NVS.
     esp_err_t ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }
@@ -103,7 +103,7 @@ The main function starts by initializing the non-volatile storage library. This
 
 ```c
 esp_err_t ret = nvs_flash_init();
-if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
     ESP_ERROR_CHECK(nvs_flash_erase());
     ret = nvs_flash_init();
 }

examples/bluetooth/gatt_security_client/main/example_ble_sec_gattc_demo.c

@@ -460,7 +460,7 @@ void app_main()
 {
     // Initialize NVS.
     esp_err_t ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/gatt_security_server/main/example_ble_sec_gatts_demo.c

@@ -469,7 +469,7 @@ void app_main()
 
     // Initialize NVS.
     ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/gatt_server/main/gatts_demo.c

@@ -679,7 +679,7 @@ void app_main()
 
     // Initialize NVS.
     ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/gatt_server/tutorial/Gatt_Server_Example_Walkthrough.md

@@ -44,7 +44,7 @@ The entry point to this example is the app_main() function:
 
     // Initialize NVS.
     ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/gatt_server_service_table/main/gatts_table_creat_demo.c

@@ -517,7 +517,7 @@ void app_main()
 
     /* Initialize NVS. */
     ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/gatt_server_service_table/tutorial/Gatt_Server_Service_Table_Example_Walkthrough.md

@@ -81,7 +81,7 @@ void app_main()
 
     // Initialize NVS.
     ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/bluetooth/gattc_multi_connect/main/gattc_multi_connect.c

@@ -886,7 +886,7 @@ static void esp_gattc_cb(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp
 void app_main()
 {
     esp_err_t ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         ret = nvs_flash_init();
     }

examples/protocols/aws_iot/subscribe_publish/main/subscribe_publish_sample.c

@@ -321,7 +321,7 @@ void app_main()
 {
     // Initialize NVS.
     esp_err_t err = nvs_flash_init();
-    if (err == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (err == ESP_ERR_NVS_NO_FREE_PAGES || err == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         err = nvs_flash_init();
     }

examples/protocols/aws_iot/thing_shadow/main/thing_shadow_sample.c

@@ -356,7 +356,7 @@ static void initialise_wifi(void)
 void app_main()
 {
     esp_err_t err = nvs_flash_init();
-    if (err == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (err == ESP_ERR_NVS_NO_FREE_PAGES || err == ESP_ERR_NVS_NEW_VERSION_FOUND) {
         ESP_ERROR_CHECK(nvs_flash_erase());
         err = nvs_flash_init();
     }

examples/protocols/esp_http_client/main/esp_http_client_example.c

@@ -351,7 +351,7 @@ static void http_test_task(void *pvParameters)
 void app_main()
 {
     esp_err_t ret = nvs_flash_init();
-    if (ret == ESP_ERR_NVS_NO_FREE_PAGES) {
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
       ESP_ERROR_CHECK(nvs_flash_erase());
       ret = nvs_flash_init();
     }