ci(performance): Add PSRAM speed test

Author: lucasssvaz

.github/scripts/tests_run.sh

@@ -8,6 +8,11 @@ function run_test() {
     local sketchdir=$(dirname $sketch)
     local sketchname=$(basename $sketchdir)
 
+    if [[ -n $target ]] && [[ -f "$sketchdir/.skip.$target" ]]; then
+        echo "Skipping $sketchname for target $target"
+        exit 0
+    fi
+
     if [ $options -eq 0 ] && [ -f $sketchdir/cfg.json ]; then
         len=`jq -r --arg chip $target '.targets[] | select(.name==$chip) | .fqbn | length' $sketchdir/cfg.json`
     else

.github/workflows/hil.yml

@@ -79,7 +79,7 @@ jobs:
           path: |
              ~/.arduino/tests/**/build*.tmp/*.bin
              ~/.arduino/tests/**/build*.tmp/*.json
-          if-no-files-found: error
+
   Test:
     needs: [gen_chunks, Build]
     name: ${{matrix.chip}}-Test#${{matrix.chunks}}

tests/performance/psramspeed/.skip.esp32c3

@@ -0,0 +1,22 @@
+{
+  "targets": [
+    {
+      "name": "esp32",
+      "fqbn":[
+        "espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app"
+      ]
+    },
+    {
+      "name": "esp32s2",
+      "fqbn": [
+        "espressif:esp32:esp32s2:PSRAM=enabled,PartitionScheme=huge_app"
+      ]
+    },
+    {
+      "name": "esp32s3",
+      "fqbn": [
+        "espressif:esp32:esp32s3:PSRAM=opi,USBMode=default,PartitionScheme=huge_app"
+      ]
+    }
+  ]
+}

tests/performance/psramspeed/.skip.esp32c6

@@ -0,0 +1,283 @@
+/*
+  Based on the ramspeed test from NuttX.
+  https://github.com/apache/nuttx-apps/blob/master/benchmarks/ramspeed/ramspeed_main.c
+  Modified for Arduino and ESP32 by Lucas Saavedra Vaz, 2024
+*/
+
+#include <Arduino.h>
+
+// Test settings
+
+// Number of runs to average
+#define N_RUNS 3
+
+// Value to fill the memory with
+#define FILL_VALUE 0x00
+
+// Number of copies to be performed in each test
+#define N_COPIES 200
+
+// Start size for the tests. Value must be a power of 2.
+// Values lower or equal than 32 KB may cause the operations to use the cache instead of the PSRAM.
+#define START_SIZE 65536
+
+// Max size to be copied. Must be bigger than 32 and it will be floored to the nearest power of 2
+#define MAX_TEST_SIZE 1 * 1024 * 1024 // 1MB
+
+// Implementation macros
+
+#if defined(UINTPTR_MAX) && UINTPTR_MAX > 0xFFFFFFFF
+#  define MEM_UNIT         uint64_t
+#  define ALIGN_MASK       0x7
+#else
+#  define MEM_UNIT         uint32_t
+#  define ALIGN_MASK       0x3
+#endif
+
+#define COPY32 *d32 = *s32; d32++; s32++;
+#define COPY8 *d8 = *s8; d8++; s8++;
+#define SET32(x) *d32 = x; d32++;
+#define SET8(x) *d8 = x; d8++;
+#define REPEAT8(expr) expr expr expr expr expr expr expr expr
+
+/* Functions */
+
+static void *mock_memcpy(void *dst, const void *src, size_t len)
+{
+  uint8_t *d8 = (uint8_t *) dst;
+  const uint8_t *s8 = (uint8_t *) src;
+
+  uintptr_t d_align = (uintptr_t)d8 & ALIGN_MASK;
+  uintptr_t s_align = (uintptr_t)s8 & ALIGN_MASK;
+  uint32_t *d32;
+  const uint32_t *s32;
+
+  /* Byte copy for unaligned memories */
+
+  if (s_align != d_align)
+    {
+      while (len > 32)
+        {
+          REPEAT8(COPY8);
+          REPEAT8(COPY8);
+          REPEAT8(COPY8);
+          REPEAT8(COPY8);
+          len -= 32;
+        }
+
+      while (len)
+        {
+          COPY8;
+          len--;
+        }
+
+      return dst;
+    }
+
+  /* Make the memories aligned */
+
+  if (d_align)
+    {
+      d_align = ALIGN_MASK + 1 - d_align;
+      while (d_align && len)
+        {
+          COPY8;
+          d_align--;
+          len--;
+        }
+    }
+
+  d32 = (uint32_t *)d8;
+  s32 = (uint32_t *)s8;
+  while (len > 32)
+    {
+      REPEAT8(COPY32);
+      len -= 32;
+    }
+
+  while (len > 4)
+    {
+      COPY32;
+      len -= 4;
+    }
+
+  d8 = (uint8_t *)d32;
+  s8 = (const uint8_t *)s32;
+  while (len)
+    {
+      COPY8;
+      len--;
+    }
+
+  return dst;
+}
+
+static void mock_memset(void *dst, uint8_t v, size_t len)
+{
+  uint8_t *d8 = (uint8_t *)dst;
+  uintptr_t d_align = (uintptr_t) d8 & ALIGN_MASK;
+  uint32_t v32;
+  uint32_t *d32;
+
+  /* Make the address aligned */
+
+  if (d_align)
+    {
+      d_align = ALIGN_MASK + 1 - d_align;
+      while (d_align && len)
+        {
+          SET8(v);
+          len--;
+          d_align--;
+        }
+    }
+
+  v32 = (uint32_t)v + ((uint32_t)v << 8)
+        + ((uint32_t)v << 16) + ((uint32_t)v << 24);
+
+  d32 = (uint32_t *)d8;
+
+  while (len > 32)
+    {
+      REPEAT8(SET32(v32));
+      len -= 32;
+    }
+
+  while (len > 4)
+    {
+      SET32(v32);
+      len -= 4;
+    }
+
+  d8 = (uint8_t *)d32;
+  while (len)
+    {
+      SET8(v);
+      len--;
+    }
+}
+
+static void print_rate(const char *name, uint64_t bytes, uint32_t cost_time)
+{
+  uint32_t rate;
+  if (cost_time == 0)
+    {
+      Serial.println("Error: Too little time taken, please increase N_COPIES");
+      return;
+    }
+
+  rate = bytes * 1000 / cost_time / 1024;
+  Serial.printf("%s Rate = %" PRIu32 " KB/s Time: %" PRIu32 " ms\n", name, rate, cost_time);
+}
+
+static void memcpy_speed_test(void *dest, const void *src, size_t size, uint32_t repeat_cnt)
+{
+  uint32_t start_time;
+  uint32_t cost_time_system;
+  uint32_t cost_time_mock;
+  uint32_t cnt;
+  uint32_t step;
+  uint64_t total_size;
+
+  for (step = START_SIZE; step <= size; step <<= 1)
+    {
+      total_size = (uint64_t)step * (uint64_t)repeat_cnt;
+
+      Serial.printf("Memcpy %" PRIu32 " Bytes test\n", step);
+
+      start_time = millis();
+
+      for (cnt = 0; cnt < repeat_cnt; cnt++)
+        {
+          memcpy(dest, src, step);
+        }
+
+      cost_time_system = millis() - start_time;
+
+      start_time = millis();
+
+      for (cnt = 0; cnt < repeat_cnt; cnt++)
+        {
+          mock_memcpy(dest, src, step);
+        }
+
+      cost_time_mock = millis() - start_time;
+
+      print_rate("System memcpy():", total_size, cost_time_system);
+      print_rate("Mock memcpy():", total_size, cost_time_mock);
+    }
+}
+
+static void memset_speed_test(void *dest, uint8_t value, size_t size, uint32_t repeat_num)
+{
+  uint32_t start_time;
+  uint32_t cost_time_system;
+  uint32_t cost_time_mock;
+  uint32_t cnt;
+  uint32_t step;
+  uint64_t total_size;
+
+  for (step = START_SIZE; step <= size; step <<= 1)
+    {
+      total_size = (uint64_t)step * (uint64_t)repeat_num;
+
+      Serial.printf("Memset %" PRIu32 " Bytes test\n", step);
+
+      start_time = millis();
+
+      for (cnt = 0; cnt < repeat_num; cnt++)
+        {
+          memset(dest, value, step);
+        }
+
+      cost_time_system = millis() - start_time;
+
+      start_time = millis();
+
+      for (cnt = 0; cnt < repeat_num; cnt++)
+        {
+          mock_memset(dest, value, step);
+        }
+
+      cost_time_mock = millis() - start_time;
+
+      print_rate("System memset():", total_size, cost_time_system);
+      print_rate("Mock memset():", total_size, cost_time_mock);
+    }
+}
+
+/* Main */
+
+void setup()
+{
+  Serial.begin(115200);
+  while (!Serial) delay(10);
+
+  void *dest = ps_malloc(MAX_TEST_SIZE);
+  const void *src = ps_malloc(MAX_TEST_SIZE);
+
+  if (!dest || !src)
+    {
+      Serial.println("Memory allocation failed");
+      return;
+    }
+
+  log_d("Starting PSRAM speed test");
+  Serial.printf("Runs: %d\n", N_RUNS);
+  Serial.printf("Copies: %d\n", N_COPIES);
+  Serial.printf("Max test size: %d\n", MAX_TEST_SIZE);
+  Serial.flush();
+  for (int i = 0; i < N_RUNS; i++) {
+    log_d("Run %d", i);
+    memcpy_speed_test(dest, src, MAX_TEST_SIZE, N_COPIES);
+    Serial.flush();
+    memset_speed_test(dest, FILL_VALUE, MAX_TEST_SIZE, N_COPIES);
+    Serial.flush();
+  }
+  log_d("PSRAM speed test done");
+}
+
+void loop()
+{
+  vTaskDelete(NULL);
+}

tests/performance/psramspeed/.skip.esp32h2

@@ -0,0 +1,90 @@
+import json
+import logging
+import os
+
+from collections import defaultdict
+
+def test_psramspeed(dut, request):
+    LOGGER = logging.getLogger(__name__)
+
+    runs_results = []
+
+    # Match "Runs: %d"
+    res = dut.expect(r"Runs: (\d+)", timeout=60)
+    runs = int(res.group(0).decode("utf-8").split(" ")[1])
+    LOGGER.info("Number of runs: {}".format(runs))
+
+    # Match "Copies: %d"
+    res = dut.expect(r"Copies: (\d+)", timeout=60)
+    copies = int(res.group(0).decode("utf-8").split(" ")[1])
+    LOGGER.info("Number of copies in each test: {}".format(copies))
+
+    # Match "Max test size: %lu"
+    res = dut.expect(r"Max test size: (\d+)", timeout=60)
+    max_test_size = int(res.group(0).decode("utf-8").split(" ")[3])
+    LOGGER.info("Max test size: {}".format(max_test_size))
+
+    for i in range(runs):
+        LOGGER.info("Run {}".format(i))
+        for j in range(2):
+            while True:
+                # Match "Memcpy/Memtest %d Bytes test"
+                res = dut.expect(r"(Memcpy|Memset) (\d+) Bytes test", timeout=60)
+                current_test = res.group(0).decode("utf-8").split(" ")[0].lower()
+                current_test_size = int(res.group(0).decode("utf-8").split(" ")[1])
+                LOGGER.info("Current {} test size: {}".format(current_test, current_test_size))
+
+                for k in range(2):
+                    # Match "System/Mock memcpy/memtest(): Rate = %d KB/s Time: %d ms" or "Error: %s"
+                    res = dut.expect(r"((System|Mock) (memcpy|memset)\(\): Rate = (\d+) KB/s Time: (\d+) ms|^Error)", timeout=90)
+                    implementation = res.group(0).decode("utf-8").split(" ")[0].lower()
+                    assert implementation != "error:", "Error detected in test output"
+                    test_type = res.group(0).decode("utf-8").split(" ")[1].lower()[:-3]
+                    rate = int(res.group(0).decode("utf-8").split(" ")[4])
+                    time = int(res.group(0).decode("utf-8").split(" ")[7])
+                    assert test_type == current_test, "Missing test output"
+                    LOGGER.info("{} {}: Rate = {} KB/s. Time = {} ms".format(implementation, test_type, rate, time))
+
+                    runs_results.append(((current_test, str(current_test_size), implementation), (rate, time)))
+
+                if current_test_size == max_test_size:
+                    break
+
+            LOGGER.info("=============================================================")
+
+    # Calculate average rate and time for each test size
+    sums = defaultdict(lambda: {'rate_sum': 0, 'time_sum': 0})
+
+    for (test, size, impl), (rate, time) in runs_results:
+        sums[(test, size, impl)]['rate_sum'] += rate
+        sums[(test, size, impl)]['time_sum'] += time
+
+    avg_results = {}
+    for (test, size, impl) in sums:
+        rate_avg = round(sums[(test, size, impl)]['rate_sum'] / runs, 2)
+        time_avg = round(sums[(test, size, impl)]['time_sum'] / runs, 2)
+        LOGGER.info("Test: {}-{}-{}: Average rate = {} KB/s. Average time = {} ms".format(test, size, impl, rate_avg, time_avg))
+        if test not in avg_results:
+            avg_results[test] = {}
+        if size not in avg_results[test]:
+            avg_results[test][size] = {}
+        avg_results[test][size][impl] = {"avg_rate": rate_avg, "avg_time": time_avg}
+
+
+    # Create JSON with results and write it to file
+    # Always create a JSON with this format (so it can be merged later on):
+    # { TEST_NAME_STR: TEST_RESULTS_DICT }
+    results = {"psramspeed": {"runs": runs, "copies": copies, "max_test_size": max_test_size, "results": avg_results}}
+
+    current_folder = os.path.dirname(request.path)
+    file_index = 0
+    report_file = os.path.join(current_folder, "result_psramspeed" + str(file_index) + ".json")
+    while os.path.exists(report_file):
+        report_file = report_file.replace(str(file_index) + ".json", str(file_index + 1) + ".json")
+        file_index += 1
+
+    with open(report_file, "w") as f:
+        try:
+            f.write(json.dumps(results))
+        except Exception as e:
+            LOGGER.warning("Failed to write results to file: {}".format(e))