Gives a speed boost to collecting the submodules

Since all of the submodules are only there to provide files and not for the purposes of being able to update them using git there is no reason to clone the full depth of the submodules repositories. The updates that are done now are shallow updates and only include 1 level of commit data. This has greatly improved the in which it takes to collect the submodules.

Author: kdschlosser

builder/__init__.py

@@ -196,6 +196,7 @@ def get_lvgl():
         'submodule',
         'update',
         '--init',
+        '--depth=1',
         '--',
         f'lib/lvgl'
     ]
@@ -214,7 +215,7 @@ def get_micropython():
         'submodule',
         'update',
         '--init',
-        '--remote',
+        '--depth=1',
         '--',
         f'lib/micropython'
     ]
@@ -224,13 +225,27 @@ def get_micropython():
     if result != 0:
         sys.exit(result)
 
+    mkrules_path = 'lib/micropython/py/mkrules.mk'
+    with open(mkrules_path, 'rb') as f:
+        data = f.read().decode('utf-8')
+
+    pattern = '$(Q)git submodule update --init $(addprefix $(TOP)/,$(GIT_SUBMODULES))'
+    if pattern in data:
+        data = data.replace(
+            pattern,
+            '$(Q)git submodule update --init --depth=1 $(addprefix $(TOP)/,$(GIT_SUBMODULES))'
+        )
+        with open(mkrules_path, 'wb') as f:
+            f.write(data.encode('utf-8'))
+
 
 def get_pycparser():
     cmd_ = [
         'git',
         'submodule',
         'update',
         '--init',
+        '--depth=1',
         '--',
         f'lib/pycparser'
     ]
@@ -303,6 +318,11 @@ def process_output(myproc, out_to_screen, spinner, cmpl, out_queue):
     line_updated = False
     err_updated = False
 
+    event = threading.Event()
+
+    os.set_blocking(myproc.stdout.fileno(), False)
+    os.set_blocking(myproc.stderr.fileno(), False)
+
     event = threading.Event()
     spinner_lock = threading.Lock()
     if spinner:
@@ -408,6 +428,7 @@ def process_output(myproc, out_to_screen, spinner, cmpl, out_queue):
         if not err_updated and not line_updated:
             if myproc.poll() is not None:
                 break
+            event.wait(0.1)
 
         err_updated = False
         line_updated = False

builder/esp32.py

@@ -115,11 +115,12 @@ def get_espidf():
     cmd = [
         [
             'git', 'submodule', 'update', '--init',
-            f'--jobs {os.cpu_count()}', '--', 'lib/esp-idf'
+            f'--jobs {os.cpu_count()}', '--depth=1',
+            '--', 'lib/esp-idf'
         ],
-        ['cd', 'lib/esp-idf'],
+        ['cd lib/esp-idf'],
         [
-            'git', 'submodule', 'update', '--init',
+            'git', 'submodule', 'update', '--init', '--depth=1',
             f'--jobs {os.cpu_count()}', '--',
             'components/bt/host/nimble/nimble',
             'components/esp_wifi',
@@ -473,7 +474,10 @@ def has_correct_idf():
                 cached_idf_version = version
 
     return (
-        cached_idf_version is not None and cached_idf_version == IDF_VER
+        cached_idf_version is not None and (
+            cached_idf_version == IDF_VER or
+            cached_idf_version == IDF_VER.rsplit('.', 1)[0]
+        )
     )
 
 
@@ -580,10 +584,34 @@ def setup_idf_environ():
     # There were some modifications made with how the environment gets set up
     # @cheops put quite a bit of time in to research the best solution
     # and also with the testing of the code.
-    if IDF_ENVIRON_SET or (not IDF_ENVIRON_SET and has_correct_idf()):
+
+    if IDF_ENVIRON_SET:
         env = os.environ
         IDF_ENVIRON_SET = True
-    elif not IDF_ENVIRON_SET:
+    elif has_correct_idf():
+        idf_path = get_idf_path()
+
+        cmd = [
+            ['cd', idf_path],
+            [
+                'git', 'submodule', 'update', '--init', '--depth=1',
+                f'--jobs {os.cpu_count()}', '--',
+                'components/bt/host/nimble/nimble',
+                'components/esp_wifi',
+                'components/esptool_py/esptool',
+                'components/lwip/lwip',
+                'components/mbedtls/mbedtls',
+                'components/bt/controller/lib_esp32',
+                'components/bt/controller/lib_esp32c3_family'
+            ]
+        ]
+        env = os.environ
+        result, _ = spawn(cmd, spinner=True, env=env)
+        if result != 0:
+            sys.exit(result)
+
+        IDF_ENVIRON_SET = True
+    else:
         print('Getting ESP-IDF build Environment')
         idf_path = 'lib/esp-idf'
 
@@ -641,9 +669,6 @@ def setup_idf_environ():
 
         env = os.environ
         IDF_ENVIRON_SET = True
-    else:
-        # this is a sanity check and should never actually run
-        env = os.environ
 
     if 'GITHUB_RUN_ID' in os.environ:
         idf_path = os.path.abspath(env["IDF_PATH"])
@@ -678,7 +703,7 @@ def submodules():
     env = {k: v for k, v in os.environ.items()}
     env['IDF_PATH'] = os.path.abspath(idf_path)
 
-    result, _ = spawn(cmds, env=env)
+    result, _ = spawn(cmds, spinner=True, env=env)
     if result != 0:
         sys.exit(result)