Move x86 Linux setup to prereqs script (#945)

* Move install_x86_support_libraries to utils.

* Update packaging x86 to use new prereqs script option

* Fix x86 CI builds to retry prereqs as well

* Add call to install_x86_support_libraries to prereqs script if needed.

* Fix installation of openssl

* Add option to prereqs script to skip openssl, for boringssl builds.

* Use 'with' syntax for file i/o

Author: jonsimantov

.github/workflows/cpp-packaging.yml

@@ -408,9 +408,13 @@ jobs:
             # binutils, used by older version of homebrew for hosting packages.
             brew update
 
-      - name: Install prerequisites
-        run: |
-          python scripts/gha/install_prereqs_desktop.py
+      - name: Install Desktop SDK prerequisites
+        uses: nick-invision/retry@v2
+        with:
+          timeout_minutes: 15
+          max_attempts: 3
+          command: |
+            python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.architecture }}' --ssl boringssl
 
       - name: Export verbose flag
         shell: bash
@@ -538,7 +542,7 @@ jobs:
       - name: Install prerequisites
         run: |
           cd sdk-src
-          python scripts/gha/install_prereqs_desktop.py
+          python scripts/gha/install_prereqs_desktop.py --ssl boringssl
           cd ..
 
       - name: postprocess and package built SDK

.github/workflows/desktop.yml

@@ -176,9 +176,13 @@ jobs:
             # binutils, used by older version of homebrew for hosting packages.
             brew update
 
-      - name: Install prerequisites
-        run: |
-          python scripts/gha/install_prereqs_desktop.py
+      - name: Install Desktop SDK prerequisites
+        uses: nick-invision/retry@v2
+        with:
+          timeout_minutes: 15
+          max_attempts: 3
+          command: |
+            python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.architecture }}'
 
       - name: Build SDK
         shell: bash

.github/workflows/integration_tests.yml

@@ -278,7 +278,7 @@ jobs:
           timeout_minutes: 2
           max_attempts: 3
           command: |
-            python scripts/gha/install_prereqs_desktop.py
+            python scripts/gha/install_prereqs_desktop.py --ssl '${{ matrix.ssl_variant }}'
             pip install -r scripts/gha/requirements.txt
             python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
       - name: Install OpenSSL (Windows)

scripts/gha/build_desktop.py

@@ -281,7 +281,7 @@ def main():
 
   # To build x86 on x86_64 linux hosts, we also need x86 support libraries
   if args.arch == 'x86' and utils.is_linux_os():
-    install_x86_support_libraries(args.gha_build)
+    utils.install_x86_support_libraries(args.gha_build)
 
   # Install C++ dependencies using vcpkg
   if not args.disable_vcpkg:

scripts/gha/install_prereqs_desktop.py

@@ -29,62 +29,77 @@
 
 """
 
+import argparse
 import utils
 
 def main():
-  # Install protobuf on linux/mac if its not installed already
-  if not utils.is_command_installed('protoc'):
-    if utils.is_linux_os():
-        # sudo apt install protobuf-compiler
-        utils.run_command(['apt', 'install', '-y','protobuf-compiler'], as_root=True)
-    elif utils.is_mac_os():
-        # brew install protobuf
-        utils.run_command(['brew', 'install', 'protobuf'])
-    
-  # Install go on linux/mac if its not installed already
-  if not utils.is_command_installed('go'):
-    if utils.is_linux_os():
-        # sudo apt install -y golang
-        utils.run_command(['apt', 'install', '-y','golang'], as_root=True)
-    elif utils.is_mac_os():
-        # brew install protobuf
-        utils.run_command(['brew', 'install', 'go'])
-
-  # Install openssl on linux/mac if its not installed already
-  if not utils.is_command_installed('go'):
-    if utils.is_linux_os():
-        # sudo apt install -y openssl
-        utils.run_command(['apt', 'install', '-y','openssl'], as_root=True)
-    elif utils.is_mac_os():
-        # brew install protobuf
-        utils.run_command(['brew', 'install', 'openssl'])
-
-  # Install ccache on linux/mac if its not installed already
-  if not utils.is_command_installed('ccache'):
-    if utils.is_linux_os():
-        # sudo apt install ccache
-        utils.run_command(['apt', 'install', '-y', 'ccache'], as_root=True)
-    elif utils.is_mac_os():
-        # brew install ccache
-        utils.run_command(['brew', 'install', 'ccache'])
-  
-  # Install clang-format on linux/mac if its not installed already
-  if not utils.is_command_installed('clang-format'):
-    if utils.is_linux_os():
-        # sudo apt install clang-format
-        utils.run_command(['apt', 'install', '-y','clang-format'], as_root=True)
-    elif utils.is_mac_os():
-        # brew install protobuf
-        utils.run_command(['brew', 'install', 'clang-format'])
-
-  # Install required python dependencies. 
-  # On Catalina, python2 in installed as default python.
-  # Example command:
-  # python3 -m pip install -r external/pip_requirements.txt --user
-  utils.run_command( 
-     ['python3' if utils.is_command_installed('python3') else 'python', '-m', 
-          'pip', 'install', '-r', 'external/pip_requirements.txt', '--user'] )
+  args = parse_cmdline_args()
 
+  if not args.running_only:
+    # Install protobuf on linux/mac if its not installed already
+    if not utils.is_command_installed('protoc'):
+      if utils.is_linux_os():
+          # sudo apt install protobuf-compiler
+          utils.run_command(['apt', 'install', '-y','protobuf-compiler'], as_root=True)
+      elif utils.is_mac_os():
+          # brew install protobuf
+          utils.run_command(['brew', 'install', 'protobuf'])
+
+    # Install go on linux/mac if its not installed already
+    if not utils.is_command_installed('go'):
+      if utils.is_linux_os():
+          # sudo apt install -y golang
+          utils.run_command(['apt', 'install', '-y','golang'], as_root=True)
+      elif utils.is_mac_os():
+          # brew install go
+          utils.run_command(['brew', 'install', 'go'])
+
+    # Install openssl on linux/mac if its not installed already
+    if args.ssl == 'openssl' and not utils.is_command_installed('openssl'):
+      if utils.is_linux_os():
+          # sudo apt install -y openssl
+          utils.run_command(['apt', 'install', '-y','openssl'], as_root=True)
+      elif utils.is_mac_os():
+          # brew install openssl
+          utils.run_command(['brew', 'install', 'openssl'])
+
+    # Install ccache on linux/mac if its not installed already
+    if not utils.is_command_installed('ccache'):
+      if utils.is_linux_os():
+          # sudo apt install ccache
+          utils.run_command(['apt', 'install', '-y', 'ccache'], as_root=True)
+      elif utils.is_mac_os():
+          # brew install ccache
+          utils.run_command(['brew', 'install', 'ccache'])
+
+    # Install clang-format on linux/mac if its not installed already
+    if not utils.is_command_installed('clang-format'):
+      if utils.is_linux_os():
+          # sudo apt install clang-format
+          utils.run_command(['apt', 'install', '-y','clang-format'], as_root=True)
+      elif utils.is_mac_os():
+          # brew install protobuf
+          utils.run_command(['brew', 'install', 'clang-format'])
+
+    # Install required python dependencies.
+    # On Catalina, python2 in installed as default python.
+    # Example command:
+    # python3 -m pip install -r external/pip_requirements.txt --user
+    utils.run_command(
+       ['python3' if utils.is_command_installed('python3') else 'python', '-m',
+            'pip', 'install', '-r', 'external/pip_requirements.txt', '--user'] )
+
+  if args.arch == 'x86':
+    utils.install_x86_support_libraries(args.gha_build)
+
+def parse_cmdline_args():
+  parser = argparse.ArgumentParser(description='Install prerequisites for building cpp sdk')
+  parser.add_argument('--arch', default=None, help='Install support libraries to build a specific architecture (currently supported: x86)')
+  parser.add_argument('--running_only', action='store_true', help='Only install prerequisites for running, not for building')
+  parser.add_argument('--gha_build', action='store_true', default=None, help='Set this option when building on GitHub, changing some prerequisite installation behavior')
+  parser.add_argument('--ssl', default='openssl', help='Which SSL is this build using (supported: openssl, boringssl)')
+  args = parser.parse_args()
+  return args
 
 if __name__ == '__main__':
   main()

scripts/gha/utils.py

@@ -218,3 +218,45 @@ def clean_vcpkg_temp_data():
   for directory_to_remove in directories_to_remove:
     abspath = os.path.join(vcpkg_root_dir_path, directory_to_remove)
     delete_directory(abspath)
+
+
+def install_x86_support_libraries(gha_build=False):
+  """Install support libraries needed to build x86 on x86_64 hosts.
+
+  Args:
+    gha_build: Pass in True if running on a GitHub runner; this will activate
+               workarounds that might be undesirable on a personal system (e.g.
+               downgrading Ubuntu packages).
+  """
+  if is_linux_os():
+    packages = ['gcc-multilib', 'g++-multilib', 'libglib2.0-dev:i386',
+                'libsecret-1-dev:i386', 'libpthread-stubs0-dev:i386',
+                'libssl-dev:i386']
+    if gha_build:
+      # Workaround for GitHub runners, which have an incompatibility between the
+      # 64-bit and 32-bit versions of the Ubuntu package libpcre2-8-0. Downgrade
+      # the installed 64-bit version of the library to get around this issue.
+      # This will presumably be fixed in a future Ubuntu update. (If you remove
+      # it, remove the workaround further down this function as well.)
+      packages = ['--allow-downgrades'] + packages + ['libpcre2-8-0=10.34-7']
+
+    # First check if these packages exist on the machine already
+    with open(os.devnull, "w") as devnull:
+      process = subprocess.run(["dpkg", "-s"] + packages, stdout=devnull, stderr=subprocess.STDOUT)
+
+    if process.returncode != 0:
+      # This implies not all of the required packages are already installed on user's machine
+      # Install them.
+      run_command(['dpkg', '--add-architecture', 'i386'], as_root=True, check=True)
+      run_command(['apt', 'update'], as_root=True, check=True)
+      run_command(['apt', 'install', '-V', '-y'] + packages, as_root=True, check=True)
+
+    if gha_build:
+      # One more workaround: downgrading libpcre2-8-0 above may have uninstalled
+      # libsecret, which is required for the Linux build. Force it to be
+      # reinstalled, but do it as a separate command to ensure that held
+      # packages aren't modified. (Once the workaround above is removed, this can
+      # be removed as well.)
+      # Note: "-f" = "fix" - let apt do what it needs to do to fix dependencies.
+      run_command(['apt', 'install', '-f', '-V', '-y', 'libsecret-1-dev'],
+                  as_root=True, check=True)