Merge branch 'master' into firestore-held-write-acks

Author: schmidt-sebastian

ci/fireci/fireci/__init__.py

@@ -11,3 +11,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from .internal import ci_command

ci/fireci/fireci/commands.py

@@ -16,7 +16,7 @@
 import os
 
 from . import gradle
-from .internal import ci_command
+from . import ci_command
 
 
 @click.argument('task', required=True, nargs=-1)

ci/fireci/fireci/emulator.py

@@ -12,19 +12,127 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import datetime
 import logging
+import os
+import signal
+import subprocess
+import time
 
 _logger = logging.getLogger('fireci.emulator')
 
+EMULATOR_BINARY = 'emulator'
+ADB_BINARY = 'adb'
+EMULATOR_NAME = 'test'
+
+EMULATOR_FLAGS = ['-no-audio', '-no-window', '-skin', '768x1280']
+
 
-# TODO(vkryachko): start/shutdown the emulator.
 class EmulatorHandler:
+  """
+     Context manager that launches an android emulator for the duration of its execution.
 
-  def __init__(self, artifacts_dir):
+     As part of its run it:
+     * Launches the emulator
+     * Waits for it to boot
+     * Starts logcat to store on-device logs
+     * Produces stdout.log, stderr.log, logcat.log in the artifacts directory
+  """
+
+  def __init__(
+      self,
+      artifacts_dir,
+      *,
+      name=EMULATOR_NAME,
+      emulator_binary=EMULATOR_BINARY,
+      adb_binary=ADB_BINARY,
+      # for testing only
+      emulator_stdin=None,
+      wait_for_device_stdin=None,
+      logcat_stdin=None):
     self._artifacts_dir = artifacts_dir
 
+    log_dir = '{}_emulator'.format(name)
+    self._stdout = self._open(log_dir, 'stdout.log')
+    self._stderr = self._open(log_dir, 'stderr.log')
+    self._adb_log = self._open(log_dir, 'logcat.log')
+    self._name = name
+
+    self._emulator_binary = emulator_binary
+    self._adb_binary = adb_binary
+
+    self._emulator_stdin = emulator_stdin
+    self._wait_for_device_stdin = wait_for_device_stdin
+    self._logcat_stdin = logcat_stdin
+
   def __enter__(self):
-    _logger.debug('Pretend to start the emulator(TODO)')
+    _logger.info('Starting avd "{}..."'.format(self._name))
+    self._process = subprocess.Popen(
+        [self._emulator_binary, '-avd', self._name] + EMULATOR_FLAGS,
+        env=os.environ,
+        stdin=self._emulator_stdin,
+        stdout=self._stdout,
+        stderr=self._stderr)
+    try:
+      self._wait_for_boot(datetime.timedelta(minutes=5))
+    except:
+      self._kill(self._process)
+      self._close_files()
+      raise
+
+    self._logcat = subprocess.Popen(
+        [self._adb_binary, 'logcat'],
+        stdin=self._logcat_stdin,
+        stdout=self._adb_log,
+    )
 
   def __exit__(self, exception_type, exception_value, traceback):
-    _logger.debug('Pretend to stop the emulator(TODO)')
+    _logger.info('Shutting down avd "{}"...'.format(self._name))
+    self._kill(self._process)
+    _logger.info('Avd "{}" shut down.'.format(self._name))
+    self._kill(self._logcat)
+    self._close_files()
+
+  def _open(self, dirname, filename):
+    """Opens a file in a given directory, creates directory if required."""
+    dirname = os.path.join(self._artifacts_dir, dirname)
+    if (not os.path.exists(dirname)):
+      os.makedirs(dirname)
+    return open(os.path.join(dirname, filename), 'w')
+
+  def _wait_for_boot(self, timeout: datetime.timedelta):
+    _logger.info('Waiting for avd to boot...')
+    wait = subprocess.Popen(
+        [self._adb_binary, 'wait-for-device'],
+        stdin=self._wait_for_device_stdin,
+        stdout=self._stdout,
+        stderr=self._stderr,
+    )
+
+    start = datetime.datetime.now()
+    while self._process.poll() is None:
+      wait_exitcode = wait.poll()
+      if wait_exitcode is not None:
+        if wait_exitcode == 0:
+          _logger.info('Emulator booted successfully.')
+          return
+        raise RuntimeError("Waiting for emulator failed.")
+
+      time.sleep(0.1)
+      now = datetime.datetime.now()
+      if now - start >= timeout:
+        self._kill(wait, sig=signal.SIGKILL)
+        raise RuntimeError("Emulator startup timed out.")
+
+    self._kill(wait)
+    raise RuntimeError(
+        "Emulator failed to launch. See emulator logs for details.")
+
+  def _kill(self, process, sig=signal.SIGTERM):
+    process.send_signal(sig)
+    process.wait()
+
+  def _close_files(self):
+    for f in (self._stdout, self._stderr, self._adb_log):
+      if f is not None:
+        f.close()

ci/fireci/fireci/gradle.py

@@ -22,6 +22,11 @@
 ADB_INSTALL_TIMEOUT = '5'
 
 
+def P(name, value):
+  """Returns name and value in the format of gradle's project property cli argument."""
+  return '-P{}={}'.format(name, value)
+
+
 def run(*args, gradle_opts='', workdir=None):
   """Invokes gradle with specified args and gradle_opts."""
   new_env = dict(os.environ)

ci/fireci/fireci/internal.py

@@ -51,12 +51,12 @@ def _artifact_handler(target_directory, artifact_patterns):
 
 
 @contextlib.contextmanager
-def _emulator_handler(enabled, target_directory):
+def _emulator_handler(enabled, *args, **kwargs):
   if not enabled:
     yield
     return
 
-  with emulator.EmulatorHandler(target_directory):
+  with emulator.EmulatorHandler(*args, **kwargs):
     yield
 
 
@@ -71,14 +71,14 @@ class _CommonOptions:
 @click.option(
     '--artifact-target-dir',
     default='_artifacts',
-    help='Directory where artifacts will be symlinked to.',
+    help='Directory where artifacts will be copied to.',
     type=click.Path(dir_okay=True, resolve_path=True),
 )
 @click.option(
     '--artifact-patterns',
     default=('**/build/test-results', '**/build/reports'),
     help=
-    'Shell-style artifact patterns that are symlinked into `artifact-target-dir`.'\
+    'Shell-style artifact patterns that are copied into `artifact-target-dir`.'\
         'Can be specified multiple times.',
     multiple=True,
     type=str,
@@ -89,6 +89,21 @@ class _CommonOptions:
     help='Specifies whether to start an Android emulator a command executes.',
     is_flag=True,
 )
+@click.option(
+    '--emulator-name',
+    default='test',
+    help='Specifies the AVD name to launch the emulator with.',
+)
+@click.option(
+    '--emulator-binary',
+    default='emulator',
+    help='Specifies the name/full path to the emulator binary.',
+)
+@click.option(
+    '--adb-binary',
+    default='adb',
+    help='Specifies the name/full path to the adb binary.',
+)
 @_pass_options
 def main(options, **kwargs):
   """Main command group.
@@ -120,7 +135,10 @@ def new_func(ctx, options, *args, **kwargs):
       with _artifact_handler(options.artifact_target_dir,
                              options.artifact_patterns), _emulator_handler(
                                  options.with_emulator,
-                                 options.artifact_target_dir):
+                                 options.artifact_target_dir,
+                                 name=options.emulator_name,
+                                 emulator_binary=options.emulator_binary,
+                                 adb_binary=options.adb_binary):
         return ctx.invoke(f, *args, **kwargs)
 
     return functools.update_wrapper(new_func, f)

ci/fireci/fireci/main.py

@@ -15,11 +15,14 @@
 import logging
 
 from . import commands
+from . import plugins
 from .internal import main
 
 logging.basicConfig(
     format='%(name)s: [%(levelname)s] %(message)s',
     level=logging.DEBUG,
 )
 
+plugins.discover()
+
 cli = main

ci/fireci/fireci/plugins.py

@@ -0,0 +1,33 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+import pkgutil
+import fireciplugins
+
+
+def discover():
+  """Discovers fireci plugins available on PYTHONPATH under firebaseplugins subpackages.
+
+     Discovery works by importing all direct subpackages of firebaseplugins and importing them,
+     plugins are supposed to register ci_command's with fireci in their __init__.py files directly
+     or by importing from their own subpackages.
+
+     Note: plugins *must* define the `firebaseplugins` package as a namespace package.
+           See: https://packaging.python.org/guides/packaging-namespace-packages/
+  """
+  modules = pkgutil.iter_modules(fireciplugins.__path__,
+                                 fireciplugins.__name__ + ".")
+  for _, name, _ in modules:
+    importlib.import_module(name)

ci/fireci/fireciplugins/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)