feat: tests for playground samples used in docs (#63)

New:
- Tests for playground samples used in docs
- Introduce new `run_preview.py` to run tests based on preview feature (it copy preview apps in sut folder once before tests)

Refactor:
- Chrome: Add implicit wait of 60sec
- Make `run_adb_command` and `run_simctl_command` public
- Preview.run_url() now takes DeviceInfo object as param (instead of id and platform)
- Escape playground url now happens in `Preview.run_url()` instead of `Preview.get_url()`

Fix:
- Retry `xcrun simctl install` in case of failure (known issue of Xcode 10+)

Author: dtopuzov

core/utils/chrome.py

@@ -17,6 +17,7 @@ def __init__(self, kill_old=True):
         path = ChromeDriverManager().install()
         Log.info('Starting Google Chrome ...')
         self.driver = webdriver.Chrome(executable_path=path)
+        self.driver.implicitly_wait(60)
         Log.info('Google Chrome started!')
 
     def open(self, url):

core/utils/device/adb.py

@@ -15,7 +15,7 @@
 
 class Adb(object):
     @staticmethod
-    def __run_adb_command(command, device_id=None, wait=True, timeout=60, fail_safe=False, log_level=logging.DEBUG):
+    def run_adb_command(command, device_id=None, wait=True, timeout=60, fail_safe=False, log_level=logging.DEBUG):
         if device_id is None:
             command = '{0} {1}'.format(ADB_PATH, command)
         else:
@@ -28,7 +28,7 @@ def __get_ids(include_emulator=False):
         Get IDs of available android devices.
         """
         devices = []
-        output = Adb.__run_adb_command('devices -l').output
+        output = Adb.run_adb_command('devices -l').output
         # Example output:
         # emulator-5554          device product:sdk_x86 model:Android_SDK_built_for_x86 device:generic_x86
         # HT46BWM02644           device usb:336592896X product:m8_google model:HTC_One_M8 device:htc_m8
@@ -42,9 +42,9 @@ def __get_ids(include_emulator=False):
     @staticmethod
     def restart():
         Log.info("Restart adb.")
-        Adb.__run_adb_command('kill-server')
+        Adb.run_adb_command('kill-server')
         Process.kill(proc_name='adb')
-        Adb.__run_adb_command('start-server')
+        Adb.run_adb_command('start-server')
 
     @staticmethod
     def get_devices(include_emulators=False):
@@ -63,7 +63,7 @@ def is_running(device_id):
             command = "shell dumpsys window windows | findstr mCurrentFocus"
         else:
             command = "shell dumpsys window windows | grep -E 'mCurrentFocus'"
-        result = Adb.__run_adb_command(command=command, device_id=device_id, timeout=10, fail_safe=True)
+        result = Adb.run_adb_command(command=command, device_id=device_id, timeout=10, fail_safe=True)
         return bool('Window' in result.output)
 
     @staticmethod
@@ -87,7 +87,7 @@ def wait_until_boot(device_id, timeout=180, check_interval=3):
 
     @staticmethod
     def reboot(device_id):
-        Adb.__run_adb_command(command='reboot', device_id=device_id)
+        Adb.run_adb_command(command='reboot', device_id=device_id)
         Adb.wait_until_boot(device_id=device_id)
 
     @staticmethod
@@ -96,18 +96,18 @@ def prevent_screen_lock(device_id):
         Disable screen lock after time of inactivity.
         :param device_id: Device identifier.
         """
-        Adb.__run_adb_command(command='shell settings put system screen_off_timeout -1', device_id=device_id)
+        Adb.run_adb_command(command='shell settings put system screen_off_timeout -1', device_id=device_id)
 
     @staticmethod
     def pull(device_id, source, target):
-        return Adb.__run_adb_command(command='pull {0} {1}'.format(source, target), device_id=device_id)
+        return Adb.run_adb_command(command='pull {0} {1}'.format(source, target), device_id=device_id)
 
     @staticmethod
     def get_page_source(device_id):
         temp_file = os.path.join(Settings.TEST_OUT_HOME, 'window_dump.xml')
         File.delete(temp_file)
-        Adb.__run_adb_command(command='shell rm /sdcard/window_dump.xml', device_id=device_id)
-        result = Adb.__run_adb_command(command='shell uiautomator dump', device_id=device_id)
+        Adb.run_adb_command(command='shell rm /sdcard/window_dump.xml', device_id=device_id)
+        result = Adb.run_adb_command(command='shell uiautomator dump', device_id=device_id)
         if 'UI hierchary dumped to' in result.output:
             time.sleep(1)
             Adb.pull(device_id=device_id, source='/sdcard/window_dump.xml', target=temp_file)
@@ -145,20 +145,20 @@ def is_text_visible(device_id, text, case_sensitive=False):
     def get_screen(device_id, file_path):
         File.delete(path=file_path)
         if Settings.HOST_OS == OSType.WINDOWS:
-            Adb.__run_adb_command(command='exec-out screencap -p > ' + file_path,
-                                  device_id=device_id,
-                                  log_level=logging.DEBUG)
+            Adb.run_adb_command(command='exec-out screencap -p > ' + file_path,
+                                device_id=device_id,
+                                log_level=logging.DEBUG)
         else:
-            Adb.__run_adb_command(command="shell screencap -p | perl -pe 's/\\x0D\\x0A/\\x0A/g' > " + file_path,
-                                  device_id=device_id)
+            Adb.run_adb_command(command="shell screencap -p | perl -pe 's/\\x0D\\x0A/\\x0A/g' > " + file_path,
+                                device_id=device_id)
         if File.exists(file_path):
             return
         else:
             raise Exception('Failed to get screen of {0}.'.format(device_id))
 
     @staticmethod
     def get_device_version(device_id):
-        result = Adb.__run_adb_command(command='shell getprop ro.build.version.release', device_id=device_id)
+        result = Adb.run_adb_command(command='shell getprop ro.build.version.release', device_id=device_id)
         if result.exit_code == 0:
             return result.output
         else:
@@ -167,7 +167,7 @@ def get_device_version(device_id):
     @staticmethod
     def open_home(device_id):
         cmd = 'shell am start -a android.intent.action.MAIN -c android.intent.category.HOME'
-        Adb.__run_adb_command(command=cmd, device_id=device_id)
+        Adb.run_adb_command(command=cmd, device_id=device_id)
         Log.info('Open home screen of {0}.'.format(str(device_id)))
 
     @staticmethod
@@ -177,6 +177,6 @@ def install(apk_path, device_id):
         :param apk_path: File path to .apk.
         :param device_id: Device id.
         """
-        result = Adb.__run_adb_command(command='-s {0} install -r {1}'.format(device_id, apk_path), timeout=60)
+        result = Adb.run_adb_command(command='-s {0} install -r {1}'.format(device_id, apk_path), timeout=60)
         assert 'Success' in result.output, 'Failed to install {0}. Output: {1}'.format(apk_path, result.output)
         Log.info('{0} installed successfully on {1}.'.format(apk_path, device_id))

core/utils/device/device.py

@@ -76,6 +76,12 @@ def get_text(self):
         return text
 
     def wait_for_text(self, text, timeout=60, retry_delay=1):
+        """
+        Wait until text is visible on device.
+        :param text: Text as string.
+        :param timeout: Timeout in seconds.
+        :param retry_delay: Retry interval in seconds.
+        """
         t_end = time.time() + timeout
         found = False
         error_msg = '{0} NOT found on {1}.'.format(text, self.name)

core/utils/device/simctl.py

@@ -11,14 +11,14 @@
 class Simctl(object):
 
     @staticmethod
-    def __run_simctl_command(command, wait=True, timeout=60):
+    def run_simctl_command(command, wait=True, timeout=60):
         command = '{0} {1}'.format('xcrun simctl', command)
         return run(cmd=command, wait=wait, timeout=timeout)
 
     # noinspection PyBroadException
     @staticmethod
     def __get_simulators():
-        result = Simctl.__run_simctl_command(command='list --json devices')
+        result = Simctl.run_simctl_command(command='list --json devices')
         try:
             return json.loads(result.output)
         except ValueError:
@@ -28,7 +28,7 @@ def __get_simulators():
     @staticmethod
     def start(simulator_info):
         if simulator_info.id is not None:
-            Simctl.__run_simctl_command(command='boot {0}'.format(simulator_info.id))
+            Simctl.run_simctl_command(command='boot {0}'.format(simulator_info.id))
             Simctl.wait_until_boot(simulator_info)
             return simulator_info
         else:
@@ -43,7 +43,7 @@ def is_running(simulator_info):
                 simulator_info.id = str(sim['udid'])
                 command = 'spawn {0} launchctl print system | grep com.apple.springboard.services'.format(
                     simulator_info.id)
-                service_state = Simctl.__run_simctl_command(command=command)
+                service_state = Simctl.run_simctl_command(command=command)
                 if "M   A   com.apple.springboard.services" in service_state.output:
                     Log.info('Simulator "{0}" booted.'.format(simulator_info.name))
                     return simulator_info
@@ -78,7 +78,7 @@ def is_available(simulator_info):
 
     @staticmethod
     def stop_application(simulator_info, app_id):
-        return Simctl.__run_simctl_command('terminate {0} {1}'.format(simulator_info.id, app_id))
+        return Simctl.run_simctl_command('terminate {0} {1}'.format(simulator_info.id, app_id))
 
     @staticmethod
     def stop_all(simulator_info):
@@ -87,14 +87,19 @@ def stop_all(simulator_info):
 
     @staticmethod
     def install(simulator_info, path):
-        result = Simctl.__run_simctl_command('install {0} {1}'.format(simulator_info.id, path))
-        assert result.exit_code == 0, 'Failed to install {0} on {1}'.format(path, simulator_info.name)
-        assert 'Failed to install the requested application' not in result.output, \
-            'Failed to install {0} on {1}'.format(path, simulator_info.name)
+        result = Simctl.run_simctl_command('install {0} {1}'.format(simulator_info.id, path))
+        if result.exit_code != 0:
+            # Since Xcode 10 sometimes xcrun simctl install fails first time (usually with iPhone X* devices).
+            Log.info('Failed to install {0} on {1}.'.format(path, simulator_info.name))
+            Log.info('Retry...')
+            result = Simctl.run_simctl_command('install {0} {1}'.format(simulator_info.id, path))
+            assert result.exit_code == 0, 'Failed to install {0} on {1}'.format(path, simulator_info.name)
+            assert 'Failed to install the requested application' not in result.output, \
+                'Failed to install {0} on {1}'.format(path, simulator_info.name)
 
     @staticmethod
     def uninstall(simulator_info, app_id):
-        result = Simctl.__run_simctl_command('uninstall {0} {1}'.format(simulator_info.id, app_id))
+        result = Simctl.run_simctl_command('uninstall {0} {1}'.format(simulator_info.id, app_id))
         assert result.exit_code == 0, 'Failed to uninstall {0} on {1}'.format(app_id, simulator_info.name)
         assert 'Failed to uninstall the requested application' not in result.output, \
             'Failed to uninstall {0} on {1}'.format(app_id, simulator_info.name)
@@ -109,19 +114,19 @@ def uninstall_all(simulator_info):
     @staticmethod
     def get_screen(sim_id, file_path):
         File.delete(file_path)
-        result = Simctl.__run_simctl_command('io {0} screenshot {1}'.format(sim_id, file_path))
+        result = Simctl.run_simctl_command('io {0} screenshot {1}'.format(sim_id, file_path))
         assert result.exit_code == 0, 'Failed to get screenshot of {0}'.format(sim_id)
         assert File.exists(file_path), 'Failed to get screenshot of {0}'.format(sim_id)
 
     @staticmethod
     def erase(simulator_info):
-        result = Simctl.__run_simctl_command('erase {0}'.format(simulator_info.id))
+        result = Simctl.run_simctl_command('erase {0}'.format(simulator_info.id))
         assert result.exit_code == 0, 'Failed to erase {0}'.format(simulator_info.name)
         Log.info('Erase {0}.'.format(simulator_info.name))
 
     @staticmethod
     def erase_all():
-        result = Simctl.__run_simctl_command('erase all')
+        result = Simctl.run_simctl_command('erase all')
         assert result.exit_code == 0, 'Failed to erase all iOS Simulators.'
         Log.info('Erase all iOS Simulators.')
 

core_tests/unit/product/test_preview_helpers.py

@@ -12,7 +12,7 @@ class PreviewHelperTests(unittest.TestCase):
     def test_01_constants(self):
         text = File.read(path=os.path.join(self.current_folder, 'preview.log'))
         url = Preview.get_url(output=text)
-        assert 'nsplay://boot\\?instanceId=' in url
+        assert 'nsplay://boot?instanceId=' in url
 
 
 if __name__ == '__main__':

products/nativescript/preview_helpers.py

@@ -1,10 +1,13 @@
 import os
 import re
 import time
+
+from core.enums.device_type import DeviceType
 from core.enums.platform_type import Platform
+from core.log.log import Log
 from core.settings import Settings
 from core.settings.Settings import TEST_SUT_HOME, TEST_RUN_HOME
-from core.utils.device.adb import Adb, ADB_PATH
+from core.utils.device.adb import Adb
 from core.utils.device.simctl import Simctl
 from core.utils.file_utils import File
 from core.utils.run import run
@@ -61,6 +64,8 @@ def get_url(output):
         """
         Get preview URL form tns log.
         This is the url you need to load in Preview app in order to see and sync your project.
+        :param output: Output of `tns preview` command.
+        :return: Playground url.
         """
         url = re.findall(r"(nsplay[^\s']+)", output)[0]
         if Settings.PYTHON_VERSION < 3:
@@ -69,24 +74,30 @@ def get_url(output):
         else:
             from urllib.parse import unquote
             url = unquote(url, 'UTF-8')
-        url = url.replace(r'?', r'\?')
-        url = url.replace(r'&', r'\&')
         return url
 
     @staticmethod
-    def run_url(url, device_id, platform):
+    def run_url(url, device):
         """
-        Runs your project in the Preview App on simulator or emulator
+        Runs project in the Preview App.
+        :param url: Playground url.
+        :param device: DeviceInfo object.
         """
-        if platform is Platform.IOS:
-            cmd = "xcrun simctl openurl {0} {1}.".format(device_id, url)
-            result = run(cmd)
+        # Url needs to be escaped before open with adb or simctl
+        url = url.replace(r'?', r'\?')
+        url = url.replace(r'&', r'\&')
+
+        # Run url
+        Log.info('Open "{0}" on {1}.'.format(url, device.name))
+        if device.type == DeviceType.EMU or device.type == DeviceType.ANDROID:
+            cmd = 'shell am start -a android.intent.action.VIEW -d "{0}" org.nativescript.preview'.format(url)
+            result = Adb.run_adb_command(command=cmd, device_id=device.id)
             assert 'error' not in result.output
-        elif platform is Platform.ANDROID:
-            cmd = '{0} -s {1} shell am start -a android.intent.action.VIEW -d "{2}" org.nativescript.preview' \
-                .format(ADB_PATH, device_id, url)
-            result = run(cmd)
+        elif device.type == DeviceType.SIM:
+            result = Simctl.run_simctl_command(command='openurl {0} {1}.'.format(device.id, url))
             assert 'error' not in result.output
+        else:
+            raise NotImplementedError('Open url not implemented for real iOS devices.')
 
     @staticmethod
     def dismiss_simulator_alert():
@@ -104,7 +115,7 @@ def run_app(app_name, platform, device, bundle=False, hmr=False, uglify=False, a
         # Read the log and extract the url to load the app on emulator
         log = File.read(result.log_file)
         url = Preview.get_url(log)
-        Preview.run_url(url, device.id, platform)
+        Preview.run_url(url=url, device=device)
         # When you run preview on ios simulator on first run confirmation dialog is showh. This script will dismiss it
         if platform == Platform.IOS:
             time.sleep(2)

run_common.py

@@ -11,6 +11,7 @@
 from core.utils.gradle import Gradle
 from core.utils.npm import Npm
 from data.templates import Template
+from products.nativescript.preview_helpers import Preview
 from products.nativescript.tns import Tns
 
 
@@ -129,7 +130,7 @@ def __install_schematics():
     Npm.install(package=Settings.Packages.NS_SCHEMATICS, folder=Settings.TEST_RUN_HOME)
 
 
-def prepare(clone_templates=True, install_ng_cli=False):
+def prepare(clone_templates=True, install_ng_cli=False, get_preivew_packages=False):
     Log.info('================== Prepare Test Run ==================')
     __cleanup()
     __install_ns_cli()
@@ -139,5 +140,7 @@ def prepare(clone_templates=True, install_ng_cli=False):
         __install_schematics()
     if clone_templates:
         __get_templates()
+    if get_preivew_packages:
+        Preview.get_app_packages()
 
     Log.settings()

run_preview.py

@@ -0,0 +1,14 @@
+import sys
+
+import nose
+
+import run_common
+from core.log.log import Log
+
+if __name__ == '__main__':
+    run_common.prepare(clone_templates=False, install_ng_cli=False, get_preivew_packages=True)
+    Log.info("Running tests...")
+    arguments = ['nosetests', '-v', '-s', '--nologcapture', '--logging-filter=nose', '--with-xunit', '--with-flaky']
+    for i in sys.argv:
+        arguments.append(str(i))
+    nose.run(argv=arguments)