chore: fix flaky tests (#452)

dtopuzov · web-flow · commit 91623db0895c · 2019-12-07T22:22:30.000+02:00
- Update timeouts
- Fix values for app size (for 6.3 we include new css parser in modules and new ios webkit also add ~2%)
- Get host os screenshot on test fail
- Log trace on some tests to debug flakyness
- Minor refactoring
- Fix Travis CI
diff --git a/.travis.yml b/.travis.yml
@@ -6,6 +6,8 @@ language: python
 python:
   - "2.7"
 before_install:
+  - export DISPLAY=:0
+  - sudo apt-get install  python-xlib -y
   - pip install -U pip
 install:
   - python -m pip install --upgrade pip
diff --git a/core/base_test/tns_test.py b/core/base_test/tns_test.py
@@ -108,9 +108,18 @@ def kill_emulators():
 
     @staticmethod
     def get_screenshots():
+        # get host snapshot
+        base_path = os.path.join(Settings.TEST_OUT_IMAGES, TestContext.CLASS_NAME, TestContext.TEST_NAME)
+        try:
+            import pyautogui
+            png_path = os.path.join(base_path, 'host.png')
+            pyautogui.screenshot().save(png_path)
+        except Exception:
+            Log.warning('Failed to take screenshot of host os.')
+
+        # get device screenshots
         for device in TestContext.STARTED_DEVICES:
             try:
-                base_path = os.path.join(Settings.TEST_OUT_IMAGES, TestContext.CLASS_NAME, TestContext.TEST_NAME)
                 png_path = os.path.join(base_path, device.name + '.png')
                 File.delete(png_path)
                 device.get_screen(png_path)
diff --git a/data/sync/master_detail_vue.py b/data/sync/master_detail_vue.py
@@ -6,8 +6,8 @@
 from selenium.webdriver.common.by import By
 
 from core.enums.app_type import AppType
-from core.enums.platform_type import Platform
 from core.enums.os_type import OSType
+from core.enums.platform_type import Platform
 from core.settings import Settings
 from core.utils.appium.appium_driver import AppiumDriver
 from core.utils.wait import Wait
@@ -19,11 +19,15 @@
 from products.nativescript.tns_paths import TnsPaths
 
 
-def sync_master_detail_vue(app_name, platform, device, bundle=True, hmr=True):
+def sync_master_detail_vue(app_name, platform, device):
     # Execute tns command
-    result = Tns.run(app_name=app_name, platform=platform, emulator=True, wait=False, bundle=bundle, hmr=hmr)
-    strings = TnsLogs.run_messages(app_name=app_name, platform=platform, run_type=RunType.FULL, bundle=bundle,
-                                   hmr=hmr, app_type=AppType.VUE, transfer_all=True)
+    log_trace = False
+    if platform == Platform.IOS:
+        # Temporary add log trace on iOS to debug an issue
+        log_trace = True
+    result = Tns.run(app_name=app_name, platform=platform, emulator=True, log_trace=log_trace, wait=False)
+    strings = TnsLogs.run_messages(app_name=app_name, platform=platform, run_type=RunType.FULL, app_type=AppType.VUE,
+                                   transfer_all=True)
     TnsLogs.wait_for_log(log_file=result.log_file, string_list=strings, timeout=360)
 
     # start appium driver (need it on iOS only)
@@ -38,23 +42,23 @@ def sync_master_detail_vue(app_name, platform, device, bundle=True, hmr=True):
     device.get_screen(path=initial_state)
 
     # Verify that application is not restarted on file changes when hmr=true
-    if hmr and Settings.HOST_OS != OSType.WINDOWS:
+    if Settings.HOST_OS != OSType.WINDOWS:
         not_existing_string_list = ['Restarting application']
     else:
         not_existing_string_list = None
 
     # Edit template in .vue file
     Sync.replace(app_name=app_name, change_set=Changes.MasterDetailVUE.VUE_TEMPLATE)
     strings = TnsLogs.run_messages(app_name=app_name, platform=platform, run_type=RunType.INCREMENTAL,
-                                   bundle=bundle, hmr=hmr, app_type=AppType.VUE, file_name='CarList.vue')
+                                   app_type=AppType.VUE, file_name='CarList.vue')
     TnsLogs.wait_for_log(log_file=result.log_file, string_list=strings,
                          not_existing_string_list=not_existing_string_list)
     device.wait_for_text(text=Changes.MasterDetailVUE.VUE_TEMPLATE.new_text)
 
     # Edit styling in .vue file
     Sync.replace(app_name=app_name, change_set=Changes.MasterDetailVUE.VUE_STYLE)
     strings = TnsLogs.run_messages(app_name=app_name, platform=platform, run_type=RunType.INCREMENTAL,
-                                   bundle=bundle, hmr=hmr, app_type=AppType.VUE, file_name='CarList.vue')
+                                   app_type=AppType.VUE, file_name='CarList.vue')
     TnsLogs.wait_for_log(log_file=result.log_file, string_list=strings,
                          not_existing_string_list=not_existing_string_list)
     style_applied = Wait.until(lambda: device.get_pixels_by_color(Colors.LIGHT_BLUE) > 200)
@@ -63,7 +67,7 @@ def sync_master_detail_vue(app_name, platform, device, bundle=True, hmr=True):
     # Revert styling in .vue file
     Sync.revert(app_name=app_name, change_set=Changes.MasterDetailVUE.VUE_STYLE)
     strings = TnsLogs.run_messages(app_name=app_name, platform=platform, run_type=RunType.INCREMENTAL,
-                                   bundle=bundle, hmr=hmr, app_type=AppType.VUE, file_name='CarList.vue')
+                                   app_type=AppType.VUE, file_name='CarList.vue')
     TnsLogs.wait_for_log(log_file=result.log_file, string_list=strings,
                          not_existing_string_list=not_existing_string_list)
     style_applied = Wait.until(lambda: device.get_pixels_by_color(Colors.LIGHT_BLUE) < 200)
@@ -79,7 +83,7 @@ def sync_master_detail_vue(app_name, platform, device, bundle=True, hmr=True):
     device.wait_for_text(text="Price")
     Sync.replace(app_name=app_name, change_set=Changes.MasterDetailVUE.VUE_DETAIL_PAGE_TEMPLATE)
     strings = TnsLogs.run_messages(app_name=app_name, platform=platform, run_type=RunType.INCREMENTAL,
-                                   bundle=bundle, hmr=hmr, app_type=AppType.VUE, file_name='CarDetails.vue')
+                                   app_type=AppType.VUE, file_name='CarDetails.vue')
     TnsLogs.wait_for_log(log_file=result.log_file, string_list=strings,
                          not_existing_string_list=not_existing_string_list)
     device.wait_for_text(text=Changes.MasterDetailVUE.VUE_DETAIL_PAGE_TEMPLATE.new_text)
diff --git a/products/nativescript/app.py b/products/nativescript/app.py
@@ -84,7 +84,7 @@ def update(app_name, modules=True, angular=True, typescript=False, web_pack=True
 
         if vue and App.is_dependency(app_name=app_name, dependency='nativescript-vue'):
             Npm.uninstall(package='nativescript-vue', option='--save', folder=app_path)
-            Npm.install(package='nativescript-vue@next', option='--save --save-exact', folder=app_path)
+            Npm.install(package='nativescript-vue', option='--save --save-exact', folder=app_path)
         # on win when `npm i` is executed with path to .tgz it saves in package.json unix style path
         # which cannot be resolved on win
         if Settings.HOST_OS == OSType.WINDOWS:
diff --git a/requirements.txt b/requirements.txt
@@ -17,4 +17,5 @@ subprocess32>=3.5.3
 requests>=2.21.0
 pytz>=2018.9
 Appium-Python-Client>=0.28
-pyautogui==0.9.39
+pyautogui==0.9.39
+python-xlib==0.25
diff --git a/requirements_darwin.txt b/requirements_darwin.txt
@@ -19,4 +19,5 @@ subprocess32>=3.5.3
 requests>=2.21.0
 pytz>=2018.9
 Appium-Python-Client>=0.28
-pyautogui==0.9.39
+pyautogui==0.9.39
+python-xlib==0.25
diff --git a/run_ns.py b/run_ns.py
@@ -8,7 +8,7 @@
 if __name__ == '__main__':
     run_common.prepare(clone_templates=True, install_ng_cli=False)
     Log.info("Running tests...")
-    arguments = ['nosetests', '-v', '-s', '--nologcapture', '--with-doctest', '--with-xunit']
+    arguments = ['nosetests', '-v', '-s', '--nologcapture', '--with-doctest', '--with-xunit', '--with-flaky']
     for i in sys.argv:
         arguments.append(str(i))
     nose.run(argv=arguments)
diff --git a/tests/cli/build/android_app_bundle_tests.py b/tests/cli/build/android_app_bundle_tests.py
@@ -3,9 +3,9 @@
 from core.base_test.tns_run_android_test import TnsRunAndroidTest
 from core.settings.Settings import TEST_SUT_HOME, TEST_RUN_HOME, AppName, Android
 from core.utils.device.adb import Adb
+from core.utils.docker import Docker
 from core.utils.file_utils import Folder, File
 from core.utils.run import run
-from core.utils.docker import Docker
 from data.templates import Template
 from products.nativescript.tns import Tns
 from products.nativescript.tns_logs import TnsLogs
@@ -60,9 +60,9 @@ def bundletool_build(bundletool_path, path_to_aab, path_to_apks):
 
     @staticmethod
     def bundletool_deploy(bundletool_path, path_to_apks, device_id):
-        deploy_command = ('java -jar {0} install-apks --apks="{1}" --device-id={2}').format(bundletool_path,
-                                                                                            path_to_apks,
-                                                                                            device_id)
+        deploy_command = 'java -jar {0} install-apks --apks="{1}" --device-id={2}'.format(bundletool_path,
+                                                                                          path_to_apks,
+                                                                                          device_id)
         result = run(deploy_command)
         assert "Error" not in result.output, "deploy of app failed"
         assert "The APKs have been extracted in the directory:" in result.output, "deploy of app failed"
@@ -81,11 +81,11 @@ def test_100_run_android_app_bundle_compile_snapshot(self):
                                  uglify=True, verify=False, compile_snapshot=True)
         strings = ['Successfully generated snapshots',
                    'The build result is located at: {0}'.format(path_to_aab)]
-        TnsLogs.wait_for_log(log_file=result.log_file, string_list=strings, timeout=180)
+        TnsLogs.wait_for_log(log_file=result.log_file, string_list=strings, timeout=300)
 
         # Verify app can be deployed on emulator via nativescript
         # Verify app looks correct inside emulator
-        self.emu.wait_for_text(text='TAP', timeout=240)
+        self.emu.wait_for_text(text='TAP', timeout=60)
 
         # Verify that the correct .so file is included in the package
         File.unzip(path_to_apks, os.path.join(self.app_name, 'apks'))
diff --git a/tests/cli/preview/templates/hello_word_js_tests.py b/tests/cli/preview/templates/hello_word_js_tests.py
@@ -1,6 +1,8 @@
 import os
 import unittest
 
+from flaky import flaky
+
 from core.base_test.tns_run_test import TnsRunTest
 from core.base_test.tns_test import TnsTest
 from core.enums.app_type import AppType
@@ -15,8 +17,8 @@
 from data.templates import Template
 from products.nativescript.preview_helpers import Preview
 from products.nativescript.tns import Tns
-from products.nativescript.tns_logs import TnsLogs
 from products.nativescript.tns_assert import TnsAssert
+from products.nativescript.tns_logs import TnsLogs
 
 
 class TnsPreviewJSTests(TnsRunTest):
@@ -73,11 +75,13 @@ def test_100_preview_ios(self):
         preview_sync_hello_world_js_ts(app_type=AppType.JS, app_name=self.app_name,
                                        device=self.sim, click_open_alert=True)
 
+    @flaky(max_runs=3)
     def test_205_preview_android_no_hmr(self):
         """Preview project on emulator with --no-hmr. Make valid changes in JS, CSS and XML"""
         preview_sync_hello_world_js_ts(app_type=AppType.JS, app_name=self.app_name,
                                        device=self.emu, hmr=False)
 
+    @flaky(max_runs=3)
     @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
     def test_205_preview_ios_no_hmr(self):
         """Preview project on simulator with --no-hmr. Make valid changes in JS, CSS and XML"""
diff --git a/tests/cli/preview/templates/hello_word_ng_tests.py b/tests/cli/preview/templates/hello_word_ng_tests.py
@@ -1,15 +1,18 @@
 import os
 import unittest
-from core.base_test.tns_test import TnsTest
+
+from flaky import flaky
+
 from core.base_test.tns_run_test import TnsRunTest
+from core.base_test.tns_test import TnsTest
 from core.enums.os_type import OSType
 from core.enums.platform_type import Platform
 from core.settings import Settings
 from core.utils.file_utils import File, Folder
 from data.sync.hello_world_ng import preview_sync_hello_world_ng
 from data.templates import Template
-from products.nativescript.tns import Tns
 from products.nativescript.preview_helpers import Preview
+from products.nativescript.tns import Tns
 
 
 class TnsPreviewNGTests(TnsRunTest):
@@ -65,11 +68,13 @@ def test_100_preview_ios(self):
         preview_sync_hello_world_ng(app_name=self.app_name, platform=Platform.IOS,
                                     device=self.sim, instrumented=True)
 
+    @flaky(max_runs=3)
     def test_205_preview_android_no_hmr(self):
         """Preview project on emulator with --hmr. Make valid changes in TS, CSS and HTML"""
         preview_sync_hello_world_ng(app_name=self.app_name, platform=Platform.ANDROID,
                                     device=self.emu, hmr=False)
 
+    @flaky(max_runs=3)
     @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
     def test_205_preview_ios_no_hmr(self):
         """Preview project on simulator with --hmr. Make valid changes in TS, CSS and HTML"""
diff --git a/tests/perf/app_size/test_app_size.py b/tests/perf/app_size/test_app_size.py
@@ -8,10 +8,10 @@
 from core.base_test.tns_test import TnsTest
 from core.enums.os_type import OSType
 from core.settings import Settings
+from core.utils.docker import Docker
 from core.utils.file_utils import File, Folder
 from core.utils.perf_utils import PerfUtils
 from core.utils.run import run
-from core.utils.docker import Docker
 from data.templates import Template
 from products.nativescript.tns import Tns
 from products.nativescript.tns_paths import TnsPaths
@@ -52,11 +52,11 @@ def tearDownClass(cls):
 
     def test_001_js_app_app_resources(self):
         folder = os.path.join(TnsPaths.get_app_path(app_name=self.js_app), 'app')
-        assert PerfUtils.is_value_in_range(actual=Folder.get_size(folder), expected=1267420, tolerance=0.1)
+        assert PerfUtils.is_value_in_range(actual=Folder.get_size(folder), expected=1268131, tolerance=0.1)
 
     def test_002_js_app_node_modules(self):
         folder = os.path.join(TnsPaths.get_app_path(app_name=self.js_app), 'node_modules')
-        assert PerfUtils.is_value_in_range(actual=Folder.get_size(folder), expected=54883579, tolerance=0.1)
+        assert PerfUtils.is_value_in_range(actual=Folder.get_size(folder), expected=58628036, tolerance=0.1)
 
     def test_003_js_app_apk(self):
         # Extract APK
@@ -71,26 +71,26 @@ def test_003_js_app_apk(self):
         run(cmd='du -hs *', cwd=lib, wait=True, log_level=logging.INFO)
 
         # Verify content of APK
-        assert PerfUtils.is_value_in_range(actual=Folder.get_size(lib), expected=51266016, tolerance=0.05)
+        assert PerfUtils.is_value_in_range(actual=Folder.get_size(lib), expected=51992248, tolerance=0.05)
         assert PerfUtils.is_value_in_range(actual=Folder.get_size(res), expected=796627, tolerance=0.05)
-        assert PerfUtils.is_value_in_range(actual=Folder.get_size(assets_app), expected=839462, tolerance=0.05)
-        assert PerfUtils.is_value_in_range(actual=Folder.get_size(assets_snapshots), expected=10621384, tolerance=0.05)
+        assert PerfUtils.is_value_in_range(actual=Folder.get_size(assets_app), expected=1210914, tolerance=0.05)
+        assert PerfUtils.is_value_in_range(actual=Folder.get_size(assets_snapshots), expected=16023484, tolerance=0.05)
 
         # Verify final apk size
-        assert PerfUtils.is_value_in_range(actual=File.get_size(apk), expected=24304589, tolerance=0.03)
+        assert PerfUtils.is_value_in_range(actual=File.get_size(apk), expected=25826703, tolerance=0.03)
 
     @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
     def test_102_js_app_ipa(self):
         ipa = TnsPaths.get_ipa_path(app_name=self.js_app, release=True, for_device=True)
-        assert PerfUtils.is_value_in_range(actual=File.get_size(ipa), expected=14030096, tolerance=0.02)
+        assert PerfUtils.is_value_in_range(actual=File.get_size(ipa), expected=14465180, tolerance=0.02)
 
     def test_100_ng_app_app_resources(self):
         app_folder = os.path.join(TnsPaths.get_app_path(app_name=self.ng_app), 'App_Resources')
-        assert PerfUtils.is_value_in_range(actual=Folder.get_size(app_folder), expected=1262116, tolerance=0.1)
+        assert PerfUtils.is_value_in_range(actual=Folder.get_size(app_folder), expected=1262699, tolerance=0.1)
 
     def test_101_ng_app_node_modules(self):
         app_folder = os.path.join(TnsPaths.get_app_path(app_name=self.ng_app), 'node_modules')
-        assert PerfUtils.is_value_in_range(actual=Folder.get_size(app_folder), expected=206440791, tolerance=0.1)
+        assert PerfUtils.is_value_in_range(actual=Folder.get_size(app_folder), expected=210482662, tolerance=0.1)
 
     def test_102_ng_app_apk(self):
         # Extract APK
@@ -102,13 +102,13 @@ def test_102_ng_app_apk(self):
         assets_snapshots = os.path.join(extracted_apk, 'assets', 'snapshots')
 
         # No asserts for lib and res, since it is same as JS project
-        assert PerfUtils.is_value_in_range(actual=Folder.get_size(assets_app), expected=1636691, tolerance=0.05)
-        assert PerfUtils.is_value_in_range(actual=Folder.get_size(assets_snapshots), expected=21822104, tolerance=0.05)
+        assert PerfUtils.is_value_in_range(actual=Folder.get_size(assets_app), expected=1991318, tolerance=0.05)
+        assert PerfUtils.is_value_in_range(actual=Folder.get_size(assets_snapshots), expected=27041144, tolerance=0.05)
 
         # Verify final apk size
-        assert PerfUtils.is_value_in_range(actual=File.get_size(apk), expected=26988067, tolerance=0.03)
+        assert PerfUtils.is_value_in_range(actual=File.get_size(apk), expected=28482168, tolerance=0.03)
 
     @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
     def test_102_ng_app_ipa(self):
         ipa = TnsPaths.get_ipa_path(app_name=self.ng_app, release=True, for_device=True)
-        assert PerfUtils.is_value_in_range(actual=File.get_size(ipa), expected=14238596, tolerance=0.02)
+        assert PerfUtils.is_value_in_range(actual=File.get_size(ipa), expected=14672899, tolerance=0.02)
diff --git a/tests/vue/test_vue_preview.py b/tests/vue/test_vue_preview.py
@@ -1,6 +1,8 @@
 import os
 import unittest
 
+from flaky import flaky
+
 from core.base_test.tns_run_test import TnsRunTest
 from core.enums.os_type import OSType
 from core.enums.platform_type import Platform
@@ -12,6 +14,7 @@
 from products.nativescript.tns import Tns
 
 
+@flaky(max_runs=3)
 class VueJSPreviewTests(TnsRunTest):
     app_name = Settings.AppName.DEFAULT
     source_project_dir = os.path.join(Settings.TEST_RUN_HOME, app_name)
