Add tests for NativeScript/nativescript-cli#2916

Author: dtopuzov

README.md

@@ -7,9 +7,9 @@ Install [Python](https://www.python.org/downloads/) 2.7.*
 Install dependencies
 ```
 pip install psutil 
-pip install nose 
 pip install nose_parameterized
 pip install enum34
+pip install flaky
 pip install uiautomator
 pip install Pillow
 pip install pytesseract

tests/device/run_android_tests.py

@@ -17,6 +17,8 @@
 
 import os
 
+from flaky import flaky
+
 from core.base_class.BaseClass import BaseClass
 from core.device.device import Device
 from core.device.emulator import Emulator
@@ -60,6 +62,7 @@ def tearDownClass(cls):
         BaseClass.tearDownClass()
         Emulator.stop()
 
+    @flaky(max_runs=2)
     def test_001_tns_run_android_js_css_xml(self):
         """Make valid changes in JS,CSS and XML"""
 
@@ -171,15 +174,17 @@ def test_210_tns_run_android_add_remove_files_and_folders(self):
         error_message = 'Deleted folder {0} is still available on {1}'.format(new_folder_name, self.DEVICE_ID)
         assert Adb.path_does_not_exist(device_id=self.DEVICE_ID, package_id=app_id, path=path), error_message
 
+    @flaky(max_runs=2)
     def test_300_tns_run_android_emulator_should_start_emulator_even_if_device_is_connected(self):
         """
         `tns run android --emulator` should start emulator even if physical device is connected
         """
         Emulator.stop()
-        output = Tns.run_android(attributes={'--path': self.app_name, '--emulator': '', '--justlaunch': ''},
-                                 assert_success=False)
+        output = Tns.run_android(
+            attributes={'--path': self.app_name, '--timeout': '180', '--emulator': '', '--justlaunch': ''},
+            assert_success=False)
         assert 'Starting Android emulator with image' in output
-        assert Emulator.is_running(device_id=EMULATOR_ID), 'Emulator not started by `tns run android`!'
+        assert Emulator.wait(device_id=EMULATOR_ID), 'Emulator not started by `tns run android`!'
 
     def test_310_tns_run_android_emulator_should_run_only_on_emulator(self):
         """

tests/device/run_ios_tests.py

@@ -18,6 +18,8 @@
 import os
 from time import sleep
 
+from flaky import flaky
+
 from core.base_class.BaseClass import BaseClass
 from core.device.device import Device
 from core.device.emulator import Emulator
@@ -64,6 +66,7 @@ def tearDownClass(cls):
         BaseClass.tearDownClass()
         Simulator.stop()
 
+    @flaky(max_runs=2)
     def test_001_tns_run_ios_js_css_xml(self):
         """Make valid changes in JS,CSS and XML"""
 
@@ -166,13 +169,14 @@ def test_210_tns_run_ios_add_remove_files_and_folders(self):
         strings = ['Successfully transferred test2', 'Successfully transferred test.txt']
         Tns.wait_for_log(log_file=log, string_list=strings)
 
+    @flaky(max_runs=2)
     def test_300_tns_run_ios_emulator_should_start_emulator_even_if_device_is_connected(self):
         """
         `tns run ios --emulator` should start emulator even if physical device is connected
         """
         Simulator.stop()
         Tns.run_ios(attributes={'--path': self.app_name, '--emulator': '', '--justlaunch': ''}, assert_success=False)
-        assert Simulator.is_running()[0], 'iOS Simulator not started by `tns run ios`!'
+        assert Simulator.wait_for_simulator()[0], 'iOS Simulator not started by `tns run ios`!'
 
     def test_310_tns_run_ios_emulator_should_run_only_on_emulator(self):
         """

tests/device/run_ios_tests_ng.py

@@ -5,6 +5,8 @@
 import os
 from time import sleep
 
+from flaky import flaky
+
 from core.base_class.BaseClass import BaseClass
 from core.device.device import Device
 from core.device.emulator import Emulator
@@ -53,6 +55,7 @@ def tearDownClass(cls):
         BaseClass.tearDownClass()
         Simulator.stop()
 
+    @flaky(max_runs=2)
     def test_001_tns_run_ios_ts_css_html(self):
         """Make valid changes in JS,CSS and XML"""
 

tests/emulator/debug_android_tests.py

@@ -3,12 +3,13 @@
 """
 import os
 
+from flaky import flaky
+
 from core.base_class.BaseClass import BaseClass
 from core.device.device import Device
 from core.device.emulator import Emulator
 from core.osutils.file import File
 from core.osutils.folder import Folder
-from core.osutils.process import Process
 from core.settings.settings import EMULATOR_NAME, EMULATOR_ID, ANDROID_RUNTIME_PATH
 from core.tns.tns import Tns
 from core.tns.tns_platform_type import Platform
@@ -38,46 +39,12 @@ def tearDownClass(cls):
         BaseClass.tearDownClass()
         Emulator.stop()
 
-    """
-    Spec:
-
-    Usage                                                                           Synopsis                                                                                          
-    Deploy on device, run the app start Chrome DevTools, and attach the debugger    $ tns debug android                                                                               
-    Deploy on device, run the app and stop at the first code statement              $ tns debug android --debug-brk [--device <Device ID>] [--debug-port <port>] [--timeout <timeout>]
-    Deploy in the native emulator, run the app and stop at the first code statement $ tns debug android --debug-brk --emulator [<Emulator Options>] [--timeout <timeout>]             
-    Attach the debug tools to a running app on device                               $ tns debug android --start [--device <Device ID>] [--debug-port <port>] [--timeout <timeout>]    
-    Attach the debug tools to a running app in the native emulator                  $ tns debug android --start --emulator [<Emulator Options>] [--timeout <timeout>]                 
-    Detach the debug tools                                                          $ tns debug android --stop                                                                        
-
-
-    Prepares, builds and deploys the project when necessary. Debugs your project on a connected device or emulator.
-    While debugging, prints the output from the application in the console and watches for changes in your code. Once a change is detected, it synchronizes the change with all selected devices and restarts/refreshes the application.
-
-    ### Options
-
-        * --device - Specifies a connected device on which to debug the app.
-        * --emulator - Specifies that you want to debug the app in the native Android emulator from the Android SDK.
-        * --debug-brk - Prepares, builds and deploys the application package on a device or in an emulator, launches the Chrome DevTools of your Chrome browser and stops at the first code statement.
-        * --start - Attaches the debug tools to a deployed and running app.
-        * --stop - Detaches the debug tools.
-        * --debug-port - Sets a new port on which to attach the debug tools.
-        * --timeout - Sets the number of seconds that the NativeScript CLI will wait for the debugger to boot. If not set, the default timeout is 90 seconds.
-        * --no-watch - If set, changes in your code will not be reflected during the execution of this command.
-        * --clean - If set, forces rebuilding the native application.
-
-    ### Attributes
-
-        * <Device ID> is the index or name of the target device as listed by $ tns device
-        * <Port> is an accessible port on the device to which you want to attach the debugging tools.
-        * <Emulator Options> is any valid combination of options as listed by $ tns help emulate android
-    """
-
     def __verify_debugger_start(self, log):
         strings = [EMULATOR_ID, 'NativeScript Debugger started', 'To start debugging, open the following URL in Chrome',
                    'chrome-devtools', 'localhost:40000']
         if Device.get_count(platform=Platform.ANDROID) > 0:
             strings.append("Multiple devices found! Starting debugger on emulator")
-        Tns.wait_for_log(log_file=log, string_list=strings, timeout=120, check_interval=10, clean_log=False)
+        Tns.wait_for_log(log_file=log, string_list=strings, timeout=180, check_interval=10, clean_log=False)
         output = File.read(file_path=log, print_content=True)
         assert "closed" not in output
         assert "detached" not in output
@@ -87,7 +54,7 @@ def __verify_debugger_attach(self, log):
         strings = [EMULATOR_ID, 'To start debugging', 'Chrome', 'chrome-devtools', 'localhost:40000']
         if Device.get_count(platform=Platform.ANDROID) > 0:
             strings.append("Multiple devices found! Starting debugger on emulator")
-        Tns.wait_for_log(log_file=log, string_list=strings, timeout=120, check_interval=10, clean_log=False)
+        Tns.wait_for_log(log_file=log, string_list=strings, timeout=180, check_interval=10, clean_log=False)
         output = File.read(file_path=log, print_content=True)
         assert "closed" not in output
         assert "detached" not in output
@@ -131,3 +98,13 @@ def test_003_debug_android_emulator_start(self):
         # Attach debugger
         log = Tns.debug_android(attributes={'--path': self.app_name, '--start': ''})
         self.__verify_debugger_attach(log=log)
+
+    @flaky(max_runs=2)
+    def test_100_debug_android_should_start_emulator_if_there_is_no_device(self):
+        """
+        Debug android should start emulator if emulator is not running
+        """
+
+        Emulator.stop()
+        Tns.debug_android(attributes={'--path': self.app_name, '--emulator': '', '--timeout': '180'})
+        assert Emulator.wait(device_id=EMULATOR_ID)

tests/emulator/run_android_tests.py

@@ -16,6 +16,7 @@
 import unittest
 
 import nose
+from flaky import flaky
 
 from core.base_class.BaseClass import BaseClass
 from core.device.device import Device
@@ -61,6 +62,7 @@ def tearDownClass(cls):
         BaseClass.tearDownClass()
         Emulator.stop()  # We need this because of test_400_tns_run_android_respect_adb_errors
 
+    @flaky(max_runs=2)
     def test_001_tns_run_android_js_css_xml_manifest(self):
         """Make valid changes in JS,CSS and XML"""
 
@@ -395,6 +397,7 @@ def test_330_tns_run_android_sync_all_files(self):
         Device.screen_match(device_name=EMULATOR_NAME, device_id=EMULATOR_ID,
                             expected_image='livesync-hello-world_home')
 
+    @flaky(max_runs=2)
     def test_340_tns_run_should_not_sync_hidden_files(self):
         """
         Adding hidden files should not break run and they should not be transferred.

tests/simulator/debug_ios_tests.py

@@ -4,6 +4,8 @@
 import os
 import time
 
+from flaky import flaky
+
 from core.base_class.BaseClass import BaseClass
 from core.device.device import Device
 from core.device.emulator import Emulator
@@ -116,6 +118,7 @@ def test_003_debug_ios_simulator_start(self):
         log = Tns.debug_ios(attributes={'--path': self.app_name, '--emulator': '', '--start': ''})
         self.__verify_debugger_attach(log=log)
 
+    @flaky(max_runs=2)
     def test_100_debug_ios_simulator_with_livesync(self):
         """
         `tns debug ios` should be able to run with livesync

tests/simulator/run_ios_tests.py

@@ -15,6 +15,7 @@
 import time
 
 import nose
+from flaky import flaky
 
 from core.base_class.BaseClass import BaseClass
 from core.device.device import Device
@@ -59,6 +60,7 @@ def tearDownClass(cls):
         BaseClass.tearDownClass()
         Emulator.stop()
 
+    @flaky(max_runs=2)
     def test_001_tns_run_ios_js_css_xml(self):
         """Make valid changes in JS,CSS and XML"""
 
@@ -261,6 +263,7 @@ def test_330_tns_run_ios_sync_all_files(self):
         Device.screen_match(device_name=SIMULATOR_NAME, device_id=self.SIMULATOR_ID,
                             expected_image='livesync-hello-world_home')
 
+    @flaky(max_runs=2)
     def test_340_tns_run_should_not_sync_hidden_files(self):
         """
         Adding hidden files should not break run and they should not be transferred.