Support tags

Usage:

```sh
./test-run.py --tags foo
./test-run.py --tags foo,bar app/ app-tap/
```

test-run will run only those tests, which have at least one of the
provided tags.

The tags metainfo should be placed within a first comment of a test
file (see examples in the README.md file).

Unsupported features:

* Marking unit tests with tags.
* Multiline comments (use singleline ones for now).

Fixes #22

Author: Totktonada

README.md

@@ -353,6 +353,55 @@ The following files will be removed:
 * `*.inprogress`
 * `[0-9]*/`
 
+### Tags
+
+Usage:
+
+```sh
+./test-run.py --tags foo
+./test-run.py --tags foo,bar app/ app-tap/
+```
+
+test-run will run only those tests, which have at least one of the
+provided tags.
+
+The tags metainfo should be placed within a first comment of a test
+file.
+
+Examples:
+
+* .lua file:
+
+  ```lua
+  #!/usr/bin/tarantool
+
+  -- tags: foo, bar
+  -- tags: one, more
+
+  <...>
+  ```
+
+* .sql file:
+
+  ```sql
+  -- tags: foo
+  -- tags: bar
+  <...>
+  ```
+
+* .py file:
+
+  ```python
+  # tags: foo
+
+  <...>
+  ```
+
+Unsupported features:
+
+* Marking unit tests with tags.
+* Multiline comments (use singleline ones for now).
+
 ### Used By
 
 - [Tarantool](https://github.com/tarantool/tarantool) - in-memory database and application server

lib/options.py

@@ -22,6 +22,13 @@ def env_list(name, default):
     return value_list or default
 
 
+def split_list(tags_str):
+    # Accept both ', ' and ',' as a separator.
+    #
+    # It is consistent with parsing of tags within a test file.
+    return [tag.strip() for tag in tags_str.split(',')]
+
+
 class Options(object):
     """Handle options of test-runner"""
 
@@ -270,6 +277,17 @@ def __init__(self):
                 dest="memtx_allocator",
                 default=os.environ.get("MEMTX_ALLOCATOR", "small"),
                 help="""Memtx allocator type for tests""")
+        parser.add_argument(
+                '--tags',
+                dest='tags',
+                default=None,
+                type=split_list,
+                help="""Comma separated list of tags.
+
+                If tags are provided, test-run will run only those
+                tests, which are marked with ANY of the provided
+                tags.
+                """)
 
         # XXX: We can use parser.parse_intermixed_args() on
         # Python 3.7 to understand commands like

lib/server.py

@@ -14,6 +14,7 @@
 from lib.options import Options
 from lib.utils import print_tail_n
 from lib.utils import bytes_to_str
+from lib.utils import find_tags
 
 DEFAULT_CHECKPOINT_PATTERNS = ["*.snap", "*.xlog", "*.vylog", "*.inprogress",
                                "[0-9]*/"]
@@ -156,14 +157,40 @@ def print_log(self, lines=None):
 
     @staticmethod
     def exclude_tests(test_names, exclude_patterns):
-        def match_any(test_name, patterns):
+        """ Filter out test files by several criteria:
+
+            exclude_patters: a list of strings. If a string is
+            a substring of the test full name, exclude it.
+
+            If --tags option is provided, exclude tests, which
+            have no a tag from the provided list.
+        """
+
+        # TODO: Support filtering by another file (for unit
+        # tests). Transform a test name into a test source name.
+        # Don't forget about out of source build.
+        # TODO: Support multiline comments (mainly for unit
+        # tests).
+
+        def match_any_pattern(test_name, patterns):
             for pattern in patterns:
                 if pattern in test_name:
                     return True
             return False
 
+        def match_any_tag(test_name, accepted_tags):
+            tags = find_tags(test_name)
+            for tag in tags:
+                if tag in accepted_tags:
+                    return True
+            return False
+
+        accepted_tags = Options().args.tags
+
         res = []
         for test_name in test_names:
-            if not match_any(test_name, exclude_patterns):
+            if match_any_pattern(test_name, exclude_patterns):
+                continue
+            if accepted_tags is None or match_any_tag(test_name, accepted_tags):
                 res.append(test_name)
         return res

lib/tarantool_server.py

@@ -76,6 +76,7 @@ class LuaTest(Test):
     RESULT_FILE_VERSION_LINE_RE = re.compile(
         r'^-- test-run result file version (?P<version>\d+)$')
     RESULT_FILE_VERSION_TEMPLATE = '-- test-run result file version {}'
+    TAGS_LINE_RE = re.compile(r'^-- tags:')
 
     def __init__(self, *args, **kwargs):
         super(LuaTest, self).__init__(*args, **kwargs)
@@ -251,6 +252,16 @@ def exec_loop(self, ts):
             # Normalize a line.
             line = line.rstrip('\n')
 
+            # Skip metainformation (only tags at the moment).
+            #
+            # It is to reduce noise changes in result files, when
+            # tags are added or edited.
+            #
+            # TODO: Ideally we should do that only on a first
+            # comment in the file.
+            if self.TAGS_LINE_RE.match(line):
+                continue
+
             # Show empty lines / comments in a result file, but
             # don't send them to tarantool.
             line_is_empty = line.strip() == ''

lib/utils.py

@@ -376,3 +376,37 @@ def str_to_bytes(s):
     if PY2:
         return s
     return s.encode('utf-8')
+
+
+def parse_tag_line(line):
+    tags_str = line.split(':', 1)[1].strip()
+    return [tag.strip() for tag in tags_str.split(',')]
+
+
+def find_tags(filename):
+    """ Extract tags from a first comment in the file.
+    """
+    # TODO: Support multiline comments. See exclude_tests() in
+    # lib/server.py.
+    if filename.endswith('.lua') or filename.endswith('.sql'):
+        singleline_comment = '--'
+    elif filename.endswith('.py'):
+        singleline_comment = '#'
+    else:
+        return []
+
+    tags = []
+    with open(filename, 'r') as f:
+        for line in f:
+            line = line.rstrip('\n')
+            if line.startswith('#!'):
+                pass
+            elif line == '':
+                pass
+            elif line.startswith(singleline_comment + ' tags:'):
+                tags.extend(parse_tag_line(line))
+            elif line.startswith(singleline_comment):
+                pass
+            else:
+                break
+    return tags