Handle position with a custom class instead of string (#60)

* Handle position with a custom class instead of string

Add datapath class to replace the 'dotted string' used to indicate the
path in the data/schema

- Fixes issue with nested maps in an include
- Allows top level validators other than fixed map

* Fix pep8 formating and avoid mutable-default-arguments

* Add strict mode and include validators

- Strict mode will give errors if there are additional elements
present in the data that are not covered by the schema.

- The include mechanism is made a bit more flexible and now allows
arbitrary validators to be included, not just maps.

* Fix issues with strict mode commit

- Update count_exception_line to use assert.
- Fix is_optional on map/list crash.
- Add --strict flag to command line.

Author: drmull

README.md

@@ -184,6 +184,11 @@ person:
 ##### Adding external includes
 After you construct a schema you can add extra, external include definitions by calling `schema.add_include(dict)`. This method takes a dictionary and adds each key as another include.
 
+### Strict mode
+By default Yamale will not give any error for extra elements present in lists and maps that are not covered by the schema. With strict mode any additional element will give an error. Strict mode is enabled by passing the strict=True flag to the validate function.
+
+It is possible to mix strict and non-strict mode by setting the strict=True/False flag in the include validator, setting the option only for the included validators.
+
 Validators
 ----------
 Here are all the validators Yamale knows about. Every validator takes a `required` keyword telling Yamale whether or not that node must exist. By default every node is required. Example: `str(required=False)`

yamale/command_line.py

@@ -19,14 +19,14 @@
 schemas = {}
 
 
-def _validate(schema_path, data_path, parser):
+def _validate(schema_path, data_path, parser, strict):
     schema = schemas.get(schema_path)
     try:
         if not schema:
             schema = yamale.make_schema(schema_path, parser)
             schemas[schema_path] = schema
         data = yamale.make_data(data_path, parser)
-        yamale.validate(schema, data)
+        yamale.validate(schema, data, strict)
     except Exception as e:
         error = '\nError!\n'
         error += 'Schema: %s\n' % schema_path
@@ -60,15 +60,15 @@ def _find_schema(data_path, schema_name):
     return _find_data_path_schema(data_path, schema_name)
 
 
-def _validate_single(yaml_path, schema_name, parser):
+def _validate_single(yaml_path, schema_name, parser, strict):
     print('Validating %s...' % yaml_path)
     s = _find_schema(yaml_path, schema_name)
     if not s:
         raise ValueError("Invalid schema name for '{}' or schema not found.".format(schema_name))
-    _validate(s, yaml_path, parser)
+    _validate(s, yaml_path, parser, strict)
 
 
-def _validate_dir(root, schema_name, cpus, parser):
+def _validate_dir(root, schema_name, cpus, parser, strict):
     pool = Pool(processes=cpus)
     res = []
     print('Finding yaml files...')
@@ -78,7 +78,8 @@ def _validate_dir(root, schema_name, cpus, parser):
                 d = os.path.join(root, f)
                 s = _find_schema(d, schema_name)
                 if s:
-                    res.append(pool.apply_async(_validate, (s, d, parser)))
+                    res.append(pool.apply_async(_validate,
+                                                (s, d, parser, strict)))
                 else:
                     print('No schema found for: %s' % d)
 
@@ -90,12 +91,12 @@ def _validate_dir(root, schema_name, cpus, parser):
     pool.join()
 
 
-def _router(root, schema_name, cpus, parser):
+def _router(root, schema_name, cpus, parser, strict=False):
     root = os.path.abspath(root)
     if os.path.isfile(root):
-        _validate_single(root, schema_name, parser)
+        _validate_single(root, schema_name, parser, strict)
     else:
-        _validate_dir(root, schema_name, cpus, parser)
+        _validate_dir(root, schema_name, cpus, parser, strict)
 
 
 def main():
@@ -108,8 +109,10 @@ def main():
                         help='number of CPUs to use. Default is 4.')
     parser.add_argument('-p', '--parser', default='pyyaml',
                         help='YAML library to load files. Choices are "ruamel" or "pyyaml" (default).')
+    parser.add_argument('--strict', action='store_true',
+                        help='Enable strict mode, unexpected elements in the data will not be accepted.')
     args = parser.parse_args()
-    _router(args.path, args.schema, args.cpu_num, args.parser)
+    _router(args.path, args.schema, args.cpu_num, args.parser, args.strict)
     print('Validation success! 👍')
 
 

yamale/schema/__init__.py

@@ -1,2 +1 @@
 from .schema import Schema
-from .data import Data

yamale/schema/data.py

@@ -0,0 +1,15 @@
+class DataPath(object):
+
+    def __init__(self, *path):
+        self._path = path
+
+    def __add__(self, other):
+        dp = DataPath()
+        dp._path = self._path + other._path
+        return dp
+
+    def __str__(self):
+        return '.'.join(map(str, (self._path)))
+
+    def __repr__(self):
+        return 'DataPath({})'.format(repr(self._path))

yamale/schema/datapath.py

@@ -1,8 +1,7 @@
 import sys
-
+from .datapath import DataPath
 from .. import syntax, util
 from .. import validators as val
-from yamale.util import YAMALE_SEP
 
 # Fix Python 2.x.
 PY2 = sys.version_info[0] == 2
@@ -13,123 +12,149 @@ class Schema(object):
     Makes a Schema object from a schema dict.
     Still acts like a dict.
     """
-    def __init__(self, schema_dict, name='', validators=None):
+    def __init__(self, schema_dict, name='', validators=None, includes=None):
         self.validators = validators or val.DefaultValidators
         self.dict = schema_dict
         self.name = name
-        self._schema = self._process_schema(schema_dict, self.validators)
-        self.includes = {}
+        self._schema = self._process_schema(DataPath(),
+                                            schema_dict,
+                                            self.validators)
+        # if this schema is included it shares the includes with the top level
+        # schema
+        self.includes = {} if includes is None else includes
 
     def add_include(self, type_dict):
         for include_name, custom_type in type_dict.items():
             t = Schema(custom_type, name=include_name,
-                       validators=self.validators)
+                       validators=self.validators, includes=self.includes)
             self.includes[include_name] = t
 
-    def __getitem__(self, key):
-        return self._schema[key]
-
-    def _process_schema(self, schema_dict, validators):
+    def _process_schema(self, path, schema_data, validators):
         """
         Go through a schema and construct validators.
         """
-        schema_flat = util.flatten(schema_dict)
-
-        for key, expression in schema_flat.items():
-            try:
-                schema_flat[key] = syntax.parse(expression, validators)
-            except SyntaxError as e:
-                # Tack on some more context and rethrow.
-                error = str(e) + ' at node \'%s\'' % key
-                raise SyntaxError(error)
-        return schema_flat
-
-    def validate(self, data):
-        errors = []
-
-        for key, validator in self._schema.items():
-            errors += self._validate(validator, data, key=key, includes=self.includes)
+        if util.is_map(schema_data) or util.is_list(schema_data):
+            for key, data in util.get_iter(schema_data):
+                schema_data[key] = self._process_schema(path + DataPath(key),
+                                                        data,
+                                                        validators)
+        else:
+            schema_data = self._parse_schema_item(path,
+                                                  schema_data,
+                                                  validators)
+        return schema_data
+
+    def _parse_schema_item(self, path, expression, validators):
+        try:
+            return syntax.parse(expression, validators)
+        except SyntaxError as e:
+            # Tack on some more context and rethrow.
+            error = str(e) + ' at node \'%s\'' % str(path)
+            raise SyntaxError(error)
+
+    def validate(self, data, data_name, strict):
+        path = DataPath()
+        errors = self._validate(self._schema, data, path, strict)
 
         if errors:
-            header = '\nError validating data %s with schema %s' % (data.name, self.name)
+            header = '\nError validating data %s with schema %s' % (data_name,
+                                                                    self.name)
             error_str = header + '\n\t' + '\n\t'.join(errors)
             if PY2:
                 error_str = error_str.encode('utf-8')
             raise ValueError(error_str)
 
-    def _validate(self, validator, data, key, position=None, includes=None):
+    def _validate_item(self, validator, data, path, strict, key):
         """
-        Run through a schema and a data structure,
-        validating along the way.
-
-        Ignores fields that are in the data structure, but not in the schema.
+        Fetch item from data at the postion key and validate with validator.
 
         Returns an array of errors.
         """
         errors = []
-
-        if position:
-            position = '%s%s%s' % (position, util.YAMALE_SEP, key)
-        else:
-            position = key
-
+        path = path + DataPath(key)
         try:  # Pull value out of data. Data can be a map or a list/sequence
-            data_item = util.get_value(data, key)
+            data_item = data[key]
         except KeyError:  # Oops, that field didn't exist.
-            if validator.is_optional:  # Optional? Who cares.
+            # Optional? Who cares.
+            if isinstance(validator, val.Validator) and validator.is_optional:
                 return errors
             # SHUT DOWN EVERTYHING
-            errors.append('%s: Required field missing' % position.replace(util.YAMALE_SEP, '.'))
+            errors.append('%s: Required field missing' % path)
             return errors
 
-        return self._validate_item(validator, data_item, position, includes)
+        return self._validate(validator, data_item, path, strict)
 
-    def _validate_item(self, validator, data_item, position, includes):
+    def _validate(self, validator, data, path, strict):
         """
-        Validates a single data item against validator.
+        Validate data with validator.
+        Special handling of non-primitive validators.
 
         Returns an array of errors.
         """
-        errors = []
 
+        if util.is_list(validator) or util.is_map(validator):
+            return self._validate_static_map_list(validator,
+                                                  data,
+                                                  path,
+                                                  strict)
+
+        errors = []
         # Optional field with optional value? Who cares.
-        if data_item is None and validator.is_optional and validator.can_be_none:
+        if (data is None and
+                validator.is_optional and
+                validator.can_be_none):
             return errors
 
-        errors += self._validate_primitive(validator, data_item, position)
+        errors += self._validate_primitive(validator, data, path)
 
         if errors:
             return errors
 
         if isinstance(validator, val.Include):
-            errors += self._validate_include(validator, data_item,
-                                             includes, position)
+            errors += self._validate_include(validator, data, path, strict)
 
         elif isinstance(validator, (val.Map, val.List)):
-            errors += self._validate_map_list(validator, data_item,
-                                              includes, position)
+            errors += self._validate_map_list(validator, data, path, strict)
 
         elif isinstance(validator, val.Any):
-            errors += self._validate_any(validator, data_item,
-                                         includes, position)
+            errors += self._validate_any(validator, data, path, strict)
 
         return errors
 
-    def _validate_map_list(self, validator, data, includes, pos):
+    def _validate_static_map_list(self, validator, data, path, strict):
+        if util.is_map(validator) and not util.is_map(data):
+            return ["%s : '%s' is not a map" % (path, data)]
+
+        if util.is_list(validator) and not util.is_list(data):
+            return ["%s : '%s' is not a list" % (path, data)]
+
+        errors = []
+
+        if strict:
+            data_keys = set(util.get_keys(data))
+            validator_keys = set(util.get_keys(validator))
+            for key in data_keys - validator_keys:
+                error_path = path + DataPath(key)
+                errors += ['%s: Unexpected element' % error_path]
+
+        for key, sub_validator in util.get_iter(validator):
+            errors += self._validate_item(sub_validator,
+                                          data,
+                                          path,
+                                          strict,
+                                          key)
+        return errors
+
+    def _validate_map_list(self, validator, data, path, strict):
         errors = []
 
         if not validator.validators:
             return errors  # No validators, user just wanted a map.
 
-        if isinstance(validator, val.List):
-            keys = range(len(data))
-        else:
-            keys = data.keys()
-
-        for key in keys:
+        for key in util.get_keys(data):
             sub_errors = []
             for v in validator.validators:
-                err = self._validate(v, data, key, pos, includes)
+                err = self._validate_item(v, data, path, strict, key)
                 if err:
                     sub_errors.append(err)
 
@@ -140,21 +165,18 @@ def _validate_map_list(self, validator, data, includes, pos):
 
         return errors
 
-    def _validate_include(self, validator, data, includes, pos):
-        errors = []
-
-        include_schema = includes.get(validator.include_name)
+    def _validate_include(self, validator, data, path, strict):
+        include_schema = self.includes.get(validator.include_name)
         if not include_schema:
-            errors.append('Include \'%s\' has not been defined.' % validator.include_name)
-            return errors
-
-        for key, validator in include_schema._schema.items():
-            errors += include_schema._validate(
-                validator, data, includes=includes, key=key, position=pos)
-
-        return errors
-
-    def _validate_any(self, validator, data, includes, pos):
+            return [('Include \'%s\' has not been defined.'
+                     % validator.include_name)]
+        strict = strict if validator.strict is None else validator.strict
+        return include_schema._validate(include_schema._schema,
+                                        data,
+                                        path,
+                                        strict)
+
+    def _validate_any(self, validator, data, path, strict):
         errors = []
 
         if not validator.validators:
@@ -163,7 +185,7 @@ def _validate_any(self, validator, data, includes, pos):
 
         sub_errors = []
         for v in validator.validators:
-            err = self._validate_item(v, data, pos, includes)
+            err = self._validate(v, data, path, strict)
             if err:
                 sub_errors.append(err)
 
@@ -174,10 +196,10 @@ def _validate_any(self, validator, data, includes, pos):
 
         return errors
 
-    def _validate_primitive(self, validator, data, pos):
+    def _validate_primitive(self, validator, data, path):
         errors = validator.validate(data)
 
         for i, error in enumerate(errors):
-            errors[i] = '%s: ' % pos.replace(YAMALE_SEP, '.') + error
+            errors[i] = ('%s: ' % path) + error
 
         return errors

yamale/schema/schema.py

@@ -0,0 +1,3 @@
+map:
+  key: value
+  key2: value2