make fields typed as lists optional (elastic#1858)

Author: miguelgrinberg

elasticsearch_dsl/document_base.py

@@ -166,24 +166,11 @@ def __init__(self, name: str, bases: Tuple[type, ...], attrs: Dict[str, Any]):
         field_defaults = {}
         for name in fields:
             value = None
-            if name in attrs:
-                # this field has a right-side value, which can be field
-                # instance on its own or wrapped with mapped_field()
-                value = attrs[name]
-                if isinstance(value, dict):
-                    # the mapped_field() wrapper function was used so we need
-                    # to look for the field instance and also record any
-                    # dataclass-style defaults
-                    value = attrs[name].get("_field")
-                    default_value = attrs[name].get("default") or attrs[name].get(
-                        "default_factory"
-                    )
-                    if default_value:
-                        field_defaults[name] = default_value
-            if value is None:
-                # the field does not have an explicit field instance given in
-                # a right-side assignment, so we need to figure out what field
-                # type to use from the annotation
+            required = None
+            multi = None
+            if name in annotations:
+                # the field has a type annotation, so next we try to figure out
+                # what field type we can use
                 type_ = annotations[name]
                 required = True
                 multi = False
@@ -201,9 +188,11 @@ def __init__(self, name: str, bases: Tuple[type, ...], attrs: Dict[str, Any]):
                     elif type_.__origin__ in [list, List]:
                         # List[type] -> mark instance as multi
                         multi = True
+                        required = False
                         type_ = type_.__args__[0]
                     else:
                         break
+                field = None
                 field_args: List[Any] = []
                 field_kwargs: Dict[str, Any] = {}
                 if not isinstance(type_, type):
@@ -215,10 +204,39 @@ def __init__(self, name: str, bases: Tuple[type, ...], attrs: Dict[str, Any]):
                 elif type_ in self.type_annotation_map:
                     # use best field type for the type hint provided
                     field, field_kwargs = self.type_annotation_map[type_]
-                else:
-                    raise TypeError(f"Cannot map type {type_}")
-                field_kwargs = {"multi": multi, "required": required, **field_kwargs}
-                value = field(*field_args, **field_kwargs)
+
+                if field:
+                    field_kwargs = {
+                        "multi": multi,
+                        "required": required,
+                        **field_kwargs,
+                    }
+                    value = field(*field_args, **field_kwargs)
+
+            if name in attrs:
+                # this field has a right-side value, which can be field
+                # instance on its own or wrapped with mapped_field()
+                attr_value = attrs[name]
+                if isinstance(attr_value, dict):
+                    # the mapped_field() wrapper function was used so we need
+                    # to look for the field instance and also record any
+                    # dataclass-style defaults
+                    attr_value = attrs[name].get("_field")
+                    default_value = attrs[name].get("default") or attrs[name].get(
+                        "default_factory"
+                    )
+                    if default_value:
+                        field_defaults[name] = default_value
+                if attr_value:
+                    value = attr_value
+                    if required is not None:
+                        value._required = required
+                    if multi is not None:
+                        value._multi = multi
+
+            if value is None:
+                raise TypeError(f"Cannot map field {name}")
+
             self.mapping.field(name, value)
             if name in attrs:
                 del attrs[name]

tests/_async/test_document.py

@@ -707,28 +707,32 @@ class TypedDoc(AsyncDocument):
     assert doc.s4 == "foo"
     with raises(ValidationException) as exc_info:
         doc.full_clean()
-    assert set(exc_info.value.args[0].keys()) == {"st", "li", "k1"}
+    assert set(exc_info.value.args[0].keys()) == {"st", "k1", "k2", "s1", "s2", "s3"}
 
     doc.st = "s"
     doc.li = [1, 2, 3]
-    doc.k1 = "k"
+    doc.k1 = "k1"
+    doc.k2 = "k2"
+    doc.s1 = "s1"
+    doc.s2 = "s2"
+    doc.s3 = "s3"
     doc.full_clean()
 
     doc.ob = TypedInnerDoc()
     with raises(ValidationException) as exc_info:
         doc.full_clean()
     assert set(exc_info.value.args[0].keys()) == {"ob"}
-    assert set(exc_info.value.args[0]["ob"][0].args[0].keys()) == {"st", "li"}
+    assert set(exc_info.value.args[0]["ob"][0].args[0].keys()) == {"st"}
 
     doc.ob.st = "s"
     doc.ob.li = [1]
     doc.full_clean()
 
-    doc.ns.append(TypedInnerDoc(st="s"))
+    doc.ns.append(TypedInnerDoc(li=[1, 2]))
     with raises(ValidationException) as exc_info:
         doc.full_clean()
 
-    doc.ns[0].li = [1, 2]
+    doc.ns[0].st = "s"
     doc.full_clean()
 
     doc.ip = "1.2.3.4"
@@ -749,8 +753,12 @@ class TypedDoc(AsyncDocument):
             }
         ],
         "ip": "1.2.3.4",
-        "k1": "k",
+        "k1": "k1",
+        "k2": "k2",
         "k3": "foo",
+        "s1": "s1",
+        "s2": "s2",
+        "s3": "s3",
         "s4": "foo",
     }
 

tests/_sync/test_document.py

@@ -707,28 +707,32 @@ class TypedDoc(Document):
     assert doc.s4 == "foo"
     with raises(ValidationException) as exc_info:
         doc.full_clean()
-    assert set(exc_info.value.args[0].keys()) == {"st", "li", "k1"}
+    assert set(exc_info.value.args[0].keys()) == {"st", "k1", "k2", "s1", "s2", "s3"}
 
     doc.st = "s"
     doc.li = [1, 2, 3]
-    doc.k1 = "k"
+    doc.k1 = "k1"
+    doc.k2 = "k2"
+    doc.s1 = "s1"
+    doc.s2 = "s2"
+    doc.s3 = "s3"
     doc.full_clean()
 
     doc.ob = TypedInnerDoc()
     with raises(ValidationException) as exc_info:
         doc.full_clean()
     assert set(exc_info.value.args[0].keys()) == {"ob"}
-    assert set(exc_info.value.args[0]["ob"][0].args[0].keys()) == {"st", "li"}
+    assert set(exc_info.value.args[0]["ob"][0].args[0].keys()) == {"st"}
 
     doc.ob.st = "s"
     doc.ob.li = [1]
     doc.full_clean()
 
-    doc.ns.append(TypedInnerDoc(st="s"))
+    doc.ns.append(TypedInnerDoc(li=[1, 2]))
     with raises(ValidationException) as exc_info:
         doc.full_clean()
 
-    doc.ns[0].li = [1, 2]
+    doc.ns[0].st = "s"
     doc.full_clean()
 
     doc.ip = "1.2.3.4"
@@ -749,8 +753,12 @@ class TypedDoc(Document):
             }
         ],
         "ip": "1.2.3.4",
-        "k1": "k",
+        "k1": "k1",
+        "k2": "k2",
         "k3": "foo",
+        "s1": "s1",
+        "s2": "s2",
+        "s3": "s3",
         "s4": "foo",
     }