Merge pull request #44 from dfxml-working-group/update_testing_and_schema_version

Update testing and schema version

Author: ajnelson-nist

.github/workflows/continuous-integration.yml

@@ -15,10 +15,10 @@ jobs:
         python-version: [3.8, 3.9]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

dependencies/dfxml_schema

@@ -157,7 +157,7 @@ def _read_differential_annotations(annodict, element, annoset):
             annoset.add(_d[an])
     #_logger.debug("annoset, after: %r." % annoset)
 
-def _qsplit(tagname):
+def _qsplit(tagname: str) -> typing.Tuple[typing.Optional[str], str]:
     """Requires string input.  Returns namespace and local tag name as a pair.  I could've sworn this was a basic implementation gimme, but ET.QName ain't it."""
     _typecheck(tagname, str)
     if tagname[0] == "{":
@@ -2369,7 +2369,7 @@ def compare_to_other(self, other, ignore_original=False) -> typing.Set[str]:
                     diffs.add(prop)
         return diffs
 
-    def populate_from_Element(self, e):
+    def populate_from_Element(self, e: ET.Element) -> None:
         global _warned_elements
         _typecheck(e, (ET.Element, ET.ElementTree))
         #_logger.debug("e = %r" % e)
@@ -2412,23 +2412,33 @@ def populate_from_Element(self, e):
                 #_logger.debug("getattr(self, %r) = %r" % (ctn, getattr(self, ctn)))
             elif cns not in [dfxml.XMLNS_DFXML, ""]:
                 # Put all non-DFXML-namespace elements into the externals list.
+                _logger.debug("ce.tag = %r.", ce.tag)
                 self.externals.append(ce)
             else:
                 if (cns, ctn, VolumeObject) not in _warned_elements:
                     _warned_elements.add((cns, ctn, VolumeObject))
                     _logger.warning("Unsure what to do with this element in a VolumeObject: %r" % ce)
 
-    def pop_poststream_elements(self, volume_element):
+    def pop_poststream_elements(self, volume_element: ET.Element) -> typing.List[ET.Element]:
         """
         This function is a utility function for the two whole-object serialization methods, print_dfxml (to string) and to_Element (to ET.Element).
 
         This function will mutate the input argument volume_element, removing the 'poststream' elements.  volume_element is expected to be the element created by self.to_partial_Element.
 
         Returns a list of child elements in DFXML Schema order.  List might be empty.
 
-        This subroutine implements a re-serialization implementation decision: elements in extension namespaces can appear at the beginning or end of the volume XML child list, per the DFXML Schema.  All of the externals are put into the beginning of the element in to_partial_Element.  Hence, error will be the last child.
+        This subroutine implements a re-serialization implementation decision: elements in extension namespaces can appear only at the end of the volume XML child list, per the DFXML Schema.  All of the externals are put into the end of the element in to_partial_Element.
         """
-        retval = []
+        retval: typing.List[ET.Element] = []
+
+        # Find all non-DFXML-namespaced elements by working from back of list of child elements, stopping when list is empty, or when a DFXML-namespaced element is encountered.
+        while len(volume_element) > 0:
+            if _qsplit(volume_element[-1].tag)[0] in [None, dfxml.XMLNS_DFXML, ""]:
+                break
+            else:
+                # (ET.Element does not have pop().)
+                retval.insert(0, volume_element[-1])
+                del(volume_element[-1])
 
         errorel = None
         if not (self.error is None or self.error == ""):
@@ -2438,12 +2448,8 @@ def pop_poststream_elements(self, volume_element):
                 # (ET.Element does not have pop().)
                 errorel = volume_element[-1]
                 del(volume_element[-1])
-            else:
-                # This branch of code should only be reached in unit testing, as it depends on the output of self.to_partial_Element.
-                if volume_element.find("error"):
-                    raise ValueError("Inconsistent serialization state: Partial volume XML element has an immediate child named 'error', but not as the last child as expected from the schema.")
         if not errorel is None:
-            retval.append(errorel)
+            retval.insert(0, errorel)
 
         return retval
 
@@ -2494,7 +2500,7 @@ def print_dfxml(self, output_fh=sys.stdout):
 
         output_fh.write("\n")
 
-    def to_Element(self):
+    def to_Element(self) -> ET.Element:
         outel = self.to_partial_Element()
 
         poststream_elements = self.pop_poststream_elements(outel)
@@ -2515,7 +2521,7 @@ def to_Element(self):
 
         return outel
 
-    def to_partial_Element(self):
+    def to_partial_Element(self) -> ET.Element:
         """Returns the volume element with its properties, except for the child fileobjects.  Properties are appended in DFXML schema order."""
         outel = ET.Element("volume")
 
@@ -2530,13 +2536,10 @@ def to_partial_Element(self):
         if len(annos_whittle_set) > 0:
             _logger.warning("Failed to export some differential annotations: %r." % annos_whittle_set)
 
-        for e in self.externals:
-            outel.append(e)
-
         if self.byte_runs:
             outel.append(self.byte_runs.to_Element())
 
-        def _append_el(prop, value):
+        def _append_el(prop: str, value: typing.Any) -> None:
             tmpel = ET.Element(prop)
             _keep = False
             if not value is None:
@@ -2549,7 +2552,7 @@ def _append_el(prop, value):
             if _keep:
                 outel.append(tmpel)
 
-        def _append_str(prop):
+        def _append_str(prop: str) -> None:
             value = getattr(self, prop)
             _append_el(prop, value)
 
@@ -2589,9 +2592,11 @@ def _append_bool(prop):
             outel.append(tmpel)
 
         # Output the error property (which will be popped and re-appended after the file list in to_Element).
-        # The error should come last because of the two spots extended elements can be placed; this is to simplify the file-listing VolumeObject.to_Element() method.
         _append_str("error")
 
+        for e in self.externals:
+            outel.append(e)
+
         if len(diffs_whittle_set) > 0:
             _logger.warning("Did not annotate all of the differing properties of this volume.  Remaining properties:  %r." % diffs_whittle_set)
 
@@ -2649,12 +2654,12 @@ def error(self, val):
         self._error = _strcast(val)
 
     @property
-    def externals(self):
+    def externals(self) -> OtherNSElementList:
         """(This property behaves the same as FileObject.externals.)"""
         return self._externals
 
     @externals.setter
-    def externals(self, val):
+    def externals(self, val: OtherNSElementList) -> None:
         _typecheck(val, OtherNSElementList)
         self._externals = val
 
@@ -3427,7 +3432,7 @@ def populate_from_Element(self, e):
                 getattr(self, ctn).populate_from_Element(ce)
             elif ctn in FileObject._class_properties:
                 setattr(self, ctn, ce.text)
-            elif cns not in [dfxml.XMLNS_DFXML, ""]:
+            elif cns not in [None, dfxml.XMLNS_DFXML, ""]:
                 # Put all non-DFXML-namespace elements into the externals list.
                 self.externals.append(ce)
             else:

dfxml/objects.py

@@ -31,7 +31,7 @@ all: \
   all-make_differential_dfxml \
   all-walk_to_dfxml \
   check-mypy \
-  check-mypy-strict
+  check-mypy-stricter
 
 all-make_differential_dfxml: \
   .venv.done.log
@@ -75,7 +75,7 @@ check: \
 
 #TODO - Type-checking would best be done against all of ../dfxml, when someone finds some time to do so.
 check-mypy: \
-  check-mypy-strict
+  check-mypy-stricter
 	source venv/bin/activate \
 	  && mypy \
 	    ../dfxml/bin/idifference.py \
@@ -88,11 +88,10 @@ check-mypy: \
 	@echo "INFO:tests/Makefile:mypy is currently run against a subset of the dfxml directory." >&2
 
 #TODO - Strict type-checking is another long-term goal, likewise eventually done against all of ../dfxml.
-check-mypy-strict: \
+check-mypy-stricter: \
   .venv.done.log
 	source venv/bin/activate \
 	  && mypy \
-	    --strict \
 	    ../demos/demo_fiwalk_diskimage.py \
 	    ../dfxml/bin/idifference2.py \
 	    ../dfxml/bin/make_differential_dfxml.py \

tests/Makefile

@@ -100,6 +100,7 @@ def test_externals():
     assert len(vor.externals) == 3
 
 def test_prefixed_externals_round_trip():
+    _logger = logging.getLogger(os.path.basename(__file__))
     dobj = Objects.DFXMLObject(version="1.2.0")
     vobj = Objects.VolumeObject()
     dobj.append(vobj)