Bump case-utils adoption to 0.9.0; validate with case_validate

This patch does the minimum steps needed to update `case_exiftool` to
validate against CASE 1.1.0.  One generated artifact, a list of
undefined vocabulary, is removed, as its purpose is now supplanted by
the IRI typo checker in `case_validate` (`case-utils` Issue 40).

The minimum Python version is now bumped to 3.8 due to an update in
NumPy (see `case-utils` PR 99).

A follow-on patch will regenerate Make-managed files.

References:
* casework/CASE-Utilities-Python#40
* casework/CASE-Utilities-Python#99

Signed-off-by: Alex Nelson <[email protected]>

Author: ajnelson-nist

.github/workflows/ci.yml

@@ -31,10 +31,6 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install Python virtualenv for Github runner
-      run: |
-        python -m pip install --upgrade pip
-        pip install virtualenv
     - name: Start from clean state
       run: make clean
     - name: Run tests

Makefile

@@ -30,8 +30,10 @@ all:
 	cd dependencies \
 	  && git diff . \
 	    | cat
-	git submodule init
-	git submodule update
+	git submodule update --init
+	$(MAKE) \
+	  --directory dependencies/CASE-Examples-QC \
+	  .git_submodule_init.done.log
 	touch $@
 
 check: \
@@ -57,8 +59,6 @@ dependencies/CASE-Examples-QC/tests/ontology_vocabulary.txt: \
 	$(MAKE) \
 	  PYTHON3=$(PYTHON3) \
 	  --directory dependencies/CASE-Examples-QC \
-	  .git_submodule_init.done.log \
-	  .lib.done.log \
 	  .venv.done.log
 	$(MAKE) \
 	  PYTHON3=$(PYTHON3) \

README.md

@@ -50,12 +50,10 @@ This project follows [SEMVER 2.0.0](https://semver.org/) where versions are decl
 
 ## Ontology versions supported
 
-This repository supports the CASE and UCO ontology versions that are linked as submodules in the [CASE Examples QC](https://github.com/ajnelson-nist/CASE-Examples-QC) repository.  Currently, those are:
+This repository supports the CASE and UCO ontology versions that are distributed with the [CASE-Utilities-Python repository](https://github.com/casework/CASE-Utilities-Python), at the newest version below a ceiling-pin in [setup.cfg](setup.cfg).  Currently, those ontology versions are:
 
-* CASE 0.3.0
-* UCO 0.5.0
-
-Classes and properties are tested for vocabulary conformance.
+* CASE 1.1.0
+* UCO 1.1.0
 
 
 ## Repository locations

case_exiftool/__init__.py

@@ -25,6 +25,7 @@
 import rdflib.plugins.sparql
 
 import case_utils
+from case_utils.namespace import NS_RDF, NS_RDFS, NS_UCO_CORE, NS_UCO_IDENTITY, NS_UCO_LOCATION, NS_UCO_OBSERVABLE, NS_UCO_TYPES, NS_XSD
 
 _logger = logging.getLogger(os.path.basename(__file__))
 
@@ -40,14 +41,6 @@
 NS_EXIFTOOL_PREVIEWIFD = rdflib.Namespace("http://ns.exiftool.ca/MakerNotes/PreviewIFD/1.0/")
 NS_EXIFTOOL_INTEROPIFD = rdflib.Namespace("http://ns.exiftool.ca/EXIF/InteropIFD/1.0/")
 NS_EXIFTOOL_IFD1 = rdflib.Namespace("http://ns.exiftool.ca/EXIF/IFD1/1.0/")
-NS_RDF = rdflib.RDF
-NS_RDFS = rdflib.RDFS
-NS_UCO_CORE = rdflib.Namespace("https://unifiedcyberontology.org/ontology/uco/core#")
-NS_UCO_LOCATION = rdflib.Namespace("https://unifiedcyberontology.org/ontology/uco/location#")
-NS_UCO_OBSERVABLE = rdflib.Namespace("https://unifiedcyberontology.org/ontology/uco/observable#")
-NS_UCO_TYPES = rdflib.Namespace("https://unifiedcyberontology.org/ontology/uco/types#")
-NS_UCO_VOCABULARY = rdflib.Namespace("https://unifiedcyberontology.org/ontology/uco/vocabulary#")
-NS_XSD = rdflib.namespace.XSD
 
 argument_parser = argparse.ArgumentParser(epilog=__doc__)
 argument_parser.add_argument("--base-prefix", default="http://example.org/kb/")
@@ -57,8 +50,8 @@
 argument_parser.add_argument("--raw-xml", help="A file recording the output of ExifTool run against some file.  Expects exiftool was run with -binary, -duplicates, and -xmlFormat.", required=True)
 argument_parser.add_argument("out_graph", help="A self-contained RDF graph file, in the format either requested by --output-format or guessed based on extension.")
 
-def controlled_dictionary_object_to_node(graph, controlled_dict):
-    n_controlled_dictionary = rdflib.BNode()
+def controlled_dictionary_object_to_node(graph, ns_base, controlled_dict):
+    n_controlled_dictionary = ns_base["controlled-dictionary-" + case_utils.local_uuid.local_uuid()]
     graph.add((
       n_controlled_dictionary,
       NS_RDF.type,
@@ -71,7 +64,7 @@ def controlled_dictionary_object_to_node(graph, controlled_dict):
         except:
             _logger.info("v_value = %r." % v_value)
             raise
-        n_entry = rdflib.BNode()
+        n_entry = ns_base["controlled-dictionary-entry-" + case_utils.local_uuid.local_uuid()]
         graph.add((
           n_controlled_dictionary,
           NS_UCO_TYPES.entry,
@@ -94,6 +87,15 @@ def controlled_dictionary_object_to_node(graph, controlled_dict):
         ))
     return n_controlled_dictionary
 
+def manufacturer_name_to_node(graph: rdflib.Graph, ns_base: rdflib.Namespace, name: str) -> rdflib.URIRef:
+    """
+    This method is provided to be overwritten in case a mapping function exists within the user's knowledge base.
+    """
+    n_manufacturer = ns_base["identity-" + case_utils.local_uuid.local_uuid()]
+    graph.add((n_manufacturer, NS_RDF.type, NS_UCO_IDENTITY.Identity))
+    graph.add((n_manufacturer, NS_UCO_CORE.name, rdflib.Literal(name)))
+    return n_manufacturer
+
 class ExifToolRDFMapper(object):
     """
     This class maps ExifTool RDF predicates into UCO objects and Facets.
@@ -207,10 +209,11 @@ def map_raw_and_printconv_iri(self, exiftool_iri):
             self.exif_dictionary_dict[dict_key] = v_raw
         elif exiftool_iri == "http://ns.exiftool.ca/EXIF/IFD0/1.0/Make":
             (v_raw, v_printconv) = self.pop_iri(exiftool_iri)
+            n_manufacturer = manufacturer_name_to_node(self.graph, self.ns_base, v_printconv)
             self.graph.add((
               self.n_camera_object_device_facet,
               NS_UCO_OBSERVABLE.manufacturer,
-              v_printconv
+              n_manufacturer
             ))
         elif exiftool_iri == "http://ns.exiftool.ca/EXIF/IFD0/1.0/Model":
             (v_raw, v_printconv) = self.pop_iri(exiftool_iri)
@@ -275,7 +278,7 @@ def map_raw_and_printconv_iri(self, exiftool_iri):
             self.graph.add((
               self.n_content_data_facet,
               NS_UCO_OBSERVABLE.sizeInBytes,
-              rdflib.Literal(v_raw.toPython(), datatype=NS_XSD.long)
+              rdflib.Literal(v_raw.toPython(), datatype=NS_XSD.integer)
             ))
         else:
             # Somewhat in the name of information preservation, somewhat as a progress marker on converting data: Attach all remaining unconverted properties directly to the ObservableObject.  Provide both values to assist with mapping decisions.
@@ -438,7 +441,7 @@ def n_camera_object_device_facet(self):
         Initialized on first access.
         """
         if self._n_camera_object_device_facet is None:
-            self._n_camera_object_device_facet = rdflib.BNode()
+            self._n_camera_object_device_facet = self.ns_base["device-facet-" + case_utils.local_uuid.local_uuid()]
             self.graph.add((
               self._n_camera_object_device_facet,
               NS_RDF.type,
@@ -457,7 +460,7 @@ def n_content_data_facet(self):
         Initialized on first access.
         """
         if self._n_content_data_facet is None:
-            self._n_content_data_facet = rdflib.BNode()
+            self._n_content_data_facet = self.ns_base["content-data-facet-" + case_utils.local_uuid.local_uuid()]
             self.graph.add((
               self._n_content_data_facet,
               NS_RDF.type,
@@ -476,7 +479,7 @@ def n_exif_dictionary_object(self):
         Initialized on first access.
         """
         if self._n_exif_dictionary_object is None:
-            self._n_exif_dictionary_object = controlled_dictionary_object_to_node(self.graph, self.exif_dictionary_dict)
+            self._n_exif_dictionary_object = controlled_dictionary_object_to_node(self.graph, self.ns_base, self.exif_dictionary_dict)
             self.graph.add((
               self.n_exif_facet,
               NS_UCO_OBSERVABLE.exifData,
@@ -490,7 +493,7 @@ def n_exif_facet(self):
         Initialized on first access.
         """
         if self._n_exif_facet is None:
-            self._n_exif_facet = rdflib.BNode()
+            self._n_exif_facet = self.ns_base["exif-facet-" + case_utils.local_uuid.local_uuid()]
             self.graph.add((
               self._n_exif_facet,
               NS_RDF.type,
@@ -509,7 +512,7 @@ def n_file_facet(self):
         Initialized on first access.
         """
         if self._n_file_facet is None:
-            self._n_file_facet = rdflib.BNode()
+            self._n_file_facet = self.ns_base["file-facet-" + case_utils.local_uuid.local_uuid()]
             self.graph.add((
               self._n_file_facet,
               NS_RDF.type,
@@ -542,11 +545,11 @@ def n_location_object_latlong_facet(self):
         Initialized on first access.
         """
         if self._n_location_object_latlong_facet is None:
-            self._n_location_object_latlong_facet = rdflib.BNode()
+            self._n_location_object_latlong_facet = self.ns_base["lat-long-coordinates-facet-" + case_utils.local_uuid.local_uuid()]
             self.graph.add((
               self._n_location_object_latlong_facet,
               NS_RDF.type,
-              NS_UCO_LOCATION.LatLongCoordinates
+              NS_UCO_LOCATION.LatLongCoordinatesFacet
             ))
             self.graph.add((
               self.n_location_object,
@@ -576,7 +579,7 @@ def n_raster_picture_facet(self):
         Initialized on first access.
         """
         if self._n_raster_picture_facet is None:
-            self._n_raster_picture_facet = rdflib.BNode()
+            self._n_raster_picture_facet = self.ns_base["raster-picture-facet-" + case_utils.local_uuid.local_uuid()]
             self.graph.add((
               self._n_raster_picture_facet,
               NS_RDF.type,
@@ -611,10 +614,15 @@ def n_relationship_object_location(self):
               NS_UCO_CORE.target,
               self.n_observable_object
             ))
+            self.graph.add((
+              self._n_relationship_object_location,
+              NS_UCO_CORE.isDirectional,
+              rdflib.Literal(True)
+            ))
             self.graph.add((
               self._n_relationship_object_location,
               NS_UCO_CORE.kindOfRelationship,
-              rdflib.Literal("Extracted_From", datatype=NS_UCO_VOCABULARY.ObservableObjectRelationshipVocab)
+              rdflib.Literal("Extracted_From")
             ))
         return self._n_relationship_object_location
 
@@ -624,7 +632,7 @@ def n_unix_file_permissions_facet(self):
         Initialized on first access.
         """
         if self._n_unix_file_permissions_facet is None:
-            self._n_unix_file_permissions_facet = rdflib.BNode()
+            self._n_unix_file_permissions_facet = self.ns_base["unix-file-permissions-facet-" + case_utils.local_uuid.local_uuid()]
             self.graph.add((
               self._n_unix_file_permissions_facet,
               NS_RDF.type,
@@ -680,10 +688,10 @@ def main():
     out_graph.namespace_manager.bind("exiftool-IFD1", NS_EXIFTOOL_IFD1)
     out_graph.namespace_manager.bind("kb", NS_BASE)
     out_graph.namespace_manager.bind("uco-core", NS_UCO_CORE)
+    out_graph.namespace_manager.bind("uco-identity", NS_UCO_IDENTITY)
     out_graph.namespace_manager.bind("uco-location", NS_UCO_LOCATION)
     out_graph.namespace_manager.bind("uco-observable", NS_UCO_OBSERVABLE)
     out_graph.namespace_manager.bind("uco-types", NS_UCO_TYPES)
-    out_graph.namespace_manager.bind("uco-vocabulary", NS_UCO_VOCABULARY)
 
     exiftool_rdf_mapper = ExifToolRDFMapper(out_graph, NS_BASE)
     exiftool_rdf_mapper.map_raw_and_printconv_rdf(args.raw_xml, args.print_conv_xml)

dependencies/CASE-Examples-QC

@@ -17,15 +17,10 @@ classifiers =
 
 [options]
 install_requires =
-    case_utils
-    # TODO - This constraint on pyparsing can be removed when rdflib Issue #1190 is resolved.
-    # https://github.com/RDFLib/rdflib/issues/1190
-    pyparsing < 3.0.0
+    case_utils >= 0.9.0, < 0.10.0
     python-dateutil
-    rdflib
-    requests
 packages = find:
-python_requires = >=3.6
+python_requires = >=3.8
 
 [options.entry_points]
 console_scripts =

dependencies/CASE-Utilities-Python

@@ -34,14 +34,14 @@ all: \
   $(top_srcdir)/setup.cfg \
   $(top_srcdir)/setup.py
 	rm -rf venv
-	$(PYTHON3) -m virtualenv \
-	  --python=$(PYTHON3) \
+	$(PYTHON3) -m venv \
 	  venv
 	source venv/bin/activate \
 	  && pip install \
 	    --upgrade \
 	    pip \
-	    setuptools
+	    setuptools \
+	    wheel
 	source venv/bin/activate \
 	  && cd $(top_srcdir)/dependencies/CASE-Utilities-Python \
 	    && pip install \

setup.cfg

@@ -17,11 +17,9 @@ top_srcdir := $(shell cd ../../../../.. ; pwd)
 
 qc_srcdir := $(top_srcdir)/dependencies/CASE-Examples-QC
 
-case_srcdir := $(top_srcdir)/dependencies/CASE-Examples-QC/dependencies/CASE-Examples/dependencies/CASE
-
 tests_srcdir := $(top_srcdir)/tests
 
-RDF_TOOLKIT_JAR := $(case_srcdir)/lib/rdf-toolkit.jar
+RDF_TOOLKIT_JAR := $(qc_srcdir)/dependencies/CASE-Examples/dependencies/UCO-develop/lib/rdf-toolkit.jar
 
 COMM ?= $(shell which gcomm 2>/dev/null || which comm)
 ifeq ($(COMM),)
@@ -34,7 +32,7 @@ $(error sort not found)
 endif
 
 all: \
-  undefined_vocabulary.txt
+  analysis.json
 
 analysis.json: \
   analysis.ttl \
@@ -51,6 +49,9 @@ analysis.json: \
 	  && python3 $(tests_srcdir)/src/compact.py \
 	    __$@ \
 	    _$@
+	source $(top_srcdir)/tests/venv/bin/activate \
+	  && case_validate \
+	    _$@
 	rm __$@
 	mv _$@ $@
 
@@ -60,13 +61,16 @@ analysis.ttl: \
   $(top_srcdir)/case_exiftool/__init__.py \
   799987_printConv.xml \
   799987_raw.xml
-	export DEMO_UUID_REQUESTING_NONRANDOM=NONRANDOM_REQUESTED \
+	export CASE_DEMO_NONRANDOM_UUID_BASE=$(top_srcdir) \
 	  && source $(top_srcdir)/tests/venv/bin/activate \
 	    && case_exiftool \
 	      --output-format turtle \
 	      --print-conv-xml 799987_printConv.xml \
 	      --raw-xml 799987_raw.xml \
 	      __$@
+	source $(top_srcdir)/tests/venv/bin/activate \
+	  && case_validate \
+	    __$@
 	java -jar $(RDF_TOOLKIT_JAR) \
 	  --infer-base-iri \
 	  --inline-blank-nodes \
@@ -106,36 +110,11 @@ analysis.ttl: \
 	mv _$@ $@
 
 check: \
-  undefined_vocabulary.txt
-	test ! -s undefined_vocabulary.txt \
-	  || (echo "ERROR:tests/govdocs1/files/799/987/Makefile:The output in analysis.json has undefined CASE or UCO terms.  The first few are:" >&2 && head undefined_vocabulary.txt >&2 && exit 1)
+  analysis.json
 	source $(top_srcdir)/tests/venv/bin/activate \
 	  && pytest
 
 clean:
 	@rm -f \
 	  analysis.json \
-	  analysis.ttl \
-	  undefined_vocabulary.txt
-
-undefined_vocabulary.txt: \
-  analysis.json \
-  $(qc_srcdir)/src/vocabulary_used.py \
-  $(qc_srcdir)/tests/ontology_vocabulary.txt
-	source $(qc_srcdir)/venv/bin/activate \
-	  && python3 $(qc_srcdir)/src/vocabulary_used.py \
-	    $< \
-	    > $@___
-	grep ':' $@___ \
-	  | egrep -v '(ns.exiftool.ca|w3.org)' \
-	    | grep -v '/kb/' \
-	      > $@__
-	rm $@___
-	LC_ALL=C \
-	  $(COMM) \
-	    -13 \
-	    $(qc_srcdir)/tests/ontology_vocabulary.txt \
-	    <(LC_ALL=C $(SORT) $@__) \
-	    > $@_
-	rm $@__
-	mv $@_ $@
+	  analysis.ttl