Delete validation failures

_delphi_utils_python/delphi_utils/utils.py

@@ -104,23 +104,24 @@ def delete_move_files():
     1. Delete files in export-dir if delivery-dir is specified and is different
        from export_dir (aka only delete files produced by the most recent run)
     2. If validation-failures-dir is specified, move failed files there instead
+    If dry-run tag is True, then this function should not (and currently does not) get called
     """
     params = read_params()
     export_dir = params["common"].get("export_dir", None)
     delivery_dir = params["delivery"].get("delivery_dir", None)
     validation_failure_dir = params["validation"]["common"].get("validation_failure_dir", None)
-
     # Create validation_failure_dir if it doesn't exist
     if (validation_failure_dir is not None) and (not os.path.exists(validation_failure_dir)):
         os.mkdir(validation_failure_dir)
     # Double-checking that export-dir is not delivery-dir
-    assert(export_dir is not None and export_dir != delivery_dir)
+    # Throw assertion error if delivery_dir or export_dir is unspecified
+    assert(delivery_dir is not None and export_dir is not None)
+    assert export_dir != delivery_dir
     files_to_delete = os.listdir(export_dir)
-    if delivery_dir is not None:
-        for file_name in files_to_delete:
-            if file_name.endswith(".csv") or file_name.endswith(".CSV"):
-                if validation_failure_dir is not None:
-                    move(os.path.join(export_dir, file_name),
-                        os.path.join(validation_failure_dir, file_name))
-                else:
-                    os.remove(os.path.join(export_dir, file_name))
+    for file_name in files_to_delete:
+        if file_name.endswith(".csv") or file_name.endswith(".CSV"):
+            if validation_failure_dir is not None:
+                move(os.path.join(export_dir, file_name),
+                    os.path.join(validation_failure_dir, file_name))
+            else:
+                os.remove(os.path.join(export_dir, file_name))

_delphi_utils_python/delphi_utils/validator/README.md

@@ -43,6 +43,27 @@ deactivate
 rm -r env
 ```
 
+### Using the runner
+
+The validator can also be called using the runner, which performs the entire pipeline:
+
+* Calling the indicator's `run` module
+* Running the validator on newly obtained CSV files, if validator is specified
+* If validation is unsuccessful, remove or transfer current files in the run
+* Archive files if archiver is specified and there are no validation failures
+* Transfer files to delivery directory if specified in params
+
+The following truth table describes behavior of the third bullet point:
+
+| case | dry_run | validation failure | export dir | delivery dir | v failure dir | action |
+| - | - | - | - | - | - | - |
+| 1 | true | never | * | * | * | no effect |
+| 2 | false | no | * | * | * | no effect |
+| 3 | false | yes | X | X | * | throw assertion exception |
+| 4 | false | yes | X | None | * | throw assertion exception |
+| 5 | false | yes | X | Y != X | None | delete CSV from X |
+| 6 | false | yes | X | Y != X | Z | move CSV from X to Z |
+
 ### Customization
 
 All of the user-changable parameters are stored in the `validation` field of the indicator's `params.json` file. If `params.json` does not already include a `validation` field, please copy that provided in this module's `params.json.template`.