Merge pull request #1273 from cmu-delphi/validation_gating_v2

Implement second pass at validation gating

Author: krivard

_delphi_utils_python/delphi_utils/runner.py

@@ -4,7 +4,7 @@
 from typing import Any, Callable, Dict, Optional
 from .archive import ArchiveDiffer, archiver_from_params
 from .logger import get_structured_logger
-from .utils import read_params
+from .utils import read_params, transfer_files
 from .validator.validate import Validator
 from .validator.run import validator_from_params
 
@@ -44,8 +44,11 @@ def run_indicator_pipeline(indicator_fn:  Callable[[Params], None],
         validation_report.log(get_structured_logger(
             name = indicator_fn.__module__,
             filename=params["common"].get("log_filename", None)))
-    if archiver and (not validator or validation_report.success()):
-        archiver.run()
+    if (not validator or validation_report.success()):
+        if archiver:
+            archiver.run()
+        if "delivery" in params:
+            transfer_files()
 
 
 if __name__ == "__main__":

_delphi_utils_python/delphi_utils/utils.py

@@ -1,8 +1,8 @@
 """Read parameter files containing configuration information."""
 # -*- coding: utf-8 -*-
 from json import load,dump
-from os.path import exists
-from shutil import copyfile
+from shutil import copyfile, move
+import os
 import sys
 
 def read_params():
@@ -11,7 +11,7 @@ def read_params():
     If the file does not exist, it copies the file 'params.json.template' to
     'params.json' and then reads the file.
     """
-    if not exists("params.json"):
+    if not os.path.exists("params.json"):
         copyfile("params.json.template", "params.json")
 
     with open("params.json", "r") as json_file:
@@ -87,3 +87,12 @@ def params_run():
         with open("params.json", "w") as f:
             dump(params, f, sort_keys=True, indent=2)
         print(f"Updated {n} items")
+
+def transfer_files():
+    """Transfer files to prepare for acquisition."""
+    params = read_params()
+    export_dir = params["common"].get("export_dir", None)
+    delivery_dir = params["delivery"].get("delivery_dir", None)
+    files_to_export = os.listdir(export_dir)
+    for file_name in files_to_export:
+        move(os.path.join(export_dir, file_name), delivery_dir)