Readme edits

Author: amartyabasu

validator/README.md

@@ -1,5 +1,15 @@
 # Validator
 
+The validator performs two main tasks:
+1) Sanity checks on daily data generated from a pipeline of specific data
+   source. 
+2) Its does a comparative analysis with recent data from the API 
+   to detect any anomalies such as spikes, significant value differences
+
+The validator validates against daily data thats already written in the disk
+making the execution of the validator independent of the pipeline execution.
+This creates an additional advantage of running the validation against multiple
+days of daily data and have a better cummulative analysis.
 
 
 ## Running the Indicator

validator/delphi_validator/driver.py

@@ -3,14 +3,14 @@
 
 # Defining start date and end date for the last fb-survey pipeline execution
 survey_sdate = "2020-06-13"
-survey_edate = "2020-06-19"
+survey_edate = "2020-06-15"
 dtobj_sdate = datetime.strptime(survey_sdate, '%Y-%m-%d')
 dtobj_edate = datetime.strptime(survey_edate, '%Y-%m-%d')
 print(dtobj_sdate.date())
 print(dtobj_edate.date())
 
 
 # Collecting all filenames
-daily_filnames = read_filenames("./data")
+daily_filnames = read_filenames("../data")
 
 fbsurvey_validation(daily_filnames, dtobj_sdate, dtobj_edate)

validator/delphi_validator/fbsurveyvalidation.py

@@ -147,6 +147,7 @@ def check_avg_val_diffs(recent_df, recent_api_df, smooth_option):
     #         dplyr::mutate(mean.stddiff.high = abs(mean.stddiff) > thresholds[["mean.stddiff"]] |
     #                           variable=="val" & abs(mean.stddiff) > thresholds[["val.mean.stddiff"]],
     #                       mean.stdabsdiff.high = mean.stdabsdiff > thresholds[["mean.stdabsdiff"]]) %>>%
+    # TOdo - Check whats the purpose of variable=="val" in the above statement
 
     switcher = {
     'raw': raw_thresholds,
@@ -242,8 +243,8 @@ def fbsurvey_validation(daily_filenames, sdate, edate, max_check_lookbehind = ti
             if (recent_df["se"].isnull().mean() > 0.5):
                 print('Recent se values are >50% NA')
 
-            #if sanity_check_rows_per_day:
-            #    check_rapid_change(checking_date, recent_df, recent_api_df, date_list, sig, geo)
+            if sanity_check_rows_per_day:
+                check_rapid_change(checking_date, recent_df, recent_api_df, date_list, sig, geo)
 
             if sanity_check_value_diffs:
                 check_avg_val_diffs(recent_df, recent_api_df, smooth_option)

validator/params.json.template

@@ -1,4 +1,5 @@
 {
+  "data_source": "fb_survey"
   "start_date": "2020-06-13",
   "end_date": "2020-06-19",
   "ref_window_size": 7