adding fake data to the common_api

Addressing Issues 458 & 425

Adds deployment strategy: Recreate

This is instead of the default Rollling deployment,
which fails because the DB is RWO, so we have to
kill old server container before starting new.

Updates exec_status every 100 rows

Author: mrcrnkovich

.gitignore

@@ -22,4 +22,4 @@ start_env.sh
 /src/server/local_files/
 .mypy_cache/
 *secrets*
-*kustomization*
+*kustomization*

src/helm-chart/templates/deployment.yaml

@@ -8,6 +8,8 @@ spec:
   {{- if not .Values.autoscaling.enabled }}
   replicas: {{ .Values.replicaCount }}
   {{- end }}
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       {{- include "helm-chart.selectorLabels" . | nindent 6 }}

src/server/api/common_api.py

@@ -6,25 +6,22 @@
 import time
 from datetime import datetime
 
+from api.fake_data import sl_mock_data
+
 try:
     from secrets_dict import SHELTERLUV_SECRET_TOKEN
 except ImportError:
     # Not running locally
     print("Couldn't get SHELTERLUV_SECRET_TOKEN from file, trying environment **********")
-    from os import environ
-
-    try:
-        SHELTERLUV_SECRET_TOKEN = environ['SHELTERLUV_SECRET_TOKEN']
-    except KeyError:
-        # Nor in environment
-        # You're SOL for now
-        print("Couldn't get secrets from file or environment")
-
+    from os import getenv
 
+    SHELTERLUV_SECRET_TOKEN = getenv('SHELTERLUV_SECRET_TOKEN')
+    if not SHELTERLUV_SECRET_TOKEN:
+        print("Couldn't get secrets from file or environment",
+            "Defaulting to Fake Data")
 
 from api import jwt_ops
 
-
 @common_api.route('/api/timeout_test/<duration>', methods=['GET'])
 def get_timeout(duration):
     start = datetime.now().strftime("%H:%M:%S");
@@ -178,7 +175,6 @@ def get_360(matching_id):
 
     return jsonify({'result': result})
 
-
 @common_api.route('/api/person/<matching_id>/animals', methods=['GET'])
 @jwt_ops.jwt_required()
 def get_animals(matching_id):
@@ -187,6 +183,9 @@ def get_animals(matching_id):
         "animal_details": {}
     }
 
+    if not SHELTERLUV_SECRET_TOKEN:
+        return jsonify(sl_mock_data('animals'))
+
     with engine.connect() as connection:
         query = text("select * from pdp_contacts where matching_id = :matching_id and source_type = 'shelterluvpeople' and archived_date is null")
         query_result = connection.execute(query, matching_id=matching_id)
@@ -212,6 +211,10 @@ def get_animals(matching_id):
 def get_person_animal_events(matching_id, animal_id):
     result = {}
     events = []
+
+    if not SHELTERLUV_SECRET_TOKEN:
+        return jsonify(sl_mock_data('events'))
+
     with engine.connect() as connection:
         query = text("select * from pdp_contacts where matching_id = :matching_id and source_type = 'shelterluvpeople' and archived_date is null")
         query_result = connection.execute(query, matching_id=matching_id)

src/server/api/fake_data.py

@@ -0,0 +1,48 @@
+""" Fake data that can be returned when an API token is missing for local
+    development, or for running pytest
+
+    Shelterluv Data contains:
+        Matched: Animal & Event End point
+"""
+
+shelterluv_data = {
+    'animals': {
+        "animal_details": {
+            '12345': {
+                "Age": 24,
+                "DOBUnixTime": 1568480456,
+                "Name": "Lola aka Fake Cat",
+                "Type": "Cat",
+                "Photos":
+                ["https://images.unsplash.com/photo-1456926631375-92c8ce872def?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxzZWFyY2h8OHx8YW5pbWFsfGVufDB8fDB8fA%3D%3D&w=1000&q=80"],
+                "Status": "Healthy In Home",
+            },
+        },
+        "person_details": {
+            "shelterluv_short_id": 2,
+        },
+    },
+    'events': {
+        '12345':[
+            {
+            'AssociatedRecords': [
+                {'Id': 12345, 'Type': 'Animal' },
+                {'Id': 12345, 'Type': 'Person'},
+            ],
+            'Subtype': 'Foster Home',
+            'Time': '1602694822',
+            'Type': 'Outcome.Adoption',
+            'User': 'Fake User',
+            },
+        ]
+    },
+}
+
+
+def sl_mock_data(end_point: str)-> dict:
+    """ Shelterluv mock data.
+        Takes the end_point as a str of `animals` or `events` and  returns
+        a dict representing a test data for that end_point.
+    """
+
+    return shelterluv_data.get(end_point)

src/server/pipeline/match_data.py

@@ -50,12 +50,15 @@ def start(connection, added_or_updated_rows, manual_matches_df, job_id):
 
     rows = items_to_update.to_dict(orient="records")
     row_print_freq = 1000 
+    db_update_freq = 100   # update db after this many rows
 
     for row_num, row in enumerate(rows):
-        if row_num % row_print_freq == 0:
+        if row_num % row_print_freq == 0:   # Write to log
             current_app.logger.info("- Matching rows {}-{} of {}".format(
                 row_num + 1, min(len(rows), row_num + row_print_freq), len(rows))
             )
+
+        if row_num % db_update_freq == 0:  # Update execution_status table
             log_db.log_exec_status(job_id, 'matching', 'executing', str({'at_row': row_num + 1, 'of_rows': len(rows)  }) )
 
         # Exact matches based on specified columns

src/server/test_api.py

@@ -31,6 +31,16 @@
     SERVER_URL = "http://server:5000"
     IS_LOCAL = False
 
+try:
+    from secrets_dict import SHELTERLUV_SECRET_TOKEN
+except ImportError:
+    SHELTERLUV_SECRET_TOKEN = os.getenv("SHELTERLUV_SECRET_TOKEN")
+
+SL_Token = True if SHELTERLUV_SECRET_TOKEN else False
+print(os.getenv("SHELTERLUV_SECRET_TOKEN"))
+
+
+
 ###  DNS lookup tests  ##############################
 
 def test_bad_dns():
@@ -139,8 +149,90 @@ def test_inact_userblocked(state: State):
     assert response.status_code == 401
 
 
+###   Shelterluv API tests ######################################
+
+@pytest.mark.skipif(SL_Token, reason="Not run when SL_Token Present")
+def test_user_get_person_animal_events(state: State):
+    """ Test that the api returns mock data if the Shelterluv Token
+        is missing from secrets
+    """
+
+    # Build auth string value including token from state
+    b_string = 'Bearer ' + state.state['base_user']
+
+    assert len(b_string) > 24
+
+    auth_hdr = {'Authorization' : b_string}
+    url = SERVER_URL + "/api/person/12345/animal/12345/events"
+    response = requests.get(url, headers = auth_hdr)
+
+    assert response.status_code == 200
+
+    from api.fake_data import sl_mock_data
+    assert response.json() == sl_mock_data("events")
 
 
+@pytest.mark.skipif(SL_Token, reason="Not run when SL_Token Present")
+def test_user_get_animals(state: State):
+    """ Test that the api returns mock data if the Shelterluv Token
+        is missing from secrets
+    """
+    
+    # Build auth string value including token from state
+    b_string = 'Bearer ' + state.state['base_user']
+
+    assert len(b_string) > 24
+
+    auth_hdr = {'Authorization' : b_string}
+    url = SERVER_URL + "/api/person/12345/animals"
+    response = requests.get(url, headers = auth_hdr)
+
+    assert response.status_code == 200
+
+    from api.fake_data import sl_mock_data
+    assert response.json() == sl_mock_data("animals")
+
+
+@pytest.mark.skipif(not SL_Token, reason="Run when SL_Token Present")
+def test_user_get_animals_sl_token(state: State):
+    """ Test to confirm api does not return mock values if the Shelterluv Token
+        is present in the secrets_dict file.
+        Note this works on the assumption the SL token is not valid, and returns
+        a default empty value
+    """
+
+    # Build auth string value including token from state
+    b_string = 'Bearer ' + state.state['base_user']
+
+    assert len(b_string) > 24
+
+    auth_hdr = {'Authorization' : b_string}
+    url = SERVER_URL + "/api/person/12345/animals"
+    response = requests.get(url, headers = auth_hdr)
+
+    assert response.status_code == 200
+    assert response.json() == {'person_details': {}, 'animal_details': {}}
+
+
+@pytest.mark.skipif(not SL_Token, reason="Run when SL_Token Present")
+def test_user_get_person_animal_events_sl_token(state: State):
+    """ Test to confirm api does not return mock values if the Shelterluv Token
+        is present in the secrets_dict file.
+        Note this works on the assumption the SL token is not valid, and returns
+        a default empty value
+    """
+
+    # Build auth string value including token from state
+    b_string = 'Bearer ' + state.state['base_user']
+
+    assert len(b_string) > 24
+
+    auth_hdr = {'Authorization' : b_string}
+    url = SERVER_URL + "/api/person/12345/animal/12345/events"
+    response = requests.get(url, headers = auth_hdr)
+
+    assert response.status_code == 200
+    assert response.json() == {} 
 
 
 ###   Admin-level tests ######################################
@@ -261,4 +353,5 @@ def test_statistics(state: State):
     auth_hdr = {'Authorization' : b_string}
 
     response = requests.get(SERVER_URL + "/api/statistics", headers=auth_hdr)
-    assert response.status_code == 200
+    assert response.status_code == 200
+