Foster event import

.gitignore

@@ -23,4 +23,4 @@ start_env.sh
 .mypy_cache/
 *secrets*
 *kustomization*
-src/.venv/
+src/.venv/

src/server/alembic/insert_rfm_edges.sql

@@ -0,0 +1,8 @@
+INSERT INTO "public"."kv_unique"(   "keycol",  "valcol") VALUES 
+(    'rfm_edges',
+    '{
+        "r":{"5": 0, "4": 262, "3": 1097, "2": 1910, "1": 2851}, 
+        "f": {"1": 0, "2": 1, "3": 2, "4": 3, "5": 4}, 
+        "m": {"1": 0.0, "2": 50.0, "3": 75.0, "4": 100.0, "5": 210.0}
+        }'
+ );

src/server/alembic/versions/90f471ac445c_create_sl_events.py

@@ -0,0 +1,41 @@
+"""Shelterluv animal events table
+
+Revision ID: 90f471ac445c
+Revises: 9687db7928ee
+Create Date: 2022-09-04 17:21:51.511030
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '90f471ac445c'
+down_revision = '9687db7928ee'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table (
+    "sl_event_types",
+    sa.Column("id", sa.Integer, autoincrement=True, primary_key=True),
+    sa.Column("event_name", sa.Text, nullable=False),
+    )
+
+    op.create_table (
+    "sl_animal_events",
+    sa.Column("id", sa.Integer, autoincrement=True, primary_key=True),
+    sa.Column("person_id", sa.Integer, nullable=False),
+    sa.Column("animal_id", sa.Integer, nullable=False),
+    sa.Column("event_type", sa.Integer, sa.ForeignKey('sl_event_types.id')),
+    sa.Column("time", sa.BigInteger, nullable=False)
+    )
+
+    op.create_index('sla_idx', 'sl_animal_events', ['person_id'])
+
+
+
+def downgrade():
+    op.drop_table("sl_animal_events")
+    op.drop_table("sl_event_types")

src/server/alembic/versions/9687db7928ee_shelterluv_animals.py

@@ -0,0 +1,33 @@
+"""Create SL_animals table
+
+Revision ID: 9687db7928ee
+Revises:  45a668fa6325
+Create Date: 2021-12-24 21:15:33.399197
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9687db7928ee'
+down_revision = '45a668fa6325'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+        op.create_table (
+        "shelterluv_animals",
+        sa.Column("id", sa.BigInteger, primary_key=True),
+        sa.Column("local_id", sa.BigInteger, nullable=False),
+        sa.Column("name", sa.Text, nullable=False),
+        sa.Column("type",  sa.Text, nullable=False),
+        sa.Column("dob", sa.BigInteger, nullable=False),
+        sa.Column("update_stamp", sa.BigInteger, nullable=False),
+        sa.Column("photo", sa.Text, nullable=False)
+    )
+
+
+def downgrade():
+    op.drop_table("shelterluv_animals")

src/server/api/API_ingest/ingest_sources_from_api.py

@@ -1,9 +1,22 @@
-from api.API_ingest import shelterluv_api_handler
 import structlog
 logger = structlog.get_logger()
 
+from api.API_ingest import shelterluv_api_handler, sl_animal_events
+
 def start(conn):
-    logger.debug("Start Fetching raw data from different API sources")
+    logger.debug("Start fetching raw data from different API sources")
+
+    logger.debug("   Fetching Shelterluv people")
+    #Run each source to store the output in dropbox and in the container as a CSV
+    slp_count = shelterluv_api_handler.store_shelterluv_people_all(conn)
+    logger.debug("   Finished fetching Shelterluv people - %d records" , slp_count)
+
+    logger.debug("   Fetching Shelterluv events")
     #Run each source to store the output in dropbox and in the container as a CSV
-    shelterluv_api_handler.store_shelterluv_people_all(conn)
-    logger.debug("Finish Fetching raw data from different API sources")
+    sle_count = sl_animal_events.slae_test()
+    logger.debug("   Finished fetching Shelterluv events - %d records" , sle_count)
+
+    logger.debug("Finished fetching raw data from different API sources")
+
+
+    #TODO:   Return object with count for each data source?

src/server/api/API_ingest/shelterluv_animals.py

@@ -0,0 +1,216 @@
+import os, time, json
+import posixpath as path
+
+import requests
+
+from api.API_ingest import shelterluv_db
+from server.api.API_ingest.shelterluv_db import insert_animals
+
+
+# from config import engine
+# from flask import current_app
+# from sqlalchemy.sql import text
+
+BASE_URL = 'http://shelterluv.com/api/'
+MAX_COUNT = 100  # Max records the API will return for one call
+
+try:
+    from secrets_dict import SHELTERLUV_SECRET_TOKEN
+except ImportError:
+    # Not running locally
+    from os import environ
+
+    try:
+        SHELTERLUV_SECRET_TOKEN = environ['SHELTERLUV_SECRET_TOKEN']
+    except KeyError:
+        # Not in environment
+        # You're SOL for now
+        print("Couldn't get SHELTERLUV_SECRET_TOKEN from file or environment")
+
+
+
+headers = {
+    "Accept": "application/json",
+    "X-API-Key": SHELTERLUV_SECRET_TOKEN
+}
+
+logger = print
+
+def get_animal_count():
+    """Test that server is operational and get total animal count."""
+    animals = 'v1/animals&offset=0&limit=1'
+    URL = path.join(BASE_URL,animals)
+
+    try:
+        response = requests.request("GET",URL, headers=headers)
+    except Exception as e:
+        logger('get_animal_count failed with ', e)
+        return -2
+
+    if response.status_code != 200:
+        logger("get_animal_count ", response.status_code, "code")
+        return -3
+
+    try:
+        decoded = json.loads(response.text)
+    except json.decoder.JSONDecodeError as e:
+        logger("get_animal_count JSON decode failed with", e)
+        return -4
+
+    if decoded['success']:
+        return decoded['total_count']
+    else:
+        return -5   # AFAICT, this means URL was bad
+
+
+def get_updated_animal_count(last_update):
+    """Test that server is operational and get total animal count."""
+    animals = 'v1/animals&offset=0&limit=1&sort=updated_at&since=' + str(last_update)
+    URL = path.join(BASE_URL,animals)
+
+    try:
+        response = requests.request("GET",URL, headers=headers)
+    except Exception as e:
+        logger('get_updated_animal_count failed with ', e)
+        return -2
+
+    if response.status_code != 200:
+        logger("get_updated_animal_count ", response.status_code, "code")
+        return -3
+
+    try:
+        decoded = json.loads(response.text)
+    except json.decoder.JSONDecodeError as e:
+        logger("get_updated_animal_count JSON decode failed with", e)
+        return -4
+
+    if decoded['success']:
+        return decoded['total_count']
+    else:
+        return -5   # AFAICT, this means URL was bad
+
+
+
+
+def filter_animals(raw_list):
+    """Given a list of animal records as returned by SL, return a list of records with only the fields we care about."""
+
+    good_keys = ['ID', 'Internal-ID', 'Name', 'Type', 'DOBUnixTime', 'CoverPhoto','LastUpdatedUnixTime']
+
+    filtered = []
+
+    for r in raw_list:
+        f = {}
+        for k in good_keys:
+            try:
+                f[k] = r[k]
+            except:
+                if k in ('DOBUnixTime','LastUpdatedUnixTime'):
+                    f[k] = 0
+                else:
+                    f[k] = ''
+        filtered.append(f)
+
+    return filtered
+
+
+
+
+def get_animals_bulk(total_count):
+    """Pull all animal records from SL """
+
+    # 'Great' API design - animal record 0 is the newest, so we need to start at the end, 
+    # back up MAX_COUNT rows, make our request, then keep backing up. We need to keep checking
+    # the total records to ensure one wasn't added in the middle of the process.
+    # Good news, the API is robust and won't blow up if you request past the end.
+
+    raw_url = path.join(BASE_URL, 'v1/animals&offset={0}&limit={1}')
+
+    start_record = int(total_count) 
+    offset = (start_record - MAX_COUNT)  if  (start_record - MAX_COUNT) > -1  else 0
+    limit = MAX_COUNT
+
+    while offset > -1 :
+
+        logger("getting at offset", offset)
+        url = raw_url.format(offset,limit)
+
+        try:
+            response = requests.request("GET",url, headers=headers)
+        except Exception as e:
+            logger('get_animals failed with ', e)
+            return -2
+
+        if response.status_code != 200:
+            logger("get_animal_count ", response.status_code, "code")
+            return -3
+
+        try:
+            decoded = json.loads(response.text)
+        except json.decoder.JSONDecodeError as e:
+            logger("get_animal_count JSON decode failed with", e)
+            return -4
+
+        if decoded['success']:
+            insert_animals( filter_animals(decoded['animals']) )
+            if offset == 0:
+                break
+            offset -= MAX_COUNT 
+            if offset < 0 :
+                limit = limit + offset
+                offset = 0
+        else:
+            return -5   # AFAICT, this means URL was bad
+
+    return 'zero'
+
+
+def update_animals(last_update):
+    """Get the animals inserted or updated since last check, insert/update db records. """
+
+    updated_records = get_updated_animal_count(last_update)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+def sla_test():
+    total_count = get_animal_count()
+    print('Total animals:',total_count)
+
+    b = get_animals_bulk(total_count)
+    print(len(b))
+
+    # f = filter_animals(b)
+    # print(f)
+
+    # count = shelterluv_db.insert_animals(f)
+    return len(b)
+
+# if __name__ == '__main__' :    
+
+#     total_count = get_animal_count()
+#     print('Total animals:',total_count)
+
+#     b = get_animals_bulk(9)
+#     print(len(b))
+
+#     f = filter_animals(b)
+#     print(f)
+
+#     count = shelterluv_db.insert_animals(f)

src/server/api/API_ingest/shelterluv_api_handler.py

@@ -10,6 +10,9 @@
 import structlog
 logger = structlog.get_logger()
 
+
+TEST_MODE = os.getenv("TEST_MODE")
+
 try:
     from secrets_dict import SHELTERLUV_SECRET_TOKEN
 except ImportError:
@@ -80,7 +83,14 @@ def store_shelterluv_people_all(conn):
         has_more = response["has_more"]
         offset += 100
 
-    logger.debug("Finish getting shelterluv contacts from people table")
+        if offset % 1000 == 0:
+            print("Reading offset ", str(offset))
+            if TEST_MODE and offset > 1000:
+                has_more=False  # Break out early 
+
+
+
+    print("Finish getting shelterluv contacts from people table")
 
     logger.debug("Start storing latest shelterluvpeople results to container")
     if os.listdir(RAW_DATA_PATH):
@@ -100,3 +110,5 @@ def store_shelterluv_people_all(conn):
 
     logger.debug("Uploading shelterluvpeople csv to database")
     ShelterluvPeople.insert_from_df(pd.read_csv(file_path, dtype="string"), conn)
+
+    return offset