From 35d4bd303b8ed7ee06954b8c39e1de07f3f3dd57 Mon Sep 17 00:00:00 2001 From: c-simpson Date: Fri, 24 Dec 2021 17:27:35 -0500 Subject: [PATCH 01/14] test endpoint and DB inserts --- .../api/API_ingest/shelterluv_animals.py | 151 ++++++++++++++++++ src/server/api/API_ingest/shelterluv_db.py | 48 ++++++ src/server/api/admin_api.py | 11 ++ 3 files changed, 210 insertions(+) create mode 100644 src/server/api/API_ingest/shelterluv_animals.py create mode 100644 src/server/api/API_ingest/shelterluv_db.py diff --git a/src/server/api/API_ingest/shelterluv_animals.py b/src/server/api/API_ingest/shelterluv_animals.py new file mode 100644 index 00000000..016ee8f3 --- /dev/null +++ b/src/server/api/API_ingest/shelterluv_animals.py @@ -0,0 +1,151 @@ +import os, time, json +import posixpath as path + +import requests + +from api.API_ingest import shelterluv_db + + +# from config import engine +# from flask import current_app +# from sqlalchemy.sql import text + +BASE_URL = 'http://shelterluv.com/api/' + + +try: + from secrets_dict import SHELTERLUV_SECRET_TOKEN +except ImportError: + # Not running locally + from os import environ + + try: + SHELTERLUV_SECRET_TOKEN = environ['SHELTERLUV_SECRET_TOKEN'] + except KeyError: + # Not in environment + # You're SOL for now + print("Couldn't get SHELTERLUV_SECRET_TOKEN from file or environment") + + + +headers = { + "Accept": "application/json", + "X-API-Key": SHELTERLUV_SECRET_TOKEN +} + +logger = print + +def get_animal_count(): + """Test that server is operational and get total animal count.""" + animals = 'v1/animals&offset=0&limit=1' + URL = path.join(BASE_URL,animals) + + try: + response = requests.request("GET",URL, headers=headers) + except Exception as e: + logger('get_animal_count failed with ', e) + return -2 + + if response.status_code != 200: + logger("get_animal_count ", response.status_code, "code") + return -3 + + try: + decoded = json.loads(response.text) + except json.decoder.JSONDecodeError as e: + logger("get_animal_count JSON decode failed with", e) + return -4 + + if decoded['success']: + return decoded['total_count'] + else: + return -5 # AFAICT, this means URL was bad + + +def filter_animals(raw_list): + """Given a list of animal records as returned by SL, return a list of records with only the fields we care about.""" + + good_keys = ['ID', 'Internal-ID', 'Name', 'Type', 'DOBUnixTime', 'CoverPhoto','LastUpdatedUnixTime'] + + filtered = [] + + for r in raw_list: + f = {} + for k in good_keys: + f[k] = r[k] + filtered.append(f) + + return filtered + + + + +def get_animals_bulk(total_count): + """Pull all animal records from SL """ + + MAX_COUNT = 100 # Max records the API will return for one call + + # 'Great' API design - animal record 0 is the newest, so we need to start at the end, + # back up MAX_COUNT rows, make our request, then keep backing up. We need to keep checking + # the total records to ensure one wasn't added in the middle of the process. + # Good news, the API is robust and won't blow up if you request past the end. + + raw_url = path.join(BASE_URL, 'v1/animals&offset={0}&limit={1}') + + start_record = total_count -1 + offset = (start_record - MAX_COUNT) if (start_record - MAX_COUNT) > -1 else 0 + + while offset > -1 : + + url = raw_url.format(offset,MAX_COUNT) + + try: + response = requests.request("GET",url, headers=headers) + except Exception as e: + logger('get_animals failed with ', e) + return -2 + + if response.status_code != 200: + logger("get_animal_count ", response.status_code, "code") + return -3 + + try: + decoded = json.loads(response.text) + except json.decoder.JSONDecodeError as e: + logger("get_animal_count JSON decode failed with", e) + return -4 + + if decoded['success']: + return decoded['animals'] + else: + return -5 # AFAICT, this means URL was bad + + + + +def sla_test(): + total_count = get_animal_count() + print('Total animals:',total_count) + + b = get_animals_bulk(200) + print(len(b)) + + f = filter_animals(b) + print(f) + + count = shelterluv_db.insert_animals(f) + return count + + +# if __name__ == '__main__' : + +# total_count = get_animal_count() +# print('Total animals:',total_count) + +# b = get_animals_bulk(9) +# print(len(b)) + +# f = filter_animals(b) +# print(f) + +# count = shelterluv_db.insert_animals(f) \ No newline at end of file diff --git a/src/server/api/API_ingest/shelterluv_db.py b/src/server/api/API_ingest/shelterluv_db.py new file mode 100644 index 00000000..6c6806e1 --- /dev/null +++ b/src/server/api/API_ingest/shelterluv_db.py @@ -0,0 +1,48 @@ +from api.api import common_api +from config import engine +from flask import jsonify, current_app +from sqlalchemy.sql import text +import requests +import time +from datetime import datetime + +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy import Table, MetaData +from pipeline import flow_script +from config import engine +from flask import request, redirect, jsonify, current_app +from api.file_uploader import validate_and_arrange_upload +from sqlalchemy.orm import Session, sessionmaker + + +def insert_animals(animal_list): + """Insert animal records into shelterluv_animals table and return row count. """ + + Session = sessionmaker(engine) + session = Session() + metadata = MetaData() + sla = Table("shelterluv_animals", metadata, autoload=True, autoload_with=engine) + + # From Shelterluv: ['ID', 'Internal-ID', 'Name', 'Type', 'DOBUnixTime', 'CoverPhoto', 'LastUpdatedUnixTime'] + # In db: ['local_id', 'id' (PK), 'name', 'type', 'dob', 'photo', 'updatestamp'] + + ins_list = [] # Create a list of per-row dicts + for rec in animal_list: + ins_list.append( + { + "id": rec["Internal-ID"], + "local_id": rec["ID"] if rec["ID"] else 0, # Sometimes there's no local id + "name": rec["Name"], + "type": rec["Type"], + "dob": rec["DOBUnixTime"], + "updatestamp": rec["LastUpdatedUnixTime"], + "photo": rec["CoverPhoto"], + } + ) + + ret = session.execute(sla.insert(ins_list)) + + session.commit() # Commit all inserted rows + session.close() + + return ret.rowcount diff --git a/src/server/api/admin_api.py b/src/server/api/admin_api.py index 6e660463..94195ba6 100644 --- a/src/server/api/admin_api.py +++ b/src/server/api/admin_api.py @@ -398,6 +398,17 @@ def hit_gdrs(): return jsonify({"scores added" : num_scores}) + + +@admin_api.route("/api/admin/test_sla", methods=["GET"]) +def trigger_sla_pull(): + + import api.API_ingest.shelterluv_animals + + num_rows = api.API_ingest.shelterluv_animals.sla_test() + return jsonify({"rows added" : num_rows}) + + # def pdfr(): # dlist = pull_donations_for_rfm() # print("Returned " + str(len(dlist)) + " rows") From 0f0d613c8f95802b8b0082fb673b2a388d31ca18 Mon Sep 17 00:00:00 2001 From: c-simpson Date: Fri, 24 Dec 2021 21:43:46 -0500 Subject: [PATCH 02/14] Migration for shelterluv_animals table --- .../9687db7928ee_shelterluv_animals.py | 33 +++++++++++++++++++ src/server/api/API_ingest/shelterluv_db.py | 4 +-- 2 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 src/server/alembic/versions/9687db7928ee_shelterluv_animals.py diff --git a/src/server/alembic/versions/9687db7928ee_shelterluv_animals.py b/src/server/alembic/versions/9687db7928ee_shelterluv_animals.py new file mode 100644 index 00000000..6e231d2e --- /dev/null +++ b/src/server/alembic/versions/9687db7928ee_shelterluv_animals.py @@ -0,0 +1,33 @@ +"""empty message + +Revision ID: 9687db7928ee +Revises: a3ba63dee8f4 +Create Date: 2021-12-24 21:15:33.399197 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '9687db7928ee' +down_revision = 'a3ba63dee8f4' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table ( + "shelterluv_animals", + sa.Column("id", sa.BigInteger, primary_key=True), + sa.Column("local_id", sa.BigInteger, nullable=False), + sa.Column("name", sa.Text, nullable=False), + sa.Column("type", sa.Text, nullable=False), + sa.Column("dob", sa.BigInteger, nullable=False), + sa.Column("update_stamp", sa.BigInteger, nullable=False), + sa.Column("photo", sa.Text, nullable=False) + ) + + +def downgrade(): + op.drop_table("shelterluv_animals") diff --git a/src/server/api/API_ingest/shelterluv_db.py b/src/server/api/API_ingest/shelterluv_db.py index 6c6806e1..65c5bc2e 100644 --- a/src/server/api/API_ingest/shelterluv_db.py +++ b/src/server/api/API_ingest/shelterluv_db.py @@ -24,7 +24,7 @@ def insert_animals(animal_list): sla = Table("shelterluv_animals", metadata, autoload=True, autoload_with=engine) # From Shelterluv: ['ID', 'Internal-ID', 'Name', 'Type', 'DOBUnixTime', 'CoverPhoto', 'LastUpdatedUnixTime'] - # In db: ['local_id', 'id' (PK), 'name', 'type', 'dob', 'photo', 'updatestamp'] + # In db: ['local_id', 'id' (PK), 'name', 'type', 'dob', 'photo', 'update_stamp'] ins_list = [] # Create a list of per-row dicts for rec in animal_list: @@ -35,7 +35,7 @@ def insert_animals(animal_list): "name": rec["Name"], "type": rec["Type"], "dob": rec["DOBUnixTime"], - "updatestamp": rec["LastUpdatedUnixTime"], + "update_stamp": rec["LastUpdatedUnixTime"], "photo": rec["CoverPhoto"], } ) From a93385fb17ac02ead526427f676ebb8cf3e51b09 Mon Sep 17 00:00:00 2001 From: c-simpson Date: Mon, 17 Jan 2022 21:33:41 -0500 Subject: [PATCH 03/14] Added truncate --- src/server/api/API_ingest/shelterluv_db.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/server/api/API_ingest/shelterluv_db.py b/src/server/api/API_ingest/shelterluv_db.py index 65c5bc2e..0de65ab5 100644 --- a/src/server/api/API_ingest/shelterluv_db.py +++ b/src/server/api/API_ingest/shelterluv_db.py @@ -46,3 +46,22 @@ def insert_animals(animal_list): session.close() return ret.rowcount + + +def truncate_animals(): + """Truncate the shelterluv_animals table""" + + + Session = sessionmaker(engine) + session = Session() + metadata = MetaData() + sla = Table("shelterluv_animals", metadata, autoload=True, autoload_with=engine) + + + truncate = "TRUNCATE table shelterluv_animals;" + result = session.execute(truncate) + + session.commit() # Commit all inserted rows + session.close() + + return 0 From 0dc6bc05dd54119dae5f3aa7001e23a6662ede63 Mon Sep 17 00:00:00 2001 From: c-simpson Date: Mon, 17 Jan 2022 21:34:02 -0500 Subject: [PATCH 04/14] Tweaks --- .../api/API_ingest/shelterluv_animals.py | 91 ++++++++++++++++--- 1 file changed, 78 insertions(+), 13 deletions(-) diff --git a/src/server/api/API_ingest/shelterluv_animals.py b/src/server/api/API_ingest/shelterluv_animals.py index 016ee8f3..4d7b9f9f 100644 --- a/src/server/api/API_ingest/shelterluv_animals.py +++ b/src/server/api/API_ingest/shelterluv_animals.py @@ -4,6 +4,7 @@ import requests from api.API_ingest import shelterluv_db +from server.api.API_ingest.shelterluv_db import insert_animals # from config import engine @@ -11,7 +12,7 @@ # from sqlalchemy.sql import text BASE_URL = 'http://shelterluv.com/api/' - +MAX_COUNT = 100 # Max records the API will return for one call try: from secrets_dict import SHELTERLUV_SECRET_TOKEN @@ -62,6 +63,35 @@ def get_animal_count(): return -5 # AFAICT, this means URL was bad +def get_updated_animal_count(last_update): + """Test that server is operational and get total animal count.""" + animals = 'v1/animals&offset=0&limit=1&sort=updated_at&since=' + str(last_update) + URL = path.join(BASE_URL,animals) + + try: + response = requests.request("GET",URL, headers=headers) + except Exception as e: + logger('get_updated_animal_count failed with ', e) + return -2 + + if response.status_code != 200: + logger("get_updated_animal_count ", response.status_code, "code") + return -3 + + try: + decoded = json.loads(response.text) + except json.decoder.JSONDecodeError as e: + logger("get_updated_animal_count JSON decode failed with", e) + return -4 + + if decoded['success']: + return decoded['total_count'] + else: + return -5 # AFAICT, this means URL was bad + + + + def filter_animals(raw_list): """Given a list of animal records as returned by SL, return a list of records with only the fields we care about.""" @@ -72,7 +102,13 @@ def filter_animals(raw_list): for r in raw_list: f = {} for k in good_keys: - f[k] = r[k] + try: + f[k] = r[k] + except: + if k in ('DOBUnixTime','LastUpdatedUnixTime'): + f[k] = 0 + else: + f[k] = '' filtered.append(f) return filtered @@ -83,8 +119,6 @@ def filter_animals(raw_list): def get_animals_bulk(total_count): """Pull all animal records from SL """ - MAX_COUNT = 100 # Max records the API will return for one call - # 'Great' API design - animal record 0 is the newest, so we need to start at the end, # back up MAX_COUNT rows, make our request, then keep backing up. We need to keep checking # the total records to ensure one wasn't added in the middle of the process. @@ -92,12 +126,14 @@ def get_animals_bulk(total_count): raw_url = path.join(BASE_URL, 'v1/animals&offset={0}&limit={1}') - start_record = total_count -1 + start_record = int(total_count) offset = (start_record - MAX_COUNT) if (start_record - MAX_COUNT) > -1 else 0 + limit = MAX_COUNT while offset > -1 : - url = raw_url.format(offset,MAX_COUNT) + logger("getting at offset", offset) + url = raw_url.format(offset,limit) try: response = requests.request("GET",url, headers=headers) @@ -116,10 +152,40 @@ def get_animals_bulk(total_count): return -4 if decoded['success']: - return decoded['animals'] + insert_animals( filter_animals(decoded['animals']) ) + if offset == 0: + break + offset -= MAX_COUNT + if offset < 0 : + limit = limit + offset + offset = 0 else: return -5 # AFAICT, this means URL was bad + return 'zero' + + +def update_animals(last_update): + """Get the animals inserted or updated since last check, insert/update db records. """ + + updated_records = get_updated_animal_count(last_update) + + + + + + + + + + + + + + + + + @@ -127,15 +193,14 @@ def sla_test(): total_count = get_animal_count() print('Total animals:',total_count) - b = get_animals_bulk(200) + b = get_animals_bulk(total_count) print(len(b)) - f = filter_animals(b) - print(f) - - count = shelterluv_db.insert_animals(f) - return count + # f = filter_animals(b) + # print(f) + # count = shelterluv_db.insert_animals(f) + return len(b) # if __name__ == '__main__' : From e9273ce0745fd485c7e620ee100b94108870368b Mon Sep 17 00:00:00 2001 From: cris_gk Date: Wed, 22 Jun 2022 19:05:27 -0400 Subject: [PATCH 05/14] Add endpoint for starting SL people ingest --- src/server/api/admin_api.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/server/api/admin_api.py b/src/server/api/admin_api.py index 94195ba6..875685f6 100644 --- a/src/server/api/admin_api.py +++ b/src/server/api/admin_api.py @@ -409,6 +409,17 @@ def trigger_sla_pull(): return jsonify({"rows added" : num_rows}) +@admin_api.route("/api/admin/test_slp", methods=["GET"]) +def trigger_slp_pull(): + + import api.API_ingest.shelterluv_api_handler + + num_rows = api.API_ingest.shelterluv_api_handler.store_shelterluv_people_all() + return jsonify({"rows added" : num_rows}) + + + + # def pdfr(): # dlist = pull_donations_for_rfm() # print("Returned " + str(len(dlist)) + " rows") From 2695d927f0548d9bb98e01246f87b8bb2be5c1d9 Mon Sep 17 00:00:00 2001 From: cris_gk Date: Sun, 21 Aug 2022 10:57:42 -0400 Subject: [PATCH 06/14] First pass at SL events pull --- .../9687db7928ee_shelterluv_animals.py | 4 +- src/server/api/API_ingest/shelterluv_db.py | 81 ++++++++- src/server/api/API_ingest/sl_animal_events.py | 161 ++++++++++++++++++ src/server/api/admin_api.py | 21 ++- 4 files changed, 254 insertions(+), 13 deletions(-) create mode 100644 src/server/api/API_ingest/sl_animal_events.py diff --git a/src/server/alembic/versions/9687db7928ee_shelterluv_animals.py b/src/server/alembic/versions/9687db7928ee_shelterluv_animals.py index 6e231d2e..51ccc31a 100644 --- a/src/server/alembic/versions/9687db7928ee_shelterluv_animals.py +++ b/src/server/alembic/versions/9687db7928ee_shelterluv_animals.py @@ -1,4 +1,4 @@ -"""empty message +"""Create SL_animals table Revision ID: 9687db7928ee Revises: a3ba63dee8f4 @@ -11,7 +11,7 @@ # revision identifiers, used by Alembic. revision = '9687db7928ee' -down_revision = 'a3ba63dee8f4' +down_revision = 'fc7325372396' branch_labels = None depends_on = None diff --git a/src/server/api/API_ingest/shelterluv_db.py b/src/server/api/API_ingest/shelterluv_db.py index 0de65ab5..1794382c 100644 --- a/src/server/api/API_ingest/shelterluv_db.py +++ b/src/server/api/API_ingest/shelterluv_db.py @@ -51,17 +51,88 @@ def insert_animals(animal_list): def truncate_animals(): """Truncate the shelterluv_animals table""" - - Session = sessionmaker(engine) - session = Session() + Session = sessionmaker(engine) + session = Session() metadata = MetaData() sla = Table("shelterluv_animals", metadata, autoload=True, autoload_with=engine) - truncate = "TRUNCATE table shelterluv_animals;" result = session.execute(truncate) - session.commit() # Commit all inserted rows + session.commit() # Commit all inserted rows + session.close() + + return 0 + + +def truncate_events(): + """Truncate the shelterluv_events table""" + + Session = sessionmaker(engine) + session = Session() + metadata = MetaData() + sla = Table("sl_animal_events", metadata, autoload=True, autoload_with=engine) + + truncate = "TRUNCATE table sl_animal_events;" + result = session.execute(truncate) + + session.commit() # Commit all inserted rows session.close() return 0 + + +def insert_events(event_list): + """Insert event records into sl_animal_events table and return row count. """ + + # Always a clean insert + truncate_events() + + Session = sessionmaker(engine) + session = Session() + metadata = MetaData() + sla = Table("sl_animal_events", metadata, autoload=True, autoload_with=engine) + + # TODO: Pull from DB + event_map = { + "Outcome.Adoption": 1, + "Outcome.Foster": 2, + "Outcome.ReturnToOwner": 3, + "Intake.AdoptionReturn": 4, + } + + # Event record: [ AssociatedRecords[Type = Person]["Id"]', + # AssociatedRecords[Type = Animal]["Id"]', + # "Type", + # "Time" + # ] + # + # In db: ['id', + # 'person_id', + # 'animal_id', + # 'event_type', + # 'time'] + + ins_list = [] # Create a list of per-row dicts + for rec in event_list: + ins_list.append( + { + "person_id": next( + filter(lambda x: x["Type"] == "Person", rec["AssociatedRecords"]) + )["Id"], + "animal_id": next( + filter(lambda x: x["Type"] == "Animal", rec["AssociatedRecords"]) + )["Id"], + "event_type": event_map[rec["Type"]], + "time": rec["Time"], + } + ) + + # TODO: Wrap with try/catch + ret = session.execute(sla.insert(ins_list)) + + session.commit() # Commit all inserted rows + session.close() + + return ret.rowcount + diff --git a/src/server/api/API_ingest/sl_animal_events.py b/src/server/api/API_ingest/sl_animal_events.py new file mode 100644 index 00000000..abac0f8b --- /dev/null +++ b/src/server/api/API_ingest/sl_animal_events.py @@ -0,0 +1,161 @@ +import os, time, json +import posixpath as path + +import requests + +from api.API_ingest import shelterluv_db +from server.api.API_ingest.shelterluv_db import insert_animals + +# There are a number of different record types. These are the ones we care about. +keep_record_types = [ + "Outcome.Adoption", + "Outcome.Foster", + "Outcome.ReturnToOwner", + "Intake.AdoptionReturn", +] + +# from config import engine +# from flask import current_app +# from sqlalchemy.sql import text + +BASE_URL = "http://shelterluv.com/api/" +MAX_COUNT = 100 # Max records the API will return for one call + +# Get the API key +try: + from secrets_dict import SHELTERLUV_SECRET_TOKEN +except ImportError: + # Not running locally + from os import environ + + try: + SHELTERLUV_SECRET_TOKEN = environ["SHELTERLUV_SECRET_TOKEN"] + except KeyError: + # Not in environment + # You're SOL for now + print("Couldn't get SHELTERLUV_SECRET_TOKEN from file or environment") + + +headers = {"Accept": "application/json", "X-API-Key": SHELTERLUV_SECRET_TOKEN} + +logger = print # print to console for testing + + +# Sample response from events request: + +# { +# "success": 1, +# "events": [ +# { +# "Type": "Outcome.Adoption", +# "Subtype": "PAC", +# "Time": "1656536900", +# "User": "phlp_mxxxx", +# "AssociatedRecords": [ +# { +# "Type": "Animal", +# "Id": "5276xxxx" +# }, +# { +# "Type": "Person", +# "Id": "5633xxxx" +# } +# ] +# }, +# {...} +# ], +# "has_more": true, +# "total_count": 67467 +# } + + +def get_event_count(): + """Test that server is operational and get total event count.""" + events = "v1/events&offset=0&limit=1" + URL = path.join(BASE_URL, events) + + try: + response = requests.request("GET", URL, headers=headers) + except Exception as e: + logger("get_event_count failed with ", e) + return -2 + + if response.status_code != 200: + logger("get_event_count ", response.status_code, "code") + return -3 + + try: + decoded = json.loads(response.text) + except json.decoder.JSONDecodeError as e: + logger("get_event_count JSON decode failed with", e) + return -4 + + if decoded["success"]: + return decoded["total_count"] + else: + return -5 # AFAICT, this means URL was bad + + +def get_events_bulk(): + """Pull all event records from SL """ + + # Interesting API design - event record 0 is the newest. But since we pull all records each time it doesn't + # really matter which direction we go. Simplest to count up, and we can pull until 'has_more' goes false. + # Good news, the API is robust and won't blow up if you request past the end. + # At 100 per request, API returns about 5000 records/minute + + event_records = [] + + raw_url = path.join(BASE_URL, "v1/events&offset={0}&limit={1}") + offset = 0 + limit = MAX_COUNT + more_records = True + + while more_records: + + url = raw_url.format(offset, limit) + + try: + response = requests.request("GET", url, headers=headers) + except Exception as e: + logger("get_events failed with ", e) + return -2 + + if response.status_code != 200: + logger("get_event_count ", response.status_code, "code") + return -3 + + try: + decoded = json.loads(response.text) + except json.decoder.JSONDecodeError as e: + logger("get_event_count JSON decode failed with", e) + return -4 + + if decoded["success"]: + for evrec in decoded["events"]: + if evrec["Type"] in keep_record_types: + event_records.append(evrec) + + more_records = decoded["has_more"] # if so, we'll make another pass + offset += limit + if offset % 1000 == 0: + print("Reading offset ", str(offset)) + + else: + return -5 # AFAICT, this means URL was bad + + return event_records + + +def slae_test(): + total_count = get_event_count() + print("Total events:", total_count) + + b = get_events_bulk() + print("Records:", len(b)) + + # f = filter_events(b) + # print(f) + + count = shelterluv_db.insert_events(b) + return count diff --git a/src/server/api/admin_api.py b/src/server/api/admin_api.py index 875685f6..bcf33040 100644 --- a/src/server/api/admin_api.py +++ b/src/server/api/admin_api.py @@ -388,18 +388,17 @@ def generate_dummy_rfm_scores(): return count +# ########### Test API endpoints +# TODO: Remove for production - -# Use this as a way to trigger functions for testing -# TODO: Remove when not needed +# trigger rfm scoring process @admin_api.route("/api/admin/test_endpoint_gdrs", methods=["GET"]) def hit_gdrs(): num_scores = generate_dummy_rfm_scores() return jsonify({"scores added" : num_scores}) - - +# trigger pull of SL animals @admin_api.route("/api/admin/test_sla", methods=["GET"]) def trigger_sla_pull(): @@ -408,7 +407,7 @@ def trigger_sla_pull(): num_rows = api.API_ingest.shelterluv_animals.sla_test() return jsonify({"rows added" : num_rows}) - +# trigger pull of SL people @admin_api.route("/api/admin/test_slp", methods=["GET"]) def trigger_slp_pull(): @@ -417,6 +416,16 @@ def trigger_slp_pull(): num_rows = api.API_ingest.shelterluv_api_handler.store_shelterluv_people_all() return jsonify({"rows added" : num_rows}) +# trigger pull of SL animal events +@admin_api.route("/api/admin/test_slae", methods=["GET"]) +def trigger_slae_pull(): + + import api.API_ingest.sl_animal_events + + num_rows = api.API_ingest.sl_animal_events.slae_test() + return jsonify({"rows added" : num_rows}) + + From 6296ded149c5f653303df03bd1a633305b1aac81 Mon Sep 17 00:00:00 2001 From: cris_gk Date: Sun, 21 Aug 2022 14:21:35 -0400 Subject: [PATCH 07/14] Sample rfm_edges --- src/server/alembic/insert_rfm_edges.sql | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 src/server/alembic/insert_rfm_edges.sql diff --git a/src/server/alembic/insert_rfm_edges.sql b/src/server/alembic/insert_rfm_edges.sql new file mode 100644 index 00000000..d22c5187 --- /dev/null +++ b/src/server/alembic/insert_rfm_edges.sql @@ -0,0 +1,8 @@ +INSERT INTO "public"."kv_unique"( "keycol", "valcol") VALUES +( 'rfm_edges', + '{ + "r":{"5": 0, "4": 262, "3": 1097, "2": 1910, "1": 2851}, + "f": {"1": 0, "2": 1, "3": 2, "4": 3, "5": 4}, + "m": {"1": 0.0, "2": 50.0, "3": 75.0, "4": 100.0, "5": 210.0} + }' + ); From 0e79c1f8dc85d74e8aa44a01e2d83fdfe25b5f71 Mon Sep 17 00:00:00 2001 From: c-simpson Date: Tue, 6 Sep 2022 19:47:55 -0400 Subject: [PATCH 08/14] Alembic to create sl_events tables --- .../versions/90f471ac445c_create_sl_events.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/server/alembic/versions/90f471ac445c_create_sl_events.py diff --git a/src/server/alembic/versions/90f471ac445c_create_sl_events.py b/src/server/alembic/versions/90f471ac445c_create_sl_events.py new file mode 100644 index 00000000..f329069f --- /dev/null +++ b/src/server/alembic/versions/90f471ac445c_create_sl_events.py @@ -0,0 +1,41 @@ +"""Shelterluv animal events table + +Revision ID: 90f471ac445c +Revises: 9687db7928ee +Create Date: 2022-09-04 17:21:51.511030 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '90f471ac445c' +down_revision = '9687db7928ee' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table ( + "sl_event_types", + sa.Column("id", sa.Integer, autoincrement=True, primary_key=True), + sa.Column("event_name", sa.Text, nullable=False), + ) + + op.create_table ( + "sl_animal_events", + sa.Column("id", sa.Integer, autoincrement=True, primary_key=True), + sa.Column("person_id", sa.Integer, nullable=False), + sa.Column("animal_id", sa.Integer, nullable=False), + sa.Column("event_type", sa.Integer, sa.ForeignKey('sl_event_types.id')), + sa.Column("time", sa.BigInteger, nullable=False) + ) + + op.create_index('sla_idx', 'sl_animal_events', ['person_id']) + + + +def downgrade(): + op.drop_table("sl_animal_events") + op.drop_table("sl_event_types") \ No newline at end of file From aa22a3424cc434adc7e4340aaedf71288f245e15 Mon Sep 17 00:00:00 2001 From: c-simpson Date: Tue, 4 Oct 2022 19:36:07 -0400 Subject: [PATCH 09/14] Comment w/ queries --- src/server/api/API_ingest/sl_animal_events.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/server/api/API_ingest/sl_animal_events.py b/src/server/api/API_ingest/sl_animal_events.py index abac0f8b..89074d04 100644 --- a/src/server/api/API_ingest/sl_animal_events.py +++ b/src/server/api/API_ingest/sl_animal_events.py @@ -1,6 +1,7 @@ import os, time, json import posixpath as path + import requests from api.API_ingest import shelterluv_db @@ -159,3 +160,30 @@ def slae_test(): count = shelterluv_db.insert_events(b) return count + + +# Query to get last adopt/foster event: + +# """ +# select +# person_id as sl_person_id, max(to_timestamp(time)::date) as last_fosteradopt_event +# from +# sl_animal_events +# where event_type < 4 -- check this +# group by +# person_id +# order by +# person_id asc; +# """ +# Volgistics last shift + +# """ +# select +# volg_id, max(from_date) as last_shift +# from +# volgisticsshifts +# group by +# volg_id +# order by +# volg_id ; +# """ \ No newline at end of file From 2e713e3b06329969921e12497a7cef8ce58b92b7 Mon Sep 17 00:00:00 2001 From: c-simpson Date: Fri, 25 Nov 2022 18:02:51 -0500 Subject: [PATCH 10/14] SL animal events bumped panda & psycopg versions --- src/server/api/API_ingest/shelterluv_db.py | 11 +++++++++++ src/server/api/API_ingest/sl_animal_events.py | 2 ++ src/server/requirements.txt | 5 +++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/server/api/API_ingest/shelterluv_db.py b/src/server/api/API_ingest/shelterluv_db.py index 1794382c..db7710af 100644 --- a/src/server/api/API_ingest/shelterluv_db.py +++ b/src/server/api/API_ingest/shelterluv_db.py @@ -99,8 +99,19 @@ def insert_events(event_list): "Outcome.Foster": 2, "Outcome.ReturnToOwner": 3, "Intake.AdoptionReturn": 4, + "Intake.FosterReturn":5 } + # """ INSERT INTO "sl_event_types" ("id","event_name") VALUES + # ( 1,'Outcome.Adoption' ), + # ( 2,'Outcome.Foster' ), + # ( 3,'Outcome.ReturnToOwner' ), + # ( 4,'Intake.AdoptionReturn' ), + # ( 5,'Intake.FosterReturn' ) """ + + + + # Event record: [ AssociatedRecords[Type = Person]["Id"]', # AssociatedRecords[Type = Animal]["Id"]', # "Type", diff --git a/src/server/api/API_ingest/sl_animal_events.py b/src/server/api/API_ingest/sl_animal_events.py index 89074d04..01228e8d 100644 --- a/src/server/api/API_ingest/sl_animal_events.py +++ b/src/server/api/API_ingest/sl_animal_events.py @@ -13,6 +13,7 @@ "Outcome.Foster", "Outcome.ReturnToOwner", "Intake.AdoptionReturn", + "Intake.FosterReturn" ] # from config import engine @@ -94,6 +95,7 @@ def get_event_count(): if decoded["success"]: return decoded["total_count"] else: + logger(decoded['error_message']) return -5 # AFAICT, this means URL was bad diff --git a/src/server/requirements.txt b/src/server/requirements.txt index f9b60f7f..27430472 100644 --- a/src/server/requirements.txt +++ b/src/server/requirements.txt @@ -1,8 +1,9 @@ +numpy==1.19.5 Flask==1.1.2 pandas==1.3.2 -numpy==1.18.1 + sqlalchemy==1.4.15 -psycopg2-binary==2.8.4 +psycopg2-binary==2.9.1 xlrd==1.2.0 # currently used for xlsx, but we should consider adjusting code to openpyxl for xlsx openpyxl requests From 4d11c9142d76ce4a593ab2d1b0b48e65f0a82945 Mon Sep 17 00:00:00 2001 From: c-simpson Date: Sun, 27 Nov 2022 16:54:13 -0500 Subject: [PATCH 11/14] Pull and insert SL animal events Includes test endpoint at end of admin_api.py --- .../9687db7928ee_shelterluv_animals.py | 4 ++-- src/server/api/API_ingest/shelterluv_db.py | 2 +- src/server/config.py | 9 ++++---- src/server/{user_mgmt => db_setup}/README.md | 0 .../{user_mgmt => db_setup}/__init__.py | 0 .../{user_mgmt => db_setup}/base_users.py | 21 +++++++++++++++++-- 6 files changed, 27 insertions(+), 9 deletions(-) rename src/server/{user_mgmt => db_setup}/README.md (100%) rename src/server/{user_mgmt => db_setup}/__init__.py (100%) rename src/server/{user_mgmt => db_setup}/base_users.py (82%) diff --git a/src/server/alembic/versions/9687db7928ee_shelterluv_animals.py b/src/server/alembic/versions/9687db7928ee_shelterluv_animals.py index 51ccc31a..7ae5de69 100644 --- a/src/server/alembic/versions/9687db7928ee_shelterluv_animals.py +++ b/src/server/alembic/versions/9687db7928ee_shelterluv_animals.py @@ -1,7 +1,7 @@ """Create SL_animals table Revision ID: 9687db7928ee -Revises: a3ba63dee8f4 +Revises: 45a668fa6325 Create Date: 2021-12-24 21:15:33.399197 """ @@ -11,7 +11,7 @@ # revision identifiers, used by Alembic. revision = '9687db7928ee' -down_revision = 'fc7325372396' +down_revision = '45a668fa6325' branch_labels = None depends_on = None diff --git a/src/server/api/API_ingest/shelterluv_db.py b/src/server/api/API_ingest/shelterluv_db.py index db7710af..b3518cf5 100644 --- a/src/server/api/API_ingest/shelterluv_db.py +++ b/src/server/api/API_ingest/shelterluv_db.py @@ -93,7 +93,7 @@ def insert_events(event_list): metadata = MetaData() sla = Table("sl_animal_events", metadata, autoload=True, autoload_with=engine) - # TODO: Pull from DB + # TODO: Pull from DB - inserted in db_setup/base_users.py/populate_sl_event_types() event_map = { "Outcome.Adoption": 1, "Outcome.Foster": 2, diff --git a/src/server/config.py b/src/server/config.py index d39667c4..56f53ce7 100644 --- a/src/server/config.py +++ b/src/server/config.py @@ -39,10 +39,11 @@ # command.stamp(alembic_cfg, "head") with engine.connect() as connection: - import user_mgmt.base_users - user_mgmt.base_users.create_base_roles() # IFF there are no roles already - user_mgmt.base_users.create_base_users() # IFF there are no users already - user_mgmt.base_users.populate_rfm_mapping_table() # Set to True to force loading latest version of populate script + import db_setup.base_users + db_setup.base_users.create_base_roles() # IFF there are no roles already + db_setup.base_users.create_base_users() # IFF there are no users already + db_setup.base_users.populate_sl_event_types() # IFF there are no event types already + db_setup.base_users.populate_rfm_mapping_table() # Set to True to force loading latest version of populate script # found in the server/alembic directory # Create these directories only one time - when initializing diff --git a/src/server/user_mgmt/README.md b/src/server/db_setup/README.md similarity index 100% rename from src/server/user_mgmt/README.md rename to src/server/db_setup/README.md diff --git a/src/server/user_mgmt/__init__.py b/src/server/db_setup/__init__.py similarity index 100% rename from src/server/user_mgmt/__init__.py rename to src/server/db_setup/__init__.py diff --git a/src/server/user_mgmt/base_users.py b/src/server/db_setup/base_users.py similarity index 82% rename from src/server/user_mgmt/base_users.py rename to src/server/db_setup/base_users.py index 1bda2098..08218438 100644 --- a/src/server/user_mgmt/base_users.py +++ b/src/server/db_setup/base_users.py @@ -3,7 +3,7 @@ import sqlalchemy as sa import os - +# This started as the place to create base users but a couple of other setup taks added. try: from secrets_dict import BASEUSER_PW, BASEEDITOR_PW, BASEADMIN_PW @@ -135,4 +135,21 @@ def table_empty(): else: print("rfm_mapping table already populated; overwrite not True so not changing.") - return \ No newline at end of file + return + + +def populate_sl_event_types(): + """If not present, insert values for shelterluv animal event types.""" + with engine.connect() as connection: + result = connection.execute("select id from sl_event_types") + type_count = len(result.fetchall()) + if type_count == 0: + print("Inserting SL event types") + connection.execute("""INSERT into sl_event_types values + (1, 'Outcome.Adoption'), + (2, 'Outcome.Foster'), + (3, 'Outcome.ReturnToOwner'), + (4, 'Intake.AdoptionReturn'), + (5, 'Intake.FosterReturn'); """) + else: + print(type_count, " event types already present in DB, not creating") From 8e480d5f096cbdf6d95c24e30d9c7b9350ef3180 Mon Sep 17 00:00:00 2001 From: c-simpson Date: Tue, 27 Dec 2022 14:24:59 -0500 Subject: [PATCH 12/14] Add venv to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6ec237c0..7df44560 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ start_env.sh .mypy_cache/ *secrets* *kustomization* +src/.venv/ From 753485baf052ea5690b352f16466432ffba6793a Mon Sep 17 00:00:00 2001 From: c-simpson Date: Tue, 27 Dec 2022 15:08:42 -0500 Subject: [PATCH 13/14] Adjust ingest endpoint (internal) Add TEST_MODE check on import --- .gitignore | 1 + .../api/API_ingest/ingest_sources_from_api.py | 20 +++++++++++++++---- .../api/API_ingest/shelterluv_api_handler.py | 12 +++++++++++ src/server/api/API_ingest/sl_animal_events.py | 6 +++++- src/server/api/internal_api.py | 2 +- 5 files changed, 35 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 6ec237c0..7df44560 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ start_env.sh .mypy_cache/ *secrets* *kustomization* +src/.venv/ diff --git a/src/server/api/API_ingest/ingest_sources_from_api.py b/src/server/api/API_ingest/ingest_sources_from_api.py index 25e92f8c..63429bcd 100644 --- a/src/server/api/API_ingest/ingest_sources_from_api.py +++ b/src/server/api/API_ingest/ingest_sources_from_api.py @@ -1,7 +1,19 @@ -from api.API_ingest import shelterluv_api_handler +from api.API_ingest import shelterluv_api_handler, sl_animal_events def start(conn): - print("Start Fetching raw data from different API sources") + print("Start fetching raw data from different API sources") + + print(" Fetching Shelterluv people") + #Run each source to store the output in dropbox and in the container as a CSV + slp_count = shelterluv_api_handler.store_shelterluv_people_all(conn) + print(" Finished fetching Shelterluv people - %d records" % slp_count) + + print(" Fetching Shelterluv events") #Run each source to store the output in dropbox and in the container as a CSV - shelterluv_api_handler.store_shelterluv_people_all(conn) - print("Finish Fetching raw data from different API sources") \ No newline at end of file + sle_count = sl_animal_events.slae_test() + print(" Finished fetching Shelterluv events - %d records" % sle_count) + + print("Finished fetching raw data from different API sources") + + + #TODO: Return object with count for each data source? \ No newline at end of file diff --git a/src/server/api/API_ingest/shelterluv_api_handler.py b/src/server/api/API_ingest/shelterluv_api_handler.py index e757d10a..1d316b0c 100644 --- a/src/server/api/API_ingest/shelterluv_api_handler.py +++ b/src/server/api/API_ingest/shelterluv_api_handler.py @@ -8,6 +8,9 @@ from constants import RAW_DATA_PATH from models import ShelterluvPeople + +TEST_MODE = os.getenv("TEST_MODE") + try: from secrets_dict import SHELTERLUV_SECRET_TOKEN except ImportError: @@ -78,6 +81,13 @@ def store_shelterluv_people_all(conn): has_more = response["has_more"] offset += 100 + if offset % 1000 == 0: + print("Reading offset ", str(offset)) + if TEST_MODE and offset > 1000: + has_more=False # Break out early + + + print("Finish getting shelterluv contacts from people table") print("Start storing latest shelterluvpeople results to container") @@ -98,3 +108,5 @@ def store_shelterluv_people_all(conn): print("Uploading shelterluvpeople csv to database") ShelterluvPeople.insert_from_df(pd.read_csv(file_path, dtype="string"), conn) + + return offset diff --git a/src/server/api/API_ingest/sl_animal_events.py b/src/server/api/API_ingest/sl_animal_events.py index 01228e8d..f39bfb7c 100644 --- a/src/server/api/API_ingest/sl_animal_events.py +++ b/src/server/api/API_ingest/sl_animal_events.py @@ -38,6 +38,8 @@ print("Couldn't get SHELTERLUV_SECRET_TOKEN from file or environment") +TEST_MODE=os.getenv("TEST_MODE") # if not present, has value None + headers = {"Accept": "application/json", "X-API-Key": SHELTERLUV_SECRET_TOKEN} logger = print # print to console for testing @@ -143,6 +145,8 @@ def get_events_bulk(): offset += limit if offset % 1000 == 0: print("Reading offset ", str(offset)) + if TEST_MODE and offset > 1000: + more_records=False # Break out early else: return -5 # AFAICT, this means URL was bad @@ -155,7 +159,7 @@ def slae_test(): print("Total events:", total_count) b = get_events_bulk() - print("Records:", len(b)) + print("Strored records:", len(b)) # f = filter_events(b) # print(f) diff --git a/src/server/api/internal_api.py b/src/server/api/internal_api.py index 81e1ee89..dc0921e0 100644 --- a/src/server/api/internal_api.py +++ b/src/server/api/internal_api.py @@ -22,7 +22,7 @@ def user_test2(): return jsonify(("OK from INTERNAL test/test @ " + str(datetime.now()))) -@internal_api.route("/api/ingestRawData", methods=["GET"]) +@internal_api.route("/api/internal/ingestRawData", methods=["GET"]) def ingest_raw_data(): try: with engine.begin() as conn: From a6caed5bac3d345c991a4e0ad6e74d6263990096 Mon Sep 17 00:00:00 2001 From: c-simpson Date: Wed, 28 Dec 2022 12:34:29 -0500 Subject: [PATCH 14/14] Logging tweaks --- .../api/API_ingest/ingest_sources_from_api.py | 15 ++++++----- src/server/api/API_ingest/sl_animal_events.py | 27 +++++++++---------- src/server/db_setup/base_users.py | 2 +- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/server/api/API_ingest/ingest_sources_from_api.py b/src/server/api/API_ingest/ingest_sources_from_api.py index 2c218aaa..d9915a2c 100644 --- a/src/server/api/API_ingest/ingest_sources_from_api.py +++ b/src/server/api/API_ingest/ingest_sources_from_api.py @@ -1,19 +1,22 @@ +import structlog +logger = structlog.get_logger() + from api.API_ingest import shelterluv_api_handler, sl_animal_events def start(conn): - print("Start fetching raw data from different API sources") + logger.debug("Start fetching raw data from different API sources") - print(" Fetching Shelterluv people") + logger.debug(" Fetching Shelterluv people") #Run each source to store the output in dropbox and in the container as a CSV slp_count = shelterluv_api_handler.store_shelterluv_people_all(conn) - print(" Finished fetching Shelterluv people - %d records" % slp_count) + logger.debug(" Finished fetching Shelterluv people - %d records" , slp_count) - print(" Fetching Shelterluv events") + logger.debug(" Fetching Shelterluv events") #Run each source to store the output in dropbox and in the container as a CSV sle_count = sl_animal_events.slae_test() - print(" Finished fetching Shelterluv events - %d records" % sle_count) + logger.debug(" Finished fetching Shelterluv events - %d records" , sle_count) - print("Finished fetching raw data from different API sources") + logger.debug("Finished fetching raw data from different API sources") #TODO: Return object with count for each data source? diff --git a/src/server/api/API_ingest/sl_animal_events.py b/src/server/api/API_ingest/sl_animal_events.py index f39bfb7c..d7f6a472 100644 --- a/src/server/api/API_ingest/sl_animal_events.py +++ b/src/server/api/API_ingest/sl_animal_events.py @@ -1,6 +1,8 @@ import os, time, json import posixpath as path +import structlog +logger = structlog.get_logger() import requests @@ -35,16 +37,13 @@ except KeyError: # Not in environment # You're SOL for now - print("Couldn't get SHELTERLUV_SECRET_TOKEN from file or environment") + logger.error("Couldn't get SHELTERLUV_SECRET_TOKEN from file or environment") TEST_MODE=os.getenv("TEST_MODE") # if not present, has value None headers = {"Accept": "application/json", "X-API-Key": SHELTERLUV_SECRET_TOKEN} -logger = print # print to console for testing - - # Sample response from events request: # { @@ -81,23 +80,23 @@ def get_event_count(): try: response = requests.request("GET", URL, headers=headers) except Exception as e: - logger("get_event_count failed with ", e) + logger.error("get_event_count failed with ", e) return -2 if response.status_code != 200: - logger("get_event_count ", response.status_code, "code") + logger.error("get_event_count ", response.status_code, "code") return -3 try: decoded = json.loads(response.text) except json.decoder.JSONDecodeError as e: - logger("get_event_count JSON decode failed with", e) + logger.error("get_event_count JSON decode failed with", e) return -4 if decoded["success"]: return decoded["total_count"] else: - logger(decoded['error_message']) + logger.error(decoded['error_message']) return -5 # AFAICT, this means URL was bad @@ -123,17 +122,17 @@ def get_events_bulk(): try: response = requests.request("GET", url, headers=headers) except Exception as e: - logger("get_events failed with ", e) + logger.error("get_events failed with ", e) return -2 if response.status_code != 200: - logger("get_event_count ", response.status_code, "code") + logger.error("get_event_count ", response.status_code, "code") return -3 try: decoded = json.loads(response.text) except json.decoder.JSONDecodeError as e: - logger("get_event_count JSON decode failed with", e) + logger.error("get_event_count JSON decode failed with", e) return -4 if decoded["success"]: @@ -144,7 +143,7 @@ def get_events_bulk(): more_records = decoded["has_more"] # if so, we'll make another pass offset += limit if offset % 1000 == 0: - print("Reading offset ", str(offset)) + logger.debug("Reading offset ", str(offset)) if TEST_MODE and offset > 1000: more_records=False # Break out early @@ -156,10 +155,10 @@ def get_events_bulk(): def slae_test(): total_count = get_event_count() - print("Total events:", total_count) + logger.debug("Total events:", total_count) b = get_events_bulk() - print("Strored records:", len(b)) + logger.debug("Strored records:", len(b)) # f = filter_events(b) # print(f) diff --git a/src/server/db_setup/base_users.py b/src/server/db_setup/base_users.py index a19cec36..4bd232f9 100644 --- a/src/server/db_setup/base_users.py +++ b/src/server/db_setup/base_users.py @@ -153,4 +153,4 @@ def populate_sl_event_types(): (4, 'Intake.AdoptionReturn'), (5, 'Intake.FosterReturn'); """) else: - print(type_count, " event types already present in DB, not creating") + logger.debug("%d event types already present in DB, not creating", type_count)