Merge branch 'master' into 522-flask-logging

Author: c-simpson

.gitignore

@@ -23,3 +23,6 @@ start_env.sh
 .mypy_cache/
 *secrets*
 *kustomization*
+src/.venv/
+src/server/secrets_dict.py
+

src/server/alembic/insert_rfm_edges.sql

@@ -0,0 +1,8 @@
+INSERT INTO "public"."kv_unique"(   "keycol",  "valcol") VALUES 
+(    'rfm_edges',
+    '{
+        "r":{"5": 0, "4": 262, "3": 1097, "2": 1910, "1": 2851}, 
+        "f": {"1": 0, "2": 1, "3": 2, "4": 3, "5": 4}, 
+        "m": {"1": 0.0, "2": 50.0, "3": 75.0, "4": 100.0, "5": 210.0}
+        }'
+ );

src/server/alembic/versions/90f471ac445c_create_sl_events.py

@@ -0,0 +1,41 @@
+"""Shelterluv animal events table
+
+Revision ID: 90f471ac445c
+Revises: 9687db7928ee
+Create Date: 2022-09-04 17:21:51.511030
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '90f471ac445c'
+down_revision = '9687db7928ee'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table (
+    "sl_event_types",
+    sa.Column("id", sa.Integer, autoincrement=True, primary_key=True),
+    sa.Column("event_name", sa.Text, nullable=False),
+    )
+    
+    op.create_table (
+    "sl_animal_events",
+    sa.Column("id", sa.Integer, autoincrement=True, primary_key=True),
+    sa.Column("person_id", sa.Integer, nullable=False),
+    sa.Column("animal_id", sa.Integer, nullable=False),
+    sa.Column("event_type", sa.Integer, sa.ForeignKey('sl_event_types.id')),
+    sa.Column("time", sa.BigInteger, nullable=False)
+    )
+    
+    op.create_index('sla_idx', 'sl_animal_events', ['person_id'])
+
+
+
+def downgrade():
+    op.drop_table("sl_animal_events")
+    op.drop_table("sl_event_types")

src/server/alembic/versions/9687db7928ee_shelterluv_animals.py

@@ -0,0 +1,33 @@
+"""Create SL_animals table
+
+Revision ID: 9687db7928ee
+Revises:  45a668fa6325
+Create Date: 2021-12-24 21:15:33.399197
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9687db7928ee'
+down_revision = '45a668fa6325'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+        op.create_table (
+        "shelterluv_animals",
+        sa.Column("id", sa.BigInteger, primary_key=True),
+        sa.Column("local_id", sa.BigInteger, nullable=False),
+        sa.Column("name", sa.Text, nullable=False),
+        sa.Column("type",  sa.Text, nullable=False),
+        sa.Column("dob", sa.BigInteger, nullable=False),
+        sa.Column("update_stamp", sa.BigInteger, nullable=False),
+        sa.Column("photo", sa.Text, nullable=False)
+    )
+
+
+def downgrade():
+    op.drop_table("shelterluv_animals")

src/server/api/.optic/.gitignore

@@ -1,9 +1,21 @@
-from api.API_ingest import shelterluv_api_handler
+from api.API_ingest import shelterluv_people, salesforce_contacts, sl_animal_events
 import structlog
 logger = structlog.get_logger()
 
-def start(conn):
+def start():
     logger.debug("Start Fetching raw data from different API sources")
-    #Run each source to store the output in dropbox and in the container as a CSV
-    shelterluv_api_handler.store_shelterluv_people_all(conn)
-    logger.debug("Finish Fetching raw data from different API sources")
+
+    logger.debug("   Fetching Salesforce contacts")
+    salesforce_contacts.store_contacts_all()
+    logger.debug("   Finished fetching Salesforce contacts")
+
+    logger.debug("   Fetching Shelterluv people")
+    slp_count = shelterluv_people.store_shelterluv_people_all()
+    logger.debug("   Finished fetching Shelterluv people - %d records" , slp_count)
+
+    logger.debug("   Fetching Shelterluv events")
+    sle_count = sl_animal_events.store_all_animals_and_events()
+    logger.debug("   Finished fetching Shelterluv events - %d records" , sle_count)
+
+    logger.debug("Finished fetching raw data from different API sources")
+

src/server/api/API_ingest/ingest_sources_from_api.py

@@ -0,0 +1,44 @@
+import os
+
+from sqlalchemy.orm import  sessionmaker
+from simple_salesforce import Salesforce
+from config import engine
+from models import SalesForceContacts
+
+import structlog
+logger = structlog.get_logger()
+
+def store_contacts_all():
+    Session = sessionmaker(engine)
+    with Session() as session:
+
+        logger.debug("truncating table salesforcecontacts")
+        session.execute("TRUNCATE TABLE salesforcecontacts")
+
+        logger.debug("retrieving the latest salesforce contacts data")
+        sf = Salesforce(domain=os.getenv('SALESFORCE_DOMAIN'), password=os.getenv('SALESFORCE_PW'), username=os.getenv('SALESFORCE_USERNAME'), security_token=os.getenv('SALESFORCE_SECURITY_TOKEN'))
+        results = sf.query("SELECT Contact_ID_18__c, FirstName, LastName, Contact.Account.Name, MailingCountry, MailingStreet, MailingCity, MailingState, MailingPostalCode, Phone, MobilePhone, Email FROM Contact")
+        logger.debug("Query returned %d Salesforce contact records", len(results['records']) )
+        
+        done = False
+        while not done:
+            for row in results['records']:
+                account_name = row['Account']['Name'] if row['Account'] is not None else None
+                contact = SalesForceContacts(contact_id=row['Contact_ID_18__c'],
+                                             first_name=row['FirstName'],
+                                             last_name=row['LastName'],
+                                             account_name=account_name,
+                                             mailing_country=row['MailingCountry'],
+                                             mailing_street=row['MailingStreet'],
+                                             mailing_city=row['MailingCity'],
+                                             mailing_state_province=row['MailingState'],
+                                             mailing_zip_postal_code=row['MailingPostalCode'],
+                                             phone=row['Phone'],
+                                             mobile=row['MobilePhone'],
+                                             email=['Email'])
+                session.add(contact)
+                done = results['done']
+                if not done:
+                    results = sf.query_more(results['nextRecordsUrl'])
+        session.commit()
+    logger.debug("finished downloading latest salesforce contacts data")

src/server/api/API_ingest/salesforce_contacts.py

@@ -0,0 +1,216 @@
+import os, time, json
+import posixpath as path
+
+import requests
+
+from api.API_ingest import shelterluv_db
+from server.api.API_ingest.shelterluv_db import insert_animals
+
+
+# from config import engine
+# from flask import current_app
+# from sqlalchemy.sql import text
+
+BASE_URL = 'http://shelterluv.com/api/'
+MAX_COUNT = 100  # Max records the API will return for one call
+
+try:
+    from secrets_dict import SHELTERLUV_SECRET_TOKEN
+except ImportError:
+    # Not running locally
+    from os import environ
+
+    try:
+        SHELTERLUV_SECRET_TOKEN = environ['SHELTERLUV_SECRET_TOKEN']
+    except KeyError:
+        # Not in environment
+        # You're SOL for now
+        print("Couldn't get SHELTERLUV_SECRET_TOKEN from file or environment")
+
+
+
+headers = {
+    "Accept": "application/json",
+    "X-API-Key": SHELTERLUV_SECRET_TOKEN
+}
+
+logger = print
+
+def get_animal_count():
+    """Test that server is operational and get total animal count."""
+    animals = 'v1/animals&offset=0&limit=1'
+    URL = path.join(BASE_URL,animals)
+
+    try:
+        response = requests.request("GET",URL, headers=headers)
+    except Exception as e:
+        logger('get_animal_count failed with ', e)
+        return -2
+
+    if response.status_code != 200:
+        logger("get_animal_count ", response.status_code, "code")
+        return -3
+
+    try:
+        decoded = json.loads(response.text)
+    except json.decoder.JSONDecodeError as e:
+        logger("get_animal_count JSON decode failed with", e)
+        return -4
+
+    if decoded['success']:
+        return decoded['total_count']
+    else:
+        return -5   # AFAICT, this means URL was bad
+
+
+def get_updated_animal_count(last_update):
+    """Test that server is operational and get total animal count."""
+    animals = 'v1/animals&offset=0&limit=1&sort=updated_at&since=' + str(last_update)
+    URL = path.join(BASE_URL,animals)
+
+    try:
+        response = requests.request("GET",URL, headers=headers)
+    except Exception as e:
+        logger('get_updated_animal_count failed with ', e)
+        return -2
+
+    if response.status_code != 200:
+        logger("get_updated_animal_count ", response.status_code, "code")
+        return -3
+
+    try:
+        decoded = json.loads(response.text)
+    except json.decoder.JSONDecodeError as e:
+        logger("get_updated_animal_count JSON decode failed with", e)
+        return -4
+
+    if decoded['success']:
+        return decoded['total_count']
+    else:
+        return -5   # AFAICT, this means URL was bad
+
+
+
+
+def filter_animals(raw_list):
+    """Given a list of animal records as returned by SL, return a list of records with only the fields we care about."""
+
+    good_keys = ['ID', 'Internal-ID', 'Name', 'Type', 'DOBUnixTime', 'CoverPhoto','LastUpdatedUnixTime']
+
+    filtered = []
+
+    for r in raw_list:
+        f = {}
+        for k in good_keys:
+            try:
+                f[k] = r[k]
+            except:
+                if k in ('DOBUnixTime','LastUpdatedUnixTime'):
+                    f[k] = 0
+                else:
+                    f[k] = ''
+        filtered.append(f)
+
+    return filtered
+
+
+
+
+def get_animals_bulk(total_count):
+    """Pull all animal records from SL """
+
+    # 'Great' API design - animal record 0 is the newest, so we need to start at the end, 
+    # back up MAX_COUNT rows, make our request, then keep backing up. We need to keep checking
+    # the total records to ensure one wasn't added in the middle of the process.
+    # Good news, the API is robust and won't blow up if you request past the end.
+
+    raw_url = path.join(BASE_URL, 'v1/animals&offset={0}&limit={1}')
+
+    start_record = int(total_count) 
+    offset = (start_record - MAX_COUNT)  if  (start_record - MAX_COUNT) > -1  else 0
+    limit = MAX_COUNT
+
+    while offset > -1 :
+
+        logger("getting at offset", offset)
+        url = raw_url.format(offset,limit)
+
+        try:
+            response = requests.request("GET",url, headers=headers)
+        except Exception as e:
+            logger('get_animals failed with ', e)
+            return -2
+
+        if response.status_code != 200:
+            logger("get_animal_count ", response.status_code, "code")
+            return -3
+
+        try:
+            decoded = json.loads(response.text)
+        except json.decoder.JSONDecodeError as e:
+            logger("get_animal_count JSON decode failed with", e)
+            return -4
+
+        if decoded['success']:
+            insert_animals( filter_animals(decoded['animals']) )
+            if offset == 0:
+                break
+            offset -= MAX_COUNT 
+            if offset < 0 :
+                limit = limit + offset
+                offset = 0
+        else:
+            return -5   # AFAICT, this means URL was bad
+
+    return 'zero'
+
+
+def update_animals(last_update):
+    """Get the animals inserted or updated since last check, insert/update db records. """
+
+    updated_records = get_updated_animal_count(last_update)
+
+    
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+def sla_test():
+    total_count = get_animal_count()
+    print('Total animals:',total_count)
+
+    b = get_animals_bulk(total_count)
+    print(len(b))
+
+    # f = filter_animals(b)
+    # print(f)
+
+    # count = shelterluv_db.insert_animals(f)
+    return len(b)
+
+# if __name__ == '__main__' :    
+
+#     total_count = get_animal_count()
+#     print('Total animals:',total_count)
+
+#     b = get_animals_bulk(9)
+#     print(len(b))
+
+#     f = filter_animals(b)
+#     print(f)
+
+#     count = shelterluv_db.insert_animals(f)