api endpoint to download full datasets from salesforce and shelterluv

.gitignore

@@ -24,3 +24,5 @@ start_env.sh
 *secrets*
 *kustomization*
 src/.venv/
+src/server/secrets_dict.py
+

src/server/api/API_ingest/ingest_sources_from_api.py

@@ -1,22 +1,21 @@
+from api.API_ingest import shelterluv_people, salesforce_contacts, sl_animal_events
 import structlog
 logger = structlog.get_logger()
 
-from api.API_ingest import shelterluv_api_handler, sl_animal_events
+def start():
+    logger.debug("Start Fetching raw data from different API sources")
 
-def start(conn):
-    logger.debug("Start fetching raw data from different API sources")
+    logger.debug("   Fetching Salesforce contacts")
+    salesforce_contacts.store_contacts_all()
+    logger.debug("   Finished fetching Salesforce contacts")
 
     logger.debug("   Fetching Shelterluv people")
-    #Run each source to store the output in dropbox and in the container as a CSV
-    slp_count = shelterluv_api_handler.store_shelterluv_people_all(conn)
+    slp_count = shelterluv_people.store_shelterluv_people_all()
     logger.debug("   Finished fetching Shelterluv people - %d records" , slp_count)
 
     logger.debug("   Fetching Shelterluv events")
-    #Run each source to store the output in dropbox and in the container as a CSV
-    sle_count = sl_animal_events.slae_test()
+    sle_count = sl_animal_events.store_all_animals_and_events()
     logger.debug("   Finished fetching Shelterluv events - %d records" , sle_count)
 
     logger.debug("Finished fetching raw data from different API sources")
 
-
-    #TODO:   Return object with count for each data source?

src/server/api/API_ingest/salesforce_contacts.py

@@ -0,0 +1,44 @@
+import os
+
+from sqlalchemy.orm import  sessionmaker
+from simple_salesforce import Salesforce
+from config import engine
+from models import SalesForceContacts
+
+import structlog
+logger = structlog.get_logger()
+
+def store_contacts_all():
+    Session = sessionmaker(engine)
+    with Session() as session:
+
+        logger.debug("truncating table salesforcecontacts")
+        session.execute("TRUNCATE TABLE salesforcecontacts")
+
+        logger.debug("retrieving the latest salesforce contacts data")
+        sf = Salesforce(domain=os.getenv('SALESFORCE_DOMAIN'), password=os.getenv('SALESFORCE_PW'), username=os.getenv('SALESFORCE_USERNAME'), security_token=os.getenv('SALESFORCE_SECURITY_TOKEN'))
+        results = sf.query("SELECT Contact_ID_18__c, FirstName, LastName, Contact.Account.Name, MailingCountry, MailingStreet, MailingCity, MailingState, MailingPostalCode, Phone, MobilePhone, Email FROM Contact")
+        logger.debug("Query returned %d Salesforce contact records", len(results['records']) )
+
+        done = False
+        while not done:
+            for row in results['records']:
+                account_name = row['Account']['Name'] if row['Account'] is not None else None
+                contact = SalesForceContacts(contact_id=row['Contact_ID_18__c'],
+                                             first_name=row['FirstName'],
+                                             last_name=row['LastName'],
+                                             account_name=account_name,
+                                             mailing_country=row['MailingCountry'],
+                                             mailing_street=row['MailingStreet'],
+                                             mailing_city=row['MailingCity'],
+                                             mailing_state_province=row['MailingState'],
+                                             mailing_zip_postal_code=row['MailingPostalCode'],
+                                             phone=row['Phone'],
+                                             mobile=row['MobilePhone'],
+                                             email=['Email'])
+                session.add(contact)
+                done = results['done']
+                if not done:
+                    results = sf.query_more(results['nextRecordsUrl'])
+        session.commit()
+    logger.debug("finished downloading latest salesforce contacts data")

src/server/api/API_ingest/shelterluv_db.py

@@ -1,18 +1,10 @@
-from api.api import common_api
-from config import engine
-from flask import jsonify, current_app
-from sqlalchemy.sql import text
-import requests
-import time
-from datetime import datetime
-
-from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy import Table, MetaData
-from pipeline import flow_script
+from sqlalchemy.orm import sessionmaker
+
 from config import engine
-from flask import request, redirect, jsonify, current_app
-from api.file_uploader import validate_and_arrange_upload
-from sqlalchemy.orm import Session, sessionmaker
+
+import structlog
+logger = structlog.get_logger()
 
 
 def insert_animals(animal_list):
@@ -53,11 +45,9 @@ def truncate_animals():
 
     Session = sessionmaker(engine)
     session = Session()
-    metadata = MetaData()
-    sla = Table("shelterluv_animals", metadata, autoload=True, autoload_with=engine)
 
     truncate = "TRUNCATE table shelterluv_animals;"
-    result = session.execute(truncate)
+    session.execute(truncate)
 
     session.commit()  # Commit all inserted rows
     session.close()
@@ -69,81 +59,74 @@ def truncate_events():
     """Truncate the shelterluv_events table"""
 
     Session = sessionmaker(engine)
-    session = Session()
-    metadata = MetaData()
-    sla = Table("sl_animal_events", metadata, autoload=True, autoload_with=engine)
-
-    truncate = "TRUNCATE table sl_animal_events;"
-    result = session.execute(truncate)
-
-    session.commit()  # Commit all inserted rows
-    session.close()
+    with Session() as session:
+        truncate = "TRUNCATE table sl_animal_events;"
+        session.execute(truncate)
+        session.commit()
 
     return 0
 
-
 def insert_events(event_list):
     """Insert event records into sl_animal_events table and return row count. """
 
     # Always a clean insert
     truncate_events()
 
     Session = sessionmaker(engine)
-    session = Session()
-    metadata = MetaData()
-    sla = Table("sl_animal_events", metadata, autoload=True, autoload_with=engine)
-
-    # TODO: Pull from DB - inserted in db_setup/base_users.py/populate_sl_event_types()
-    event_map = {
-        "Outcome.Adoption": 1,
-        "Outcome.Foster": 2,
-        "Outcome.ReturnToOwner": 3,
-        "Intake.AdoptionReturn": 4,
-        "Intake.FosterReturn":5
-    }
-
-    # """ INSERT INTO "sl_event_types" ("id","event_name") VALUES 
-    # ( 1,'Outcome.Adoption' ),
-    # ( 2,'Outcome.Foster' ),
-    # ( 3,'Outcome.ReturnToOwner' ),
-    # ( 4,'Intake.AdoptionReturn' ),
-    # ( 5,'Intake.FosterReturn' ) """
-
-
-
-
-    # Event record:    [ AssociatedRecords[Type = Person]["Id"]',
-    #                    AssociatedRecords[Type = Animal]["Id"]',
-    #                     "Type",
-    #                     "Time"
-    #                     ]
-    #
-    #  In db:           ['id',
-    #                     'person_id',
-    #                     'animal_id',
-    #                     'event_type',
-    #                     'time']
-
-    ins_list = []  # Create a list of per-row dicts
-    for rec in event_list:
-        ins_list.append(
-            {
-                "person_id": next(
-                    filter(lambda x: x["Type"] == "Person", rec["AssociatedRecords"])
-                )["Id"],
-                "animal_id": next(
-                    filter(lambda x: x["Type"] == "Animal", rec["AssociatedRecords"])
-                )["Id"],
-                "event_type": event_map[rec["Type"]],
-                "time": rec["Time"],
-            }
-        )
-
-    # TODO: Wrap with try/catch
-    ret = session.execute(sla.insert(ins_list))
-
-    session.commit()  # Commit all inserted rows
-    session.close()
+    with Session() as session:
+        metadata = MetaData()
+        sla = Table("sl_animal_events", metadata, autoload=True, autoload_with=engine)
+
+        # TODO: Pull from DB - inserted in db_setup/base_users.py/populate_sl_event_types()
+        event_map = {
+            "Outcome.Adoption": 1,
+            "Outcome.Foster": 2,
+            "Outcome.ReturnToOwner": 3,
+            "Intake.AdoptionReturn": 4,
+            "Intake.FosterReturn":5
+        }
+
+        # """ INSERT INTO "sl_event_types" ("id","event_name") VALUES
+        # ( 1,'Outcome.Adoption' ),
+        # ( 2,'Outcome.Foster' ),
+        # ( 3,'Outcome.ReturnToOwner' ),
+        # ( 4,'Intake.AdoptionReturn' ),
+        # ( 5,'Intake.FosterReturn' ) """
+
+
+
+
+        # Event record:    [ AssociatedRecords[Type = Person]["Id"]',
+        #                    AssociatedRecords[Type = Animal]["Id"]',
+        #                     "Type",
+        #                     "Time"
+        #                     ]
+        #
+        #  In db:           ['id',
+        #                     'person_id',
+        #                     'animal_id',
+        #                     'event_type',
+        #                     'time']
+
+        ins_list = []  # Create a list of per-row dicts
+        for rec in event_list:
+            ins_list.append(
+                {
+                    "person_id": next(
+                        filter(lambda x: x["Type"] == "Person", rec["AssociatedRecords"])
+                    )["Id"],
+                    "animal_id": next(
+                        filter(lambda x: x["Type"] == "Animal", rec["AssociatedRecords"])
+                    )["Id"],
+                    "event_type": event_map[rec["Type"]],
+                    "time": rec["Time"],
+                }
+            )
+
+        # TODO: Wrap with try/catch
+        ret = session.execute(sla.insert(ins_list))
+        session.commit()
+        logger.debug("finished inserting events")
 
     return ret.rowcount