Fallback token and other fixes (#33)

Fallback token and other fixes

Add Github fallback token to avoid exceeding API calls rate limit
Fix Squad and Service Name titles issues for github-related PRs
Fix failed_zuul.py hanging during process empty repo

Reviewed-by: Vladimir Vshivkov

Author: YustinaKvr

failed_zuul.py

@@ -19,7 +19,6 @@
 db_host = os.getenv("DB_HOST")
 db_port = os.getenv("DB_PORT")
 db_name = os.getenv("DB_ZUUL")  # here we're using dedicated postgres db 'zuul' since Failed Zuul PRs panel should be placed on a same dashboard such as Open PRs
-db_csv = os.getenv("DB_CSV")  # here rtc service table is located
 db_user = os.getenv("DB_USER")
 db_password = os.getenv("DB_PASSWORD")
 
@@ -72,7 +71,28 @@ def create_prs_table(conn_zuul, cur_zuul, table_name):
         print(f"Create table: an error occurred while trying to create a table {table_name} in the database: {e}")
 
 
+def is_repo_empty(org, repo, gitea_token):
+    try:
+        commits_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/commits?token={gitea_token}")
+        commits_resp.raise_for_status()
+
+        commits_data = json.loads(commits_resp.content.decode())
+        if not commits_data:
+            return True
+        return False
+    except requests.exceptions.HTTPError as e:
+        if e.response.status_code == 409:  # Conflict error which might mean empty repo, skip this repo to avoid script hangs
+            print(f"Repo {repo} is empty, skipping")
+            return True
+        print(f"Check repo: an error occurred while trying to get commits for repo {repo}: {e}")
+        return False
+    except requests.exceptions.RequestException as e:
+        print(f"Check repo: an error occurred while trying to get commits for repo {repo}: {e}")
+        return False
+
+
 def get_repos(org, gitea_token):
+    print("Gathering repos...")
     repos = []
     page = 1
     while True:
@@ -90,7 +110,8 @@ def get_repos(org, gitea_token):
             break
 
         for repo in repos_dict:
-            repos.append(repo["name"])
+            if not is_repo_empty(org, repo["name"], gitea_token):  # Skipping empty repos
+                repos.append(repo["name"])
 
         link_header = repos_resp.headers.get("Link")
         if link_header is None or "rel=\"next\"" not in link_header:
@@ -121,14 +142,16 @@ def get_f_pr_commits(org, repo, f_pr_number, gitea_token):
         created_at = None
         days_passed = None
 
-        pull_request_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls/{f_pr_number}/commits?token={gitea_token}")
+        pull_request_resp = session.get(
+            f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls/{f_pr_number}/commits?token={gitea_token}")
         pull_request_resp.raise_for_status()
 
         f_pr_info = json.loads(pull_request_resp.content.decode("utf-8"))
 
         if len(f_pr_info) > 0:
             f_commit_sha = f_pr_info[0]["sha"]
-            commit_status_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/statuses/{f_commit_sha}?token={gitea_token}")
+            commit_status_resp = session.get(
+                f"{gitea_api_endpoint}/repos/{org}/{repo}/statuses/{f_commit_sha}?token={gitea_token}")
             commit_status_resp.raise_for_status()
 
             commit_info = json.loads(commit_status_resp.content.decode("utf-8"))
@@ -143,113 +166,114 @@ def get_f_pr_commits(org, repo, f_pr_number, gitea_token):
             return zuul_url, status, created_at, days_passed
 
     except requests.exceptions.RequestException as e:
-        print(f"Get failed PR commits: an error occurred while trying to get pull requests of {repo} repo for {org} org: {e}")
+        print(
+            f"Get failed PR commits: an error occurred while trying to get pull requests of {repo} repo for {org} org: {e}")
 
 
 def get_failed_prs(org, repo, gitea_token, conn_zuul, cur_zuul, table_name):
-
+    # print(f"Processing {repo}...")  # Debug print, uncomment in case of script hangs
     try:
-        if repo != "doc-exports" and repo != "dsf":
+        if repo != "doc-exports":
             page = 1
             while True:
-                    repo_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls?state=open&page={page}&limit=1000&token={gitea_token}")
-                    pull_requests = []
-                    if repo_resp.status_code == 200:
-                        try:
-                            pull_requests = json.loads(repo_resp.content.decode("utf-8"))
-                        except json.JSONDecodeError as e:
-                            print(f"Get parent PR: an error occurred while decoding JSON: {e}")
-                        if not pull_requests:
-                            break
-
-                        for pull_req in pull_requests:
-                            body = pull_req["body"]
-                            if body.startswith("This is an automatically created Pull Request"):
-                                if pull_req["merged"] is True:
-                                    continue
-                                else:
-                                    f_par_pr_num = extract_number_from_body(body)
-                                    f_pr_number = pull_req["number"]
-                                    service_name = repo
-                                    squad = ""
-                                    title = pull_req["title"]
-                                    f_pr_url = pull_req["url"]
-                                    f_pr_state = pull_req["state"]
-                                    zuul_url, status, created_at, days_passed = get_f_pr_commits(org, repo, f_pr_number, gitea_token)
-                                try:
-                                    if all(item is not None for item in [zuul_url, status, created_at, days_passed]):
-                                        cur_zuul.execute(f"""
-
-                                            INSERT INTO public.{table_name}
-                                            ("Service Name", "Failed PR Title", "Failed PR URL", "Squad", "Failed PR State", "Zuul URL", "Zuul Check Status", "Days Passed",  "Parent PR Number")
-                                            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
-                                        """,
-                                        (service_name, title, f_pr_url, squad, f_pr_state, zuul_url, status, days_passed, f_par_pr_num)
-                                                    )
-                                        conn_zuul.commit()
-                                except Exception as e:
-                                    print(f"Failed PRs: an error occurred while inserting into {table_name} table: {e}")
-                            else:
-                                continue
-                    elif org == "docs-swiss" and repo_resp.status_code != 200:
+                # print(f"Fetching PRs for {repo}, page {page}...")  # Debug print, uncomment in case of script hangs
+                repo_resp = session.get(
+                    f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls?state=open&page={page}&limit=1000&token={gitea_token}")
+                pull_requests = []
+                if repo_resp.status_code == 200:
+                    try:
+                        pull_requests = json.loads(repo_resp.content.decode("utf-8"))
+                    except json.JSONDecodeError as e:
+                        print(f"Get parent PR: an error occurred while decoding JSON: {e}")
+                    if not pull_requests:
                         break
-                    page += 1
+
+                    for pull_req in pull_requests:
+                        body = pull_req["body"]
+                        if body.startswith("This is an automatically created Pull Request"):
+                            if pull_req["merged"] is True:
+                                continue
+                            else:
+                                f_par_pr_num = extract_number_from_body(body)
+                                f_pr_number = pull_req["number"]
+                                service_name = repo
+                                squad = ""
+                                title = pull_req["title"]
+                                f_pr_url = pull_req["url"]
+                                f_pr_state = pull_req["state"]
+                                zuul_url, status, created_at, days_passed = get_f_pr_commits(org, repo, f_pr_number,
+                                                                                             gitea_token)
+                            try:
+                                if all(item is not None for item in [zuul_url, status, created_at, days_passed]):
+                                    cur_zuul.execute(f"""
+                                        INSERT INTO public.{table_name}
+                                        ("Service Name", "Failed PR Title", "Failed PR URL", "Squad", "Failed PR State", "Zuul URL", "Zuul Check Status", "Days Passed",  "Parent PR Number")
+                                        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+                                    """,
+                                     (service_name, title, f_pr_url, squad, f_pr_state, zuul_url, status, days_passed, f_par_pr_num)
+                                     )
+                                    conn_zuul.commit()
+                            except Exception as e:
+                                print(f"Failed PRs: an error occurred while inserting into {table_name} table: {e}")
+                        else:
+                            continue
+                elif org == "docs-swiss" and repo_resp.status_code != 200:
+                    break
+                page += 1
 
     except Exception as e:
         print('Failed PRs: an error occurred:', e)
 
 
-def update_squad_and_title(cur_dict, conn_table, cur_table, rtctable, opentable):
+def update_squad_and_title(conn_zuul, cur_zuul, rtctable, opentable):
     print(f"Updating squads and titles in {opentable}...")
     try:
-        cur_table.execute(f"SELECT * FROM {opentable};")
-        failed_prs_rows = cur_table.fetchall()
+        cur_zuul.execute(f"SELECT * FROM {opentable};")
+        failed_prs_rows = cur_zuul.fetchall()
 
         for row in failed_prs_rows:
             service_name_index = 1
             id_index = 0
 
-            cur_dict.execute(
+            cur_zuul.execute(
                 f"""SELECT "Title", "Category"
                     FROM {rtctable}
                     WHERE "Repository" = %s;""",
                 (row[service_name_index],)
             )
-            rtc_row = cur_dict.fetchone()
+            rtc_row = cur_zuul.fetchone()
 
             if rtc_row:
-                cur_table.execute(
+                cur_zuul.execute(
                     f"""UPDATE {opentable}
                         SET "Service Name" = %s, "Squad" = %s
                         WHERE id = %s;""",
                     (rtc_row[0], rtc_row[1], row[id_index])
                 )
 
             if row[service_name_index] in ('doc-exports', 'docs_on_docs', 'docsportal'):
-                cur_table.execute(
+                cur_zuul.execute(
                     f"""UPDATE {opentable}
                         SET "Squad" = 'Other'
                         WHERE id = %s;""",
                     (row[id_index],)
                 )
 
-            conn_table.commit()
+            conn_zuul.commit()
 
     except Exception as e:
         print(f"Error updating squad and title: {e}")
-        conn_table.rollback()
+        conn_zuul.rollback()
 
 
 def main(org, table_name, rtc):
     check_env_variables()
 
     conn_zuul = connect_to_db(db_name)
     cur_zuul = conn_zuul.cursor()
-    conn_csv = connect_to_db(db_csv)
-    cur_csv = conn_csv.cursor()
 
-    cur_csv.execute(f"DROP TABLE IF EXISTS {table_name}")
-    conn_csv.commit()
+    cur_zuul.execute(f"DROP TABLE IF EXISTS {table_name}")
+    conn_zuul.commit()
 
     create_prs_table(conn_zuul, cur_zuul, table_name)
 
@@ -259,10 +283,8 @@ def main(org, table_name, rtc):
     for repo in repos:
         get_failed_prs(org, repo, gitea_token, conn_zuul, cur_zuul, table_name)
 
-    update_squad_and_title(cur_csv, conn_zuul, cur_zuul, rtc, table_name)
+    update_squad_and_title(conn_zuul, cur_zuul, rtc, failed_table)
 
-    cur_csv.close()
-    conn_csv.close()
     cur_zuul.close()
     conn_zuul.close()
 
@@ -275,18 +297,7 @@ def main(org, table_name, rtc):
     main(org_string, failed_table, rtc_table)
     main(f"{org_string}-swiss", f"{failed_table}_swiss", f"{rtc_table}_swiss")
 
-
-if __name__ == "__main__":
-    org_string = "docs"
-    failed_table = "failed_zuul_prs"
-    rtc_table = "repo_title_category"
-
-    main(org_string, failed_table, rtc_table)
-    main(f"{org_string}-swiss", f"{failed_table}_swiss", f"{rtc_table}_swiss")
-
-
     end_time = time.time()
     execution_time = end_time - start_time
     minutes, seconds = divmod(execution_time, 60)
     print(f"Script failed_zuul.py executed in {int(minutes)} minutes {int(seconds)} seconds! Let's go drink some beer :)")
-