Fix fetch anime and play and mark as unbroken

DIRECTORY.md

@@ -1212,6 +1212,7 @@
   * [Daily Horoscope](web_programming/daily_horoscope.py)
   * [Download Images From Google Query](web_programming/download_images_from_google_query.py)
   * [Emails From Url](web_programming/emails_from_url.py)
+  * [Fetch Anime And Play](web_programming/fetch_anime_and_play.py)
   * [Fetch Bbc News](web_programming/fetch_bbc_news.py)
   * [Fetch Github Info](web_programming/fetch_github_info.py)
   * [Fetch Jobs](web_programming/fetch_jobs.py)

web_programming/fetch_anime_and_play.py.BROKEN → web_programming/fetch_anime_and_play.py

@@ -1,7 +1,5 @@
-from xml.dom import NotFoundErr
-
 import requests
-from bs4 import BeautifulSoup, NavigableString
+from bs4 import BeautifulSoup, NavigableString, Tag
 from fake_useragent import UserAgent
 
 BASE_URL = "https://ww1.gogoanime2.org"
@@ -13,7 +11,7 @@ def search_scraper(anime_name: str) -> list:
     Take an url and
     return list of anime after scraping the site.
 
-    >>> type(search_scraper("demon_slayer"))
+    >>> type(search_scraper("naruto"))
     <class 'list'>
 
     Args:
@@ -41,25 +39,23 @@ def search_scraper(anime_name: str) -> list:
 
     # get list of anime
     anime_ul = soup.find("ul", {"class": "items"})
+    if anime_ul is None or isinstance(anime_ul, NavigableString):
+        msg = f"Could not find and anime with name {anime_name}"
+        raise ValueError(msg)
     anime_li = anime_ul.children
 
     # for each anime, insert to list. the name and url.
     anime_list = []
     for anime in anime_li:
-        if not isinstance(anime, NavigableString):
-            try:
-                anime_url, anime_title = (
-                    anime.find("a")["href"],
-                    anime.find("a")["title"],
-                )
-                anime_list.append(
-                    {
-                        "title": anime_title,
-                        "url": anime_url,
-                    }
-                )
-            except (NotFoundErr, KeyError):
-                pass
+        if isinstance(anime, Tag):
+            anime_url = anime.find("a")
+            if anime_url is None or isinstance(anime_url, NavigableString):
+                continue
+            anime_title = anime.find("a")
+            if anime_title is None or isinstance(anime_title, NavigableString):
+                continue
+
+            anime_list.append({"title": anime_title["title"], "url": anime_url["href"]})
 
     return anime_list
 
@@ -93,22 +89,24 @@ def search_anime_episode_list(episode_endpoint: str) -> list:
 
     # With this id. get the episode list.
     episode_page_ul = soup.find("ul", {"id": "episode_related"})
+    if episode_page_ul is None or isinstance(episode_page_ul, NavigableString):
+        msg = f"Could not find any anime eposiodes with name {anime_name}"
+        raise ValueError(msg)
     episode_page_li = episode_page_ul.children
 
     episode_list = []
     for episode in episode_page_li:
-        try:
-            if not isinstance(episode, NavigableString):
-                episode_list.append(
-                    {
-                        "title": episode.find("div", {"class": "name"}).text.replace(
-                            " ", ""
-                        ),
-                        "url": episode.find("a")["href"],
-                    }
-                )
-        except (KeyError, NotFoundErr):
-            pass
+        if isinstance(episode, Tag):
+            url = episode.find("a")
+            if url is None or isinstance(url, NavigableString):
+                continue
+            title = episode.find("div", {"class": "name"})
+            if title is None or isinstance(title, NavigableString):
+                continue
+
+            episode_list.append(
+                {"title": title.text.replace(" ", ""), "url": url["href"]}
+            )
 
     return episode_list
 
@@ -140,11 +138,16 @@ def get_anime_episode(episode_endpoint: str) -> list:
 
     soup = BeautifulSoup(response.text, "html.parser")
 
-    try:
-        episode_url = soup.find("iframe", {"id": "playerframe"})["src"]
-        download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"
-    except (KeyError, NotFoundErr) as e:
-        raise e
+    url = soup.find("iframe", {"id": "playerframe"})
+    if url is None or isinstance(url, NavigableString):
+        msg = f"Could not find url and download url from {episode_endpoint}"
+        raise RuntimeError(msg)
+
+    episode_url = url["src"]
+    if not isinstance(episode_url, str):
+        msg = f"Could not find url and download url from {episode_endpoint}"
+        raise RuntimeError(msg)
+    download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"
 
     return [f"{BASE_URL}{episode_url}", f"{BASE_URL}{download_url}"]
 

web_programming/get_amazon_product_data.py

@@ -19,11 +19,13 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
     """
     url = f"https://www.amazon.in/laptop/s?k={product}"
     header = {
-        "User-Agent": """Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36
-        (KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36""",
+        "User-Agent": (
+            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
+            "(KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36"
+        ),
         "Accept-Language": "en-US, en;q=0.5",
     }
-    soup = BeautifulSoup(requests.get(url, headers=header).text)
+    soup = BeautifulSoup(requests.get(url, headers=header).text, features="lxml")
     # Initialize a Pandas dataframe with the column titles
     data_frame = DataFrame(
         columns=[
@@ -74,8 +76,8 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
             except ValueError:
                 discount = float("nan")
         except AttributeError:
-            pass
-        data_frame.loc[len(data_frame.index)] = [
+            continue
+        data_frame.loc[str(len(data_frame.index))] = [
             product_title,
             product_link,
             product_price,