From 732ba41226a58a02c27802530d1481f8ad7c00ba Mon Sep 17 00:00:00 2001 From: yongjunhong Date: Tue, 20 Aug 2024 00:19:26 +0900 Subject: [PATCH 1/3] Fix issue with missing library detection Signed-off-by: yongjunhong --- requirements.txt | 5 ++- .../package_manager/Gradle.py | 40 ++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 90ce9e36..8e7459c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,7 @@ requirements-parser defusedxml packageurl-python igraph -matplotlib \ No newline at end of file +matplotlib +requests +cloudscraper +pyuseragents \ No newline at end of file diff --git a/src/fosslight_dependency/package_manager/Gradle.py b/src/fosslight_dependency/package_manager/Gradle.py index 1cc5bc5f..245b9528 100644 --- a/src/fosslight_dependency/package_manager/Gradle.py +++ b/src/fosslight_dependency/package_manager/Gradle.py @@ -8,8 +8,12 @@ import json import fosslight_util.constant as constant import fosslight_dependency.constant as const +import cloudscraper from fosslight_dependency._package_manager import PackageManager from fosslight_dependency._package_manager import version_refine, get_url_to_purl +from requests import exceptions +from bs4 import BeautifulSoup as bs +from pyuseragents import random as random_user_agent logger = logging.getLogger(constant.LOGGER_NAME) @@ -62,7 +66,9 @@ def parse_oss_information(self, f_name): purl = '' try: for licenses in d['licenses']: - if licenses['name'] != '': + if licenses['name'] == 'No license found': + license_names.append(parse_oss_name_version_in_maven(self.dn_url, group_id, artifact_id)) + elif licenses['name'] != '': license_names.append(licenses['name'].replace(",", "")) license_name = ', '.join(license_names) except Exception: @@ -115,3 +121,35 @@ def parse_oss_name_version_in_artifactid(name): oss_version = artifact_comp[2] return group_id, artifact_id, oss_version + + +def parse_oss_name_version_in_maven(dn_url, group_id, artifact_id): + url = f"{dn_url}{group_id}/{artifact_id}" + headers = { + 'Content-Type': 'text/html;', + 'User-Agent': random_user_agent() + } + try: + scraper = cloudscraper.create_scraper() + res = scraper.get(url=url, headers=headers) + res.raise_for_status() + except exceptions.HTTPError as http_err: + logger.error(f"HTTP error occurred: {http_err}") + return "No license found" + except exceptions.RequestException as req_err: + logger.error(f"Request exception occurred: {req_err}") + return "No license found" + except Exception as e: + logger.error(f"Unexpected error occurred: {e}") + return "No license found" + + soup = bs(res.content, 'html.parser') + license_th = soup.find('th', text='License') + if not license_th: + return "No license found" + + license_td = license_th.find_next_sibling('td') + license_span = license_td.find('span', {'class': 'b lic'}) if license_td else None + license_name = license_span.text.strip() + + return license_name From 6fa6575a23c2a4163cf49fa4ec7c9a52f35ddc48 Mon Sep 17 00:00:00 2001 From: yongjunhong Date: Sun, 25 Aug 2024 16:18:28 +0900 Subject: [PATCH 2/3] Change exception message Signed-off-by: yongjunhong --- .../package_manager/Gradle.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/fosslight_dependency/package_manager/Gradle.py b/src/fosslight_dependency/package_manager/Gradle.py index 245b9528..bc8101c6 100644 --- a/src/fosslight_dependency/package_manager/Gradle.py +++ b/src/fosslight_dependency/package_manager/Gradle.py @@ -17,6 +17,7 @@ logger = logging.getLogger(constant.LOGGER_NAME) +NO_LICENSE_MESSAGE = "NO_LICENSE_DETECTED" class Gradle(PackageManager): package_manager_name = const.GRADLE @@ -134,19 +135,19 @@ def parse_oss_name_version_in_maven(dn_url, group_id, artifact_id): res = scraper.get(url=url, headers=headers) res.raise_for_status() except exceptions.HTTPError as http_err: - logger.error(f"HTTP error occurred: {http_err}") - return "No license found" + logger.debug(f"HTTP error occurred: {http_err}") + return NO_LICENSE_MESSAGE except exceptions.RequestException as req_err: - logger.error(f"Request exception occurred: {req_err}") - return "No license found" + logger.debug(f"Request exception occurred: {req_err}") + return NO_LICENSE_MESSAGE except Exception as e: - logger.error(f"Unexpected error occurred: {e}") - return "No license found" + logger.debug(f"Unexpected error occurred: {e}") + return NO_LICENSE_MESSAGE soup = bs(res.content, 'html.parser') license_th = soup.find('th', text='License') if not license_th: - return "No license found" + return NO_LICENSE_MESSAGE license_td = license_th.find_next_sibling('td') license_span = license_td.find('span', {'class': 'b lic'}) if license_td else None From 35c0bdbfafbd7f25e9643b7e8e1d858c83634185 Mon Sep 17 00:00:00 2001 From: yongjunhong Date: Sun, 25 Aug 2024 16:24:27 +0900 Subject: [PATCH 3/3] Add blank line Signed-off-by: yongjunhong --- src/fosslight_dependency/package_manager/Gradle.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fosslight_dependency/package_manager/Gradle.py b/src/fosslight_dependency/package_manager/Gradle.py index bc8101c6..82354554 100644 --- a/src/fosslight_dependency/package_manager/Gradle.py +++ b/src/fosslight_dependency/package_manager/Gradle.py @@ -19,6 +19,7 @@ NO_LICENSE_MESSAGE = "NO_LICENSE_DETECTED" + class Gradle(PackageManager): package_manager_name = const.GRADLE