Merge pull request #99 from cmu-delphi/ds/fetch

Consolidate `fetch` interfaces

Author: brookslogan

DESCRIPTION

@@ -27,11 +27,18 @@ Imports:
     readr,
     MMWRweek,
     rlang,
-    magrittr
+    magrittr,
+    tibble,
+    xml2
 RoxygenNote: 7.2.3
 Suggests:
+    dplyr,
     knitr,
+    mockery,
+    mockr,
     rmarkdown,
-    testthat (>= 3.1.5)
+    testthat (>= 3.1.5),
+    withr
 VignetteBuilder: knitr
 Language: en-US
+Config/testthat/edition: 3

NAMESPACE

@@ -18,10 +18,7 @@ export(delphi)
 export(dengue_nowcast)
 export(ecdc_ili)
 export(epirange)
-export(fetch_classic)
-export(fetch_csv)
-export(fetch_json)
-export(fetch_tbl)
+export(fetch)
 export(flusurv)
 export(fluview)
 export(fluview_clinical)
@@ -34,7 +31,6 @@ export(nidss_dengue)
 export(nidss_flu)
 export(nowcast)
 export(paho_dengue)
-export(parse_timeset_input)
 export(pvt_cdc)
 export(pvt_dengue_sensors)
 export(pvt_ght)
@@ -43,7 +39,6 @@ export(pvt_norostat)
 export(pvt_quidel)
 export(pvt_sensors)
 export(pvt_twitter)
-export(request_url)
 export(wiki)
 export(with_base_url)
 importFrom(MMWRweek,MMWRweek2Date)
@@ -64,9 +59,14 @@ importFrom(checkmate,test_list)
 importFrom(httr,RETRY)
 importFrom(httr,content)
 importFrom(httr,http_error)
+importFrom(httr,http_type)
 importFrom(httr,modify_url)
 importFrom(httr,stop_for_status)
 importFrom(jsonlite,fromJSON)
 importFrom(magrittr,"%>%")
 importFrom(readr,read_csv)
 importFrom(rlang,abort)
+importFrom(tibble,as_tibble)
+importFrom(xml2,read_html)
+importFrom(xml2,xml_find_all)
+importFrom(xml2,xml_text)

R/auth.R

@@ -1,8 +1,8 @@
 #' Get the API key
 #'
-#' Get the API key from the environment variable DELPHI_EPIDATA_KEY or getOption("delphi.epidata.key")
+#' Get the API key from the environment variable `DELPHI_EPIDATA_KEY` or
+#' `getOption("delphi.epidata.key")`.
 #'
-#' @param verbose Will print the source of the key if TRUE (e.g. options, environment, or config file)
 #' @return The API key
 #'
 #' @export

R/check.R

@@ -28,7 +28,8 @@ assert_date_param <- function(name, value, len = NULL, required = TRUE) {
   )
 }
 
-#' Allows a timeset param: a date vector, a character vector, an integer-like vector, or a single EpiRange
+#' Allows a timeset param: a date vector, a character vector, an integer-like
+#' vector, or a single EpiRange
 #' @importFrom checkmate assert check_character check_date check_integerish check_class check_list check_names
 assert_timeset_param <- function(name, value, len = NULL, required = TRUE) {
   null.ok <- !required

R/constants.R

@@ -1,3 +1,3 @@
-version <- "1.0.0"
+version <- "0.5.0"
 http_headers <- httr::add_headers("User-Agent" = paste0("epidatr/", version), "Accept-Encoding" = "gzip")
 global_base_url <- "https://api.delphi.cmu.edu/epidata/"

R/covidcast.R

@@ -100,20 +100,33 @@ print.covidcast_data_source <- function(source, ...) {
 #' print(smoothed_cli)
 #' df <- smoothed_cli$call("nation", "us", epirange(20210405, 20210410))
 #' @param base_url optional alternative API base url
-#' @importFrom httr RETRY stop_for_status content
+#' @importFrom httr stop_for_status content http_type
 #' @importFrom jsonlite fromJSON
+#' @importFrom xml2 read_html xml_find_all xml_text
 #' @return an instance of covidcast_epidata
 #'
 #' @export
 covidcast_epidata <- function(base_url = global_base_url) {
   url <- join_url(base_url, "covidcast/meta")
-  res <- do_request(url, list())
+  response <- do_request(url, list())
 
-  httr::stop_for_status(res)
-  r <- httr::content(res, "text", encoding = "UTF-8")
-  meta <- jsonlite::fromJSON(r, simplifyVector = FALSE)
+  if (response$status_code != 200) {
+    # 500, 429, 401 are possible
+    msg <- "fetch data from API"
+    if (httr::http_type(response) == "text/html") {
+      # grab the error information out of the returned HTML document
+      msg <- paste(msg, ":", xml2::xml_text(xml2::xml_find_all(
+        xml2::read_html(content(response, "text")),
+        "//p"
+      )))
+    }
+    httr::stop_for_status(response, task = msg)
+  }
+
+  response_content <- httr::content(response, "text", encoding = "UTF-8")
+  response_content <- jsonlite::fromJSON(response_content, simplifyVector = FALSE)
 
-  sources <- do.call(c, lapply(meta, parse_source, base_url = base_url))
+  sources <- do.call(c, lapply(response_content, parse_source, base_url = base_url))
   class(sources) <- c("covidcast_data_source_list", class(sources))
 
   all_signals <- do.call(c, lapply(sources, function(x) {