Merge pull request #51 from tidy-intelligence/50-add-retry-logic-in-c…

…ase-undefined-rate-limit-is-reached

Extend perform_request() to handle retries

R/perform_request.R

@@ -10,7 +10,7 @@
 #' @param language A character string specifying the language code for the API
 #'  response. Defaults to NULL.
 #' @param per_page An integer specifying the number of results per page for the
-#'  API. Defaults to 1000.
+#'  API. Defaults to 10,000.
 #' Must be a value between 1 and 32,500.
 #' @param date A character string specifying the date range of the data to be
 #'  retrieved (e.g., "2000:2020"). If NULL (default), no date filtering is
@@ -22,6 +22,8 @@
 #'  for paginated requests. Defaults to FALSE.
 #' @param base_url A character string specifying the base URL of the World Bank
 #'  API (default is "https://api.worldbank.org/v2").
+#' @param max_tries An integer specifying the number of attempts for the API
+#'  request.
 #'
 #' @return A list containing the parsed JSON response from the API. If the
 #'  response contains multiple pages, the results are combined into a single
@@ -40,18 +42,21 @@
 perform_request <- function(
   resource,
   language = NULL,
-  per_page = 1000,
+  per_page = 10000L,
   date = NULL,
   source = NULL,
   progress = FALSE,
-  base_url = "https://api.worldbank.org/v2/"
+  base_url = "https://api.worldbank.org/v2/",
+  max_tries = 10L
 ) {
 
   validate_per_page(per_page)
+  validate_max_tries(max_tries)
 
   req <- create_request(
     base_url, resource, language, per_page, date, source
-  )
+  ) |>
+    req_retry(max_tries = max_tries)
 
   resp <- req_perform(req)
 
@@ -86,6 +91,12 @@ validate_per_page <- function(per_page) {
   }
 }
 
+validate_max_tries <- function(max_tries) {
+  if (!is.numeric(max_tries) || max_tries %% 1L != 0 || max_tries < 2L) {
+    cli::cli_abort("{.arg max_tries} must be an integer larger than 1.")
+  }
+}
+
 create_request <- function(
   base_url, resource, language, per_page, date, source
 ) {

R/wdi_get.R

@@ -17,7 +17,7 @@
 #' @param language A character string specifying the language for the request,
 #'  see \link{wdi_get_languages}. Defaults to `"en"`.
 #' @param per_page An integer specifying the number of results per page for the
-#'  API. Defaults to 1000.
+#'  API. Defaults to 10,000.
 #' @param progress A logical value indicating whether to show progress messages
 #'  during the data download and parsing. Defaults to `TRUE`.
 #' @param source An integer value specifying the data source, see
@@ -90,7 +90,7 @@ wdi_get <- function(
   end_date = NULL,
   frequency = "annual",
   language = "en",
-  per_page = 1000,
+  per_page = 10000L,
   progress = TRUE,
   source = NULL,
   format = "long"

R/wdi_get_indicators.R

@@ -43,7 +43,8 @@
 #' # Download all supported indicators in Spanish
 #' wdi_get_indicators(language = "es")
 #'}
-wdi_get_indicators <- function(language = "en", per_page = 32500) {
+#'
+wdi_get_indicators <- function(language = "en", per_page = 32500L) {
 
   indicators_raw <- perform_request(
     "indicators", language = language, per_page = per_page

tests/testthat/tests-perform_request.R

@@ -9,6 +9,7 @@ test_that("perform_request handles error responses", {
 
   with_mocked_bindings(
     create_request = function(...) NULL,
+    req_retry = function(...) NULL,
     req_perform = function(...) mock_error_response,
     is_request_error = function(...) TRUE,
     handle_request_error = function(resp) stop("API error: Invalid indicator"),
@@ -24,6 +25,11 @@ test_that("perform_request validates per_page parameter", {
   expect_silent(perform_request("countries", per_page = 1000))
 })
 
+test_that("perform_request validates max_tries parameter", {
+  expect_error(perform_request("countries", max_tries = -100))
+  expect_silent(perform_request("countries", max_tries = 2))
+})
+
 test_that("validate_per_page handles valid per_page values", {
   expect_silent(validate_per_page(1000))
   expect_silent(validate_per_page(1))
@@ -41,6 +47,15 @@ test_that("validate_per_page throws an error for invalid per_page values", {
                "must be an integer between 1 and 32,500")
 })
 
+test_that("validate_max_tries throws an error for invalid max_tries values", {
+  expect_error(validate_max_tries(0),
+               "must be an integer larger than 1")
+  expect_error(validate_max_tries(-1),
+               "must be an integer larger than 1")
+  expect_error(validate_max_tries(1000.5),
+               "must be an integer larger than 1")
+})
+
 test_that("create_request constructs a request with default parameters", {
   req <- create_request(
     "https://api.worldbank.org/v2", "incomeLevels", NULL, 1000, NULL, NULL