Save pages returned by oa_request() and do not process further

.gitignore

@@ -11,3 +11,4 @@ vignettes/*.R
 vignettes/*.html
 concept-abbre.csv
 dev/
+.lintr

R/oa_fetch.R

@@ -171,6 +171,14 @@
 #' See https://docs.openalex.org/how-to-use-the-api/get-lists-of-entities/paging.
 #' @param pages Integer vector.
 #' The range of pages to return. If NULL, return all pages.
+#' @param output_pages_to Character.
+#' If NULL, the individual pages will be downloaded and processed in memory.
+#' If not NULL, the individual pages
+#' downloaded will be saved in the directory specified by `output_pages_to`.
+#' The directory will be created if it does not exist.
+#' **The function will overwrite existing files in the directory without
+#' warning!**
+#' Defaults to NULL.
 #' @param count_only Logical.
 #' If TRUE, the function returns only the number of item matching the query.
 #' Defaults to FALSE.
@@ -181,7 +189,9 @@
 #' @param verbose Logical.
 #' If TRUE, print information about the querying process. Defaults to TRUE.
 #'
-#' @return a data.frame or a list of bibliographic records.
+#' @return a data.frame or a list of bibliographic records. If `output_pages_to` is
+#' not NULL a character vector containing the names of the saved pages
+#' is returned.
 #'
 #' For more extensive information about OpenAlex API, please visit:
 #' <https://docs.openalex.org>
@@ -312,6 +322,7 @@
                        per_page = 200,
                        paging = "cursor",
                        pages = NULL,
+                       output_pages_to = NULL,
                        count_only = FALSE,
                        mailto = oa_email(),
                        api_key = oa_apikey(),
@@ -379,6 +390,17 @@
   # Setting items per page
   query_ls[["per-page"]] <- per_page
 
+  # Setup output_pages_to if not NULL
+  if (!is.null(output_pages_to)) {
+    output_pages_to <- normalizePath(output_pages_to, mustWork = FALSE)
+    if (!dir.exists(output_pages_to)) {
+      dir.create(output_pages_to)
+    }
+    result <- character(n_pages)
+
+    save_no_pages <- 20
+  }
+
   # Activation of cursor pagination
   data <- vector("list", length = n_pages)
   res <- NULL
@@ -388,10 +410,24 @@
     next_page <- get_next_page(paging, i, res)
     query_ls[[paging]] <- next_page
     res <- api_request(query_url, ua, query = query_ls)
-    if (!is.null(res$results)) data[[i]] <- res$results
+    next_page <- get_next_page(paging, i + 1, res)
+    if (!is.null(output_pages_to)) {
+      fn <- file.path(output_pages_to, paste0("page_", i, ".rds"))
+      saveRDS(unlist(
+        res$results,
+        recursive = FALSE
+      ), file.path(output_pages_to, paste0("page_", i, ".rds")))
+      result[[i]] <- fn
+    } else {
+      if (!is.null(res$results)) data[[i]] <- res$results
+    }
   }
 
-  unlist(data, recursive = FALSE)
+  if (is.null(output_pages_to)) {
+    return(unlist(data, recursive = FALSE))
+  } else {
+    return(result)
+  }
 }
 
 get_next_page <- function(paging, i, res = NULL) {