From 7356e993e51c62ae941c38db6788d92dfbb94c8b Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Wed, 18 Feb 2026 07:13:31 -0500 Subject: [PATCH 01/24] add return to dlw_country_catalog() --- R/dlw_country_catalog.R | 1 + 1 file changed, 1 insertion(+) diff --git a/R/dlw_country_catalog.R b/R/dlw_country_catalog.R index f8a5e71..6d2ff82 100644 --- a/R/dlw_country_catalog.R +++ b/R/dlw_country_catalog.R @@ -41,4 +41,5 @@ dlw_country_catalog <- function(country_code, ctl <- handle_resp(req) set_in_dlwenv(key, ctl, verbose) + return(ctl) } From 447b7c3ca39fbc8e267d2c75780a80cd40614836 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Wed, 18 Feb 2026 08:27:58 -0500 Subject: [PATCH 02/24] add qs2 format --- R/zzz.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/R/zzz.R b/R/zzz.R index 6efa81b..9e475f5 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -1,11 +1,11 @@ op.dlw <- list( - dlw.verbose = TRUE, + dlw.verbose = TRUE, dlw.default_api_version = "v1", - dlw.output_method = cli::ansi_has_hyperlink_support(), + dlw.output_method = cli::ansi_has_hyperlink_support(), cli.ignore_unknown_rstudio_theme = TRUE, dlw.local_dir = Sys.getenv("DLW_local_dir"), dlw.board_type = c("folder", "local"), - dlw.format = c("qs", "parquet"), + dlw.format = c("qs2", "qs", "parquet"), dlw.download_formats = c("dta") # we could add more later. ) From fba2a31f812c28452e81f734ea2560f6bdf899a0 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Wed, 18 Feb 2026 08:50:02 -0500 Subject: [PATCH 03/24] implement {stamp} to load and save dlw data and update documentation --- R/dlw_get_data.R | 138 ++++++++++++++++++++++++++++++----------------- 1 file changed, 89 insertions(+), 49 deletions(-) diff --git a/R/dlw_get_data.R b/R/dlw_get_data.R index c45dc5f..3e4d74e 100644 --- a/R/dlw_get_data.R +++ b/R/dlw_get_data.R @@ -1,5 +1,21 @@ #' Get data from datalibweb (refactored) #' +#' @description +#' `dlw_get_data()` function is the main user-facing function for +#' retrieving datasets from the Datalibweb (DLW) API. It handles +#' downloading, caching, and reading datasets, supporting both local +#' and temporary storage. +#' +#' **How it works:** +#' 1. Checks if the requested file exists locally (unless `local_overwrite = TRUE`). +#' 2. If it exists, reads it using `dlw_read()`. +#' 3. If not, downloads the data from the DLW API using `dlw_download()`, saves +#' it in the specified format, and returns it as a `data.table`. +#' 4. Handles directory management and caching via `get_wrk_board()`. +#' +#' This function streamlines access to DLW datasets, automatically +#' managing download, storage, and retrieval. +#' #' @param filename character: Name of the file to save/read (required) #' @inheritParams dlw_country_catalog #' @inheritParams dlw_server_catalog @@ -10,7 +26,7 @@ #' @param local logical: whether or not to save and read data locally. default #' is TRUE if `local_dir` exists. #' @param format character: File format to use for pinning data ('parquet' -#' [default] or 'qs') +#' [default] or 'qs2') #' @param local_overwrite logical. Whether to overwrite any saved data. Default #' is FALSE #' @param version numeric: Version of the pin to read (for pinning data @@ -33,50 +49,53 @@ dlw_get_data <- function(country_code, cli::cli_abort("{.arg filename} is a required argument.") } - # Construct board and pin_name for reading - board <- get_wrk_board(local = local, - local_dir = local_dir) + # Construct directory and id_name for reading + dlw_dir <- get_wrk_board(local = local, local_dir = local_dir) - pin_name <- filename |> + id_name <- filename |> fs::path_ext_remove() |> fs::path(ext = format) # set in dlwenv - set_in_dlwenv("current_board", board) - set_in_dlwenv("current_pin", pin_name) + set_in_dlwenv("current_board", dlw_dir) + set_in_dlwenv("current_pin", id_name) + + files_in_dir <- basename(list.files(dlw_dir)) - if (!local_overwrite && pin_name %in% pins::pin_list(board)) { + if (!local_overwrite && id_name %in% files_in_dir) { # Only read the requested version, do not download - out <- dlw_read(board = board, - pin_name = pin_name, - version = version) + out <- dlw_read(dlw_dir = dlw_dir, + id_name = id_name, + version = version) return(out) } - dlw_download(country_code = country_code, - server = server, - filename = filename, - format = format, - board = board, - pin_name = pin_name, - ..., - verbose = verbose) + dlw_download( + country_code = country_code, + server = server, + filename = filename, + format = format, + dlw_dir = dlw_dir, + id_name = id_name, + ..., + verbose = verbose + ) } -#' Download data from datalibweb and save as a pin +#' Download data from datalibweb and save using {stamp} framework #' #' @inheritParams dlw_get_data #' @inheritParams dlw_read #' @param filename character: Name of the file to save/read (required) -#' @param format character: File format to use for pinning data ('qs' +#' @param format character: File format to use for saving data ('qs2' #' [default] or 'parquet') #' @returns A list with the board and pin_name used #' @keywords internal dlw_download <- function(country_code, filename, - board, - pin_name, + dlw_dir, + id_name, format, server = NULL, ..., @@ -104,33 +123,41 @@ dlw_download <- function(country_code, tmpfile <- fs::file_temp(ext = "dta") writeBin(raw_data, tmpfile) - # Read .dta and pin as parquet or qs + # Read .dta and save it as qs2 using {stamp} dt <- haven::read_dta(tmpfile, encoding = "latin1") |> setDT() unlink(tmpfile) - pins::pin_write(board = board, - x = dt, - name = pin_name, - type = format, - versioned = TRUE) + + stamp::st_save(dt, fs::path(dlw_dir, id_name), alias = "dlw") dt } #' Read data from a pin (local or temp) #' -#' @param board A pins board object (as returned by dlw_download) -#' @param pin_name The name of the pin (as returned by dlw_download) -#' @param version numeric: Version of the pin to read (for pinning data +#' Reads a dataset from a specified directory and file name, returning it as a `data.table`. +#' +#' - Lists files in `dlw_dir` and checks if `id_name` exists. +#' - If not found, aborts with an error. +#' - Removes any extension from `id_name`. +#' - Loads the data using `stamp::st_load()` from a `.qs2` file in the directory. +#' +#' This function is used internally to retrieve previously saved or downloaded datasets in a fast, versioned format. +#' +#' @param dlw_dir A folder object (as returned by dlw_download) +#' @param id_name The name of the a dataset (as returned by dlw_download) +#' @param version numeric: Version of the data to read (for versioning data #' retrieval only) #' @returns data.table #' @keywords internal -dlw_read <- function(board, pin_name, version = NULL) { +dlw_read <- function(dlw_dir, id_name, version = NULL) { - if (!(pin_name %in% pins::pin_list(board))) { - cli::cli_abort("File {.file {pin_name}} not found in the provided board.") + files_in_dir <- basename(list.files(dlw_dir)) + + if (!(id_name %in% files_in_dir)) { + cli::cli_abort("File {.file {id_name}} not found in the provided directory.") } - pins::pin_read(board, pin_name, version = version) |> - setDT() + + stamp::st_load(fs::path(dlw_dir, id_name), alias = "dlw") } @@ -148,30 +175,43 @@ get_raw_data <- \(req) { set_in_dlwenv(key = "last_raw_data", value = raw_data) -} - + # return raw data so callers receive the bytes + raw_data +} -#' Get workign pips board +#' Get working folder for saving or reading data +#' +#' Determines the directory to use for saving or reading data, depending on whether you want to use a local directory or a temporary one. +#' +#' - If `local` is `TRUE`, it checks if `local_dir` exists. If not, it creates it (using `fs::dir_create()`). It then returns this directory path. +#' - If `local` is `FALSE`, it tries to get a temporary directory path from the package environment (`get_from_dlwenv("temp_dir")`). If this does not exist, it creates a new temporary directory, stores its path in the environment, and returns it. +#' +#' This function ensures that data is always saved to a valid directory, either user-specified or temporary. #' #' @inheritParams dlw_get_data -#' @returns board from (pins) package +#' @returns Folder path #' @keywords internal get_wrk_board <- function(local, local_dir) { if (local) { - if (fs::is_dir(local_dir)) { - pins::board_folder(local_dir) + if (!fs::is_dir(local_dir)) { + wrk_dir <- fs::dir_create(local_dir) } else { - pins::board_local(local_dir) + wrk_dir <- fs::dir_create(local_dir) } } else { - brd <- get_from_dlwenv("temp_board") - if (is.null(brd)) { - brd <- pins::board_temp() - set_in_dlwenv(key = "temp_board", value = brd) + wrk_dir <- get_from_dlwenv("temp_dir") + + if (is.null(wrk_dir)) { + #wrk_dir <- fs::path_temp("wrk_dir") + #fs::dir_create(wrk_dir) + wrk_dir <- fs::path(tempdir()) + stamp::st_init(wrk_dir, alias = "dlw") + + set_in_dlwenv(key = "temp_dir", value = wrk_dir) } - brd + wrk_dir } } From 16b2879473dc14e80ab5227272ce3dd005a7c19a Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Wed, 18 Feb 2026 09:31:49 -0500 Subject: [PATCH 04/24] rename helper functions with dot prefix --- R/dlw_get_data.R | 28 ++++++++++++++-------------- R/zzz.R | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/R/dlw_get_data.R b/R/dlw_get_data.R index 3e4d74e..8e35a73 100644 --- a/R/dlw_get_data.R +++ b/R/dlw_get_data.R @@ -8,10 +8,10 @@ #' #' **How it works:** #' 1. Checks if the requested file exists locally (unless `local_overwrite = TRUE`). -#' 2. If it exists, reads it using `dlw_read()`. -#' 3. If not, downloads the data from the DLW API using `dlw_download()`, saves +#' 2. If it exists, reads it using `.dlw_read()`. +#' 3. If not, downloads the data from the DLW API using `.dlw_download()`, saves #' it in the specified format, and returns it as a `data.table`. -#' 4. Handles directory management and caching via `get_wrk_board()`. +#' 4. Handles directory management and caching via `.get_wrk_board()`. #' #' This function streamlines access to DLW datasets, automatically #' managing download, storage, and retrieval. @@ -50,7 +50,7 @@ dlw_get_data <- function(country_code, } # Construct directory and id_name for reading - dlw_dir <- get_wrk_board(local = local, local_dir = local_dir) + dlw_dir <- .get_wrk_board(local = local, local_dir = local_dir) id_name <- filename |> fs::path_ext_remove() |> @@ -64,13 +64,13 @@ dlw_get_data <- function(country_code, if (!local_overwrite && id_name %in% files_in_dir) { # Only read the requested version, do not download - out <- dlw_read(dlw_dir = dlw_dir, + out <- .dlw_read(dlw_dir = dlw_dir, id_name = id_name, version = version) return(out) } - dlw_download( + .dlw_download( country_code = country_code, server = server, filename = filename, @@ -86,13 +86,13 @@ dlw_get_data <- function(country_code, #' Download data from datalibweb and save using {stamp} framework #' #' @inheritParams dlw_get_data -#' @inheritParams dlw_read +#' @inheritParams .dlw_read #' @param filename character: Name of the file to save/read (required) #' @param format character: File format to use for saving data ('qs2' #' [default] or 'parquet') #' @returns A list with the board and pin_name used #' @keywords internal -dlw_download <- function(country_code, +.dlw_download <- function(country_code, filename, dlw_dir, id_name, @@ -117,7 +117,7 @@ dlw_download <- function(country_code, dots) raw_data <- do.call("build_request", args) |> - get_raw_data() + .get_raw_data() # Save raw data to a temp file for reading tmpfile <- fs::file_temp(ext = "dta") @@ -143,13 +143,13 @@ dlw_download <- function(country_code, #' #' This function is used internally to retrieve previously saved or downloaded datasets in a fast, versioned format. #' -#' @param dlw_dir A folder object (as returned by dlw_download) -#' @param id_name The name of the a dataset (as returned by dlw_download) +#' @param dlw_dir A folder object (as returned by .dlw_download) +#' @param id_name The name of the a dataset (as returned by .dlw_download) #' @param version numeric: Version of the data to read (for versioning data #' retrieval only) #' @returns data.table #' @keywords internal -dlw_read <- function(dlw_dir, id_name, version = NULL) { + .dlw_read <- function(dlw_dir, id_name, version = NULL) { files_in_dir <- basename(list.files(dlw_dir)) @@ -169,7 +169,7 @@ dlw_read <- function(dlw_dir, id_name, version = NULL) { #' #' @returns raw data from [resp_body_raw] #' @keywords internal -get_raw_data <- \(req) { + .get_raw_data <- \(req) { raw_data <- handle_resp(req) @@ -194,7 +194,7 @@ get_raw_data <- \(req) { #' @inheritParams dlw_get_data #' @returns Folder path #' @keywords internal -get_wrk_board <- function(local, local_dir) { + .get_wrk_board <- function(local, local_dir) { if (local) { if (!fs::is_dir(local_dir)) { wrk_dir <- fs::dir_create(local_dir) diff --git a/R/zzz.R b/R/zzz.R index 9e475f5..a260820 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -4,7 +4,7 @@ op.dlw <- list( dlw.output_method = cli::ansi_has_hyperlink_support(), cli.ignore_unknown_rstudio_theme = TRUE, dlw.local_dir = Sys.getenv("DLW_local_dir"), - dlw.board_type = c("folder", "local"), + #dlw.board_type = c("folder", "local"), dlw.format = c("qs2", "qs", "parquet"), dlw.download_formats = c("dta") # we could add more later. ) From 7bb8af0230cf2073d48f6cf23aa151a0e7d0859f Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Wed, 18 Feb 2026 09:37:52 -0500 Subject: [PATCH 05/24] {stamp} add to the description file --- DESCRIPTION | 4 ++- man/dlw_get_data.Rd | 17 +++++++++-- man/dlw_read.Rd | 23 --------------- man/{dlw_download.Rd => dot-dlw_download.Rd} | 20 ++++++------- man/dot-dlw_read.Rd | 31 ++++++++++++++++++++ man/{get_raw_data.Rd => dot-get_raw_data.Rd} | 6 ++-- man/dot-get_wrk_board.Rd | 28 ++++++++++++++++++ man/get_wrk_board.Rd | 22 -------------- 8 files changed, 90 insertions(+), 61 deletions(-) delete mode 100644 man/dlw_read.Rd rename man/{dlw_download.Rd => dot-dlw_download.Rd} (62%) create mode 100644 man/dot-dlw_read.Rd rename man/{get_raw_data.Rd => dot-get_raw_data.Rd} (83%) create mode 100644 man/dot-get_wrk_board.Rd delete mode 100644 man/get_wrk_board.Rd diff --git a/DESCRIPTION b/DESCRIPTION index dcb2b83..15aa8d2 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -22,7 +22,7 @@ Imports: haven, httr2, keyring, - pins, + stamp, rlang, stats Suggests: @@ -32,3 +32,5 @@ Suggests: rmarkdown Config/testthat/edition: 3 VignetteBuilder: knitr +Remotes: + github::randrescastaneda/stamp diff --git a/man/dlw_get_data.Rd b/man/dlw_get_data.Rd index f53e932..195f344 100644 --- a/man/dlw_get_data.Rd +++ b/man/dlw_get_data.Rd @@ -31,7 +31,7 @@ in option dlw.local_dir which is set initially as "".} is TRUE if `local_dir` exists.} \item{format}{character: File format to use for pinning data ('parquet' -[default] or 'qs')} +[default] or 'qs2')} \item{local_overwrite}{logical. Whether to overwrite any saved data. Default is FALSE} @@ -48,5 +48,18 @@ vermast, veralt, collection, module)} data base request as data.table } \description{ -Get data from datalibweb (refactored) +`dlw_get_data()` function is the main user-facing function for +retrieving datasets from the Datalibweb (DLW) API. It handles +downloading, caching, and reading datasets, supporting both local +and temporary storage. + +**How it works:** +1. Checks if the requested file exists locally (unless `local_overwrite = TRUE`). +2. If it exists, reads it using `.dlw_read()`. +3. If not, downloads the data from the DLW API using `.dlw_download()`, saves + it in the specified format, and returns it as a `data.table`. +4. Handles directory management and caching via `.get_wrk_board()`. + +This function streamlines access to DLW datasets, automatically +managing download, storage, and retrieval. } diff --git a/man/dlw_read.Rd b/man/dlw_read.Rd deleted file mode 100644 index e3919a1..0000000 --- a/man/dlw_read.Rd +++ /dev/null @@ -1,23 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/dlw_get_data.R -\name{dlw_read} -\alias{dlw_read} -\title{Read data from a pin (local or temp)} -\usage{ -dlw_read(board, pin_name, version = NULL) -} -\arguments{ -\item{board}{A pins board object (as returned by dlw_download)} - -\item{pin_name}{The name of the pin (as returned by dlw_download)} - -\item{version}{numeric: Version of the pin to read (for pinning data -retrieval only)} -} -\value{ -data.table -} -\description{ -Read data from a pin (local or temp) -} -\keyword{internal} diff --git a/man/dlw_download.Rd b/man/dot-dlw_download.Rd similarity index 62% rename from man/dlw_download.Rd rename to man/dot-dlw_download.Rd index 773906f..cf96f35 100644 --- a/man/dlw_download.Rd +++ b/man/dot-dlw_download.Rd @@ -1,14 +1,14 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/dlw_get_data.R -\name{dlw_download} -\alias{dlw_download} -\title{Download data from datalibweb and save as a pin} +\name{.dlw_download} +\alias{.dlw_download} +\title{Download data from datalibweb and save using {stamp} framework} \usage{ -dlw_download( +.dlw_download( country_code, filename, - board, - pin_name, + dlw_dir, + id_name, format, server = NULL, ..., @@ -20,11 +20,11 @@ dlw_download( \item{filename}{character: Name of the file to save/read (required)} -\item{board}{A pins board object (as returned by dlw_download)} +\item{dlw_dir}{A folder object (as returned by .dlw_download)} -\item{pin_name}{The name of the pin (as returned by dlw_download)} +\item{id_name}{The name of the a dataset (as returned by .dlw_download)} -\item{format}{character: File format to use for pinning data ('qs' +\item{format}{character: File format to use for saving data ('qs2' [default] or 'parquet')} \item{server}{character: in case we have more than one server. default is GMD} @@ -38,6 +38,6 @@ vermast, veralt, collection, module)} A list with the board and pin_name used } \description{ -Download data from datalibweb and save as a pin +Download data from datalibweb and save using {stamp} framework } \keyword{internal} diff --git a/man/dot-dlw_read.Rd b/man/dot-dlw_read.Rd new file mode 100644 index 0000000..ce2c1e5 --- /dev/null +++ b/man/dot-dlw_read.Rd @@ -0,0 +1,31 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/dlw_get_data.R +\name{.dlw_read} +\alias{.dlw_read} +\title{Read data from a pin (local or temp)} +\usage{ +.dlw_read(dlw_dir, id_name, version = NULL) +} +\arguments{ +\item{dlw_dir}{A folder object (as returned by .dlw_download)} + +\item{id_name}{The name of the a dataset (as returned by .dlw_download)} + +\item{version}{numeric: Version of the data to read (for versioning data +retrieval only)} +} +\value{ +data.table +} +\description{ +Reads a dataset from a specified directory and file name, returning it as a `data.table`. +} +\details{ +- Lists files in `dlw_dir` and checks if `id_name` exists. +- If not found, aborts with an error. +- Removes any extension from `id_name`. +- Loads the data using `stamp::st_load()` from a `.qs2` file in the directory. + +This function is used internally to retrieve previously saved or downloaded datasets in a fast, versioned format. +} +\keyword{internal} diff --git a/man/get_raw_data.Rd b/man/dot-get_raw_data.Rd similarity index 83% rename from man/get_raw_data.Rd rename to man/dot-get_raw_data.Rd index c8729ab..060edbb 100644 --- a/man/get_raw_data.Rd +++ b/man/dot-get_raw_data.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/dlw_get_data.R -\name{get_raw_data} -\alias{get_raw_data} +\name{.get_raw_data} +\alias{.get_raw_data} \title{perform request and get raw data} \usage{ -get_raw_data(req) +.get_raw_data(req) } \arguments{ \item{req}{request from [build_request]} diff --git a/man/dot-get_wrk_board.Rd b/man/dot-get_wrk_board.Rd new file mode 100644 index 0000000..ac038e2 --- /dev/null +++ b/man/dot-get_wrk_board.Rd @@ -0,0 +1,28 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/dlw_get_data.R +\name{.get_wrk_board} +\alias{.get_wrk_board} +\title{Get working folder for saving or reading data} +\usage{ +.get_wrk_board(local, local_dir) +} +\arguments{ +\item{local}{logical: whether or not to save and read data locally. default +is TRUE if `local_dir` exists.} + +\item{local_dir}{character: Local directory to save data. Default available +in option dlw.local_dir which is set initially as "".} +} +\value{ +Folder path +} +\description{ +Determines the directory to use for saving or reading data, depending on whether you want to use a local directory or a temporary one. +} +\details{ +- If `local` is `TRUE`, it checks if `local_dir` exists. If not, it creates it (using `fs::dir_create()`). It then returns this directory path. +- If `local` is `FALSE`, it tries to get a temporary directory path from the package environment (`get_from_dlwenv("temp_dir")`). If this does not exist, it creates a new temporary directory, stores its path in the environment, and returns it. + +This function ensures that data is always saved to a valid directory, either user-specified or temporary. +} +\keyword{internal} diff --git a/man/get_wrk_board.Rd b/man/get_wrk_board.Rd deleted file mode 100644 index d54635e..0000000 --- a/man/get_wrk_board.Rd +++ /dev/null @@ -1,22 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/dlw_get_data.R -\name{get_wrk_board} -\alias{get_wrk_board} -\title{Get workign pips board} -\usage{ -get_wrk_board(local, local_dir) -} -\arguments{ -\item{local}{logical: whether or not to save and read data locally. default -is TRUE if `local_dir` exists.} - -\item{local_dir}{character: Local directory to save data. Default available -in option dlw.local_dir which is set initially as "".} -} -\value{ -board from (pins) package -} -\description{ -Get workign pips board -} -\keyword{internal} From 7bff5567403221787539bdea25a37e7fcf8882f5 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Wed, 18 Feb 2026 09:58:28 -0500 Subject: [PATCH 06/24] update dlw.Rmd --- R/dlw_get_data.R | 2 +- man/dot-dlw_download.Rd | 4 +- vignettes/dlw.Rmd | 90 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 91 insertions(+), 5 deletions(-) diff --git a/R/dlw_get_data.R b/R/dlw_get_data.R index 8e35a73..b92d938 100644 --- a/R/dlw_get_data.R +++ b/R/dlw_get_data.R @@ -83,7 +83,7 @@ dlw_get_data <- function(country_code, } -#' Download data from datalibweb and save using {stamp} framework +#' Download data from datalibweb and save using stamp framework #' #' @inheritParams dlw_get_data #' @inheritParams .dlw_read diff --git a/man/dot-dlw_download.Rd b/man/dot-dlw_download.Rd index cf96f35..0cea1fa 100644 --- a/man/dot-dlw_download.Rd +++ b/man/dot-dlw_download.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/dlw_get_data.R \name{.dlw_download} \alias{.dlw_download} -\title{Download data from datalibweb and save using {stamp} framework} +\title{Download data from datalibweb and save using stamp framework} \usage{ .dlw_download( country_code, @@ -38,6 +38,6 @@ vermast, veralt, collection, module)} A list with the board and pin_name used } \description{ -Download data from datalibweb and save using {stamp} framework +Download data from datalibweb and save using stamp framework } \keyword{internal} diff --git a/vignettes/dlw.Rmd b/vignettes/dlw.Rmd index 1011732..4b33675 100644 --- a/vignettes/dlw.Rmd +++ b/vignettes/dlw.Rmd @@ -16,6 +16,7 @@ knitr::opts_chunk$set( ```{r setup} library(dlw) +library(data.table) ``` `{dlw}` is an R client of the internal [datalibweb](https://datalibweb2.worldbank.org/) API of the World Bank. Datalibweb is a data system designed to enable users to seamlessly access microdata and documentation in the World Bank. Users can access the most up-to-date and historical versions of harmonized data collections and raw/non-harmonized data for subsequent analysis @@ -27,7 +28,92 @@ To begin working with `{dlw}`, you must provide your datalibweb API token using You can obtain or renew your token by visiting the [datalibweb page](https://datalibweb2.worldbank.org/) and following the instructions there. Once you have your token, set it in your R session as follows: ```{r, eval = FALSE} -dlw_set_token("your_token_here") +# dlw_set_token("your_token_here") ``` -[TO COMPLETE] +## Server Catalog + +The server catalog provides a comprehensive list of all datasets available across all countries in the datalibweb system. You can retrieve the server catalog using the `dlw_server_catalog()` function. + +```{r} +server_cat <- dlw_server_catalog() +head(server_cat) +``` + +### Exploring the Server Catalog + +You can filter and explore the server catalog to find datasets of interest. + +```{r} +# For examples, +# to find list of all datasets related to HEIS surveys: +heis_datasets <- server_cat[Survey == "HEIS", ] +head(heis_datasets) + +# to find list of all datasets for a specific module: +module_gpwg <- server_cat[Module == "GPWG", ] +head(module_gpwg) + +# to find list of all datasets for a specific year: +year_2023 <- server_cat[Year == 2023, ] +head(year_2023) + +# to find list of all datasets for a specific country and year: +country_year <- server_cat[Country == "PRY" & Year == 2020, ] +head(country_year) +``` + +## Country Catalog + +The country catalog provides detailed metadata for datasets specific to a single country. You can retrieve the country catalog using the `dlw_country_catalog()` function by specifying the desired country code. + +```{r} +country_pry <- dlw_country_catalog(country_code = "PRY") +head(country_pry) +``` + +### Exploring the Country Catalog + +You can filter and explore the country catalog to find list datasets of interest. +```{r} +# For examples +# to find list of all datasets related to EPH survey in Paraguay: +eph_pry <- country_pry[Survey == "EPH", ] +head(eph_pry) + +# to find list of all datasets for a specific year: +year_2020_pry <- country_pry[Year == 2020, ] +head(year_2020_pry) +``` + +## Downloading Datasets +Once you have identified datasets of interest from either the server or country catalog, you can download them using the `dlw_get_data()` function. +```{r} +dlw_get_data( + country_code = "PRY", + year = 2020L, + server = "GMD", + survey = "EPH", + module = "GPWG", + filename = "PRY_2020_EPH_V01_M_V03_A_GMD_GPWG.dta", + collection = "GMD" +) +``` + +Alternatively, you can use `dlw_get_gmd()`, which is a convenient wrapper around `dlw_get_data()`. To download the most recent version of a dataset, you only need to provide the country code, year, and module. +```{r} +pry20 <- dlw_get_gmd(country_code = "PRY", year = 2020, module = "GPWG") +``` + +To download a specific version of a dataset, you can provide additional arguments such as `survey`, `vermast`, and `veralt` to `dlw_get_gmd()`. +```{r} +# this downloads version V02 of the EPH survey for Paraguay in 2020 within the GPWG module. +pry20_v2 <- dlw_get_gmd( + country_code = "PRY", + year = 2020, + module = "GPWG", + survey = "EPH", + vermast = "V01", + veralt = "V02" +) +``` From 58d03e34a79522f47c1e43e5c7fa27a7a022df40 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Wed, 18 Feb 2026 10:02:56 -0500 Subject: [PATCH 07/24] country and server catalogs vignettes added --- vignettes/dlw_country_catalog.Rmd | 300 ++++++++++++++++++++++++++++++ vignettes/dlw_server_catalog.Rmd | 284 ++++++++++++++++++++++++++++ 2 files changed, 584 insertions(+) create mode 100644 vignettes/dlw_country_catalog.Rmd create mode 100644 vignettes/dlw_server_catalog.Rmd diff --git a/vignettes/dlw_country_catalog.Rmd b/vignettes/dlw_country_catalog.Rmd new file mode 100644 index 0000000..d3ad81a --- /dev/null +++ b/vignettes/dlw_country_catalog.Rmd @@ -0,0 +1,300 @@ +--- +title: "Country Catalog" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Country Catalog} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>", + eval = TRUE +) +``` + +## Introduction + +The **dlw** package provides granular access to country-specific dataset information from the Datalibweb (DLW) API. While `dlw_server_catalog()` gives you a complete list of all datasets across all countries, `dlw_country_catalog()` provides detailed, country-level metadata for a specific nation. + +This vignette covers: + +- Understanding the difference between server and country catalogs +- Retrieving country-specific catalog information +- Filtering and exploring country-level datasets +- Leveraging caching for performance + +## Prerequisites + +Before using `dlw_country_catalog()`, ensure you have: + +1. Installed the **dlw** package +2. Set your API token using `dlw_set_token()` + +```{r setup} +# Install dlw (if not already installed) +# devtools::install_github("worldbank/dlw") + +# devtools::load_all(".") +library(dlw) +library(data.table) + +# Set your API token +# dlw_set_token("your_api_token_here") +``` + +## Server Catalog vs. Country Catalog + +Before diving into `dlw_country_catalog()`, understand the difference: + +| Aspect | Server Catalog | Country Catalog | +|--------|---|---| +| **Scope** | All datasets across all countries | Datasets for a single country | +| **Function** | `dlw_server_catalog()` | `dlw_country_catalog()` | +| **Detail Level** | More information | Country-specific info | +| **Use Case** | Overview, across countries | Deep dive into a specific country | +| **Size** | Larger dataset | Smaller, focused dataset | + +**When to use each:** + +Use country catalog function to explore all available data for a particular country, e.g., Colombia +```{r} +country_cat <- dlw_country_catalog("COL") +head(country_cat) +``` + +Use server catalog function to get the list of datasets available across all countries. +```{r comparison} +server_cat <- dlw_server_catalog() +head(server_cat) + +``` + +## Basic Usage: Retrieve Country Catalog + +The simplest way to get a country catalog is: + +```{r basic-usage} +# Retrieve the public catalog for Colombia +catalog_col <- dlw_country_catalog(country_code = "COL") + +# View the first few rows +head(catalog_col) + +# Check the dimensions +dim(catalog_col) +``` + +This returns a `data.table` with detailed metadata about all datasets available for Colombia. + +### Understanding the Catalog Structure + +```{r catalog-structure} +# View column names +colnames(catalog_col) + +# Check the structure +str(catalog_col) + +``` + +Key columns typically include: + +| Column | Type | Description | +|--------|------|-------------| +| **ServerAlias** | character | Server name | +| **Country** | character | ISO3 country code | +| **Year** | integer | Year of the survey | +| **FilePath** | character | Full file path on the server | +| **Ext** | character | File extension | +| **FileSize** | number | File size | +| **Timestamp** | character | Date of data compilation | +| **Checksum** | character | hash | + +## Exploring Country Catalog + +### Get Summary Statistics + +```{r summary-stats} +# How many surveys for this country? +unique_surveys <- catalog_col[, .(count = .N), by = Survey] +print(unique_surveys) + +# Year range of surveys +year_range <- catalog_col[, .( + min_year = min(Year, na.rm = TRUE), + max_year = max(Year, na.rm = TRUE), + num_years = uniqueN(Year) +)] +print(year_range) + +``` + +### Filter by Survey Type + +```{r filter-survey} +# Find all GEIH surveys +geih_surveys <- catalog_col[Survey == "GEIH"] +head(geih_surveys) + +# Find all surveys from 2010 onwards +recent_surveys <- catalog_col[Year >= 2010] +nrow(recent_surveys) + +# Find HBS surveys +hbs_surveys <- catalog_col[Survey == "HBS"] +if (nrow(hbs_surveys) > 0) { + print(hbs_surveys) +} else { + cat("No HBS surveys found for this country\n") +} +``` + +### Find Latest Versions + +```{r latest-versions} +# For each survey, find the latest year +latest_by_survey <- catalog_col[, .SD[which.max(Year)], by = Survey] +print(latest_by_survey[, .(Survey, Year)]) + +# Find the absolute latest survey +latest_overall <- catalog_col[order(-Year)][1] +print(latest_overall) +``` + +## Working with Multiple Countries + +Compare catalogs across different countries: + +```{r multiple-countries} +# Get catalogs for multiple countries +countries <- c("COL", "BRA", "IND", "MEX") + +# Create a list to store catalogs +catalogs <- lapply(countries, dlw_country_catalog) +names(catalogs) <- countries + +# Compare dataset counts +dataset_counts <- data.table( + Country = countries, + Datasets = sapply(catalogs, nrow) +) +print(dataset_counts) + +# Find common surveys across countries +surveys_by_country <- lapply(catalogs, function(x) unique(x$Survey)) +common_surveys <- Reduce(intersect, surveys_by_country) +cat("Surveys available in all countries:", paste(common_surveys, collapse = ", "), "\n") + +# Find unique surveys per country +for (country in countries) { + unique_surveys <- setdiff( + unique(catalogs[[country]]$Survey), + common_surveys + ) + if (length(unique_surveys) > 0) { + cat(country, "unique surveys:", paste(unique_surveys, collapse = ", "), "\n") + } +} +``` + +## Understanding Caching + +By default, `dlw_country_catalog()` caches country catalogs in memory during your R session. This improves performance for repeated calls: + +```{r caching} +# First call: queries the API (slower) +system.time({ + cat1 <- dlw_country_catalog("COL") +}) + +# Second call: uses cache (much faster) +system.time({ + cat2 <- dlw_country_catalog("COL") +}) + +# Verify they're identical +identical(cat1, cat2) + +# Force a fresh API call +system.time({ + cat_fresh <- dlw_country_catalog("COL", force = TRUE) +}) +``` + +### When to Use `force = TRUE` + +- **Development/testing**: When you expect the server catalog to have changed +- **Production workflows**: Occasionally refresh to catch new datasets +- **Debugging**: When troubleshooting connectivity issues +- **Multi-session workflows**: When data may have been updated + +```{r refresh-catalog} +# Refresh the catalog from the server +catalog_latest <- dlw_country_catalog( + country_code = "COL", + force = TRUE, + verbose = TRUE +) +``` + +## Troubleshooting + +### No Data Returned + +If `dlw_country_catalog()` returns an empty result: + +```{r troubleshooting-empty} +# Check if the country code is valid +catalog <- dlw_country_catalog("COL") +if (nrow(catalog) == 0) { + cat("No datasets found for this country\n") + + # Try refreshing + catalog <- dlw_country_catalog("COL", force = TRUE) +} +``` + +### Authentication Errors + +If you get authentication errors: + +```{r troubleshooting-auth} +# Verify token is set +# token <- dlw_get_token() +# if (is.null(token) || token == "") { + # dlw_set_token("your_api_token_here") +# } + +``` + +### API Connection Issues + +If you cannot reach the server: + +```{r troubleshooting-connection} +# Force refresh to test connection +tryCatch({ + catalog <- dlw_country_catalog("COL", force = TRUE, verbose = TRUE) + cat("Connection successful\n") +}, error = function(e) { + cat("Connection failed:", e$message, "\n") +}) +``` + +## Related Functions + +- `dlw_server_catalog()` — List of all datasets across all countries +- `dlw_server_inventory()` — Filter server catalog by multiple criteria +- `dlw_get_data()` — Download data +- `dlw_set_token()` — Set API authentication token + +For more information, use: + +```r +?dlw_country_catalog +?dlw_get_data +?dlw_server_catalog +``` diff --git a/vignettes/dlw_server_catalog.Rmd b/vignettes/dlw_server_catalog.Rmd new file mode 100644 index 0000000..a4b1845 --- /dev/null +++ b/vignettes/dlw_server_catalog.Rmd @@ -0,0 +1,284 @@ +--- +title: "Server Catalog" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Server Catalog} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>", + eval = TRUE +) +``` + +## Introduction + +The **dlw** package provides access to datasets from the Datalibweb (DLW) API. Before downloading data, you need to explore what datasets are available. The `dlw_server_catalog()` function retrieves a comprehensive catalog of all available files on the server, making it easy to discover and filter datasets. + +This vignette walks you through: + +- Retrieving the server catalog +- Understanding the catalog structure +- Filtering and exploring datasets +- Leveraging caching for performance + +## Prerequisites + +Before using `dlw_server_catalog()`, ensure you have: + +1. Installed the **dlw** package +2. Set your API token using `dlw_set_token()` + +```{r setup} +# Install dlw (if not already installed) +# devtools::install_github("worldbank/dlw") + +# devtools::load_all(".") +library(dlw) +library(data.table) + +# Set your API token +# dlw_set_token("your_api_token_here") +``` + +## Basic Usage: Retrieve the Catalog + +The simplest way to get the server catalog is: + +```{r basic-usage} +# Retrieve the default (GMD) server catalog list of datasets +catalog <- dlw_server_catalog() + +# View the first few rows +head(catalog) +``` + +This returns a `data.table` with metadata about all `.dta` files available on the server. + +### What's in the Catalog? + +The catalog contains the following key columns: + +```{r catalog-structure} +# View column names +colnames(catalog) + +# Check the structure +str(catalog) +``` + +Key columns include: + +| Column | Description | +|--------|-------------| +| **FileName** | The file name (e.g., `COL_2022_GEIH_v01_M_v02_A_GMD_GPWG.dta`) | +| **Country_code** | ISO3 country code (e.g., `COL`, `IND`, `BRA`) | +| **Survey_year** | Year of the survey (integer) | +| **Survey_acronym** | Survey acronym (e.g., `ECH`, `ENH`, `GEIH`, `HBS`) | +| **Vermast** | Master version (e.g., `V01`, `V02`) | +| **Veralt** | Alternative version (e.g., `V01`, `V02`) | +| **Collection** | Collection name (e.g., `GMD`) | +| **Module** | Module name (e.g., `ALL`, `GPWG`, `ASPIRE`, `HIST`, `GROUP`, `BIN`) | + +## Exploring the Catalog + +### Count Datasets by Country + +```{r count-by-country} +# How many datasets per country? +catalog[, .(count = .N), by = Country_code][order(-count)] +``` + +### Find All Surveys for a Specific Country + +```{r surveys-by-country} +# Get all surveys for Colombia +colombia <- catalog[Country_code == "COL"] + +# View unique surveys and years +colombia[, .(count = .N), by = .(Survey_acronym, Survey_year)][order(Survey_year)] +``` + +### Find the Latest Version of a Dataset + +```{r latest-version} +# Find the most recent survey for Colombia +latest_col <- catalog[Country_code == "COL"][order(-Survey_year)][1] + +# View details +latest_col[, .(Country_code, Survey_year, Survey_acronym, Vermast, Veralt, FileName)] +``` + +### Filter by Multiple Criteria + +```{r multiple-criteria} +# Find all GEIH surveys from 2010 onwards +geih_recent <- catalog[ + Survey_acronym == "GEIH" & Survey_year >= 2010 +][order(Country_code, -Survey_year)] + +head(geih_recent) +``` + +### Search by Module or Collection + +```{r search-module} +# Find all datasets containing GPWG Module +gpwg_data <- catalog[grepl("GPWG", Module, ignore.case = TRUE)] + +nrow(gpwg_data) # How many? + +# View unique modules +catalog[, .(count = .N), by = Module][order(-count)] +``` + +## Using `dlw_server_inventory()` for Easier Filtering + +While `dlw_server_catalog()` returns the full catalog, `dlw_server_inventory()` provides a convenient wrapper for filtering: + +```{r server-inventory} +# Filter by country and year +inv <- dlw_server_inventory("COL", year = 2010) + +# This is equivalent to: +inv_manual <- catalog[Country_code == "COL" & Year == 2010] + +# They should be identical +identical(inv, inv_manual) +``` + +## Understanding Caching + +By default, `dlw_server_catalog()` caches the catalog in memory during your R session. This improves performance for repeated calls: + +```{r caching} +# First call: queries the API (slower) +system.time({ + cat1 <- dlw_server_catalog() +}) + +# Second call: uses cache (much faster) +system.time({ + cat2 <- dlw_server_catalog() +}) + +# Force a fresh API call +system.time({ + cat_fresh <- dlw_server_catalog(force = TRUE) +}) +``` + +### When to Use `force = TRUE` + +- **Development/testing**: When you expect the server catalog to have changed +- **Production workflows**: Occasionally refresh to catch new datasets +- **Debugging**: When troubleshooting connectivity or data issues + +```{r refresh-catalog} +# Refresh the catalog from the server +catalog_latest <- dlw_server_catalog(force = TRUE, verbose = TRUE) +``` + +## Advanced: Understanding Filename Structure + +The dlw package follows a strict naming convention for GMD files. Understanding this helps you work with datasets programmatically. + +### Filename Pattern + +The general pattern is: + +``` +___V_M_V_A__.dta +``` + +**Example:** `COL_2022_GEIH_V01_M_V02_A_GMD_GPWG.dta` + +**Breakdown:** + +- **COL**: Country code +- **2022**: Survey year +- **GEIH**: Survey acronym +- **V01**: Master version +- **M**: Marker (always present) +- **V02**: Alternative version +- **A**: Marker (always present) +- **GMD**: Collection +- **GPWG**: Module +- **.dta**: File format (Stata) + +### Accessing Parsed Components + +The catalog automatically parses this structure into columns: + +```{r parsed-components} +# Extract key components for a file +file_info <- catalog[FileName == "COL_2022_GEIH_V01_M_V02_A_GMD_GPWG.dta"] + +data.frame( + Country = file_info$Country_code, + Year = file_info$Survey_year, + Acronym = file_info$Survey_acronym, + MasterVersion = file_info$Vermast, + AltVersion = file_info$Veralt, + Collection = file_info$Collection, + Module = file_info$Module) +``` + +### Downloading Data + +See `?dlw_get_data()` for more download options. + +## Best Practices + +1. **Cache wisely**: Use the default caching behavior for interactive work; use `force = TRUE` in automated scripts. +2. **Validate filenames**: Always check that your filtered dataset contains the expected files before downloading. +3. **Use descriptive filters**: Prefer filtering by country, year, and survey acronym rather than searching by module names. +4. **Handle errors gracefully**: Wrap `dlw_get_data()` calls in `tryCatch()` for production workflows. + +## Troubleshooting + +### No Datasets Returned + +If your filter returns zero rows, verify: + +- Country code is correct (use 3-letter ISO code) +- Survey year exists in the catalog +- Survey acronym is spelled correctly + +### API Authentication Issues + +If you get authentication errors: + +1. Verify your token with `dlw_get_token()` +2. Re-set your token with `dlw_set_token()` +3. Try `dlw_server_catalog(force = TRUE)` to refresh + + +## Summary + +The `dlw_server_catalog()` function provides a powerful way to: + +- Discover available datasets +- Filter by country, year, survey, and topic +- Understand dataset structure and versions +- Prepare for data downloads + +### Related Functions + +- `dlw_server_catalog()` — Retrieve full server catalog +- `dlw_server_inventory()` — Convenient filtering wrapper +- `dlw_get_data()` — Download data using catalog filenames +- `dlw_set_token()` — Set API authentication token +- `dlw_get_token()` — Retrieve current API token + +For more information, use: + +```r +?dlw_server_catalog +?dlw_server_inventory +?dlw_get_data +``` From 7d1979ff26e0c9cc9f7296740eedb7feeec97879 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Wed, 18 Feb 2026 10:12:28 -0500 Subject: [PATCH 08/24] unit tests added for country and server catalogs --- tests/testthat/test-dlw_country_catalog.R | 23 +++++++++++++++++++++-- tests/testthat/test-dlw_server_catalog.R | 22 ++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/tests/testthat/test-dlw_country_catalog.R b/tests/testthat/test-dlw_country_catalog.R index 8849056..ef70c9b 100644 --- a/tests/testthat/test-dlw_country_catalog.R +++ b/tests/testthat/test-dlw_country_catalog.R @@ -1,3 +1,22 @@ -test_that("multiplication works", { - expect_equal(2 * 2, 4) +test_that("dlw_country_catalog returns a data.table with expected columns", { + skip_on_cran() + skip_if_not_installed("dlw") + skip_if_not_installed("data.table") + + result <- dlw_country_catalog(country_code = "COL") + expect_s3_class(result, "data.table") + expect_true(all( + c("ServerAlias", "Country", "Survey", "Year", "Ext", "Checksum") %in% + names(result) + )) + expect_gt(nrow(result), 0) }) + +test_that("dlw_country_catalog can filter by country_code", { + skip_on_cran() + skip_if_not_installed("dlw") + skip_if_not_installed("data.table") + + result <- dlw_country_catalog(country_code = "COL") + expect_true(all(result$Country == "COL")) +}) \ No newline at end of file diff --git a/tests/testthat/test-dlw_server_catalog.R b/tests/testthat/test-dlw_server_catalog.R index c1aaf09..c1d18d2 100644 --- a/tests/testthat/test-dlw_server_catalog.R +++ b/tests/testthat/test-dlw_server_catalog.R @@ -6,3 +6,25 @@ test_that("dlw_server_catalog returns a data.table/data.frame", { test_that("dlw_server_catalog errors for unknown server", { expect_error(dlw_server_catalog("x")) }) + +test_that("dlw_server_catalog returns a data.table with expected columns", { + skip_on_cran() + skip_if_not_installed("dlw") + skip_if_not_installed("data.table") + + result <- dlw_server_catalog() + expect_s3_class(result, "data.table") + expect_true(all( + c("FileName", "Country", "Survey", "Year", "Survey_year") %in% names(result) + )) + expect_gt(nrow(result), 0) +}) + +test_that("dlw_server_catalog can filter by server name", { + skip_on_cran() + skip_if_not_installed("dlw") + skip_if_not_installed("data.table") + + result <- dlw_server_catalog(server = "GMD") + expect_s3_class(result, "data.table") +}) \ No newline at end of file From b15469815c3905bcac9c5d70db47958436c86be5 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Wed, 18 Feb 2026 10:40:14 -0500 Subject: [PATCH 09/24] add categories to _pkgdown.yml --- _pkgdown.yml | 52 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/_pkgdown.yml b/_pkgdown.yml index 43e3327..9debf2c 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -3,24 +3,36 @@ template: bootstrap: 5 reference: -- title: GMD data - desc: Utilities for GMD data -- contents: - - has_concept("GMD utilities") -- title: Get data -- contents: - - dlw_get_data - - has_concept("catalogs") -- title: Token -- contents: - - has_concept("token utilities") -- title: dlw repository -- contents: - - starts_with("repo") -- title: .dlwenv environment -- contents: - - has_concept("dlwenv utilities") -- title: Misc -- contents: - - starts_with("print") +- title: Configuration and Utility + desc: > + Functions for setting up authentication, managing environment variables, and providing general utilities required for interacting with the Datalibweb API. + contents: + - dlw_set_token + - has_concept("token utilities") + - has_concept("dlwenv utilities") + - repo + - dlw_list_env + +- title: Preparing Input Arguments for Downloading DLW Data + desc: > + Functions for constructing API requests and retrieving metadata catalogs. These help users filter, select, and specify the datasets they want to download. + contents: + - build_request + - dlw_country_catalog + - dlw_server_catalog + - has_concept("catalogs") + +- title: Downloading Data + desc: > + Core functions for downloading, caching, and reading datasets from Datalibweb. These handle the full data retrieval workflow, including checking for local copies, downloading from the API, and reading data into R. + contents: + - dlw_get_data + - dlw_get_gmd + - dlw_get_gmd_support + +- title: Miscellaneous + desc: > + Miscellaneous functions, including print methods and other helpers. + contents: + - starts_with("print") From a577790944d97178855c8444d0dd3d21a845000d Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Thu, 19 Feb 2026 11:58:10 -0500 Subject: [PATCH 10/24] updated _pkgdown.yml --- _pkgdown.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/_pkgdown.yml b/_pkgdown.yml index 9debf2c..ed4cb75 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -1,7 +1,9 @@ url: https://worldbank.github.io/dlw/ + template: bootstrap: 5 - + bootswatch: materia + reference: - title: Configuration and Utility desc: > @@ -34,5 +36,4 @@ reference: desc: > Miscellaneous functions, including print methods and other helpers. contents: - - starts_with("print") - + - starts_with("print") \ No newline at end of file From abd1b2b07f2c0b3a9cf74d7cdaa5981a11fd6ba4 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Thu, 19 Feb 2026 12:14:13 -0500 Subject: [PATCH 11/24] fix the issue in yaml file --- _pkgdown.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_pkgdown.yml b/_pkgdown.yml index ed4cb75..16c77a6 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -3,7 +3,7 @@ url: https://worldbank.github.io/dlw/ template: bootstrap: 5 bootswatch: materia - + reference: - title: Configuration and Utility desc: > @@ -36,4 +36,4 @@ reference: desc: > Miscellaneous functions, including print methods and other helpers. contents: - - starts_with("print") \ No newline at end of file + - starts_with("print") From c79bbd681336626bb3d1c65a820b80622e7be621 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Thu, 26 Feb 2026 15:48:14 -0500 Subject: [PATCH 12/24] update pkgdown yaml file --- .github/workflows/pkgdown.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index bfc9f4d..1ecd8b3 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -2,7 +2,7 @@ # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help on: push: - branches: [main, master] + branches: [main, master, dev] pull_request: release: types: [published] @@ -29,12 +29,12 @@ jobs: - uses: r-lib/actions/setup-r@v2 with: - use-public-rspm: true + r-version: 'release' - uses: r-lib/actions/setup-r-dependencies@v2 with: extra-packages: any::pkgdown, local::. - needs: website + needs: website-r-dependencies@v2` action handles system dependencies automatically. - name: Build site run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE) From c9d51403b2d1588710480d5cf5360cfe35d5ee80 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Thu, 5 Mar 2026 09:08:31 -0500 Subject: [PATCH 13/24] Update the vignettes - wrap API-dependent chunks with a condition that skips evaluation in CI --- _pkgdown.yml | 2 ++ vignettes/dlw.Rmd | 11 ++++++++++- vignettes/dlw_country_catalog.Rmd | 10 +++++++++- vignettes/dlw_server_catalog.Rmd | 10 +++++++++- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/_pkgdown.yml b/_pkgdown.yml index 16c77a6..5af327f 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -3,6 +3,8 @@ url: https://worldbank.github.io/dlw/ template: bootstrap: 5 bootswatch: materia + bslib: + pkgdown-nav-height: 56px reference: - title: Configuration and Utility diff --git a/vignettes/dlw.Rmd b/vignettes/dlw.Rmd index 4b33675..6525a38 100644 --- a/vignettes/dlw.Rmd +++ b/vignettes/dlw.Rmd @@ -8,9 +8,18 @@ vignette: > --- ```{r, include = FALSE} +has_token <- tryCatch( + { + dlw::dlw_check_token() + TRUE + }, + error = function(e) FALSE +) + knitr::opts_chunk$set( collapse = TRUE, - comment = "#>" + comment = "#>", + eval = has_token ) ``` diff --git a/vignettes/dlw_country_catalog.Rmd b/vignettes/dlw_country_catalog.Rmd index d3ad81a..bec7019 100644 --- a/vignettes/dlw_country_catalog.Rmd +++ b/vignettes/dlw_country_catalog.Rmd @@ -8,10 +8,18 @@ vignette: > --- ```{r, include = FALSE} +has_token <- tryCatch( + { + dlw::dlw_check_token() + TRUE + }, + error = function(e) FALSE +) + knitr::opts_chunk$set( collapse = TRUE, comment = "#>", - eval = TRUE + eval = has_token ) ``` diff --git a/vignettes/dlw_server_catalog.Rmd b/vignettes/dlw_server_catalog.Rmd index a4b1845..bba7a7a 100644 --- a/vignettes/dlw_server_catalog.Rmd +++ b/vignettes/dlw_server_catalog.Rmd @@ -8,10 +8,18 @@ vignette: > --- ```{r, include = FALSE} +has_token <- tryCatch( + { + dlw::dlw_check_token() + TRUE + }, + error = function(e) FALSE +) + knitr::opts_chunk$set( collapse = TRUE, comment = "#>", - eval = TRUE + eval = has_token ) ``` From ddaad845ee7374044276a6b12ea5236fb3e9fd10 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Thu, 5 Mar 2026 13:36:16 -0500 Subject: [PATCH 14/24] remove eval from the knitr setup --- vignettes/dlw.Rmd | 11 +---------- vignettes/dlw_country_catalog.Rmd | 12 ++---------- vignettes/dlw_server_catalog.Rmd | 12 ++---------- 3 files changed, 5 insertions(+), 30 deletions(-) diff --git a/vignettes/dlw.Rmd b/vignettes/dlw.Rmd index 6525a38..4b33675 100644 --- a/vignettes/dlw.Rmd +++ b/vignettes/dlw.Rmd @@ -8,18 +8,9 @@ vignette: > --- ```{r, include = FALSE} -has_token <- tryCatch( - { - dlw::dlw_check_token() - TRUE - }, - error = function(e) FALSE -) - knitr::opts_chunk$set( collapse = TRUE, - comment = "#>", - eval = has_token + comment = "#>" ) ``` diff --git a/vignettes/dlw_country_catalog.Rmd b/vignettes/dlw_country_catalog.Rmd index bec7019..867485f 100644 --- a/vignettes/dlw_country_catalog.Rmd +++ b/vignettes/dlw_country_catalog.Rmd @@ -8,18 +8,10 @@ vignette: > --- ```{r, include = FALSE} -has_token <- tryCatch( - { - dlw::dlw_check_token() - TRUE - }, - error = function(e) FALSE -) - knitr::opts_chunk$set( collapse = TRUE, - comment = "#>", - eval = has_token + comment = "#>"#, + #eval = TRUE ) ``` diff --git a/vignettes/dlw_server_catalog.Rmd b/vignettes/dlw_server_catalog.Rmd index bba7a7a..cec7974 100644 --- a/vignettes/dlw_server_catalog.Rmd +++ b/vignettes/dlw_server_catalog.Rmd @@ -8,18 +8,10 @@ vignette: > --- ```{r, include = FALSE} -has_token <- tryCatch( - { - dlw::dlw_check_token() - TRUE - }, - error = function(e) FALSE -) - knitr::opts_chunk$set( collapse = TRUE, - comment = "#>", - eval = has_token + comment = "#>"#, + #eval = TRUE ) ``` From 8088a70ebf368a53756a7425fbfb2f83770989a3 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Thu, 5 Mar 2026 13:46:20 -0500 Subject: [PATCH 15/24] rename dlw, dlw_country_catalog, and dlw_serve_catalog Rmds to .Rmd.orig --- vignettes/{dlw.Rmd => dlw.Rmd.orig} | 0 .../{dlw_country_catalog.Rmd => dlw_country_catalog.Rmd.orig} | 0 vignettes/{dlw_server_catalog.Rmd => dlw_server_catalog.Rmd.orig} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename vignettes/{dlw.Rmd => dlw.Rmd.orig} (100%) rename vignettes/{dlw_country_catalog.Rmd => dlw_country_catalog.Rmd.orig} (100%) rename vignettes/{dlw_server_catalog.Rmd => dlw_server_catalog.Rmd.orig} (100%) diff --git a/vignettes/dlw.Rmd b/vignettes/dlw.Rmd.orig similarity index 100% rename from vignettes/dlw.Rmd rename to vignettes/dlw.Rmd.orig diff --git a/vignettes/dlw_country_catalog.Rmd b/vignettes/dlw_country_catalog.Rmd.orig similarity index 100% rename from vignettes/dlw_country_catalog.Rmd rename to vignettes/dlw_country_catalog.Rmd.orig diff --git a/vignettes/dlw_server_catalog.Rmd b/vignettes/dlw_server_catalog.Rmd.orig similarity index 100% rename from vignettes/dlw_server_catalog.Rmd rename to vignettes/dlw_server_catalog.Rmd.orig From 689a62196b1ba7be5b7987353a9585be17062e4a Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Thu, 5 Mar 2026 14:08:39 -0500 Subject: [PATCH 16/24] generate Rmd with executed codes --- .github/workflows/pkgdown.yaml | 7 + _pkgdown.yml | 2 +- vignettes/dlw.Rmd | 684 ++++++++++++++++++++++++++++++ vignettes/dlw_country_catalog.Rmd | 558 ++++++++++++++++++++++++ vignettes/dlw_server_catalog.Rmd | 539 +++++++++++++++++++++++ 5 files changed, 1789 insertions(+), 1 deletion(-) create mode 100644 vignettes/dlw.Rmd create mode 100644 vignettes/dlw_country_catalog.Rmd create mode 100644 vignettes/dlw_server_catalog.Rmd diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index 1ecd8b3..6feca47 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -3,7 +3,9 @@ on: push: branches: [main, master, dev] + tags: ['*'] pull_request: + branches: [main, master, dev] release: types: [published] workflow_dispatch: @@ -23,6 +25,11 @@ jobs: permissions: contents: write steps: + - uses: insightsengineering/r-pkgdown-multiversion@v2 + with: + default-landing-page: "master" + branches-or-tags-to-list: "master|dev" + - uses: actions/checkout@v4 - uses: r-lib/actions/setup-pandoc@v2 diff --git a/_pkgdown.yml b/_pkgdown.yml index 5af327f..c094d33 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -1,4 +1,4 @@ -url: https://worldbank.github.io/dlw/ +url: https://worldbank.github.io/dlw/dev/ template: bootstrap: 5 diff --git a/vignettes/dlw.Rmd b/vignettes/dlw.Rmd new file mode 100644 index 0000000..be18602 --- /dev/null +++ b/vignettes/dlw.Rmd @@ -0,0 +1,684 @@ +--- +title: "dlw" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{dlw} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + + + + +``` r +library(dlw) +library(data.table) +#> data.table 1.18.2.1 using 7 threads (see ?getDTthreads). Latest news: r-datatable.com +``` + +`{dlw}` is an R client of the internal [datalibweb](https://datalibweb2.worldbank.org/) API of the World Bank. Datalibweb is a data system designed to enable users to seamlessly access microdata and documentation in the World Bank. Users can access the most up-to-date and historical versions of harmonized data collections and raw/non-harmonized data for subsequent analysis + + +## Token +To begin working with `{dlw}`, you must provide your datalibweb API token using the `dlw_set_token()` function. + +You can obtain or renew your token by visiting the [datalibweb page](https://datalibweb2.worldbank.org/) and following the instructions there. Once you have your token, set it in your R session as follows: + + +``` r +# dlw_set_token("your_token_here") +``` + +## Server Catalog + +The server catalog provides a comprehensive list of all datasets available across all countries in the datalibweb system. You can retrieve the server catalog using the `dlw_server_catalog()` function. + + +``` r +server_cat <- dlw_server_catalog() +#> Pruning cache +#> ℹ saving ServerCatalog_GMD in .dlwevn +head(server_cat) +#> ServerAlias Country Year Survey +#> +#> 1: GMD VEN 2006 EHM +#> 2: GMD VEN 2005 EHM +#> 3: GMD VEN 2004 EHM +#> 4: GMD VEN 2003 EHM +#> 5: GMD VEN 2002 EHM +#> 6: GMD VEN 2001 EHM +#> FilePath +#> +#> 1: VEN/VEN_2006_EHM/VEN_2006_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2006_EHM_v01_M_v01_A_GMD_HIST.dta +#> 2: VEN/VEN_2005_EHM/VEN_2005_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2005_EHM_v01_M_v01_A_GMD_HIST.dta +#> 3: VEN/VEN_2004_EHM/VEN_2004_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2004_EHM_v01_M_v01_A_GMD_HIST.dta +#> 4: VEN/VEN_2003_EHM/VEN_2003_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2003_EHM_v01_M_v01_A_GMD_HIST.dta +#> 5: VEN/VEN_2002_EHM/VEN_2002_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2002_EHM_v01_M_v01_A_GMD_HIST.dta +#> 6: VEN/VEN_2001_EHM/VEN_2001_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2001_EHM_v01_M_v01_A_GMD_HIST.dta +#> Ext FileSize Timestamp +#> +#> 1: dta 7.02 MB May 14 2025 08:32 PM +#> 2: dta 6.84 MB May 14 2025 08:31 PM +#> 3: dta 7.01 MB May 14 2025 08:31 PM +#> 4: dta 8.55 MB May 14 2025 08:31 PM +#> 5: dta 10.12 MB May 14 2025 08:31 PM +#> 6: dta 8.63 MB May 14 2025 08:31 PM +#> Checksum OnlySol +#> +#> 1: e11b225b892a75fc66bac8ec1d432603 NA +#> 2: dfe2aa7b1be7c3ad818966ea55e43f86 NA +#> 3: 17fd8ceb4dbcc39c5ab948e771e83273 NA +#> 4: 2ce1a74eaba62e9f99a3a66c22f939e5 NA +#> 5: 2ffd4868918885f4fc2394c55806dd48 NA +#> 6: 608696c2fc617f4af9d6c8ed6303cadd NA +#> FileName +#> +#> 1: VEN_2006_EHM_v01_M_v01_A_GMD_HIST.dta +#> 2: VEN_2005_EHM_v01_M_v01_A_GMD_HIST.dta +#> 3: VEN_2004_EHM_v01_M_v01_A_GMD_HIST.dta +#> 4: VEN_2003_EHM_v01_M_v01_A_GMD_HIST.dta +#> 5: VEN_2002_EHM_v01_M_v01_A_GMD_HIST.dta +#> 6: VEN_2001_EHM_v01_M_v01_A_GMD_HIST.dta +#> Country_code Survey_year Survey_acronym +#> +#> 1: VEN 2006 EHM +#> 2: VEN 2005 EHM +#> 3: VEN 2004 EHM +#> 4: VEN 2003 EHM +#> 5: VEN 2002 EHM +#> 6: VEN 2001 EHM +#> Vermast Veralt Collection Module ext +#> +#> 1: V01 V01 GMD HIST DTA +#> 2: V01 V01 GMD HIST DTA +#> 3: V01 V01 GMD HIST DTA +#> 4: V01 V01 GMD HIST DTA +#> 5: V01 V01 GMD HIST DTA +#> 6: V01 V01 GMD HIST DTA +``` + +### Exploring the Server Catalog + +You can filter and explore the server catalog to find datasets of interest. + + +``` r +# For examples, +# to find list of all datasets related to HEIS surveys: +heis_datasets <- server_cat[Survey == "HEIS", ] +head(heis_datasets) +#> ServerAlias Country Year Survey +#> +#> 1: GMD JOR 2010 HEIS +#> 2: GMD JOR 2010 HEIS +#> 3: GMD JOR 2010 HEIS +#> 4: GMD JOR 2010 HEIS +#> 5: GMD JOR 2010 HEIS +#> 6: GMD JOR 2010 HEIS +#> FilePath +#> +#> 1: JOR/JOR_2010_HEIS/JOR_2010_HEIS_V01_M_V04_A_GMD/Data/Harmonized/JOR_2010_HEIS_V01_M_V04_A_GMD_ALL.dta +#> 2: JOR/JOR_2010_HEIS/JOR_2010_HEIS_V01_M_V04_A_GMD/Data/Harmonized/JOR_2010_HEIS_V01_M_V04_A_GMD_GPWG.dta +#> 3: JOR/JOR_2010_HEIS/JOR_2010_HEIS_V01_M_V03_A_GMD/Data/Harmonized/JOR_2010_HEIS_V01_M_V03_A_GMD_ALL.dta +#> 4: JOR/JOR_2010_HEIS/JOR_2010_HEIS_V01_M_V03_A_GMD/Data/Harmonized/JOR_2010_HEIS_V01_M_V03_A_GMD_GPWG.dta +#> 5: JOR/JOR_2010_HEIS/JOR_2010_HEIS_V01_M_V01_A_GMD/Data/Harmonized/JOR_2010_HEIS_V01_M_V01_A_GMD_ALL.dta +#> 6: JOR/JOR_2010_HEIS/JOR_2010_HEIS_V01_M_V01_A_GMD/Data/Harmonized/JOR_2010_HEIS_V01_M_V01_A_GMD_ASPIRE.dta +#> Ext FileSize Timestamp +#> +#> 1: dta 24.99 MB May 14 2025 07:43 PM +#> 2: dta 5.49 MB May 14 2025 07:43 PM +#> 3: dta 13.06 MB May 14 2025 07:43 PM +#> 4: dta 5.37 MB May 14 2025 07:43 PM +#> 5: dta 12.48 MB May 14 2025 07:43 PM +#> 6: dta 787.21 KB May 14 2025 07:43 PM +#> Checksum OnlySol +#> +#> 1: 286563022fd66455e593e47be693cc47 NA +#> 2: 4ba0d20ffff465cb0f8ba2b064380dda NA +#> 3: 6a097bc4a4f1653bff66efa423791cde NA +#> 4: c9314898696f8d9b9ecc40ffb833e5af NA +#> 5: e8889ceaa2c0e47c00d51ec5cdfc2048 NA +#> 6: 20a28979bcea01bbe4ca1bffa99a898c NA +#> FileName +#> +#> 1: JOR_2010_HEIS_V01_M_V04_A_GMD_ALL.dta +#> 2: JOR_2010_HEIS_V01_M_V04_A_GMD_GPWG.dta +#> 3: JOR_2010_HEIS_V01_M_V03_A_GMD_ALL.dta +#> 4: JOR_2010_HEIS_V01_M_V03_A_GMD_GPWG.dta +#> 5: JOR_2010_HEIS_V01_M_V01_A_GMD_ALL.dta +#> 6: JOR_2010_HEIS_V01_M_V01_A_GMD_ASPIRE.dta +#> Country_code Survey_year Survey_acronym +#> +#> 1: JOR 2010 HEIS +#> 2: JOR 2010 HEIS +#> 3: JOR 2010 HEIS +#> 4: JOR 2010 HEIS +#> 5: JOR 2010 HEIS +#> 6: JOR 2010 HEIS +#> Vermast Veralt Collection Module ext +#> +#> 1: V01 V04 GMD ALL DTA +#> 2: V01 V04 GMD GPWG DTA +#> 3: V01 V03 GMD ALL DTA +#> 4: V01 V03 GMD GPWG DTA +#> 5: V01 V01 GMD ALL DTA +#> 6: V01 V01 GMD ASPIRE DTA + +# to find list of all datasets for a specific module: +module_gpwg <- server_cat[Module == "GPWG", ] +head(module_gpwg) +#> ServerAlias Country Year Survey +#> +#> 1: GMD UZB 2025 HBS +#> 2: GMD UZB 2024 HBS +#> 3: GMD UZB 2024 HBS +#> 4: GMD UZB 2023 HBS +#> 5: GMD UZB 2022 HBS +#> 6: GMD UZB 2021 HBS +#> FilePath +#> +#> 1: UZB/UZB_2025_HBS/UZB_2025_HBS_V01_M_V01_A_GMD/Data/Harmonized/UZB_2025_HBS_V01_M_V01_A_GMD_GPWG.dta +#> 2: UZB/UZB_2024_HBS/UZB_2024_HBS_V01_M_V02_A_GMD/Data/Harmonized/UZB_2024_HBS_V01_M_V02_A_GMD_GPWG.dta +#> 3: UZB/UZB_2024_HBS/UZB_2024_HBS_V01_M_V01_A_GMD/Data/Harmonized/UZB_2024_HBS_V01_M_V01_A_GMD_GPWG.dta +#> 4: UZB/UZB_2023_HBS/UZB_2023_HBS_V01_M_V01_A_GMD/Data/Harmonized/UZB_2023_HBS_V01_M_V01_A_GMD_GPWG.dta +#> 5: UZB/UZB_2022_HBS/UZB_2022_HBS_V01_M_V01_A_GMD/Data/Harmonized/UZB_2022_HBS_V01_M_V01_A_GMD_GPWG.dta +#> 6: UZB/UZB_2021_HBS/UZB_2021_HBS_V01_M_V01_A_GMD/Data/Harmonized/UZB_2021_HBS_V01_M_V01_A_GMD_GPWG.dta +#> Ext FileSize Timestamp +#> +#> 1: dta 7.14 MB Feb 20 2026 08:30 PM +#> 2: dta 9.26 MB Feb 20 2026 08:30 PM +#> 3: dta 8.37 MB Oct 24 2025 01:32 PM +#> 4: dta 6.80 MB May 14 2025 08:31 PM +#> 5: dta 5.26 MB May 14 2025 08:31 PM +#> 6: dta 3.69 MB May 14 2025 08:31 PM +#> Checksum OnlySol +#> +#> 1: d41d8cd98f00b204e9800998ecf8427e NA +#> 2: d41d8cd98f00b204e9800998ecf8427e NA +#> 3: a3781e287748b35d363fa128c9a3a19e NA +#> 4: 62c75fba8f1b40f2000c5ea3b07ded09 NA +#> 5: 43f3b80d3cfc0770ee8102a591134461 NA +#> 6: fa508041ae7aec93f0b4c6d4edeb5d6a NA +#> FileName +#> +#> 1: UZB_2025_HBS_V01_M_V01_A_GMD_GPWG.dta +#> 2: UZB_2024_HBS_V01_M_V02_A_GMD_GPWG.dta +#> 3: UZB_2024_HBS_V01_M_V01_A_GMD_GPWG.dta +#> 4: UZB_2023_HBS_V01_M_V01_A_GMD_GPWG.dta +#> 5: UZB_2022_HBS_V01_M_V01_A_GMD_GPWG.dta +#> 6: UZB_2021_HBS_V01_M_V01_A_GMD_GPWG.dta +#> Country_code Survey_year Survey_acronym +#> +#> 1: UZB 2025 HBS +#> 2: UZB 2024 HBS +#> 3: UZB 2024 HBS +#> 4: UZB 2023 HBS +#> 5: UZB 2022 HBS +#> 6: UZB 2021 HBS +#> Vermast Veralt Collection Module ext +#> +#> 1: V01 V01 GMD GPWG DTA +#> 2: V01 V02 GMD GPWG DTA +#> 3: V01 V01 GMD GPWG DTA +#> 4: V01 V01 GMD GPWG DTA +#> 5: V01 V01 GMD GPWG DTA +#> 6: V01 V01 GMD GPWG DTA + +# to find list of all datasets for a specific year: +year_2023 <- server_cat[Year == 2023, ] +head(year_2023) +#> ServerAlias Country Year Survey +#> +#> 1: GMD UZB 2023 HBS +#> 2: GMD UZB 2023 HBS +#> 3: GMD USA 2023 CPS-ASEC-LIS +#> 4: GMD USA 2023 CPS-ASEC-LIS +#> 5: GMD URY 2023 ECH +#> 6: GMD URY 2023 ECH +#> FilePath +#> +#> 1: UZB/UZB_2023_HBS/UZB_2023_HBS_V01_M_V01_A_GMD/Data/Harmonized/UZB_2023_HBS_V01_M_V01_A_GMD_ALL.dta +#> 2: UZB/UZB_2023_HBS/UZB_2023_HBS_V01_M_V01_A_GMD/Data/Harmonized/UZB_2023_HBS_V01_M_V01_A_GMD_GPWG.dta +#> 3: USA/USA_2023_CPS-ASEC-LIS/USA_2023_CPS-ASEC-LIS_V01_M_V02_A_GMD/Data/Harmonized/USA_2023_CPS-ASEC-LIS_V01_M_V02_A_GMD_BIN.dta +#> 4: USA/USA_2023_CPS-ASEC-LIS/USA_2023_CPS-ASEC-LIS_V01_M_V01_A_GMD/Data/Harmonized/USA_2023_CPS-ASEC-LIS_V01_M_V01_A_GMD_BIN.dta +#> 5: URY/URY_2023_ECH/URY_2023_ECH_V01_M_V01_A_GMD/Data/Harmonized/URY_2023_ECH_V01_M_V01_A_GMD_ALL.dta +#> 6: URY/URY_2023_ECH/URY_2023_ECH_V01_M_V01_A_GMD/Data/Harmonized/URY_2023_ECH_V01_M_V01_A_GMD_GPWG.dta +#> Ext FileSize Timestamp +#> +#> 1: dta 57.74 MB May 14 2025 08:31 PM +#> 2: dta 6.80 MB May 14 2025 08:31 PM +#> 3: dta 72.75 KB Jan 22 2026 07:28 PM +#> 4: dta 36.53 KB May 14 2025 08:31 PM +#> 5: dta 34.67 MB May 14 2025 08:20 PM +#> 6: dta 5.72 MB May 14 2025 08:20 PM +#> Checksum OnlySol +#> +#> 1: 6b79fab477546bbfd1b114215f6ba276 NA +#> 2: 62c75fba8f1b40f2000c5ea3b07ded09 NA +#> 3: d41d8cd98f00b204e9800998ecf8427e NA +#> 4: ff02cadce6a0c572b7dc63ef000e6863 NA +#> 5: 8e15ff737bf1042f1b03f5551cb405b6 NA +#> 6: 7bd7555fa5bb19580ec2946fa2279a34 NA +#> FileName +#> +#> 1: UZB_2023_HBS_V01_M_V01_A_GMD_ALL.dta +#> 2: UZB_2023_HBS_V01_M_V01_A_GMD_GPWG.dta +#> 3: USA_2023_CPS-ASEC-LIS_V01_M_V02_A_GMD_BIN.dta +#> 4: USA_2023_CPS-ASEC-LIS_V01_M_V01_A_GMD_BIN.dta +#> 5: URY_2023_ECH_V01_M_V01_A_GMD_ALL.dta +#> 6: URY_2023_ECH_V01_M_V01_A_GMD_GPWG.dta +#> Country_code Survey_year Survey_acronym +#> +#> 1: UZB 2023 HBS +#> 2: UZB 2023 HBS +#> 3: USA 2023 CPS-ASEC-LIS +#> 4: USA 2023 CPS-ASEC-LIS +#> 5: URY 2023 ECH +#> 6: URY 2023 ECH +#> Vermast Veralt Collection Module ext +#> +#> 1: V01 V01 GMD ALL DTA +#> 2: V01 V01 GMD GPWG DTA +#> 3: V01 V02 GMD BIN DTA +#> 4: V01 V01 GMD BIN DTA +#> 5: V01 V01 GMD ALL DTA +#> 6: V01 V01 GMD GPWG DTA + +# to find list of all datasets for a specific country and year: +country_year <- server_cat[Country == "PRY" & Year == 2020, ] +head(country_year) +#> ServerAlias Country Year Survey +#> +#> 1: GMD PRY 2020 EPH +#> 2: GMD PRY 2020 EPH +#> 3: GMD PRY 2020 EPH +#> 4: GMD PRY 2020 EPH +#> 5: GMD PRY 2020 EPH +#> 6: GMD PRY 2020 EPH +#> FilePath +#> +#> 1: PRY/PRY_2020_EPH/PRY_2020_EPH_V01_M_V03_A_GMD/Data/Harmonized/PRY_2020_EPH_V01_M_V03_A_GMD_ALL.dta +#> 2: PRY/PRY_2020_EPH/PRY_2020_EPH_V01_M_V03_A_GMD/Data/Harmonized/PRY_2020_EPH_V01_M_V03_A_GMD_GPWG.dta +#> 3: PRY/PRY_2020_EPH/PRY_2020_EPH_V01_M_V02_A_GMD/Data/Harmonized/PRY_2020_EPH_V01_M_V02_A_GMD_ALL.dta +#> 4: PRY/PRY_2020_EPH/PRY_2020_EPH_V01_M_V02_A_GMD/Data/Harmonized/PRY_2020_EPH_V01_M_V02_A_GMD_GPWG.dta +#> 5: PRY/PRY_2020_EPH/PRY_2020_EPH_V01_M_V01_A_GMD/Data/Harmonized/PRY_2020_EPH_V01_M_V01_A_GMD_ALL.dta +#> 6: PRY/PRY_2020_EPH/PRY_2020_EPH_V01_M_V01_A_GMD/Data/Harmonized/PRY_2020_EPH_V01_M_V01_A_GMD_GPWG.dta +#> Ext FileSize Timestamp +#> +#> 1: dta 11.51 MB May 14 2025 08:03 PM +#> 2: dta 2.68 MB May 14 2025 08:03 PM +#> 3: dta 10.70 MB May 14 2025 08:03 PM +#> 4: dta 2.53 MB May 14 2025 08:03 PM +#> 5: dta 4.46 MB May 14 2025 08:03 PM +#> 6: dta 1.76 MB May 14 2025 08:03 PM +#> Checksum OnlySol +#> +#> 1: 06a3f754dc1b3365f76a3c30528f113e NA +#> 2: 9da8378532194cf6acd8be8925dd6006 NA +#> 3: 1770c99b5680b16133cf64ce03894505 NA +#> 4: 0743ab3383791c724ce74d1750a34561 NA +#> 5: b15a72c03bc7c825ff88d4cd1f50b467 NA +#> 6: c948747e6c3654a564981823b34aa4f5 NA +#> FileName +#> +#> 1: PRY_2020_EPH_V01_M_V03_A_GMD_ALL.dta +#> 2: PRY_2020_EPH_V01_M_V03_A_GMD_GPWG.dta +#> 3: PRY_2020_EPH_V01_M_V02_A_GMD_ALL.dta +#> 4: PRY_2020_EPH_V01_M_V02_A_GMD_GPWG.dta +#> 5: PRY_2020_EPH_V01_M_V01_A_GMD_ALL.dta +#> 6: PRY_2020_EPH_V01_M_V01_A_GMD_GPWG.dta +#> Country_code Survey_year Survey_acronym +#> +#> 1: PRY 2020 EPH +#> 2: PRY 2020 EPH +#> 3: PRY 2020 EPH +#> 4: PRY 2020 EPH +#> 5: PRY 2020 EPH +#> 6: PRY 2020 EPH +#> Vermast Veralt Collection Module ext +#> +#> 1: V01 V03 GMD ALL DTA +#> 2: V01 V03 GMD GPWG DTA +#> 3: V01 V02 GMD ALL DTA +#> 4: V01 V02 GMD GPWG DTA +#> 5: V01 V01 GMD ALL DTA +#> 6: V01 V01 GMD GPWG DTA +``` + +## Country Catalog + +The country catalog provides detailed metadata for datasets specific to a single country. You can retrieve the country catalog using the `dlw_country_catalog()` function by specifying the desired country code. + + +``` r +country_pry <- dlw_country_catalog(country_code = "PRY") +#> ℹ saving CountryCatalog_PRY in .dlwevn +head(country_pry) +#> ServerAlias Country Year Survey +#> +#> 1: GMD PRY 2024 EPHC +#> 2: GMD PRY 2024 EPHC +#> 3: GMD PRY 2023 EPHC +#> 4: GMD PRY 2023 EPHC +#> 5: GMD PRY 2023 EPHC +#> 6: GMD PRY 2023 EPHC +#> FilePath +#> +#> 1: PRY/PRY_2024_EPHC/PRY_2024_EPHC_V01_M_V01_A_GMD/Data/Harmonized/PRY_2024_EPHC_V01_M_V01_A_GMD_ALL.dta +#> 2: PRY/PRY_2024_EPHC/PRY_2024_EPHC_V01_M_V01_A_GMD/Data/Harmonized/PRY_2024_EPHC_V01_M_V01_A_GMD_GPWG.dta +#> 3: PRY/PRY_2023_EPHC/PRY_2023_EPHC_V02_M_V01_A_GMD/Data/Harmonized/PRY_2023_EPHC_V02_M_V01_A_GMD_ALL.dta +#> 4: PRY/PRY_2023_EPHC/PRY_2023_EPHC_V02_M_V01_A_GMD/Data/Harmonized/PRY_2023_EPHC_V02_M_V01_A_GMD_GPWG.dta +#> 5: PRY/PRY_2023_EPHC/PRY_2023_EPHC_V01_M_V01_A_GMD/Data/Harmonized/PRY_2023_EPHC_V01_M_V01_A_GMD_ALL.dta +#> 6: PRY/PRY_2023_EPHC/PRY_2023_EPHC_V01_M_V01_A_GMD/Data/Harmonized/PRY_2023_EPHC_V01_M_V01_A_GMD_GPWG.dta +#> Ext FileSize Timestamp +#> +#> 1: dta 36.36 MB Aug 15 2025 07:40 PM +#> 2: dta 6.92 MB Aug 15 2025 07:40 PM +#> 3: dta 36.62 MB Aug 15 2025 07:40 PM +#> 4: dta 6.96 MB Aug 15 2025 07:40 PM +#> 5: dta 37.89 MB May 14 2025 08:03 PM +#> 6: dta 5.97 MB May 14 2025 08:03 PM +#> Checksum OnlySol +#> +#> 1: d41d8cd98f00b204e9800998ecf8427e NA +#> 2: d41d8cd98f00b204e9800998ecf8427e NA +#> 3: d41d8cd98f00b204e9800998ecf8427e NA +#> 4: d41d8cd98f00b204e9800998ecf8427e NA +#> 5: 2ec26933942ea00ac0815d65c513ca35 NA +#> 6: 257b606f428e16f27b2f30614f419f7a NA +``` + +### Exploring the Country Catalog + +You can filter and explore the country catalog to find list datasets of interest. + +``` r +# For examples +# to find list of all datasets related to EPH survey in Paraguay: +eph_pry <- country_pry[Survey == "EPH", ] +head(eph_pry) +#> ServerAlias Country Year Survey +#> +#> 1: GMD PRY 2022 EPH +#> 2: GMD PRY 2022 EPH +#> 3: GMD PRY 2022 EPH +#> 4: GMD PRY 2022 EPH +#> 5: GMD PRY 2021 EPH +#> 6: GMD PRY 2021 EPH +#> FilePath +#> +#> 1: PRY/PRY_2022_EPH/PRY_2022_EPH_V01_M_V02_A_GMD/Data/Harmonized/PRY_2022_EPH_V01_M_V02_A_GMD_ALL.dta +#> 2: PRY/PRY_2022_EPH/PRY_2022_EPH_V01_M_V02_A_GMD/Data/Harmonized/PRY_2022_EPH_V01_M_V02_A_GMD_GPWG.dta +#> 3: PRY/PRY_2022_EPH/PRY_2022_EPH_V01_M_V01_A_GMD/Data/Harmonized/PRY_2022_EPH_V01_M_V01_A_GMD_ALL.dta +#> 4: PRY/PRY_2022_EPH/PRY_2022_EPH_V01_M_V01_A_GMD/Data/Harmonized/PRY_2022_EPH_V01_M_V01_A_GMD_GPWG.dta +#> 5: PRY/PRY_2021_EPH/PRY_2021_EPH_V01_M_V03_A_GMD/Data/Harmonized/PRY_2021_EPH_V01_M_V03_A_GMD_ALL.dta +#> 6: PRY/PRY_2021_EPH/PRY_2021_EPH_V01_M_V03_A_GMD/Data/Harmonized/PRY_2021_EPH_V01_M_V03_A_GMD_GPWG.dta +#> Ext FileSize Timestamp +#> +#> 1: dta 10.67 MB May 14 2025 08:03 PM +#> 2: dta 1.88 MB May 14 2025 08:03 PM +#> 3: dta 4.77 MB May 14 2025 08:03 PM +#> 4: dta 1.63 MB May 14 2025 08:03 PM +#> 5: dta 10.71 MB May 14 2025 08:03 PM +#> 6: dta 2.40 MB May 14 2025 08:03 PM +#> Checksum OnlySol +#> +#> 1: 37299ca45846dc35c6d650b624ba2d68 NA +#> 2: be3752dd69d16f666be14735de52c4e2 NA +#> 3: 7f698e1667f4df29e98d2a9786520009 NA +#> 4: aea111432ee717eead2eb99a5cf29275 NA +#> 5: a90ca729e8125d4be6a5f5e894025c78 NA +#> 6: 3e6f31def4c2f19822f10753f3c77cd5 NA + +# to find list of all datasets for a specific year: +year_2020_pry <- country_pry[Year == 2020, ] +head(year_2020_pry) +#> ServerAlias Country Year Survey +#> +#> 1: GMD PRY 2020 EPH +#> 2: GMD PRY 2020 EPH +#> 3: GMD PRY 2020 EPH +#> 4: GMD PRY 2020 EPH +#> 5: GMD PRY 2020 EPH +#> 6: GMD PRY 2020 EPH +#> FilePath +#> +#> 1: PRY/PRY_2020_EPH/PRY_2020_EPH_V01_M_V03_A_GMD/Data/Harmonized/PRY_2020_EPH_V01_M_V03_A_GMD_ALL.dta +#> 2: PRY/PRY_2020_EPH/PRY_2020_EPH_V01_M_V03_A_GMD/Data/Harmonized/PRY_2020_EPH_V01_M_V03_A_GMD_GPWG.dta +#> 3: PRY/PRY_2020_EPH/PRY_2020_EPH_V01_M_V02_A_GMD/Data/Harmonized/PRY_2020_EPH_V01_M_V02_A_GMD_ALL.dta +#> 4: PRY/PRY_2020_EPH/PRY_2020_EPH_V01_M_V02_A_GMD/Data/Harmonized/PRY_2020_EPH_V01_M_V02_A_GMD_GPWG.dta +#> 5: PRY/PRY_2020_EPH/PRY_2020_EPH_V01_M_V01_A_GMD/Data/Harmonized/PRY_2020_EPH_V01_M_V01_A_GMD_ALL.dta +#> 6: PRY/PRY_2020_EPH/PRY_2020_EPH_V01_M_V01_A_GMD/Data/Harmonized/PRY_2020_EPH_V01_M_V01_A_GMD_GPWG.dta +#> Ext FileSize Timestamp +#> +#> 1: dta 11.51 MB May 14 2025 08:03 PM +#> 2: dta 2.68 MB May 14 2025 08:03 PM +#> 3: dta 10.70 MB May 14 2025 08:03 PM +#> 4: dta 2.53 MB May 14 2025 08:03 PM +#> 5: dta 4.46 MB May 14 2025 08:03 PM +#> 6: dta 1.76 MB May 14 2025 08:03 PM +#> Checksum OnlySol +#> +#> 1: 06a3f754dc1b3365f76a3c30528f113e NA +#> 2: 9da8378532194cf6acd8be8925dd6006 NA +#> 3: 1770c99b5680b16133cf64ce03894505 NA +#> 4: 0743ab3383791c724ce74d1750a34561 NA +#> 5: b15a72c03bc7c825ff88d4cd1f50b467 NA +#> 6: c948747e6c3654a564981823b34aa4f5 NA +``` + +## Downloading Datasets +Once you have identified datasets of interest from either the server or country catalog, you can download them using the `dlw_get_data()` function. + +``` r +dlw_get_data( + country_code = "PRY", + year = 2020L, + server = "GMD", + survey = "EPH", + module = "GPWG", + filename = "PRY_2020_EPH_V01_M_V03_A_GMD_GPWG.dta", + collection = "GMD" +) +#> ✔ stamp initialized +#> alias: dlw +#> root: +#> C:/Users/WB308892/AppData/Local/Temp/Rtmpu6kXN5 +#> state: +#> C:/Users/WB308892/AppData/Local/Temp/Rtmpu6kXN5/.stamp +#> ✔ Saved [qs2] → +#> ']8;;file://c:/users/wb308892/appdata/local/temp/rtmpu6kxn5/pry_2020_eph_v01_m_v03_a_gmd_gpwg.qs2c:/users/wb308892/appdata/local/temp/rtmpu6kxn5/pry_2020_eph_v01_m_v03_a_gmd_gpwg.qs2]8;;' +#> @ version 954d05a220c4e28f +#> countrycode year hhid pid +#> +#> 1: pry 2020 001060011 1 +#> 2: pry 2020 001060011 2 +#> 3: pry 2020 001060031 1 +#> 4: pry 2020 001060041 2 +#> 5: pry 2020 001060041 3 +#> --- +#> 17485: pry 2020 286880131 3 +#> 17486: pry 2020 286880181 1 +#> 17487: pry 2020 286880181 2 +#> 17488: pry 2020 286880181 3 +#> 17489: pry 2020 286880181 4 +#> hsize welfare welfaredef welfarenom +#> +#> 1: 2 12689241 12689241 12689241 +#> 2: 2 12689241 12689241 12689241 +#> 3: 1 36671200 36671200 36671200 +#> 4: 6 14402758 14402758 14402758 +#> 5: 6 14402758 14402758 14402758 +#> --- +#> 17485: 3 30408284 30408284 26440002 +#> 17486: 4 14146062 14146062 12300001 +#> 17487: 4 14146062 14146062 12300001 +#> 17488: 4 14146062 14146062 12300001 +#> 17489: 4 14146062 14146062 12300001 +#> welfareother welfareothertype +#> +#> 1: 12689241 INC +#> 2: 12689241 INC +#> 3: 36671200 INC +#> 4: 14402758 INC +#> 5: 14402758 INC +#> --- +#> 17485: 30408284 INC +#> 17486: 14146062 INC +#> 17487: 14146062 INC +#> 17488: 14146062 INC +#> 17489: 14146062 INC +#> welfshprosperity welfaretype +#> +#> 1: 12689241 INC +#> 2: 12689241 INC +#> 3: 36671200 INC +#> 4: 14402758 INC +#> 5: 14402758 INC +#> --- +#> 17485: 30408284 INC +#> 17486: 14146062 INC +#> 17487: 14146062 INC +#> 17488: 14146062 INC +#> 17489: 14146062 INC +#> subnatid1 +#> +#> 1: 1 - Area Metropolitana Asuncion +#> 2: 1 - Area Metropolitana Asuncion +#> 3: 1 - Area Metropolitana Asuncion +#> 4: 1 - Area Metropolitana Asuncion +#> 5: 1 - Area Metropolitana Asuncion +#> --- +#> 17485: 5 - Resto Rural +#> 17486: 5 - Resto Rural +#> 17487: 5 - Resto Rural +#> 17488: 5 - Resto Rural +#> 17489: 5 - Resto Rural +#> subnatid1_prev +#> +#> 1: 1 - Area Metropolitana Asuncion +#> 2: 1 - Area Metropolitana Asuncion +#> 3: 1 - Area Metropolitana Asuncion +#> 4: 1 - Area Metropolitana Asuncion +#> 5: 1 - Area Metropolitana Asuncion +#> --- +#> 17485: 5 - Resto Rural +#> 17486: 5 - Resto Rural +#> 17487: 5 - Resto Rural +#> 17488: 5 - Resto Rural +#> 17489: 5 - Resto Rural +#> subnatid2_prev subnatid3_prev +#> +#> 1: 0 - Asuncion +#> 2: 0 - Asuncion +#> 3: 0 - Asuncion +#> 4: 0 - Asuncion +#> 5: 0 - Asuncion +#> --- +#> 17485: 20 - Resto +#> 17486: 20 - Resto +#> 17487: 20 - Resto +#> 17488: 20 - Resto +#> 17489: 20 - Resto +#> gaul_adm1_code gaul_adm2_code +#> +#> 1: 2311 NA +#> 2: 2311 NA +#> 3: 2311 NA +#> 4: 2311 NA +#> 5: 2311 NA +#> --- +#> 17485: NA NA +#> 17486: NA NA +#> 17487: NA NA +#> 17488: NA NA +#> 17489: NA NA +#> gaul_adm3_code subnatid2 subnatid3 +#> +#> 1: NA . . +#> 2: NA . . +#> 3: NA . . +#> 4: NA . . +#> 5: NA . . +#> --- +#> 17485: NA . . +#> 17486: NA . . +#> 17487: NA . . +#> 17488: NA . . +#> 17489: NA . . +#> subnatidsurvey urban +#> +#> 1: 0 - Asuncion 1 +#> 2: 0 - Asuncion 1 +#> 3: 0 - Asuncion 1 +#> 4: 0 - Asuncion 1 +#> 5: 0 - Asuncion 1 +#> --- +#> 17485: 20 - Resto 0 +#> 17486: 20 - Resto 0 +#> 17487: 20 - Resto 0 +#> 17488: 20 - Resto 0 +#> 17489: 20 - Resto 0 +#> male age weight code +#> +#> 1: 1 70 180 PRY +#> 2: 0 65 180 PRY +#> 3: 1 42 407 PRY +#> 4: 1 34 451 PRY +#> 5: 0 31 451 PRY +#> --- +#> 17485: 1 18 127 PRY +#> 17486: 1 40 194 PRY +#> 17487: 0 27 194 PRY +#> 17488: 1 6 194 PRY +#> 17489: 1 3 194 PRY +``` + +Alternatively, you can use `dlw_get_gmd()`, which is a convenient wrapper around `dlw_get_data()`. To download the most recent version of a dataset, you only need to provide the country code, year, and module. + +``` r +pry20 <- dlw_get_gmd(country_code = "PRY", year = 2020, module = "GPWG") +#> +#> ── dlw_get_data Calls ──────────────────────── +#> Call 1: +#> ]8;;x-r-help:dlw::dlw_get_datadlw_get_data]8;;( +#> country_code = "PRY", +#> year = 2020L, +#> server = "GMD", +#> survey = "EPH", +#> module = "GPWG", +#> filename = "PRY_2020_EPH_V01_M_V03_A_GMD_GPWG.dta", +#> collection = "GMD" +#> ) +#> ✔ Skip save (reason: no_change_policy) for +#> ']8;;file://c:/users/wb308892/appdata/local/temp/rtmpu6kxn5/pry_2020_eph_v01_m_v03_a_gmd_gpwg.qs2c:/users/wb308892/appdata/local/temp/rtmpu6kxn5/pry_2020_eph_v01_m_v03_a_gmd_gpwg.qs2]8;;' +``` + +To download a specific version of a dataset, you can provide additional arguments such as `survey`, `vermast`, and `veralt` to `dlw_get_gmd()`. + +``` r +# this downloads version V02 of the EPH survey for Paraguay in 2020 within the GPWG module. +pry20_v2 <- dlw_get_gmd( + country_code = "PRY", + year = 2020, + module = "GPWG", + survey = "EPH", + vermast = "V01", + veralt = "V02" +) +#> +#> ── dlw_get_data Calls ──────────────────────── +#> Call 1: +#> ]8;;x-r-help:dlw::dlw_get_datadlw_get_data]8;;( +#> country_code = "PRY", +#> year = 2020L, +#> server = "GMD", +#> survey = "EPH", +#> module = "GPWG", +#> filename = "PRY_2020_EPH_V01_M_V02_A_GMD_GPWG.dta", +#> collection = "GMD" +#> ) +#> ✔ Saved [qs2] → +#> ']8;;file://c:/users/wb308892/appdata/local/temp/rtmpu6kxn5/pry_2020_eph_v01_m_v02_a_gmd_gpwg.qs2c:/users/wb308892/appdata/local/temp/rtmpu6kxn5/pry_2020_eph_v01_m_v02_a_gmd_gpwg.qs2]8;;' +#> @ version b7d6cceb360513d4 +``` diff --git a/vignettes/dlw_country_catalog.Rmd b/vignettes/dlw_country_catalog.Rmd new file mode 100644 index 0000000..eb35739 --- /dev/null +++ b/vignettes/dlw_country_catalog.Rmd @@ -0,0 +1,558 @@ +--- +title: "Country Catalog" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Country Catalog} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + + + +## Introduction + +The **dlw** package provides granular access to country-specific dataset information from the Datalibweb (DLW) API. While `dlw_server_catalog()` gives you a complete list of all datasets across all countries, `dlw_country_catalog()` provides detailed, country-level metadata for a specific nation. + +This vignette covers: + +- Understanding the difference between server and country catalogs +- Retrieving country-specific catalog information +- Filtering and exploring country-level datasets +- Leveraging caching for performance + +## Prerequisites + +Before using `dlw_country_catalog()`, ensure you have: + +1. Installed the **dlw** package +2. Set your API token using `dlw_set_token()` + + +``` r +# Install dlw (if not already installed) +# devtools::install_github("worldbank/dlw") + +# devtools::load_all(".") +library(dlw) +library(data.table) + +# Set your API token +# dlw_set_token("your_api_token_here") +``` + +## Server Catalog vs. Country Catalog + +Before diving into `dlw_country_catalog()`, understand the difference: + +| Aspect | Server Catalog | Country Catalog | +|--------|---|---| +| **Scope** | All datasets across all countries | Datasets for a single country | +| **Function** | `dlw_server_catalog()` | `dlw_country_catalog()` | +| **Detail Level** | More information | Country-specific info | +| **Use Case** | Overview, across countries | Deep dive into a specific country | +| **Size** | Larger dataset | Smaller, focused dataset | + +**When to use each:** + +Use country catalog function to explore all available data for a particular country, e.g., Colombia + +``` r +country_cat <- dlw_country_catalog("COL") +#> ℹ saving CountryCatalog_COL in .dlwevn +head(country_cat) +#> ServerAlias Country Year Survey +#> +#> 1: GMD COL 2024 GEIH +#> 2: GMD COL 2024 GEIH +#> 3: GMD COL 2023 GEIH +#> 4: GMD COL 2023 GEIH +#> 5: GMD COL 2023 GEIH +#> 6: GMD COL 2023 GEIH +#> FilePath +#> +#> 1: COL/COL_2024_GEIH/COL_2024_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2024_GEIH_V01_M_V01_A_GMD_ALL.dta +#> 2: COL/COL_2024_GEIH/COL_2024_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2024_GEIH_V01_M_V01_A_GMD_GPWG.dta +#> 3: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V02_A_GMD/Data/Harmonized/COL_2023_GEIH_V01_M_V02_A_GMD_ALL.dta +#> 4: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V02_A_GMD/Data/Harmonized/COL_2023_GEIH_V01_M_V02_A_GMD_GPWG.dta +#> 5: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2023_GEIH_v01_M_v01_A_GMD_ALL.dta +#> 6: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2023_GEIH_v01_M_v01_A_GMD_GPWG.dta +#> Ext FileSize Timestamp +#> +#> 1: dta 455.26 MB Feb 20 2026 08:30 PM +#> 2: dta 84.60 MB Feb 20 2026 08:30 PM +#> 3: dta 472.51 MB Feb 23 2026 08:20 PM +#> 4: dta 87.81 MB Feb 23 2026 08:20 PM +#> 5: dta 476.62 MB May 14 2025 07:19 PM +#> 6: dta 89.36 MB May 14 2025 07:19 PM +#> Checksum OnlySol +#> +#> 1: d41d8cd98f00b204e9800998ecf8427e NA +#> 2: d41d8cd98f00b204e9800998ecf8427e NA +#> 3: d41d8cd98f00b204e9800998ecf8427e NA +#> 4: d41d8cd98f00b204e9800998ecf8427e NA +#> 5: 4a7923297982c28cd872714a9efe1228 NA +#> 6: 447774875d1808239da4a4071aa663eb NA +``` + +Use server catalog function to get the list of datasets available across all countries. + +``` r +server_cat <- dlw_server_catalog() +#> ℹ Returning ServerCatalog_GMD from .dlwevn +head(server_cat) +#> ServerAlias Country Year Survey +#> +#> 1: GMD VEN 2006 EHM +#> 2: GMD VEN 2005 EHM +#> 3: GMD VEN 2004 EHM +#> 4: GMD VEN 2003 EHM +#> 5: GMD VEN 2002 EHM +#> 6: GMD VEN 2001 EHM +#> FilePath +#> +#> 1: VEN/VEN_2006_EHM/VEN_2006_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2006_EHM_v01_M_v01_A_GMD_HIST.dta +#> 2: VEN/VEN_2005_EHM/VEN_2005_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2005_EHM_v01_M_v01_A_GMD_HIST.dta +#> 3: VEN/VEN_2004_EHM/VEN_2004_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2004_EHM_v01_M_v01_A_GMD_HIST.dta +#> 4: VEN/VEN_2003_EHM/VEN_2003_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2003_EHM_v01_M_v01_A_GMD_HIST.dta +#> 5: VEN/VEN_2002_EHM/VEN_2002_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2002_EHM_v01_M_v01_A_GMD_HIST.dta +#> 6: VEN/VEN_2001_EHM/VEN_2001_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2001_EHM_v01_M_v01_A_GMD_HIST.dta +#> Ext FileSize Timestamp +#> +#> 1: dta 7.02 MB May 14 2025 08:32 PM +#> 2: dta 6.84 MB May 14 2025 08:31 PM +#> 3: dta 7.01 MB May 14 2025 08:31 PM +#> 4: dta 8.55 MB May 14 2025 08:31 PM +#> 5: dta 10.12 MB May 14 2025 08:31 PM +#> 6: dta 8.63 MB May 14 2025 08:31 PM +#> Checksum OnlySol +#> +#> 1: e11b225b892a75fc66bac8ec1d432603 NA +#> 2: dfe2aa7b1be7c3ad818966ea55e43f86 NA +#> 3: 17fd8ceb4dbcc39c5ab948e771e83273 NA +#> 4: 2ce1a74eaba62e9f99a3a66c22f939e5 NA +#> 5: 2ffd4868918885f4fc2394c55806dd48 NA +#> 6: 608696c2fc617f4af9d6c8ed6303cadd NA +#> FileName +#> +#> 1: VEN_2006_EHM_v01_M_v01_A_GMD_HIST.dta +#> 2: VEN_2005_EHM_v01_M_v01_A_GMD_HIST.dta +#> 3: VEN_2004_EHM_v01_M_v01_A_GMD_HIST.dta +#> 4: VEN_2003_EHM_v01_M_v01_A_GMD_HIST.dta +#> 5: VEN_2002_EHM_v01_M_v01_A_GMD_HIST.dta +#> 6: VEN_2001_EHM_v01_M_v01_A_GMD_HIST.dta +#> Country_code Survey_year Survey_acronym +#> +#> 1: VEN 2006 EHM +#> 2: VEN 2005 EHM +#> 3: VEN 2004 EHM +#> 4: VEN 2003 EHM +#> 5: VEN 2002 EHM +#> 6: VEN 2001 EHM +#> Vermast Veralt Collection Module ext +#> +#> 1: V01 V01 GMD HIST DTA +#> 2: V01 V01 GMD HIST DTA +#> 3: V01 V01 GMD HIST DTA +#> 4: V01 V01 GMD HIST DTA +#> 5: V01 V01 GMD HIST DTA +#> 6: V01 V01 GMD HIST DTA +``` + +## Basic Usage: Retrieve Country Catalog + +The simplest way to get a country catalog is: + + +``` r +# Retrieve the public catalog for Colombia +catalog_col <- dlw_country_catalog(country_code = "COL") +#> ℹ Returning CountryCatalog_COL from .dlwevn + +# View the first few rows +head(catalog_col) +#> ServerAlias Country Year Survey +#> +#> 1: GMD COL 2024 GEIH +#> 2: GMD COL 2024 GEIH +#> 3: GMD COL 2023 GEIH +#> 4: GMD COL 2023 GEIH +#> 5: GMD COL 2023 GEIH +#> 6: GMD COL 2023 GEIH +#> FilePath +#> +#> 1: COL/COL_2024_GEIH/COL_2024_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2024_GEIH_V01_M_V01_A_GMD_ALL.dta +#> 2: COL/COL_2024_GEIH/COL_2024_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2024_GEIH_V01_M_V01_A_GMD_GPWG.dta +#> 3: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V02_A_GMD/Data/Harmonized/COL_2023_GEIH_V01_M_V02_A_GMD_ALL.dta +#> 4: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V02_A_GMD/Data/Harmonized/COL_2023_GEIH_V01_M_V02_A_GMD_GPWG.dta +#> 5: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2023_GEIH_v01_M_v01_A_GMD_ALL.dta +#> 6: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2023_GEIH_v01_M_v01_A_GMD_GPWG.dta +#> Ext FileSize Timestamp +#> +#> 1: dta 455.26 MB Feb 20 2026 08:30 PM +#> 2: dta 84.60 MB Feb 20 2026 08:30 PM +#> 3: dta 472.51 MB Feb 23 2026 08:20 PM +#> 4: dta 87.81 MB Feb 23 2026 08:20 PM +#> 5: dta 476.62 MB May 14 2025 07:19 PM +#> 6: dta 89.36 MB May 14 2025 07:19 PM +#> Checksum OnlySol +#> +#> 1: d41d8cd98f00b204e9800998ecf8427e NA +#> 2: d41d8cd98f00b204e9800998ecf8427e NA +#> 3: d41d8cd98f00b204e9800998ecf8427e NA +#> 4: d41d8cd98f00b204e9800998ecf8427e NA +#> 5: 4a7923297982c28cd872714a9efe1228 NA +#> 6: 447774875d1808239da4a4071aa663eb NA + +# Check the dimensions +dim(catalog_col) +#> [1] 4824 10 +``` + +This returns a `data.table` with detailed metadata about all datasets available for Colombia. + +### Understanding the Catalog Structure + + +``` r +# View column names +colnames(catalog_col) +#> [1] "ServerAlias" "Country" "Year" +#> [4] "Survey" "FilePath" "Ext" +#> [7] "FileSize" "Timestamp" "Checksum" +#> [10] "OnlySol" + +# Check the structure +str(catalog_col) +#> Classes 'data.table' and 'data.frame': 4824 obs. of 10 variables: +#> $ ServerAlias: chr "GMD" "GMD" "GMD" "GMD" ... +#> $ Country : chr "COL" "COL" "COL" "COL" ... +#> $ Year : int 2024 2024 2023 2023 2023 2023 2022 2022 2022 2022 ... +#> $ Survey : chr "GEIH" "GEIH" "GEIH" "GEIH" ... +#> $ FilePath : chr "COL/COL_2024_GEIH/COL_2024_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2024_GEIH_V01_M_V01_A_GMD_ALL.dta" "COL/COL_2024_GEIH/COL_2024_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2024_GEIH_V01_M_V01_A_GMD_GPWG.dta" "COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V02_A_GMD/Data/Harmonized/COL_2023_GEIH_V01_M_V02_A_GMD_ALL.dta" "COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V02_A_GMD/Data/Harmonized/COL_2023_GEIH_V01_M_V02_A_GMD_GPWG.dta" ... +#> $ Ext : chr "dta" "dta" "dta" "dta" ... +#> $ FileSize : chr "455.26 MB" "84.60 MB" "472.51 MB" "87.81 MB" ... +#> $ Timestamp : chr "Feb 20 2026 08:30 PM" "Feb 20 2026 08:30 PM" "Feb 23 2026 08:20 PM" "Feb 23 2026 08:20 PM" ... +#> $ Checksum : chr "d41d8cd98f00b204e9800998ecf8427e" "d41d8cd98f00b204e9800998ecf8427e" "d41d8cd98f00b204e9800998ecf8427e" "d41d8cd98f00b204e9800998ecf8427e" ... +#> $ OnlySol : logi NA NA NA NA NA NA ... +#> - attr(*, ".internal.selfref")= +``` + +Key columns typically include: + +| Column | Type | Description | +|--------|------|-------------| +| **ServerAlias** | character | Server name | +| **Country** | character | ISO3 country code | +| **Year** | integer | Year of the survey | +| **FilePath** | character | Full file path on the server | +| **Ext** | character | File extension | +| **FileSize** | number | File size | +| **Timestamp** | character | Date of data compilation | +| **Checksum** | character | hash | + +## Exploring Country Catalog + +### Get Summary Statistics + + +``` r +# How many surveys for this country? +unique_surveys <- catalog_col[, .(count = .N), by = Survey] +print(unique_surveys) +#> Survey count +#> +#> 1: GEIH 4179 +#> 2: ECH 404 +#> 3: ENH 35 +#> 4: PISA 23 +#> 5: LLECE 6 +#> 6: PIRLS 14 +#> 7: TIMSS 12 +#> 8: ENH-FT 2 +#> 9: ENCV 13 +#> 10: ECV 14 +#> 11: GEIH-Q04 23 +#> 12: GEIH-Q03 31 +#> 13: GEIH-Q02 33 +#> 14: GEIH-Q01 35 + +# Year range of surveys +year_range <- catalog_col[, .( + min_year = min(Year, na.rm = TRUE), + max_year = max(Year, na.rm = TRUE), + num_years = uniqueN(Year) +)] +print(year_range) +#> min_year max_year num_years +#> +#> 1: 1980 2024 32 +``` + +### Filter by Survey Type + + +``` r +# Find all GEIH surveys +geih_surveys <- catalog_col[Survey == "GEIH"] +head(geih_surveys) +#> ServerAlias Country Year Survey +#> +#> 1: GMD COL 2024 GEIH +#> 2: GMD COL 2024 GEIH +#> 3: GMD COL 2023 GEIH +#> 4: GMD COL 2023 GEIH +#> 5: GMD COL 2023 GEIH +#> 6: GMD COL 2023 GEIH +#> FilePath +#> +#> 1: COL/COL_2024_GEIH/COL_2024_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2024_GEIH_V01_M_V01_A_GMD_ALL.dta +#> 2: COL/COL_2024_GEIH/COL_2024_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2024_GEIH_V01_M_V01_A_GMD_GPWG.dta +#> 3: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V02_A_GMD/Data/Harmonized/COL_2023_GEIH_V01_M_V02_A_GMD_ALL.dta +#> 4: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V02_A_GMD/Data/Harmonized/COL_2023_GEIH_V01_M_V02_A_GMD_GPWG.dta +#> 5: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2023_GEIH_v01_M_v01_A_GMD_ALL.dta +#> 6: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2023_GEIH_v01_M_v01_A_GMD_GPWG.dta +#> Ext FileSize Timestamp +#> +#> 1: dta 455.26 MB Feb 20 2026 08:30 PM +#> 2: dta 84.60 MB Feb 20 2026 08:30 PM +#> 3: dta 472.51 MB Feb 23 2026 08:20 PM +#> 4: dta 87.81 MB Feb 23 2026 08:20 PM +#> 5: dta 476.62 MB May 14 2025 07:19 PM +#> 6: dta 89.36 MB May 14 2025 07:19 PM +#> Checksum OnlySol +#> +#> 1: d41d8cd98f00b204e9800998ecf8427e NA +#> 2: d41d8cd98f00b204e9800998ecf8427e NA +#> 3: d41d8cd98f00b204e9800998ecf8427e NA +#> 4: d41d8cd98f00b204e9800998ecf8427e NA +#> 5: 4a7923297982c28cd872714a9efe1228 NA +#> 6: 447774875d1808239da4a4071aa663eb NA + +# Find all surveys from 2010 onwards +recent_surveys <- catalog_col[Year >= 2010] +nrow(recent_surveys) +#> [1] 3801 + +# Find HBS surveys +hbs_surveys <- catalog_col[Survey == "HBS"] +if (nrow(hbs_surveys) > 0) { + print(hbs_surveys) +} else { + cat("No HBS surveys found for this country\n") +} +#> No HBS surveys found for this country +``` + +### Find Latest Versions + + +``` r +# For each survey, find the latest year +latest_by_survey <- catalog_col[, .SD[which.max(Year)], by = Survey] +print(latest_by_survey[, .(Survey, Year)]) +#> Survey Year +#> +#> 1: GEIH 2024 +#> 2: ECH 2005 +#> 3: ENH 2000 +#> 4: PISA 2018 +#> 5: LLECE 2013 +#> 6: PIRLS 2011 +#> 7: TIMSS 2007 +#> 8: ENH-FT 1999 +#> 9: ENCV 2014 +#> 10: ECV 2014 +#> 11: GEIH-Q04 2024 +#> 12: GEIH-Q03 2024 +#> 13: GEIH-Q02 2024 +#> 14: GEIH-Q01 2024 + +# Find the absolute latest survey +latest_overall <- catalog_col[order(-Year)][1] +print(latest_overall) +#> ServerAlias Country Year Survey +#> +#> 1: GMD COL 2024 GEIH +#> FilePath +#> +#> 1: COL/COL_2024_GEIH/COL_2024_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2024_GEIH_V01_M_V01_A_GMD_ALL.dta +#> Ext FileSize Timestamp +#> +#> 1: dta 455.26 MB Feb 20 2026 08:30 PM +#> Checksum OnlySol +#> +#> 1: d41d8cd98f00b204e9800998ecf8427e NA +``` + +## Working with Multiple Countries + +Compare catalogs across different countries: + + +``` r +# Get catalogs for multiple countries +countries <- c("COL", "BRA", "IND", "MEX") + +# Create a list to store catalogs +catalogs <- lapply(countries, dlw_country_catalog) +#> ℹ Returning CountryCatalog_COL from .dlwevn +#> ℹ saving CountryCatalog_BRA in .dlwevn +#> ℹ saving CountryCatalog_IND in .dlwevn +#> ℹ saving CountryCatalog_MEX in .dlwevn +names(catalogs) <- countries + +# Compare dataset counts +dataset_counts <- data.table( + Country = countries, + Datasets = sapply(catalogs, nrow) +) +print(dataset_counts) +#> Country Datasets +#> +#> 1: COL 4824 +#> 2: BRA 4157 +#> 3: IND 2058 +#> 4: MEX 4363 + +# Find common surveys across countries +surveys_by_country <- lapply(catalogs, function(x) unique(x$Survey)) +common_surveys <- Reduce(intersect, surveys_by_country) +cat("Surveys available in all countries:", paste(common_surveys, collapse = ", "), "\n") +#> Surveys available in all countries: PISA + +# Find unique surveys per country +for (country in countries) { + unique_surveys <- setdiff( + unique(catalogs[[country]]$Survey), + common_surveys + ) + if (length(unique_surveys) > 0) { + cat(country, "unique surveys:", paste(unique_surveys, collapse = ", "), "\n") + } +} +#> COL unique surveys: GEIH, ECH, ENH, LLECE, PIRLS, TIMSS, ENH-FT, ENCV, ECV, GEIH-Q04, GEIH-Q03, GEIH-Q02, GEIH-Q01 +#> BRA unique surveys: PNADC-E1, PNADC-E5, PNAD, PNADC, NLA, EMIS, LLECE, POF, PNADC-Q04, PNADC-Q03, PNADC-Q02, PNADC-Q01, PME, PME-Q04, PME-Q03, PME-Q02, PME-Q01, AISH +#> IND unique surveys: HCES, CPHS, NSS-SCH2, NSS-SCH1, NSS, PLFS-URBAN-PANEL, PLFS, PLFS-Urban-Panel-thru-2021, EUS, SARPS, R75252E, R75250H, NSS68-SCH10, NSS68-SCH1.0-T1, NSS66-SCH1.0-T1, NSS61-SCH10, NSS61-SCH1.0, NSS50-SCH10, NSS50-SCH1.0, NLA, EGRA, NSS-SCH10, IHDS-II +#> MEX unique surveys: ENIGHNS, ENIGH, ENOE, ENOEN, LLECE, ENOE-Q03, ENOE-Q02, ENOE-Q01, ENOE-Q04 +``` + +## Understanding Caching + +By default, `dlw_country_catalog()` caches country catalogs in memory during your R session. This improves performance for repeated calls: + + +``` r +# First call: queries the API (slower) +system.time({ + cat1 <- dlw_country_catalog("COL") +}) +#> ℹ Returning CountryCatalog_COL from .dlwevn +#> user system elapsed +#> 0 0 0 + +# Second call: uses cache (much faster) +system.time({ + cat2 <- dlw_country_catalog("COL") +}) +#> ℹ Returning CountryCatalog_COL from .dlwevn +#> user system elapsed +#> 0.00 0.00 0.02 + +# Verify they're identical +identical(cat1, cat2) +#> [1] TRUE + +# Force a fresh API call +system.time({ + cat_fresh <- dlw_country_catalog("COL", force = TRUE) +}) +#> ℹ Returning CountryCatalog_COL from .dlwevn +#> ℹ saving CountryCatalog_COL in .dlwevn +#> user system elapsed +#> 0.15 0.00 0.59 +``` + +### When to Use `force = TRUE` + +- **Development/testing**: When you expect the server catalog to have changed +- **Production workflows**: Occasionally refresh to catch new datasets +- **Debugging**: When troubleshooting connectivity issues +- **Multi-session workflows**: When data may have been updated + + +``` r +# Refresh the catalog from the server +catalog_latest <- dlw_country_catalog( + country_code = "COL", + force = TRUE, + verbose = TRUE +) +#> ℹ Returning CountryCatalog_COL from .dlwevn +#> ℹ saving CountryCatalog_COL in .dlwevn +``` + +## Troubleshooting + +### No Data Returned + +If `dlw_country_catalog()` returns an empty result: + + +``` r +# Check if the country code is valid +catalog <- dlw_country_catalog("COL") +#> ℹ Returning CountryCatalog_COL from .dlwevn +if (nrow(catalog) == 0) { + cat("No datasets found for this country\n") + + # Try refreshing + catalog <- dlw_country_catalog("COL", force = TRUE) +} +``` + +### Authentication Errors + +If you get authentication errors: + + +``` r +# Verify token is set +# token <- dlw_get_token() +# if (is.null(token) || token == "") { + # dlw_set_token("your_api_token_here") +# } + +``` + +### API Connection Issues + +If you cannot reach the server: + + +``` r +# Force refresh to test connection +tryCatch({ + catalog <- dlw_country_catalog("COL", force = TRUE, verbose = TRUE) + cat("Connection successful\n") +}, error = function(e) { + cat("Connection failed:", e$message, "\n") +}) +#> ℹ Returning CountryCatalog_COL from .dlwevn +#> ℹ saving CountryCatalog_COL in .dlwevn +#> Connection successful +``` + +## Related Functions + +- `dlw_server_catalog()` — List of all datasets across all countries +- `dlw_server_inventory()` — Filter server catalog by multiple criteria +- `dlw_get_data()` — Download data +- `dlw_set_token()` — Set API authentication token + +For more information, use: + +```r +?dlw_country_catalog +?dlw_get_data +?dlw_server_catalog +``` diff --git a/vignettes/dlw_server_catalog.Rmd b/vignettes/dlw_server_catalog.Rmd new file mode 100644 index 0000000..54afd28 --- /dev/null +++ b/vignettes/dlw_server_catalog.Rmd @@ -0,0 +1,539 @@ +--- +title: "Server Catalog" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Server Catalog} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + + + +## Introduction + +The **dlw** package provides access to datasets from the Datalibweb (DLW) API. Before downloading data, you need to explore what datasets are available. The `dlw_server_catalog()` function retrieves a comprehensive catalog of all available files on the server, making it easy to discover and filter datasets. + +This vignette walks you through: + +- Retrieving the server catalog +- Understanding the catalog structure +- Filtering and exploring datasets +- Leveraging caching for performance + +## Prerequisites + +Before using `dlw_server_catalog()`, ensure you have: + +1. Installed the **dlw** package +2. Set your API token using `dlw_set_token()` + + +``` r +# Install dlw (if not already installed) +# devtools::install_github("worldbank/dlw") + +# devtools::load_all(".") +library(dlw) +library(data.table) + +# Set your API token +# dlw_set_token("your_api_token_here") +``` + +## Basic Usage: Retrieve the Catalog + +The simplest way to get the server catalog is: + + +``` r +# Retrieve the default (GMD) server catalog list of datasets +catalog <- dlw_server_catalog() +#> ℹ Returning ServerCatalog_GMD from .dlwevn + +# View the first few rows +head(catalog) +#> ServerAlias Country Year Survey +#> +#> 1: GMD VEN 2006 EHM +#> 2: GMD VEN 2005 EHM +#> 3: GMD VEN 2004 EHM +#> 4: GMD VEN 2003 EHM +#> 5: GMD VEN 2002 EHM +#> 6: GMD VEN 2001 EHM +#> FilePath +#> +#> 1: VEN/VEN_2006_EHM/VEN_2006_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2006_EHM_v01_M_v01_A_GMD_HIST.dta +#> 2: VEN/VEN_2005_EHM/VEN_2005_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2005_EHM_v01_M_v01_A_GMD_HIST.dta +#> 3: VEN/VEN_2004_EHM/VEN_2004_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2004_EHM_v01_M_v01_A_GMD_HIST.dta +#> 4: VEN/VEN_2003_EHM/VEN_2003_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2003_EHM_v01_M_v01_A_GMD_HIST.dta +#> 5: VEN/VEN_2002_EHM/VEN_2002_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2002_EHM_v01_M_v01_A_GMD_HIST.dta +#> 6: VEN/VEN_2001_EHM/VEN_2001_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2001_EHM_v01_M_v01_A_GMD_HIST.dta +#> Ext FileSize Timestamp +#> +#> 1: dta 7.02 MB May 14 2025 08:32 PM +#> 2: dta 6.84 MB May 14 2025 08:31 PM +#> 3: dta 7.01 MB May 14 2025 08:31 PM +#> 4: dta 8.55 MB May 14 2025 08:31 PM +#> 5: dta 10.12 MB May 14 2025 08:31 PM +#> 6: dta 8.63 MB May 14 2025 08:31 PM +#> Checksum OnlySol +#> +#> 1: e11b225b892a75fc66bac8ec1d432603 NA +#> 2: dfe2aa7b1be7c3ad818966ea55e43f86 NA +#> 3: 17fd8ceb4dbcc39c5ab948e771e83273 NA +#> 4: 2ce1a74eaba62e9f99a3a66c22f939e5 NA +#> 5: 2ffd4868918885f4fc2394c55806dd48 NA +#> 6: 608696c2fc617f4af9d6c8ed6303cadd NA +#> FileName +#> +#> 1: VEN_2006_EHM_v01_M_v01_A_GMD_HIST.dta +#> 2: VEN_2005_EHM_v01_M_v01_A_GMD_HIST.dta +#> 3: VEN_2004_EHM_v01_M_v01_A_GMD_HIST.dta +#> 4: VEN_2003_EHM_v01_M_v01_A_GMD_HIST.dta +#> 5: VEN_2002_EHM_v01_M_v01_A_GMD_HIST.dta +#> 6: VEN_2001_EHM_v01_M_v01_A_GMD_HIST.dta +#> Country_code Survey_year Survey_acronym +#> +#> 1: VEN 2006 EHM +#> 2: VEN 2005 EHM +#> 3: VEN 2004 EHM +#> 4: VEN 2003 EHM +#> 5: VEN 2002 EHM +#> 6: VEN 2001 EHM +#> Vermast Veralt Collection Module ext +#> +#> 1: V01 V01 GMD HIST DTA +#> 2: V01 V01 GMD HIST DTA +#> 3: V01 V01 GMD HIST DTA +#> 4: V01 V01 GMD HIST DTA +#> 5: V01 V01 GMD HIST DTA +#> 6: V01 V01 GMD HIST DTA +``` + +This returns a `data.table` with metadata about all `.dta` files available on the server. + +### What's in the Catalog? + +The catalog contains the following key columns: + + +``` r +# View column names +colnames(catalog) +#> [1] "ServerAlias" "Country" +#> [3] "Year" "Survey" +#> [5] "FilePath" "Ext" +#> [7] "FileSize" "Timestamp" +#> [9] "Checksum" "OnlySol" +#> [11] "FileName" "Country_code" +#> [13] "Survey_year" "Survey_acronym" +#> [15] "Vermast" "Veralt" +#> [17] "Collection" "Module" +#> [19] "ext" + +# Check the structure +str(catalog) +#> Classes 'data.table' and 'data.frame': 11906 obs. of 19 variables: +#> $ ServerAlias : chr "GMD" "GMD" "GMD" "GMD" ... +#> $ Country : chr "VEN" "VEN" "VEN" "VEN" ... +#> $ Year : int 2006 2005 2004 2003 2002 2001 1999 1998 1995 1992 ... +#> $ Survey : chr "EHM" "EHM" "EHM" "EHM" ... +#> $ FilePath : chr "VEN/VEN_2006_EHM/VEN_2006_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2006_EHM_v01_M_v01_A_GMD_HIST.dta" "VEN/VEN_2005_EHM/VEN_2005_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2005_EHM_v01_M_v01_A_GMD_HIST.dta" "VEN/VEN_2004_EHM/VEN_2004_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2004_EHM_v01_M_v01_A_GMD_HIST.dta" "VEN/VEN_2003_EHM/VEN_2003_EHM_v01_M_v01_A_GMD/Data/Harmonized/VEN_2003_EHM_v01_M_v01_A_GMD_HIST.dta" ... +#> $ Ext : chr "dta" "dta" "dta" "dta" ... +#> $ FileSize : chr "7.02 MB" "6.84 MB" "7.01 MB" "8.55 MB" ... +#> $ Timestamp : chr "May 14 2025 08:32 PM" "May 14 2025 08:31 PM" "May 14 2025 08:31 PM" "May 14 2025 08:31 PM" ... +#> $ Checksum : chr "e11b225b892a75fc66bac8ec1d432603" "dfe2aa7b1be7c3ad818966ea55e43f86" "17fd8ceb4dbcc39c5ab948e771e83273" "2ce1a74eaba62e9f99a3a66c22f939e5" ... +#> $ OnlySol : logi NA NA NA NA NA NA ... +#> $ FileName : chr "VEN_2006_EHM_v01_M_v01_A_GMD_HIST.dta" "VEN_2005_EHM_v01_M_v01_A_GMD_HIST.dta" "VEN_2004_EHM_v01_M_v01_A_GMD_HIST.dta" "VEN_2003_EHM_v01_M_v01_A_GMD_HIST.dta" ... +#> $ Country_code : chr "VEN" "VEN" "VEN" "VEN" ... +#> $ Survey_year : chr "2006" "2005" "2004" "2003" ... +#> $ Survey_acronym: chr "EHM" "EHM" "EHM" "EHM" ... +#> $ Vermast : chr "V01" "V01" "V01" "V01" ... +#> $ Veralt : chr "V01" "V01" "V01" "V01" ... +#> $ Collection : chr "GMD" "GMD" "GMD" "GMD" ... +#> $ Module : chr "HIST" "HIST" "HIST" "HIST" ... +#> $ ext : chr "DTA" "DTA" "DTA" "DTA" ... +#> - attr(*, ".internal.selfref")= +#> - attr(*, "index")= int(0) +#> ..- attr(*, "__Survey")= int [1:11906] 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 ... +#> ..- attr(*, "__Module")= int [1:11906] 42 46 47 51 52 56 57 58 64 65 ... +#> ..- attr(*, "__Year")= int [1:11906] 296 292 293 294 295 288 289 290 291 284 ... +#> ..- attr(*, "__Year__Country")= int [1:11906] 296 292 293 294 295 288 289 290 291 284 ... +``` + +Key columns include: + +| Column | Description | +|--------|-------------| +| **FileName** | The file name (e.g., `COL_2022_GEIH_v01_M_v02_A_GMD_GPWG.dta`) | +| **Country_code** | ISO3 country code (e.g., `COL`, `IND`, `BRA`) | +| **Survey_year** | Year of the survey (integer) | +| **Survey_acronym** | Survey acronym (e.g., `ECH`, `ENH`, `GEIH`, `HBS`) | +| **Vermast** | Master version (e.g., `V01`, `V02`) | +| **Veralt** | Alternative version (e.g., `V01`, `V02`) | +| **Collection** | Collection name (e.g., `GMD`) | +| **Module** | Module name (e.g., `ALL`, `GPWG`, `ASPIRE`, `HIST`, `GROUP`, `BIN`) | + +## Exploring the Catalog + +### Count Datasets by Country + + +``` r +# How many datasets per country? +catalog[, .(count = .N), by = Country_code][order(-count)] +#> Country_code count +#> +#> 1: 408 +#> 2: PER 367 +#> 3: CRI 317 +#> 4: URY 316 +#> 5: SLV 299 +#> --- +#> 172: TTO 2 +#> 173: GRD 2 +#> 174: BRB 2 +#> 175: TKM 1 +#> 176: QAT 1 +``` + +### Find All Surveys for a Specific Country + + +``` r +# Get all surveys for Colombia +colombia <- catalog[Country_code == "COL"] + +# View unique surveys and years +colombia[, .(count = .N), by = .(Survey_acronym, Survey_year)][order(Survey_year)] +#> Survey_acronym Survey_year count +#> +#> 1: ENH 1980 1 +#> 2: ENH 1988 1 +#> 3: ENH 1989 1 +#> 4: ENH 1991 1 +#> 5: ENH 1992 1 +#> 6: ENH 1996 1 +#> 7: ENH 1999 1 +#> 8: ENH 2000 1 +#> 9: ECH 2001 7 +#> 10: ECH 2002 9 +#> 11: ECH 2003 7 +#> 12: ECH 2004 7 +#> 13: ECH 2005 7 +#> 14: GEIH 2008 13 +#> 15: GEIH 2009 11 +#> 16: GEIH 2010 11 +#> 17: GEIH 2011 11 +#> 18: GEIH 2012 12 +#> 19: GEIH 2013 9 +#> 20: GEIH 2014 11 +#> 21: GEIH 2015 11 +#> 22: GEIH 2016 8 +#> 23: GEIH 2017 8 +#> 24: GEIH 2018 13 +#> 25: GEIH 2019 9 +#> 26: GEIH 2020 8 +#> 27: GEIH 2021 10 +#> 28: GEIH 2022 6 +#> 29: GEIH 2023 4 +#> 30: GEIH 2024 2 +#> Survey_acronym Survey_year count +#> +``` + +### Find the Latest Version of a Dataset + + +``` r +# Find the most recent survey for Colombia +latest_col <- catalog[Country_code == "COL"][order(-Survey_year)][1] + +# View details +latest_col[, .(Country_code, Survey_year, Survey_acronym, Vermast, Veralt, FileName)] +#> Country_code Survey_year Survey_acronym +#> +#> 1: COL 2024 GEIH +#> Vermast Veralt +#> +#> 1: V01 V01 +#> FileName +#> +#> 1: COL_2024_GEIH_V01_M_V01_A_GMD_ALL.dta +``` + +### Filter by Multiple Criteria + + +``` r +# Find all GEIH surveys from 2010 onwards +geih_recent <- catalog[ + Survey_acronym == "GEIH" & Survey_year >= 2010 +][order(Country_code, -Survey_year)] + +head(geih_recent) +#> ServerAlias Country Year Survey +#> +#> 1: GMD COL 2024 GEIH +#> 2: GMD COL 2024 GEIH +#> 3: GMD COL 2023 GEIH +#> 4: GMD COL 2023 GEIH +#> 5: GMD COL 2023 GEIH +#> 6: GMD COL 2023 GEIH +#> FilePath +#> +#> 1: COL/COL_2024_GEIH/COL_2024_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2024_GEIH_V01_M_V01_A_GMD_ALL.dta +#> 2: COL/COL_2024_GEIH/COL_2024_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2024_GEIH_V01_M_V01_A_GMD_GPWG.dta +#> 3: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V02_A_GMD/Data/Harmonized/COL_2023_GEIH_V01_M_V02_A_GMD_ALL.dta +#> 4: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V02_A_GMD/Data/Harmonized/COL_2023_GEIH_V01_M_V02_A_GMD_GPWG.dta +#> 5: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2023_GEIH_v01_M_v01_A_GMD_ALL.dta +#> 6: COL/COL_2023_GEIH/COL_2023_GEIH_V01_M_V01_A_GMD/Data/Harmonized/COL_2023_GEIH_v01_M_v01_A_GMD_GPWG.dta +#> Ext FileSize Timestamp +#> +#> 1: dta 455.26 MB Feb 20 2026 08:30 PM +#> 2: dta 84.60 MB Feb 20 2026 08:30 PM +#> 3: dta 472.51 MB Feb 23 2026 08:20 PM +#> 4: dta 87.81 MB Feb 23 2026 08:20 PM +#> 5: dta 476.62 MB May 14 2025 07:19 PM +#> 6: dta 89.36 MB May 14 2025 07:19 PM +#> Checksum OnlySol +#> +#> 1: d41d8cd98f00b204e9800998ecf8427e NA +#> 2: d41d8cd98f00b204e9800998ecf8427e NA +#> 3: d41d8cd98f00b204e9800998ecf8427e NA +#> 4: d41d8cd98f00b204e9800998ecf8427e NA +#> 5: 4a7923297982c28cd872714a9efe1228 NA +#> 6: 447774875d1808239da4a4071aa663eb NA +#> FileName +#> +#> 1: COL_2024_GEIH_V01_M_V01_A_GMD_ALL.dta +#> 2: COL_2024_GEIH_V01_M_V01_A_GMD_GPWG.dta +#> 3: COL_2023_GEIH_V01_M_V02_A_GMD_ALL.dta +#> 4: COL_2023_GEIH_V01_M_V02_A_GMD_GPWG.dta +#> 5: COL_2023_GEIH_v01_M_v01_A_GMD_ALL.dta +#> 6: COL_2023_GEIH_v01_M_v01_A_GMD_GPWG.dta +#> Country_code Survey_year Survey_acronym +#> +#> 1: COL 2024 GEIH +#> 2: COL 2024 GEIH +#> 3: COL 2023 GEIH +#> 4: COL 2023 GEIH +#> 5: COL 2023 GEIH +#> 6: COL 2023 GEIH +#> Vermast Veralt Collection Module ext +#> +#> 1: V01 V01 GMD ALL DTA +#> 2: V01 V01 GMD GPWG DTA +#> 3: V01 V02 GMD ALL DTA +#> 4: V01 V02 GMD GPWG DTA +#> 5: V01 V01 GMD ALL DTA +#> 6: V01 V01 GMD GPWG DTA +``` + +### Search by Module or Collection + + +``` r +# Find all datasets containing GPWG Module +gpwg_data <- catalog[grepl("GPWG", Module, ignore.case = TRUE)] + +nrow(gpwg_data) # How many? +#> [1] 4975 + +# View unique modules +catalog[, .(count = .N), by = Module][order(-count)] +#> Module count +#> +#> 1: GPWG 4975 +#> 2: ALL 3787 +#> 3: BIN 1486 +#> 4: 408 +#> 5: ASPIRE 366 +#> 6: GROUP 176 +#> 7: L 155 +#> 8: HIST 118 +#> 9: H3 101 +#> 10: LOC 101 +#> 11: SPAT 101 +#> 12: SSAPOV 36 +#> 13: CPIICP 14 +#> 14: CPI 13 +#> 15: INE 13 +#> 16: KEY 13 +#> 17: MPM 13 +#> 18: POV 13 +#> 19: SP 13 +#> 20: SARMD 2 +#> 21: NAT 1 +#> 22: SEDLAC 1 +#> Module count +#> +``` + +## Using `dlw_server_inventory()` for Easier Filtering + +While `dlw_server_catalog()` returns the full catalog, `dlw_server_inventory()` provides a convenient wrapper for filtering: + + +``` r +# Filter by country and year +inv <- dlw_server_inventory("COL", year = 2010) + +# This is equivalent to: +inv_manual <- catalog[Country_code == "COL" & Year == 2010] + +# They should be identical +identical(inv, inv_manual) +#> [1] TRUE +``` + +## Understanding Caching + +By default, `dlw_server_catalog()` caches the catalog in memory during your R session. This improves performance for repeated calls: + + +``` r +# First call: queries the API (slower) +system.time({ + cat1 <- dlw_server_catalog() +}) +#> ℹ Returning ServerCatalog_GMD from .dlwevn +#> user system elapsed +#> 0.01 0.00 0.00 + +# Second call: uses cache (much faster) +system.time({ + cat2 <- dlw_server_catalog() +}) +#> ℹ Returning ServerCatalog_GMD from .dlwevn +#> user system elapsed +#> 0 0 0 + +# Force a fresh API call +system.time({ + cat_fresh <- dlw_server_catalog(force = TRUE) +}) +#> ℹ Returning ServerCatalog_GMD from .dlwevn +#> ℹ saving ServerCatalog_GMD in .dlwevn +#> user system elapsed +#> 0.55 0.05 1.92 +``` + +### When to Use `force = TRUE` + +- **Development/testing**: When you expect the server catalog to have changed +- **Production workflows**: Occasionally refresh to catch new datasets +- **Debugging**: When troubleshooting connectivity or data issues + + +``` r +# Refresh the catalog from the server +catalog_latest <- dlw_server_catalog(force = TRUE, verbose = TRUE) +#> ℹ Returning ServerCatalog_GMD from .dlwevn +#> ℹ saving ServerCatalog_GMD in .dlwevn +``` + +## Advanced: Understanding Filename Structure + +The dlw package follows a strict naming convention for GMD files. Understanding this helps you work with datasets programmatically. + +### Filename Pattern + +The general pattern is: + +``` +___V_M_V_A__.dta +``` + +**Example:** `COL_2022_GEIH_V01_M_V02_A_GMD_GPWG.dta` + +**Breakdown:** + +- **COL**: Country code +- **2022**: Survey year +- **GEIH**: Survey acronym +- **V01**: Master version +- **M**: Marker (always present) +- **V02**: Alternative version +- **A**: Marker (always present) +- **GMD**: Collection +- **GPWG**: Module +- **.dta**: File format (Stata) + +### Accessing Parsed Components + +The catalog automatically parses this structure into columns: + + +``` r +# Extract key components for a file +file_info <- catalog[FileName == "COL_2022_GEIH_V01_M_V02_A_GMD_GPWG.dta"] + +data.frame( + Country = file_info$Country_code, + Year = file_info$Survey_year, + Acronym = file_info$Survey_acronym, + MasterVersion = file_info$Vermast, + AltVersion = file_info$Veralt, + Collection = file_info$Collection, + Module = file_info$Module) +#> Country Year Acronym MasterVersion +#> 1 COL 2022 GEIH V01 +#> AltVersion Collection Module +#> 1 V02 GMD GPWG +``` + +### Downloading Data + +See `?dlw_get_data()` for more download options. + +## Best Practices + +1. **Cache wisely**: Use the default caching behavior for interactive work; use `force = TRUE` in automated scripts. +2. **Validate filenames**: Always check that your filtered dataset contains the expected files before downloading. +3. **Use descriptive filters**: Prefer filtering by country, year, and survey acronym rather than searching by module names. +4. **Handle errors gracefully**: Wrap `dlw_get_data()` calls in `tryCatch()` for production workflows. + +## Troubleshooting + +### No Datasets Returned + +If your filter returns zero rows, verify: + +- Country code is correct (use 3-letter ISO code) +- Survey year exists in the catalog +- Survey acronym is spelled correctly + +### API Authentication Issues + +If you get authentication errors: + +1. Verify your token with `dlw_get_token()` +2. Re-set your token with `dlw_set_token()` +3. Try `dlw_server_catalog(force = TRUE)` to refresh + + +## Summary + +The `dlw_server_catalog()` function provides a powerful way to: + +- Discover available datasets +- Filter by country, year, survey, and topic +- Understand dataset structure and versions +- Prepare for data downloads + +### Related Functions + +- `dlw_server_catalog()` — Retrieve full server catalog +- `dlw_server_inventory()` — Convenient filtering wrapper +- `dlw_get_data()` — Download data using catalog filenames +- `dlw_set_token()` — Set API authentication token +- `dlw_get_token()` — Retrieve current API token + +For more information, use: + +```r +?dlw_server_catalog +?dlw_server_inventory +?dlw_get_data +``` From 91cd89145aa2d89c74bb51017b3fac830aeeeb86 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Thu, 5 Mar 2026 14:19:33 -0500 Subject: [PATCH 17/24] update pkgdown.yaml --- .github/workflows/pkgdown.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index 6feca47..1f68ed3 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -25,11 +25,6 @@ jobs: permissions: contents: write steps: - - uses: insightsengineering/r-pkgdown-multiversion@v2 - with: - default-landing-page: "master" - branches-or-tags-to-list: "master|dev" - - uses: actions/checkout@v4 - uses: r-lib/actions/setup-pandoc@v2 @@ -47,6 +42,11 @@ jobs: run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE) shell: Rscript {0} + - uses: insightsengineering/r-pkgdown-multiversion@v2 + with: + default-landing-page: "master" + branches-or-tags-to-list: "master|dev" + - name: Deploy to GitHub pages 🚀 if: github.event_name != 'pull_request' uses: JamesIves/github-pages-deploy-action@v4.5.0 From 4ce9cb961ab6452e574f8b7b132f6db4daf8ecce Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Thu, 5 Mar 2026 14:32:35 -0500 Subject: [PATCH 18/24] update pkgdown yaml --- .github/workflows/pkgdown.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index 1f68ed3..248eab7 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -35,8 +35,8 @@ jobs: - uses: r-lib/actions/setup-r-dependencies@v2 with: - extra-packages: any::pkgdown, local::. - needs: website-r-dependencies@v2` action handles system dependencies automatically. + extra-packages: any::pkgdown + needs: website - name: Build site run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE) @@ -44,8 +44,7 @@ jobs: - uses: insightsengineering/r-pkgdown-multiversion@v2 with: - default-landing-page: "master" - branches-or-tags-to-list: "master|dev" + refs-to-list: "main|dev" - name: Deploy to GitHub pages 🚀 if: github.event_name != 'pull_request' From 208276e9734adcddcb4c69793e5204e566b8d10a Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Thu, 5 Mar 2026 14:38:15 -0500 Subject: [PATCH 19/24] fix yaml --- .github/workflows/pkgdown.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index 248eab7..52f9153 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -44,7 +44,8 @@ jobs: - uses: insightsengineering/r-pkgdown-multiversion@v2 with: - refs-to-list: "main|dev" + default-landing-page: "main" + branches-or-tags-to-list: "main|dev" - name: Deploy to GitHub pages 🚀 if: github.event_name != 'pull_request' From ae77278f748abfdf55019aca8305ca7a1b488ce3 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Fri, 13 Mar 2026 09:33:20 -0400 Subject: [PATCH 20/24] yaml file uupdated --- .github/workflows/pkgdown.yaml | 15 ++++++++------- _pkgdown.yml | 4 +--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index 52f9153..7a948b4 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -1,5 +1,6 @@ # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help + on: push: branches: [main, master, dev] @@ -8,7 +9,6 @@ on: branches: [main, master, dev] release: types: [published] - workflow_dispatch: name: pkgdown.yaml @@ -17,7 +17,6 @@ permissions: read-all jobs: pkgdown: runs-on: ubuntu-latest - # Only restrict concurrency for non-PR jobs concurrency: group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }} env: @@ -26,6 +25,8 @@ jobs: contents: write steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: r-lib/actions/setup-pandoc@v2 @@ -42,11 +43,6 @@ jobs: run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE) shell: Rscript {0} - - uses: insightsengineering/r-pkgdown-multiversion@v2 - with: - default-landing-page: "main" - branches-or-tags-to-list: "main|dev" - - name: Deploy to GitHub pages 🚀 if: github.event_name != 'pull_request' uses: JamesIves/github-pages-deploy-action@v4.5.0 @@ -54,3 +50,8 @@ jobs: clean: false branch: gh-pages folder: docs + + - uses: insightsengineering/r-pkgdown-multiversion@v2 + with: + default-landing-page: "main" + branches-or-tags-to-list: "main|dev" diff --git a/_pkgdown.yml b/_pkgdown.yml index c094d33..8ca4cba 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -2,9 +2,7 @@ url: https://worldbank.github.io/dlw/dev/ template: bootstrap: 5 - bootswatch: materia - bslib: - pkgdown-nav-height: 56px + bootswatch: cosmo reference: - title: Configuration and Utility From ae331d8e412004e4c95b7b5676a4029b8d41b0f0 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Fri, 13 Mar 2026 10:12:14 -0400 Subject: [PATCH 21/24] fix issues in yaml file --- .github/workflows/pkgdown.yaml | 18 +++++------------- _pkgdown.yml | 2 +- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index 7a948b4..bfc9f4d 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -1,14 +1,12 @@ # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help - on: push: - branches: [main, master, dev] - tags: ['*'] + branches: [main, master] pull_request: - branches: [main, master, dev] release: types: [published] + workflow_dispatch: name: pkgdown.yaml @@ -17,6 +15,7 @@ permissions: read-all jobs: pkgdown: runs-on: ubuntu-latest + # Only restrict concurrency for non-PR jobs concurrency: group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }} env: @@ -25,18 +24,16 @@ jobs: contents: write steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - uses: r-lib/actions/setup-pandoc@v2 - uses: r-lib/actions/setup-r@v2 with: - r-version: 'release' + use-public-rspm: true - uses: r-lib/actions/setup-r-dependencies@v2 with: - extra-packages: any::pkgdown + extra-packages: any::pkgdown, local::. needs: website - name: Build site @@ -50,8 +47,3 @@ jobs: clean: false branch: gh-pages folder: docs - - - uses: insightsengineering/r-pkgdown-multiversion@v2 - with: - default-landing-page: "main" - branches-or-tags-to-list: "main|dev" diff --git a/_pkgdown.yml b/_pkgdown.yml index 8ca4cba..9f53313 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -1,4 +1,4 @@ -url: https://worldbank.github.io/dlw/dev/ +url: https://worldbank.github.io/dlw/ template: bootstrap: 5 From 835c78775c01ac13bf5c617528c0c41647c72c86 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Fri, 13 Mar 2026 12:05:54 -0400 Subject: [PATCH 22/24] README.md generated --- README.Rmd | 2 +- README.html | 737 ++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 39 ++- 3 files changed, 763 insertions(+), 15 deletions(-) create mode 100644 README.html diff --git a/README.Rmd b/README.Rmd index f78faf6..cc04652 100644 --- a/README.Rmd +++ b/README.Rmd @@ -1,7 +1,7 @@ --- output: github_document --- - +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/worldbank/dlw) ```{r, include = FALSE} diff --git a/README.html b/README.html new file mode 100644 index 0000000..17fbdf6 --- /dev/null +++ b/README.html @@ -0,0 +1,737 @@ + + + + + + + + + + + + + + + + + + + +

Ask DeepWiki

+ + +

dlw

+ + + + +

{dlw} is an R client of the internal datalibweb API of the +World Bank. Datalibweb is a data system designed to enable users to +seamlessly access microdata and documentation in the World Bank. Users +can access the most up-to-date and historical versions of harmonized +data collections and raw/non-harmonized data for subsequent analysis

+

Installation

+

Since {dlw} is hosted in the World Bank’s GitHub +organization, you need to authorize your Personal Access Token (PAT) for +SAML Single Sign-On (SSO) before you can install it. Follow these steps +to authorize your PAT:

+
    +

  1. Go to your GitHub account: Visit https://github.com and log in.

    2. +

  2. Check your PAT: Go to Settings → Developer settings → Personal +access tokens. Find the token you are using (or create a new one if +needed).

    3. +

  3. Authorize your PAT for SAML SSO: Go to https://github.com/settings/tokens. +Next to your token, You’ll see a button that says “Configure SSO.” Click +it to authorize your token to the worldbank.

    4. +

  4. Try installing

    5. +
+

You can install the development version of dlw from GitHub with:

+
# install.packages("pak")
+pak::pak("worldbank/dlw")
+# or
+remotes::github_install("worldbank/dlw")
+

Usage

+
library(dlw)
+

Token

+

To begin working with {dlw}, you must provide your +datalibweb API token using the dlw_set_token() +function.

+

You can obtain or renew your token by visiting the datalibweb page and +following the instructions there. Once you have your token, set it in +your R session as follows:

+
dlw_set_token("your_token_here")
+

server catalob

+

After setting your token, you can download the catalog for the +corresponding server using dlw_server_catalog(). By +default, this downloads the “GMD” catalog. The downloaded catalog is +saved for the current session in a hidden environment within the +{dlw} package, allowing for easy and efficient access in +subsequent operations.

+
ctl <- dlw_server_catalog()
+#> Pruning
+#> cache
+#> ℹ saving ServerCatalog_GMD in .dlwevn
+

The {dlw} package returns results as +data.table objects, enabling lightning-fast data +manipulation and filtering using concise syntax. For example, you can +instantly list all available files for Colombia in 2010 in module “ALL” +of the GMD collection with a single line of code:

+
ctl[Country_code == "COL" & Module == "ALL" & Survey_year == 2010, 
+  .(FileName, Vermast, Veralt )]
+#>                                 FileName Vermast Veralt
+#>                                   <char>  <char> <char>
+#> 1: COL_2010_GEIH_V04_M_V01_A_GMD_ALL.dta     V04    V01
+#> 2: COL_2010_GEIH_V02_M_V09_A_GMD_ALL.dta     V02    V09
+#> 3: COL_2010_GEIH_V02_M_V08_A_GMD_ALL.dta     V02    V08
+#> 4: COL_2010_GEIH_v02_M_v07_A_GMD_ALL.dta     V02    V07
+

Downloading files

+

The workhorse function to download data is +dlw_get_data(). However, it requires several pieces of +information that you may not have at hand, such as:

+
dlw_get_data(
+  country_code = "PRY",
+  year = 2011L,
+  server = "GMD",
+  survey = "EPH",
+  module = "GPWG",
+  filename = "PRY_2011_EPH_V01_M_V03_A_GMD_GPWG.dta",
+  collection = "GMD"
+)
+

To simplify this process, we have developed a wrapper function that +works only for the GMD server: dlw_get_gmd(). This function +is much easier to use:

+
pry <- dlw_get_gmd(country_code = "PRY", year = 2011, module = "GPWG", vermast = "v01", veralt = "v03")
+#> 
+#> ── dlw_get_
+#> Call 1:
+#> dlw_get_data(
+#>   country_code = "PRY",
+#>   year = 2011L,
+#>   server = "GMD",
+#>   survey = "EPH",
+#>   module = "GPWG",
+#>   filename = "PRY_2011_EPH_V01_M_V03_A_GMD_GPWG.dta",
+#>   collection = "GMD"
+#> )
+#> ✔ stamp
+#>   initialized
+#>   alias:
+#>   dlw
+#>   root:
+#>   C:/Users/WB308892/AppData/Local/Temp/RtmpEHCrN4
+#>   state:
+#>   C:/Users/WB308892/AppData/Local/Temp/RtmpEHCrN4/.stamp
+#> ✔ Saved [qs2] →
+#>   'c:/users/wb308892/appdata/local/temp/rtmpehcrn4/pry_2011_eph_v01_m_v03_a_gmd_gpwg.qs2'
+#>   @ version dfe89d4cd3898bd7
+pry[, weighted.mean(welfare, weight, na.rm = TRUE)]
+#> [1] 1099527
+

If you are interested in downloading the most recent version of a +file, you can simply omit the version arguments:

+
pry <- dlw_get_gmd(country_code = "PRY", year = 2011, module = "GPWG")
+#> 
+#> ── dlw_get_data Calls ───────────────
+#> Call 1:
+#> dlw_get_data(
+#>   country_code = "PRY",
+#>   year = 2011L,
+#>   server = "GMD",
+#>   survey = "EPH",
+#>   module = "GPWG",
+#>   filename = "PRY_2011_EPH_V02_M_V01_A_GMD_GPWG.dta",
+#>   collection = "GMD"
+#> )
+#> ✔ Saved [qs2] →
+#>   'c:/users/wb308892/appdata/local/temp/rtmpehcrn4/pry_2011_eph_v02_m_v01_a_gmd_gpwg.qs2'
+#>   @ version 3eb405fd0fc580bd
+pry[, weighted.mean(welfare, weight, na.rm = TRUE)]
+#> [1] 12675293
+

In this case, the most recent version available will be used (for +example, master version v02 and alternative version v01).

+ + + diff --git a/README.md b/README.md index 3df8a5e..b3bf3ca 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,12 @@ +[![Ask +DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/worldbank/dlw) # dlw + `{dlw}` is an R client of the internal @@ -72,7 +75,8 @@ allowing for easy and efficient access in subsequent operations. ``` r ctl <- dlw_server_catalog() -#> Pruning cache +#> Pruning +#> cache #> ℹ saving ServerCatalog_GMD in .dlwevn ``` @@ -86,9 +90,10 @@ ctl[Country_code == "COL" & Module == "ALL" & Survey_year == 2010, .(FileName, Vermast, Veralt )] #> FileName Vermast Veralt #> -#> 1: COL_2010_GEIH_V02_M_V09_A_GMD_ALL.dta V02 V09 -#> 2: COL_2010_GEIH_V02_M_V08_A_GMD_ALL.dta V02 V08 -#> 3: COL_2010_GEIH_v02_M_v07_A_GMD_ALL.dta v02 v07 +#> 1: COL_2010_GEIH_V04_M_V01_A_GMD_ALL.dta V04 V01 +#> 2: COL_2010_GEIH_V02_M_V09_A_GMD_ALL.dta V02 V09 +#> 3: COL_2010_GEIH_V02_M_V08_A_GMD_ALL.dta V02 V08 +#> 4: COL_2010_GEIH_v02_M_v07_A_GMD_ALL.dta V02 V07 ``` ### Downloading files @@ -116,7 +121,7 @@ easier to use: ``` r pry <- dlw_get_gmd(country_code = "PRY", year = 2011, module = "GPWG", vermast = "v01", veralt = "v03") #> -#> ── dlw_get_data Calls ────────────────────────────────────────────────────────── +#> ── dlw_get_ #> Call 1: #> dlw_get_data( #> country_code = "PRY", @@ -127,10 +132,17 @@ pry <- dlw_get_gmd(country_code = "PRY", year = 2011, module = "GPWG", vermast = #> filename = "PRY_2011_EPH_V01_M_V03_A_GMD_GPWG.dta", #> collection = "GMD" #> ) -#> ℹ saving last_req in .dlwevn -#> ℹ saving last_raw_data in .dlwevn -#> Creating new version '20250617T202722Z-662d2' -#> Writing to pin 'PRY_2011_EPH_V01_M_V03_A_GMD_GPWG.parquet' +#> ✔ stamp +#> initialized +#> alias: +#> dlw +#> root: +#> C:/Users/WB308892/AppData/Local/Temp/RtmpEHCrN4 +#> state: +#> C:/Users/WB308892/AppData/Local/Temp/RtmpEHCrN4/.stamp +#> ✔ Saved [qs2] → +#> 'c:/users/wb308892/appdata/local/temp/rtmpehcrn4/pry_2011_eph_v01_m_v03_a_gmd_gpwg.qs2' +#> @ version dfe89d4cd3898bd7 pry[, weighted.mean(welfare, weight, na.rm = TRUE)] #> [1] 1099527 ``` @@ -141,7 +153,7 @@ you can simply omit the version arguments: ``` r pry <- dlw_get_gmd(country_code = "PRY", year = 2011, module = "GPWG") #> -#> ── dlw_get_data Calls ────────────────────────────────────────────────────────── +#> ── dlw_get_data Calls ─────────────── #> Call 1: #> dlw_get_data( #> country_code = "PRY", @@ -152,10 +164,9 @@ pry <- dlw_get_gmd(country_code = "PRY", year = 2011, module = "GPWG") #> filename = "PRY_2011_EPH_V02_M_V01_A_GMD_GPWG.dta", #> collection = "GMD" #> ) -#> ℹ saving last_req in .dlwevn -#> ℹ saving last_raw_data in .dlwevn -#> Creating new version '20250617T202725Z-6bd54' -#> Writing to pin 'PRY_2011_EPH_V02_M_V01_A_GMD_GPWG.parquet' +#> ✔ Saved [qs2] → +#> 'c:/users/wb308892/appdata/local/temp/rtmpehcrn4/pry_2011_eph_v02_m_v01_a_gmd_gpwg.qs2' +#> @ version 3eb405fd0fc580bd pry[, weighted.mean(welfare, weight, na.rm = TRUE)] #> [1] 12675293 ``` From f7c4f1bc4d2af8b8b7406ad0ff0f4edd7534873f Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Sun, 15 Mar 2026 20:25:12 -0400 Subject: [PATCH 23/24] unit test for dlw_get_gmd with stamp --- tests/testthat/test-dlw_get_gmd.R | 87 +++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/testthat/test-dlw_get_gmd.R diff --git a/tests/testthat/test-dlw_get_gmd.R b/tests/testthat/test-dlw_get_gmd.R new file mode 100644 index 0000000..e9c0ba0 --- /dev/null +++ b/tests/testthat/test-dlw_get_gmd.R @@ -0,0 +1,87 @@ +has_token <- tryCatch( + { + dlw::dlw_check_token() + TRUE + }, + error = function(e) FALSE +) + +skip_if_no_token <- function() { + if (!has_token) skip("No DLW token available") +} + +test_that("dlw_get_gmd downloads, saves, and loads data from temp folder", { + skip_if_no_token() + + # Create and initialize temp folder + tempfld <- fs::path_temp("prj-temp-fld") + fs::dir_create(tempfld) + stamp::st_init(tempfld, alias = "dlw") + + # Download and save data + result <- dlw_get_gmd( + country_code = "COL", + year = 2010, + survey = "GEIH", + module = "GPWG", + vermast = "v02", + veralt = "v09", + local_dir = tempfld + ) + + expected_file <- fs::path(tempfld, "COL_2010_GEIH_V02_M_V09_A_GMD_GPWG.qs2") + + # File was saved + expect_true(fs::file_exists(expected_file)) + + # Returned object is a data.table + expect_s3_class(result, "data.table") + expect_gt(nrow(result), 0) + + # Load from stamp and compare + testdf <- stamp::st_load(expected_file, alias = "dlw") + + expect_s3_class(testdf, "data.table") + expect_equal(nrow(result), nrow(testdf)) + expect_equal(ncol(result), ncol(testdf)) + expect_equal(names(result), names(testdf)) + + # Cleanup + fs::dir_delete(tempfld) +}) + +test_that("dlw_get_gmd returns cached data without re-downloading", { + skip_if_no_token() + + tempfld <- fs::path_temp("prj-temp-fld-cache") + fs::dir_create(tempfld) + stamp::st_init(tempfld, alias = "dlw") + + # First call downloads the data + result1 <- dlw_get_gmd( + country_code = "COL", + year = 2010, + survey = "GEIH", + module = "GPWG", + vermast = "v02", + veralt = "v09", + local_dir = tempfld + ) + + # Second call should read from cache (no download) + result2 <- dlw_get_gmd( + country_code = "COL", + year = 2010, + survey = "GEIH", + module = "GPWG", + vermast = "v02", + veralt = "v09", + local_dir = tempfld + ) + + expect_equal(nrow(result1), nrow(result2)) + expect_equal(names(result1), names(result2)) + + # Cleanup + fs::dir_delete(tempfld) +}) From 1b0abc6e92f5305e08e7a4660c51431936485e20 Mon Sep 17 00:00:00 2001 From: Tefera19 Date: Sun, 15 Mar 2026 20:54:54 -0400 Subject: [PATCH 24/24] README.html deleted; unit test for dlw_get_gmd with stamp has created --- README.html | 737 ------------------------------ tests/testthat/test-dlw_get_gmd.R | 38 +- 2 files changed, 1 insertion(+), 774 deletions(-) delete mode 100644 README.html diff --git a/README.html b/README.html deleted file mode 100644 index 17fbdf6..0000000 --- a/README.html +++ /dev/null @@ -1,737 +0,0 @@ - - - - - - - - - - - - - - - - - - - -

Ask DeepWiki

- - -

dlw

- - - - -

{dlw} is an R client of the internal datalibweb API of the -World Bank. Datalibweb is a data system designed to enable users to -seamlessly access microdata and documentation in the World Bank. Users -can access the most up-to-date and historical versions of harmonized -data collections and raw/non-harmonized data for subsequent analysis

-

Installation

-

Since {dlw} is hosted in the World Bank’s GitHub -organization, you need to authorize your Personal Access Token (PAT) for -SAML Single Sign-On (SSO) before you can install it. Follow these steps -to authorize your PAT:

-
    -

  1. Go to your GitHub account: Visit https://github.com and log in.

    2. -

  2. Check your PAT: Go to Settings → Developer settings → Personal -access tokens. Find the token you are using (or create a new one if -needed).

    3. -

  3. Authorize your PAT for SAML SSO: Go to https://github.com/settings/tokens. -Next to your token, You’ll see a button that says “Configure SSO.” Click -it to authorize your token to the worldbank.

    4. -

  4. Try installing

    5. -
-

You can install the development version of dlw from GitHub with:

-
# install.packages("pak")
-pak::pak("worldbank/dlw")
-# or
-remotes::github_install("worldbank/dlw")
-

Usage

-
library(dlw)
-

Token

-

To begin working with {dlw}, you must provide your -datalibweb API token using the dlw_set_token() -function.

-

You can obtain or renew your token by visiting the datalibweb page and -following the instructions there. Once you have your token, set it in -your R session as follows:

-
dlw_set_token("your_token_here")
-

server catalob

-

After setting your token, you can download the catalog for the -corresponding server using dlw_server_catalog(). By -default, this downloads the “GMD” catalog. The downloaded catalog is -saved for the current session in a hidden environment within the -{dlw} package, allowing for easy and efficient access in -subsequent operations.

-
ctl <- dlw_server_catalog()
-#> Pruning
-#> cache
-#> ℹ saving ServerCatalog_GMD in .dlwevn
-

The {dlw} package returns results as -data.table objects, enabling lightning-fast data -manipulation and filtering using concise syntax. For example, you can -instantly list all available files for Colombia in 2010 in module “ALL” -of the GMD collection with a single line of code:

-
ctl[Country_code == "COL" & Module == "ALL" & Survey_year == 2010, 
-  .(FileName, Vermast, Veralt )]
-#>                                 FileName Vermast Veralt
-#>                                   <char>  <char> <char>
-#> 1: COL_2010_GEIH_V04_M_V01_A_GMD_ALL.dta     V04    V01
-#> 2: COL_2010_GEIH_V02_M_V09_A_GMD_ALL.dta     V02    V09
-#> 3: COL_2010_GEIH_V02_M_V08_A_GMD_ALL.dta     V02    V08
-#> 4: COL_2010_GEIH_v02_M_v07_A_GMD_ALL.dta     V02    V07
-

Downloading files

-

The workhorse function to download data is -dlw_get_data(). However, it requires several pieces of -information that you may not have at hand, such as:

-
dlw_get_data(
-  country_code = "PRY",
-  year = 2011L,
-  server = "GMD",
-  survey = "EPH",
-  module = "GPWG",
-  filename = "PRY_2011_EPH_V01_M_V03_A_GMD_GPWG.dta",
-  collection = "GMD"
-)
-

To simplify this process, we have developed a wrapper function that -works only for the GMD server: dlw_get_gmd(). This function -is much easier to use:

-
pry <- dlw_get_gmd(country_code = "PRY", year = 2011, module = "GPWG", vermast = "v01", veralt = "v03")
-#> 
-#> ── dlw_get_
-#> Call 1:
-#> dlw_get_data(
-#>   country_code = "PRY",
-#>   year = 2011L,
-#>   server = "GMD",
-#>   survey = "EPH",
-#>   module = "GPWG",
-#>   filename = "PRY_2011_EPH_V01_M_V03_A_GMD_GPWG.dta",
-#>   collection = "GMD"
-#> )
-#> ✔ stamp
-#>   initialized
-#>   alias:
-#>   dlw
-#>   root:
-#>   C:/Users/WB308892/AppData/Local/Temp/RtmpEHCrN4
-#>   state:
-#>   C:/Users/WB308892/AppData/Local/Temp/RtmpEHCrN4/.stamp
-#> ✔ Saved [qs2] →
-#>   'c:/users/wb308892/appdata/local/temp/rtmpehcrn4/pry_2011_eph_v01_m_v03_a_gmd_gpwg.qs2'
-#>   @ version dfe89d4cd3898bd7
-pry[, weighted.mean(welfare, weight, na.rm = TRUE)]
-#> [1] 1099527
-

If you are interested in downloading the most recent version of a -file, you can simply omit the version arguments:

-
pry <- dlw_get_gmd(country_code = "PRY", year = 2011, module = "GPWG")
-#> 
-#> ── dlw_get_data Calls ───────────────
-#> Call 1:
-#> dlw_get_data(
-#>   country_code = "PRY",
-#>   year = 2011L,
-#>   server = "GMD",
-#>   survey = "EPH",
-#>   module = "GPWG",
-#>   filename = "PRY_2011_EPH_V02_M_V01_A_GMD_GPWG.dta",
-#>   collection = "GMD"
-#> )
-#> ✔ Saved [qs2] →
-#>   'c:/users/wb308892/appdata/local/temp/rtmpehcrn4/pry_2011_eph_v02_m_v01_a_gmd_gpwg.qs2'
-#>   @ version 3eb405fd0fc580bd
-pry[, weighted.mean(welfare, weight, na.rm = TRUE)]
-#> [1] 12675293
-

In this case, the most recent version available will be used (for -example, master version v02 and alternative version v01).

- - - diff --git a/tests/testthat/test-dlw_get_gmd.R b/tests/testthat/test-dlw_get_gmd.R index e9c0ba0..708492f 100644 --- a/tests/testthat/test-dlw_get_gmd.R +++ b/tests/testthat/test-dlw_get_gmd.R @@ -1,6 +1,6 @@ has_token <- tryCatch( { - dlw::dlw_check_token() + dlw::dlw_test_token TRUE }, error = function(e) FALSE @@ -49,39 +49,3 @@ test_that("dlw_get_gmd downloads, saves, and loads data from temp folder", { # Cleanup fs::dir_delete(tempfld) }) - -test_that("dlw_get_gmd returns cached data without re-downloading", { - skip_if_no_token() - - tempfld <- fs::path_temp("prj-temp-fld-cache") - fs::dir_create(tempfld) - stamp::st_init(tempfld, alias = "dlw") - - # First call downloads the data - result1 <- dlw_get_gmd( - country_code = "COL", - year = 2010, - survey = "GEIH", - module = "GPWG", - vermast = "v02", - veralt = "v09", - local_dir = tempfld - ) - - # Second call should read from cache (no download) - result2 <- dlw_get_gmd( - country_code = "COL", - year = 2010, - survey = "GEIH", - module = "GPWG", - vermast = "v02", - veralt = "v09", - local_dir = tempfld - ) - - expect_equal(nrow(result1), nrow(result2)) - expect_equal(names(result1), names(result2)) - - # Cleanup - fs::dir_delete(tempfld) -})