From 22fd5bd70265a5a5356ec254ff9f20589085f4f6 Mon Sep 17 00:00:00 2001 From: Hadley Wickham Date: Wed, 23 Oct 2024 13:09:35 -0500 Subject: [PATCH] Signing for AWS requests (#569) * Add AWS request signing. Fixes #566 * Generalise OAuth signing to any pre-request signature. Fixes #562 --- DESCRIPTION | 1 + NAMESPACE | 1 + NEWS.md | 1 + R/multi-req.R | 2 +- R/oauth.R | 20 +-- R/req-auth-aws.R | 217 +++++++++++++++++++++++ R/req-auth-sign.R | 20 +++ R/req-body.R | 22 ++- R/req-perform-connection.R | 1 + R/req-perform.R | 4 +- man/req_auth_aws_v4.Rd | 52 ++++++ tests/testthat/_snaps/req-auth-aws.md | 28 +++ tests/testthat/test-oauth-flow-refresh.R | 10 +- tests/testthat/test-req-auth-aws.R | 59 ++++++ 14 files changed, 415 insertions(+), 23 deletions(-) create mode 100644 R/req-auth-aws.R create mode 100644 R/req-auth-sign.R create mode 100644 man/req_auth_aws_v4.Rd create mode 100644 tests/testthat/_snaps/req-auth-aws.md create mode 100644 tests/testthat/test-req-auth-aws.R diff --git a/DESCRIPTION b/DESCRIPTION index 48e64739..4c9c63c5 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -38,6 +38,7 @@ Suggests: jsonlite, knitr, later, + paws.common, promises, rmarkdown, testthat (>= 3.1.8), diff --git a/NAMESPACE b/NAMESPACE index 112fc391..7e64c629 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -50,6 +50,7 @@ export(oauth_token) export(oauth_token_cached) export(obfuscate) export(obfuscated) +export(req_auth_aws_v4) export(req_auth_basic) export(req_auth_bearer_token) export(req_body_file) diff --git a/NEWS.md b/NEWS.md index 5554d284..4425cc4e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,7 @@ # httr2 1.0.5 +* New `req_auth_aws_v4()` signs request using AWS's special format (#562, #566). * `req_perform_parallel()` and `req_perform_promise()` now correctly set up the method and body (#549). # httr2 1.0.4 diff --git a/R/multi-req.R b/R/multi-req.R index 8defad02..9cb4beb4 100644 --- a/R/multi-req.R +++ b/R/multi-req.R @@ -162,7 +162,7 @@ Performance <- R6Class("Performance", public = list( self$progress <- progress self$error_call <- error_call - req <- auth_oauth_sign(req) + req <- auth_sign(req) req <- cache_pre_fetch(req, path) if (is_response(req)) { self$resp <- req diff --git a/R/oauth.R b/R/oauth.R index b96ee0b8..ffe00b67 100644 --- a/R/oauth.R +++ b/R/oauth.R @@ -20,27 +20,25 @@ #' @export req_oauth <- function(req, flow, flow_params, cache) { # Want req object to contain meaningful objects, not just a closure - req_policies(req, - auth_oauth = list( + req <- req_auth_sign(req, + fun = auth_oauth_sign, + params = list( cache = cache, flow = flow, flow_params = flow_params ) ) + req <- req_policies(req, auth_oauth = TRUE) + req } -auth_oauth_sign <- function(req, reauth = FALSE) { - if (!req_policy_exists(req, "auth_oauth")) { - return(req) - } - +auth_oauth_sign <- function(req, cache, flow, flow_params, reauth = FALSE) { token <- auth_oauth_token_get( - cache = req$policies$auth_oauth$cache, - flow = req$policies$auth_oauth$flow, - flow_params = req$policies$auth_oauth$flow_params, + cache = cache, + flow = flow, + flow_params = flow_params, reauth = reauth ) - req_auth_bearer_token(req, token$access_token) } diff --git a/R/req-auth-aws.R b/R/req-auth-aws.R new file mode 100644 index 00000000..7d32e69f --- /dev/null +++ b/R/req-auth-aws.R @@ -0,0 +1,217 @@ +#' Sign a request with the AWS SigV4 signing protocol +#' +#' This is a custom auth protocol implemented by AWS. +#' +#' @inheritParams req_perform +#' @param aws_access_key_id,aws_secret_access_key AWS key and secret. +#' @param aws_session_token AWS session token, if required. +#' @param aws_service,aws_region The AWS service and region to use for the +#' request. If not supplied, will be automatically parsed from the URL +#' hostname. +#' @export +#' @examplesIf httr2:::has_paws_credentials() +#' creds <- paws.common::locate_credentials() +#' model_id <- "anthropic.claude-3-5-sonnet-20240620-v1:0" +#' req <- request("https://bedrock-runtime.us-east-1.amazonaws.com") +#' # https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html +#' req <- req_url_path_append(req, "model", model_id, "converse") +#' req <- req_body_json(req, list( +#' messages = list(list( +#' role = "user", +#' content = list(list(text = "What's your name?")) +#' )) +#' )) +#' req <- req_auth_aws_v4( +#' req, +#' aws_access_key_id = creds$access_key_id, +#' aws_secret_access_key = creds$secret_access_key, +#' aws_session_token = creds$session_token +#' ) +#' resp <- req_perform_connection(req) +#' str(resp_body_json(resp)) +req_auth_aws_v4 <- function(req, + aws_access_key_id, + aws_secret_access_key, + aws_session_token = NULL, + aws_service = NULL, + aws_region = NULL) { + + check_request(req) + check_string(aws_access_key_id) + check_string(aws_secret_access_key) + check_string(aws_session_token, allow_null = TRUE) + check_string(aws_service, allow_null = TRUE) + check_string(aws_region, allow_null = TRUE) + + req_auth_sign(req, + fun = auth_aws_sign, + params = list( + aws_access_key_id = aws_access_key_id, + aws_secret_access_key = aws_secret_access_key, + aws_session_token = aws_session_token, + aws_service = aws_service, + aws_region = aws_region + ) + ) +} + +auth_aws_sign <- function(req, + aws_access_key_id, + aws_secret_access_key, + aws_session_token = NULL, + aws_service = NULL, + aws_region = NULL, + reauth = FALSE) { + + current_time <- Sys.time() + + body_sha256 <- openssl::sha256(req_body_get(req)) + + # We begin by adding some necessary headers that must be added before + # canoncalization even thought they aren't documented until later + req <- req_aws_headers(req, + current_time = current_time, + aws_session_token = aws_session_token, + body_sha256 = body_sha256 + ) + + signature <- aws_v4_signature( + method = req_method_get(req), + url = url_parse(req$url), + headers = req$headers, + body_sha256 = body_sha256, + current_time = current_time, + aws_service = aws_service, + aws_region = aws_region, + aws_access_key_id = aws_access_key_id, + aws_secret_access_key = aws_secret_access_key + ) + req_headers(req, Authorization = signature$Authorization) +} + + +req_aws_headers <- function(req, current_time, aws_session_token, body_sha256) { + RequestDateTime <- format(current_time, "%Y%m%dT%H%M%SZ", tz = "UTC") + + req_headers( + req, + "x-amz-date" = RequestDateTime, + "x-amz-security-token" = aws_session_token, + .redact = "x-amz-security-token" + ) +} + +# https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html +aws_v4_signature <- function(method, + url, + headers, + body_sha256, + aws_access_key_id, + aws_secret_access_key, + current_time = Sys.time(), + aws_service = NULL, + aws_region = NULL) { + + if (is.null(aws_service) || is.null(aws_region)) { + host <- strsplit(url$hostname, ".", fixed = TRUE)[[1]] + aws_service <- aws_service %||% strsplit(host[[1]], "-", fixed = TRUE)[[1]][[1]] + aws_region <- aws_region %||% host[[2]] + } + + # 1. Create a canonical request + # https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#create-canonical-request + HTTPMethod <- method + CanonicalURI <- curl::curl_escape(url$path %||% "/") + # AWS does not want / to be encoded here + CanonicalURI <- gsub("%2F", "/", CanonicalURI, fixed = TRUE) + + if (is.null(url$query)) { + CanonicalQueryString <- "" + } else { + sorted_query <- url$query[order(names(url$query))] + CanonicalQueryString <- query_build(CanonicalQueryString) + } + + headers$host <- url$hostname + names(headers) <- tolower(names(headers)) + headers <- headers[order(names(headers))] + headers[] <- trimws(headers) + headers[] <- gsub(" {2,}", " ", headers) + CanonicalHeaders <- paste0(names(headers), ":", headers, "\n", collapse = "") + SignedHeaders <- paste0(names(headers), collapse = ";") + + CanonicalRequest <- paste0( + HTTPMethod, "\n", + CanonicalURI, "\n", + CanonicalQueryString, "\n", + CanonicalHeaders, "\n", + SignedHeaders, "\n", + body_sha256 + ) + # 2. Create the hash of the canonical request + # https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html + HashedCanonicalRequest <- openssl::sha256(CanonicalRequest) + + # 3. Create the string to sign + # https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#create-string-to-sign + + Algorithm <- "AWS4-HMAC-SHA256" + RequestDateTime <- format(current_time, "%Y%m%dT%H%M%SZ", tz = "UTC") + Date <- format(current_time, "%Y%m%d", tz = "UTC") + CredentialScope <- file.path(Date, aws_region, aws_service, "aws4_request") + + string_to_sign <- paste0( + Algorithm, "\n", + RequestDateTime, "\n", + CredentialScope, "\n", + HashedCanonicalRequest + ) + + # 4. Derive a signing key + # https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#derive-signing-key + + DateKey <- hmac_sha256(paste0("AWS4", aws_secret_access_key), Date) + DateRegionKey <- hmac_sha256(DateKey, aws_region) + DateRegionServiceKey <- hmac_sha256(DateRegionKey, aws_service) + SigningKey <- hmac_sha256(DateRegionServiceKey, "aws4_request") + + # 5. Calculate signature + # https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#calculate-signature + + signature <- hmac_sha256(SigningKey, string_to_sign) + signature <- paste0(as.character(signature), collapse = "") + + # 6. Add the signature to the request + # https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#calculate-signature + credential <- file.path(aws_access_key_id, CredentialScope) + + Authorization <- paste0( + Algorithm, ",", + "Credential=", credential, ",", + "SignedHeaders=", SignedHeaders, ",", + "Signature=", signature + ) + + list( + CanonicalRequest = CanonicalRequest, + string_to_sign = string_to_sign, + SigningKey = SigningKey, + Authorization = Authorization + ) +} + +hmac_sha256 <- function(key, value) { + openssl::sha256(charToRaw(value), key) +} + +has_paws_credentials <- function() { + tryCatch( + { + paws.common::locate_credentials() + TRUE + }, + error = function(e) { + FALSE + } + ) +} diff --git a/R/req-auth-sign.R b/R/req-auth-sign.R new file mode 100644 index 00000000..00026a54 --- /dev/null +++ b/R/req-auth-sign.R @@ -0,0 +1,20 @@ + +req_auth_sign <- function(req, fun, params) { + req_policies(req, + auth_sign = list( + fun = fun, + params = params + ) + ) +} +auth_sign <- function(req, reauth = FALSE) { + if (!req_policy_exists(req, "auth_sign")) { + return(req) + } + + exec(req$policies$auth_sign$fun, + req = req, + reauth = reauth, + !!!req$policies$auth_sign$params + ) +} diff --git a/R/req-body.R b/R/req-body.R index c53e188b..f9786731 100644 --- a/R/req-body.R +++ b/R/req-body.R @@ -217,6 +217,22 @@ req_body_info <- function(req) { } } +req_body_get <- function(req) { + if (is.null(req$body)) { + return("") + } + switch( + req$body$type, + raw = req$body$data, + form = { + data <- unobfuscate(req$body$data) + query_build(data) + }, + json = exec(jsonlite::toJSON, req$body$data, !!!req$body$params), + cli::cli_abort("Unsupported request body type {.str {req$body$type}}.") + ) +} + req_body_apply <- function(req) { if (is.null(req$body)) { return(req) @@ -243,14 +259,12 @@ req_body_apply <- function(req) { } else if (type == "raw") { req <- req_body_apply_raw(req, data) } else if (type == "json") { - json <- exec(jsonlite::toJSON, data, !!!req$body$params) - req <- req_body_apply_raw(req, json) + req <- req_body_apply_raw(req, req_body_get(req)) } else if (type == "multipart") { data <- unobfuscate(data) req$fields <- data } else if (type == "form") { - data <- unobfuscate(data) - req <- req_body_apply_raw(req, query_build(data)) + req <- req_body_apply_raw(req, req_body_get(req)) } else { cli::cli_abort("Unsupported request body {.arg type}.", .internal = TRUE) } diff --git a/R/req-perform-connection.R b/R/req-perform-connection.R index 28cc05d2..d8baf479 100644 --- a/R/req-perform-connection.R +++ b/R/req-perform-connection.R @@ -37,6 +37,7 @@ req_perform_connection <- function(req, blocking = TRUE) { check_request(req) check_bool(blocking) + req <- auth_sign(req) req_prep <- req_prepare(req) handle <- req_handle(req_prep) the$last_request <- req diff --git a/R/req-perform.R b/R/req-perform.R index 3b950eb1..5eb8915b 100644 --- a/R/req-perform.R +++ b/R/req-perform.R @@ -87,7 +87,7 @@ req_perform <- function( } req <- req_verbosity(req, verbosity) - req <- auth_oauth_sign(req) + req <- auth_sign(req) req <- cache_pre_fetch(req, path) if (is_response(req)) { @@ -131,7 +131,7 @@ req_perform <- function( signal(class = "httr2_retry", tries = tries, delay = delay) } else if (!reauth && resp_is_invalid_oauth_token(req, resp)) { reauth <- TRUE - req <- auth_oauth_sign(req, TRUE) + req <- auth_sign(req, TRUE) req_prep <- req_prepare(req) handle <- req_handle(req_prep) delay <- 0 diff --git a/man/req_auth_aws_v4.Rd b/man/req_auth_aws_v4.Rd new file mode 100644 index 00000000..32bc5580 --- /dev/null +++ b/man/req_auth_aws_v4.Rd @@ -0,0 +1,52 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/req-auth-aws.R +\name{req_auth_aws_v4} +\alias{req_auth_aws_v4} +\title{Sign a request with the AWS SigV4 signing protocol} +\usage{ +req_auth_aws_v4( + req, + aws_access_key_id, + aws_secret_access_key, + aws_session_token = NULL, + aws_service = NULL, + aws_region = NULL +) +} +\arguments{ +\item{req}{A httr2 \link{request} object.} + +\item{aws_access_key_id, aws_secret_access_key}{AWS key and secret.} + +\item{aws_session_token}{AWS session token, if required.} + +\item{aws_service, aws_region}{The AWS service and region to use for the +request. If not supplied, will be automatically parsed from the URL +hostname.} +} +\description{ +This is a custom auth protocol implemented by AWS. +} +\examples{ +\dontshow{if (httr2:::has_paws_credentials()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +creds <- paws.common::locate_credentials() +model_id <- "anthropic.claude-3-5-sonnet-20240620-v1:0" +req <- request("https://bedrock-runtime.us-east-1.amazonaws.com") +# https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html +req <- req_url_path_append(req, "model", model_id, "converse") +req <- req_body_json(req, list( + messages = list(list( + role = "user", + content = list(list(text = "What's your name?")) + )) +)) +req <- req_auth_aws_v4( + req, + aws_access_key_id = creds$access_key_id, + aws_secret_access_key = creds$secret_access_key, + aws_session_token = creds$session_token +) +resp <- req_perform_connection(req) +str(resp_body_json(resp)) +\dontshow{\}) # examplesIf} +} diff --git a/tests/testthat/_snaps/req-auth-aws.md b/tests/testthat/_snaps/req-auth-aws.md new file mode 100644 index 00000000..d0015609 --- /dev/null +++ b/tests/testthat/_snaps/req-auth-aws.md @@ -0,0 +1,28 @@ +# validates its inputs + + Code + req_auth_aws_v4(1) + Condition + Error in `req_auth_aws_v4()`: + ! `req` must be an HTTP request object, not the number 1. + Code + req_auth_aws_v4(req, 1) + Condition + Error in `req_auth_aws_v4()`: + ! `aws_access_key_id` must be a single string, not the number 1. + Code + req_auth_aws_v4(req, "", "", aws_session_token = 1) + Condition + Error in `req_auth_aws_v4()`: + ! `aws_session_token` must be a single string or `NULL`, not the number 1. + Code + req_auth_aws_v4(req, "", "", aws_service = 1) + Condition + Error in `req_auth_aws_v4()`: + ! `aws_service` must be a single string or `NULL`, not the number 1. + Code + req_auth_aws_v4(req, "", "", aws_region = 1) + Condition + Error in `req_auth_aws_v4()`: + ! `aws_region` must be a single string or `NULL`, not the number 1. + diff --git a/tests/testthat/test-oauth-flow-refresh.R b/tests/testthat/test-oauth-flow-refresh.R index a677b88c..ae16256c 100644 --- a/tests/testthat/test-oauth-flow-refresh.R +++ b/tests/testthat/test-oauth-flow-refresh.R @@ -9,8 +9,8 @@ test_that("cache considers refresh_token", { req_oauth_refresh(client, refresh_token = "rt2") # cache must be empty - expect_equal(req1$policies$auth_oauth$cache$get(), NULL) - expect_equal(req2$policies$auth_oauth$cache$get(), NULL) + expect_equal(req1$policies$auth_sign$params$cache$get(), NULL) + expect_equal(req2$policies$auth_sign$params$cache$get(), NULL) # simulate that we made a request and got back a token token <- oauth_token( @@ -21,11 +21,11 @@ test_that("cache considers refresh_token", { .date = Sys.time() ) # ... that is now cached - req1$policies$auth_oauth$cache$set(token) + req1$policies$auth_sign$params$cache$set(token) # req1 cache must be filled, but req2 cache still be empty - expect_equal(req1$policies$auth_oauth$cache$get(), token) - expect_equal(req2$policies$auth_oauth$cache$get(), NULL) + expect_equal(req1$policies$auth_sign$params$cache$get(), token) + expect_equal(req2$policies$auth_sign$params$cache$get(), NULL) }) test_that("warns if refresh token changes", { diff --git a/tests/testthat/test-req-auth-aws.R b/tests/testthat/test-req-auth-aws.R new file mode 100644 index 00000000..e024f12d --- /dev/null +++ b/tests/testthat/test-req-auth-aws.R @@ -0,0 +1,59 @@ +test_that("can correctly sign a request", { + skip_if_not(has_paws_credentials()) + creds <- paws.common::locate_credentials() + + # https://docs.aws.amazon.com/STS/latest/APIReference/API_GetCallerIdentity.html + req <- request("https://sts.amazonaws.com/") + req <- req_auth_aws_v4(req, + aws_access_key_id = creds$access_key_id, + aws_secret_access_key = creds$secret_access_key, + aws_session_token = creds$session_token, + aws_region = creds$region + ) + req <- req_body_form( + req, + Action = "GetCallerIdentity", + Version = "2011-06-15" + ) + expect_no_error(req_perform(req)) +}) + +test_that("signing agrees with glacier example", { + # Example from + # https://docs.aws.amazon.com/amazonglacier/latest/dev/amazon-glacier-signing-requests.html + + signature <- aws_v4_signature( + method = "PUT", + url = url_parse("https://glacier.us-east-1.amazonaws.com/-/vaults/examplevault"), + headers = list( + "x-amz-date" = "20120525T002453Z", + "x-amz-glacier-version" = "2012-06-01" + ), + body_sha256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + current_time = as.POSIXct("2012-05-25 00:24:53", tz = "UTC"), + aws_access_key_id = "AKIAIOSFODNN7EXAMPLE", + aws_secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + ) + signature_pieces <- strsplit(paste0("Authorization=", signature$Authorization), ",")[[1]] + + known <- list( + Authorization = "AWS4-HMAC-SHA256", + Credential = "AKIAIOSFODNN7EXAMPLE/20120525/us-east-1/glacier/aws4_request", + SignedHeaders = "host;x-amz-date;x-amz-glacier-version", + Signature = "3ce5b2f2fffac9262b4da9256f8d086b4aaf42eba5f111c21681a65a127b7c2a" + ) + known_signature <- paste0(names(known), "=", known) + + expect_equal(signature_pieces, known_signature) +}) + +test_that("validates its inputs", { + req <- request("https://sts.amazonaws.com/") + expect_snapshot(error = TRUE, { + req_auth_aws_v4(1) + req_auth_aws_v4(req, 1) + req_auth_aws_v4(req, "", "", aws_session_token = 1) + req_auth_aws_v4(req, "", "", aws_service = 1) + req_auth_aws_v4(req, "", "", aws_region = 1) + }) +})