diff --git a/changelog.d/allow_disabling_aws_request_signing.enhancement.md b/changelog.d/allow_disabling_aws_request_signing.enhancement.md new file mode 100644 index 0000000000000..9a52ec0de27f4 --- /dev/null +++ b/changelog.d/allow_disabling_aws_request_signing.enhancement.md @@ -0,0 +1,2 @@ +AWS components can now have request signing disabled by setting `auth.sign: false` on AWS +components. This can be useful, for example, when sending anonymous requests to AWS S3. diff --git a/src/aws/auth.rs b/src/aws/auth.rs index c0e65b5dc85ba..3f3899f4fd4cb 100644 --- a/src/aws/auth.rs +++ b/src/aws/auth.rs @@ -175,6 +175,14 @@ pub enum AwsAuthentication { /// [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints #[configurable(metadata(docs::examples = "us-west-2"))] region: Option, + + /// Whether to sign requests. + /// + /// Setting this to false can be useful when writing to an AWS S3 bucket that allows + /// anonymous requests. + #[serde(default = "crate::serde::default_true")] + #[derivative(Default(value = "true"))] + sign: bool, }, } @@ -239,7 +247,7 @@ impl AwsAuthentication { service_region: Region, proxy: &ProxyConfig, tls_options: &Option, - ) -> crate::Result { + ) -> crate::Result> { match self { Self::AccessKey { access_key_id, @@ -265,9 +273,9 @@ impl AwsAuthentication { let provider = builder.build_from_provider(provider).await; - return Ok(SharedCredentialsProvider::new(provider)); + return Ok(Some(SharedCredentialsProvider::new(provider))); } - Ok(provider) + Ok(Some(provider)) } AwsAuthentication::File { credentials_file, @@ -288,7 +296,7 @@ impl AwsAuthentication { .profile_name(profile) .configure(&provider_config) .build(); - Ok(SharedCredentialsProvider::new(profile_provider)) + Ok(Some(SharedCredentialsProvider::new(profile_provider))) } AwsAuthentication::Role { assume_role, @@ -313,9 +321,15 @@ impl AwsAuthentication { ) .await; - Ok(SharedCredentialsProvider::new(provider)) + Ok(Some(SharedCredentialsProvider::new(provider))) } - AwsAuthentication::Default { imds, region, .. } => Ok(SharedCredentialsProvider::new( + AwsAuthentication::Default { sign: false, .. } => Ok(None), + AwsAuthentication::Default { + sign: true, + imds, + region, + .. + } => Ok(Some(SharedCredentialsProvider::new( default_credentials_provider( region.clone().map(Region::new).unwrap_or(service_region), proxy, @@ -323,7 +337,7 @@ impl AwsAuthentication { *imds, ) .await?, - )), + ))), } } @@ -411,6 +425,7 @@ mod tests { load_timeout_secs: Some(10), imds: ImdsAuthentication { .. }, region: None, + sign: true, } )); } @@ -453,6 +468,7 @@ mod tests { connect_timeout: CONNECT_TIMEOUT, read_timeout: READ_TIMEOUT, }, + sign: true, } )); } @@ -676,4 +692,21 @@ mod tests { _ => panic!(), } } + + #[test] + fn parsing_sign_false() { + let config = toml::from_str::( + r#" + auth.sign = false + "#, + ) + .unwrap(); + + match config.auth { + AwsAuthentication::Default { sign, .. } => { + assert!(!sign); + } + _ => panic!(), + } + } } diff --git a/src/aws/mod.rs b/src/aws/mod.rs index a4a68d34a2f96..f1f9ec923001b 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -24,7 +24,7 @@ use aws_smithy_runtime_api::client::{ runtime_components::RuntimeComponents, }; use aws_smithy_types::body::SdkBody; -use aws_types::sdk_config::SharedHttpClient; +use aws_types::sdk_config::{SharedAsyncSleep, SharedHttpClient}; use bytes::Bytes; use futures_util::FutureExt; use http::HeaderMap; @@ -201,25 +201,27 @@ pub async fn create_client_and_region( }; // Build the configuration first. - let mut config_builder = SdkConfig::builder() - .http_client(connector) - .sleep_impl(Arc::new(TokioSleep::new())) - .identity_cache(auth.credentials_cache().await?) - .credentials_provider( + let mut config_builder = SdkConfig::builder(); + + config_builder + .set_http_client(Some(SharedHttpClient::new(connector))) + .set_sleep_impl(Some(SharedAsyncSleep::new(Arc::new(TokioSleep::new())))) + .set_identity_cache(Some(auth.credentials_cache().await?)) + .set_region(Some(region.clone())) + .set_retry_config(Some(retry_config.clone())) + .set_credentials_provider( auth.credentials_provider(region.clone(), proxy, tls_options) .await?, - ) - .region(region.clone()) - .retry_config(retry_config.clone()); + ); if let Some(endpoint_override) = endpoint { - config_builder = config_builder.endpoint_url(endpoint_override); + config_builder.set_endpoint_url(Some(endpoint_override)); } if let Some(use_fips) = aws_config::default_provider::use_fips::use_fips_provider(&provider_config).await { - config_builder = config_builder.use_fips(use_fips); + config_builder.set_use_fips(Some(use_fips)); } if let Some(timeout) = timeout { @@ -234,7 +236,7 @@ pub async fn create_client_and_region( .set_connect_timeout(connect_timeout.map(Duration::from_secs)) .set_read_timeout(read_timeout.map(Duration::from_secs)); - config_builder = config_builder.timeout_config(timeout_config_builder.build()); + config_builder.set_timeout_config(Some(timeout_config_builder.build())); } let config = config_builder.build(); diff --git a/src/sinks/aws_kinesis/firehose/integration_tests.rs b/src/sinks/aws_kinesis/firehose/integration_tests.rs index fc9f22e5c5a73..538a8895747b7 100644 --- a/src/sinks/aws_kinesis/firehose/integration_tests.rs +++ b/src/sinks/aws_kinesis/firehose/integration_tests.rs @@ -83,6 +83,7 @@ async fn firehose_put_records_without_partition_key() { load_timeout_secs: Some(5), imds: ImdsAuthentication::default(), region: None, + sign: true, })), endpoints: vec![elasticsearch_address()], bulk: BulkConfig { @@ -195,6 +196,7 @@ async fn firehose_put_records_with_partition_key() { load_timeout_secs: Some(5), imds: ImdsAuthentication::default(), region: None, + sign: true, })), endpoints: vec![elasticsearch_address()], bulk: BulkConfig { @@ -278,6 +280,7 @@ async fn ensure_elasticsearch_domain(domain_name: String) -> String { &None, ) .await + .unwrap() .unwrap(), ) .endpoint_url(test_region_endpoint().endpoint().unwrap()) diff --git a/src/sinks/elasticsearch/common.rs b/src/sinks/elasticsearch/common.rs index 112a5b32f0743..b16602720c3ba 100644 --- a/src/sinks/elasticsearch/common.rs +++ b/src/sinks/elasticsearch/common.rs @@ -269,10 +269,14 @@ impl ElasticsearchCommon { #[cfg(feature = "aws-core")] pub async fn sign_request( request: &mut http::Request, - credentials_provider: &aws_credential_types::provider::SharedCredentialsProvider, + credentials_provider: &Option, region: &Option, ) -> crate::Result<()> { - crate::aws::sign_request("es", request, credentials_provider, region).await + if let Some(credentials_provider) = credentials_provider { + crate::aws::sign_request("es", request, credentials_provider, region).await + } else { + Ok(()) + } } async fn get_version( diff --git a/src/sinks/elasticsearch/integration_tests.rs b/src/sinks/elasticsearch/integration_tests.rs index 1f627b198f8c8..53c9a088d1fa3 100644 --- a/src/sinks/elasticsearch/integration_tests.rs +++ b/src/sinks/elasticsearch/integration_tests.rs @@ -290,6 +290,7 @@ async fn auto_version_aws() { load_timeout_secs: Some(5), imds: ImdsAuthentication::default(), region: None, + sign: true, }, )), endpoints: vec![aws_server()], @@ -396,6 +397,7 @@ async fn insert_events_on_aws() { load_timeout_secs: Some(5), imds: ImdsAuthentication::default(), region: None, + sign: true, }, )), endpoints: vec![aws_server()], @@ -422,6 +424,7 @@ async fn insert_events_on_aws_with_compression() { load_timeout_secs: Some(5), imds: ImdsAuthentication::default(), region: None, + sign: true, }, )), endpoints: vec![aws_server()], diff --git a/src/sinks/prometheus/remote_write/service.rs b/src/sinks/prometheus/remote_write/service.rs index fb459d7c4bd8e..ef1127d84fb76 100644 --- a/src/sinks/prometheus/remote_write/service.rs +++ b/src/sinks/prometheus/remote_write/service.rs @@ -94,10 +94,14 @@ impl Service for RemoteWriteService { #[cfg(feature = "aws-core")] async fn sign_request( request: &mut http::Request, - credentials_provider: &SharedCredentialsProvider, + credentials_provider: &Option, region: &Option, ) -> crate::Result<()> { - crate::aws::sign_request("aps", request, credentials_provider, region).await + if let Some(credentials_provider) = credentials_provider { + crate::aws::sign_request("aps", request, credentials_provider, region).await + } else { + Ok(()) + } } pub(super) async fn build_request( diff --git a/src/sinks/util/auth.rs b/src/sinks/util/auth.rs index 5ea99b79b7a61..a3ef231d301de 100644 --- a/src/sinks/util/auth.rs +++ b/src/sinks/util/auth.rs @@ -3,7 +3,7 @@ pub enum Auth { Basic(crate::http::Auth), #[cfg(feature = "aws-core")] Aws { - credentials_provider: aws_credential_types::provider::SharedCredentialsProvider, + credentials_provider: Option, region: aws_types::region::Region, }, } diff --git a/src/sources/aws_sqs/integration_tests.rs b/src/sources/aws_sqs/integration_tests.rs index eec750b00566c..23bf85b41f0de 100644 --- a/src/sources/aws_sqs/integration_tests.rs +++ b/src/sources/aws_sqs/integration_tests.rs @@ -55,6 +55,7 @@ async fn get_sqs_client() -> aws_sdk_sqs::Client { AwsAuthentication::test_auth() .credentials_provider(Region::new("custom"), &Default::default(), &None) .await + .unwrap() .unwrap(), ) .endpoint_url(sqs_address()) diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue index 3c26ae7e3455c..6a8f1fc6544bb 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue @@ -127,6 +127,16 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } batch: { diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue index ff50dfea28990..a9df1e2c14e55 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue @@ -127,6 +127,16 @@ base: components: sinks: aws_cloudwatch_metrics: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } batch: { diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue index 45f999b2aa76e..21c47ceecf1be 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue @@ -127,6 +127,16 @@ base: components: sinks: aws_kinesis_firehose: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } batch: { diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue index 8d924f6b1a84e..e765d1de310c8 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue @@ -127,6 +127,16 @@ base: components: sinks: aws_kinesis_streams: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } batch: { diff --git a/website/cue/reference/components/sinks/base/aws_s3.cue b/website/cue/reference/components/sinks/base/aws_s3.cue index 8d9f0b31144b3..10d522ea724e6 100644 --- a/website/cue/reference/components/sinks/base/aws_s3.cue +++ b/website/cue/reference/components/sinks/base/aws_s3.cue @@ -202,6 +202,16 @@ base: components: sinks: aws_s3: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } batch: { diff --git a/website/cue/reference/components/sinks/base/aws_sns.cue b/website/cue/reference/components/sinks/base/aws_sns.cue index 4b5f15b81d68c..4999f7ec6e307 100644 --- a/website/cue/reference/components/sinks/base/aws_sns.cue +++ b/website/cue/reference/components/sinks/base/aws_sns.cue @@ -127,6 +127,16 @@ base: components: sinks: aws_sns: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } encoding: { diff --git a/website/cue/reference/components/sinks/base/aws_sqs.cue b/website/cue/reference/components/sinks/base/aws_sqs.cue index 5b33a0b9f374d..6bf12faea43ec 100644 --- a/website/cue/reference/components/sinks/base/aws_sqs.cue +++ b/website/cue/reference/components/sinks/base/aws_sqs.cue @@ -127,6 +127,16 @@ base: components: sinks: aws_sqs: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } encoding: { diff --git a/website/cue/reference/components/sinks/base/elasticsearch.cue b/website/cue/reference/components/sinks/base/elasticsearch.cue index 68943aa186fac..d6cb20ea0c764 100644 --- a/website/cue/reference/components/sinks/base/elasticsearch.cue +++ b/website/cue/reference/components/sinks/base/elasticsearch.cue @@ -165,6 +165,17 @@ base: components: sinks: elasticsearch: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + relevant_when: "strategy = \"aws\"" + required: false + type: bool: default: true + } strategy: { description: "The authentication strategy to use." required: true diff --git a/website/cue/reference/components/sinks/base/prometheus_remote_write.cue b/website/cue/reference/components/sinks/base/prometheus_remote_write.cue index 340e12742396d..d73d16e06b3cf 100644 --- a/website/cue/reference/components/sinks/base/prometheus_remote_write.cue +++ b/website/cue/reference/components/sinks/base/prometheus_remote_write.cue @@ -142,6 +142,17 @@ base: components: sinks: prometheus_remote_write: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + relevant_when: "strategy = \"aws\"" + required: false + type: bool: default: true + } strategy: { description: "The authentication strategy to use." required: true diff --git a/website/cue/reference/components/sources/base/aws_s3.cue b/website/cue/reference/components/sources/base/aws_s3.cue index 4538062a02679..6d9edc1fcd0a8 100644 --- a/website/cue/reference/components/sources/base/aws_s3.cue +++ b/website/cue/reference/components/sources/base/aws_s3.cue @@ -122,6 +122,16 @@ base: components: sources: aws_s3: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } compression: { diff --git a/website/cue/reference/components/sources/base/aws_sqs.cue b/website/cue/reference/components/sources/base/aws_sqs.cue index 0d4377a71eab8..d78454fff5874 100644 --- a/website/cue/reference/components/sources/base/aws_sqs.cue +++ b/website/cue/reference/components/sources/base/aws_sqs.cue @@ -122,6 +122,16 @@ base: components: sources: aws_sqs: configuration: { required: true type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } + sign: { + description: """ + Whether to sign requests. + + Setting this to false can be useful when writing to an AWS S3 bucket that allows + anonymous requests. + """ + required: false + type: bool: default: true + } } } client_concurrency: {