From 08ff0069f00d0a84c8ef6cd3e2f55eefc098b2fb Mon Sep 17 00:00:00 2001 From: Constantin M Adam Date: Fri, 25 Oct 2024 10:10:01 -0400 Subject: [PATCH] Allow input of s3 credentials on command line Signed-off-by: Constantin M Adam --- .../fdedup/python/src/fuzzy_dedup_python.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/transforms/universal/fdedup/python/src/fuzzy_dedup_python.py b/transforms/universal/fdedup/python/src/fuzzy_dedup_python.py index acb1be3bb..054447e70 100644 --- a/transforms/universal/fdedup/python/src/fuzzy_dedup_python.py +++ b/transforms/universal/fdedup/python/src/fuzzy_dedup_python.py @@ -1,4 +1,5 @@ import argparse +import ast import os import sys @@ -119,8 +120,17 @@ def get_arguments(self, in_args: argparse.Namespace, service_name: str) -> list: "output_folder": output_folder, } if in_args.use_s3: - sys_argv.append("--data_s3_cred") - sys_argv.append(ParamsUtils.convert_to_ast(s3_creds)) + if in_args.s3_cred is not None: + s3_cred_ast = ParamsUtils.convert_to_ast(in_args.s3_cred) + sys_argv.append("--data_s3_cred") + sys_argv.append(s3_cred_ast) + elif ( + s3_creds.get("access_key") is not None + and s3_creds.get("secret_key") is not None + and s3_creds.get("url") is not None + ): + sys_argv.append("--data_s3_cred") + sys_argv.append(ParamsUtils.convert_to_ast(s3_creds)) sys_argv.append("--data_s3_config") else: sys_argv.append("--data_local_config") @@ -207,6 +217,13 @@ def parse_args() -> argparse.Namespace: help="use s3", ) + parser.add_argument( + "--s3_cred", + type=ast.literal_eval, + default=None, + help="ast string of options for s3 credentials", + ) + return parser.parse_args()