Introduced built-in functions to apply Fernet encryption for values i…

…n YAML configs

CHANGELOG.md

@@ -1,8 +1,15 @@
 # Changelog
 
+## [0.30.0] - 2024-07-14
+
+- Introduced built-in Fernet encryption for values in YAML configs, which is mostly useful for user passwords and various secrets.
+- Added YAML tags `!encrypt` and `!decrypt`.
+- Added ability to rotate keys for all config values encrypted with Fernet.
+- Made `business_roles` optional for `USER` object type.
+
 ## [0.29.2] - 2024-07-11
 
-- Fix parsing error of `secrets` parameter for `PROCEDURE`.
+- Fixed parsing error of `secrets` parameter for `PROCEDURE`.
 
 ## [0.29.1] - 2024-07-08
 

setup.cfg

@@ -47,6 +47,7 @@ console_scripts =
     snowddl = snowddl.app.base:entry_point
     snowddl-convert = snowddl.app.convert:entry_point
     snowddl-singledb = snowddl.app.singledb:entry_point
+    snowddl-fernet = snowddl.fernet.cli:entry_point
 
 [options.package_data]
 snowddl = _config/**/*.yaml

snowddl/app/base.py

@@ -37,6 +37,7 @@ def __init__(self):
 
     def init_arguments_parser(self):
         formatter = lambda prog: HelpFormatter(prog, max_help_position=36)
+
         parser = ArgumentParser(
             prog="snowddl", description="Object management automation tool for Snowflake", formatter_class=formatter
         )

snowddl/app/convert.py

@@ -13,8 +13,9 @@
 class ConvertApp(BaseApp):
     def init_arguments_parser(self):
         formatter = lambda prog: HelpFormatter(prog, max_help_position=32)
+
         parser = ArgumentParser(
-            prog="snowddlconv",
+            prog="snowddl-convert",
             description="Convert existing objects in Snowflake account to SnowDDL config",
             formatter_class=formatter,
         )

snowddl/app/singledb.py

@@ -30,6 +30,7 @@ def __init__(self):
 
     def init_arguments_parser(self):
         formatter = lambda prog: HelpFormatter(prog, max_help_position=32)
+
         parser = ArgumentParser(
             prog="snowddl-singledb",
             description="Special SnowDDL mode to process schema objects of single database only",

snowddl/fernet/cli.py

@@ -0,0 +1,263 @@
+from argparse import ArgumentParser, HelpFormatter
+from logging import getLogger, StreamHandler, Formatter
+from os import environ, getcwd
+from pathlib import Path
+from re import compile
+
+from snowddl.fernet.wrapper import FernetWrapper
+
+
+def init_logger():
+    logger = getLogger("snowddl")
+    logger.setLevel("INFO")
+
+    formatter = Formatter("%(asctime)s - %(levelname)s - %(message)s")
+    formatter.default_msec_format = "%s.%03d"
+
+    handler = StreamHandler()
+    handler.setFormatter(formatter)
+
+    logger.addHandler(handler)
+
+    return logger
+
+
+def entry_point():
+    parser = build_parser()
+    args = vars(parser.parse_args())
+    wrapper = FernetWrapper(args.get("k"))
+
+    if args["action"] == "generate-key":
+        action_generate_key(args, wrapper)
+    elif args["action"] == "encrypt":
+        action_encrypt(args, wrapper)
+    elif args["action"] == "decrypt":
+        action_decrypt(args, wrapper)
+    elif args["action"] == "rotate":
+        action_rotate(args, wrapper)
+    elif args["action"] == "config-encrypt":
+        action_config_encrypt(args, wrapper)
+    elif args["action"] == "config-decrypt":
+        action_config_decrypt(args, wrapper)
+    elif args["action"] == "config-rotate":
+        action_config_rotate(args, wrapper)
+
+
+def build_parser():
+    formatter = lambda prog: HelpFormatter(prog, max_help_position=32)
+
+    parser = ArgumentParser(
+        prog="snowddl-fernet",
+        description="Utility functions to encrypt secrets for SnowDDL config with Fernet",
+        formatter_class=formatter,
+    )
+
+    subparsers = parser.add_subparsers(dest="action")
+    subparsers.required = True
+
+    # Action: generate-key
+    generate_key_subparser = subparsers.add_parser("generate-key", help="Generate a fresh encryption key")
+
+    generate_key_subparser.add_argument(
+        "-k",
+        help="All current encryption keys separated by comma (<key1>,<key2>,<key3>)",
+        metavar="ENCRYPTION_KEYS",
+        default=environ.get(FernetWrapper.ENV_ENCRYPTION_KEYS),
+    )
+
+    generate_key_subparser.add_argument(
+        "--prepend",
+        help="Prepend newly generated key to existing keys",
+        default=False,
+        action="store_true",
+    )
+
+    generate_key_subparser.add_argument(
+        "--export",
+        help="Output key(s) as export env command for CLI",
+        default=False,
+        action="store_true",
+    )
+
+    # Action: encrypt
+    encrypt_subparser = subparsers.add_parser("encrypt", help="Encrypt value with first key")
+
+    encrypt_subparser.add_argument(
+        "-k",
+        help="All current encryption keys separated by comma (<key1>,<key2>,<key3>)",
+        metavar="ENCRYPTION_KEYS",
+        default=environ.get(FernetWrapper.ENV_ENCRYPTION_KEYS),
+    )
+
+    encrypt_subparser.add_argument(
+        "value",
+        help="String value",
+        metavar="VALUE",
+    )
+
+    # Action: decrypt
+    decrypt_subparser = subparsers.add_parser("decrypt", help="Decrypt value with any key")
+
+    decrypt_subparser.add_argument(
+        "-k",
+        help="All current encryption keys separated by comma (<key1>,<key2>,<key3>)",
+        metavar="ENCRYPTION_KEYS",
+        default=environ.get(FernetWrapper.ENV_ENCRYPTION_KEYS),
+    )
+
+    decrypt_subparser.add_argument(
+        "value",
+        help="Encrypted value",
+        metavar="VALUE",
+    )
+
+    # Action: rotate
+    rotate_subparser = subparsers.add_parser("rotate", help="Rotate value encrypted with any key")
+
+    rotate_subparser.add_argument(
+        "-k",
+        help="All current encryption keys separated by comma (<key1>,<key2>,<key3>)",
+        metavar="ENCRYPTION_KEYS",
+        default=environ.get(FernetWrapper.ENV_ENCRYPTION_KEYS),
+    )
+
+    rotate_subparser.add_argument(
+        "value",
+        help="Encrypted value to rotate",
+        metavar="VALUE",
+    )
+
+    # Action: config-encrypt
+    config_encrypt_subparser = subparsers.add_parser(
+        "config-encrypt",
+        help="Encrypy all values in config"
+    )
+
+    config_encrypt_subparser.add_argument(
+        "-k",
+        help="All current encryption keys separated by comma (<key1>,<key2>,<key3>)",
+        metavar="ENCRYPTION_KEYS",
+        default=environ.get(FernetWrapper.ENV_ENCRYPTION_KEYS),
+    )
+
+    config_encrypt_subparser.add_argument(
+        "-c",
+        help="Path to config directory (default: current directory)",
+        metavar="CONFIG_PATH",
+        default=getcwd(),
+    )
+
+    # Action: config-decrypt
+    config_decrypt_subparser = subparsers.add_parser(
+        "config-decrypt",
+        help="Decrypt all values in config"
+    )
+
+    config_decrypt_subparser.add_argument(
+        "-k",
+        help="All current encryption keys separated by comma (<key1>,<key2>,<key3>)",
+        metavar="ENCRYPTION_KEYS",
+        default=environ.get(FernetWrapper.ENV_ENCRYPTION_KEYS),
+    )
+
+    config_decrypt_subparser.add_argument(
+        "-c",
+        help="Path to config directory (default: current directory)",
+        metavar="CONFIG_PATH",
+        default=getcwd(),
+    )
+
+    # Action: config-rotate
+    config_rotate_subparser = subparsers.add_parser(
+        "config-rotate",
+        help="Rotate all values in config"
+    )
+
+    config_rotate_subparser.add_argument(
+        "-k",
+        help="All current encryption keys separated by comma (<key1>,<key2>,<key3>)",
+        metavar="ENCRYPTION_KEYS",
+        default=environ.get(FernetWrapper.ENV_ENCRYPTION_KEYS),
+    )
+
+    config_rotate_subparser.add_argument(
+        "-c",
+        help="Path to config directory (default: current directory)",
+        metavar="CONFIG_PATH",
+        default=getcwd(),
+    )
+
+    return parser
+
+
+def action_generate_key(args, wrapper: FernetWrapper):
+    all_keys = [wrapper.generate_key()]
+
+    if args.get("prepend"):
+        all_keys.extend(wrapper.key_sequence)
+
+    if args.get("export"):
+        print(f"export {wrapper.ENV_ENCRYPTION_KEYS}={','.join(all_keys)}")
+    else:
+        print(",".join(all_keys))
+
+
+def action_encrypt(args, wrapper: FernetWrapper):
+    print(wrapper.encrypt(args["value"]))
+
+
+def action_decrypt(args, wrapper: FernetWrapper):
+    print(wrapper.decrypt(args["value"]))
+
+
+def action_rotate(args, wrapper: FernetWrapper):
+    print(wrapper.rotate(args["value"]))
+
+
+def action_config_encrypt(args, wrapper: FernetWrapper):
+    logger = init_logger()
+    regexp = compile(r"!encrypt\s+(.+?)\n")
+
+    for file in get_config_files_generator(args):
+        original_text = file.read_text(encoding="utf-8")
+        updated_text, number_of_sub = regexp.subn(lambda m: f"!decrypt {wrapper.encrypt(m[1])}\n", original_text)
+
+        if number_of_sub > 0:
+            file.write_text(updated_text, encoding="utf-8")
+            logger.info(f"Updated file [{file}] with [{number_of_sub}] substitutions")
+
+
+def action_config_decrypt(args, wrapper: FernetWrapper):
+    logger = init_logger()
+    regexp = compile(r"!decrypt\s+(.+?)\n")
+
+    for file in get_config_files_generator(args):
+        original_text = file.read_text(encoding="utf-8")
+        updated_text, number_of_sub = regexp.subn(lambda m: f"!encrypt {wrapper.decrypt(m[1])}\n", original_text)
+
+        if number_of_sub > 0:
+            file.write_text(updated_text, encoding="utf-8")
+            logger.info(f"Updated file [{file}] with [{number_of_sub}] substitutions")
+
+
+def action_config_rotate(args, wrapper: FernetWrapper):
+    logger = init_logger()
+    regexp = compile(r"!decrypt\s+(.+?)\n")
+
+    for file in get_config_files_generator(args):
+        original_text = file.read_text(encoding="utf-8")
+        updated_text, number_of_sub = regexp.subn(lambda m: f"!decrypt {wrapper.rotate(m[1])}\n", original_text)
+
+        if number_of_sub > 0:
+            file.write_text(updated_text, encoding="utf-8")
+            logger.info(f"Updated file [{file}] with [{number_of_sub}] substitutions")
+
+
+def get_config_files_generator(args):
+    path = Path(args["c"])
+
+    if not path.is_dir():
+        raise ValueError(f"Config path [{path}] is not a directory")
+
+    for file in path.glob("**/*.yaml"):
+        yield file

snowddl/fernet/wrapper.py

@@ -0,0 +1,53 @@
+from cryptography.fernet import Fernet, MultiFernet
+from os import environ
+from typing import List, Optional
+
+
+class FernetWrapper:
+    ENV_ENCRYPTION_KEYS = "SNOWFLAKE_CONFIG_ENCRYPTION_KEYS"
+
+    def __init__(self, explicit_encryption_keys: Optional[str] = None):
+        self.key_sequence: List[str] = []
+        self.multi_fernet: Optional[MultiFernet] = None
+
+        if explicit_encryption_keys:
+            self.key_sequence = self._parse_encryption_keys(explicit_encryption_keys)
+        elif self.ENV_ENCRYPTION_KEYS in environ:
+            self.key_sequence = self._parse_encryption_keys(environ[self.ENV_ENCRYPTION_KEYS])
+
+        if self.key_sequence:
+            self.multi_fernet = MultiFernet(Fernet(key) for key in self.key_sequence)
+
+    def generate_key(self):
+        return Fernet.generate_key().decode("ascii")
+
+    def encrypt(self, value: str):
+        if not self.multi_fernet:
+            raise RuntimeError("Cannot encrypt value due to missing Fernet encryption keys")
+
+        return self.multi_fernet.encrypt(value.encode("utf-8")).decode("ascii")
+
+    def decrypt(self, value: str):
+        if not self.multi_fernet:
+            raise RuntimeError("Cannot decrypt value due to missing Fernet encryption keys")
+
+        return self.multi_fernet.decrypt(value.encode("ascii")).decode("utf-8")
+
+    def rotate(self, value: str):
+        if not self.multi_fernet:
+            raise RuntimeError("Cannot rotate value due to missing Fernet encryption keys")
+
+        return self.multi_fernet.rotate(value.encode("ascii")).decode("ascii")
+
+    def _parse_encryption_keys(self, encryption_keys: str):
+        key_sequence = []
+
+        for key in encryption_keys.split(","):
+            key = key.strip()
+
+            if not key:
+                continue
+
+            key_sequence.append(key)
+
+        return key_sequence

snowddl/parser/_yaml.py

@@ -1,6 +1,11 @@
 from pathlib import Path
 from yaml import SafeLoader, Node, add_constructor
 
+from snowddl.fernet.wrapper import FernetWrapper
+
+
+fernet_wrapper = FernetWrapper()
+
 
 class SnowDDLLoader(SafeLoader):
     pass
@@ -25,4 +30,16 @@ def include_constructor(loader: SnowDDLLoader, node: Node):
     return include_path.read_text(encoding="utf-8")
 
 
+def encrypt_constructor(loader: SnowDDLLoader, node: Node):
+    yaml_path = Path(loader.name)
+
+    raise ValueError(f"Detected non-encrypted value in file [{yaml_path}]")
+
+
+def decrypt_constructor(loader: SnowDDLLoader, node: Node):
+    return fernet_wrapper.decrypt(str(node.value))
+
+
 add_constructor("!include", include_constructor, Loader=SnowDDLLoader)
+add_constructor("!encrypt", encrypt_constructor, Loader=SnowDDLLoader)
+add_constructor("!decrypt", decrypt_constructor, Loader=SnowDDLLoader)