From 46ceca4c2a35afd8612fcf2a3de434b097d5b73b Mon Sep 17 00:00:00 2001 From: jberends Date: Fri, 31 May 2019 11:00:43 +0200 Subject: [PATCH] build and sign artifacts feature (#9) * adding sign command * creating custom trust store/keystore for gnupg for kecpkg exclusive * refactoring a lot, moving util functions from command/utils to utils * adding gpg.py for gpg support * adding platform aware gpg support (find binary) * creating sign create key command! * more rubust implementation of get_pacakge_dir * adding export key and tests * adding package verification * improvements in sign --build. * Will show the list of keys available to sign with and autofill emal addres when available. * added python 3.6 and 3.7 option * added get and set keys in config. Better display of configsettings. --- .idea/inspectionProfiles/Project_Default.xml | 4 +- .idea/kecpkg-tools.iml | 2 +- .idea/misc.xml | 4 +- .idea/runConfigurations/kecpkg_new.xml | 19 +- .idea/runConfigurations/kecpkg_sign.xml | 33 +++ .idea/runConfigurations/kecpkg_upload.xml | 19 +- .travis.yml | 1 + CHANGELOG.md | 10 + MANIFEST.in | 6 +- Pipfile | 24 ++ README.md | 49 ++++ README.rst | 9 +- build_release.sh | 4 + kecpkg/__init__.py | 2 +- kecpkg/cli.py | 2 + kecpkg/commands/build.py | 121 +++++++- kecpkg/commands/config.py | 26 +- kecpkg/commands/new.py | 4 +- kecpkg/commands/prune.py | 4 +- kecpkg/commands/purge.py | 4 +- kecpkg/commands/sign.py | 286 +++++++++++++++++++ kecpkg/commands/upload.py | 5 +- kecpkg/commands/utils.py | 53 +--- kecpkg/create.py | 7 +- kecpkg/files/templates/.gitignore.template | 9 + kecpkg/files/templates/script.py.template | 1 + kecpkg/gpg.py | 122 ++++++++ kecpkg/settings.py | 22 +- kecpkg/utils.py | 119 ++++++-- pyproject.toml => pyproject.toml.depr | 2 +- requirements.txt | 6 +- setup.py | 15 +- tests/commands/test_build.py | 13 +- tests/commands/test_sign.py | 110 +++++++ tests/commands/test_upload.py | 12 +- tests/utils.py | 18 +- tox.ini | 5 +- 37 files changed, 1021 insertions(+), 131 deletions(-) create mode 100644 .idea/runConfigurations/kecpkg_sign.xml create mode 100644 Pipfile create mode 100644 README.md create mode 100755 build_release.sh create mode 100644 kecpkg/commands/sign.py create mode 100644 kecpkg/gpg.py rename pyproject.toml => pyproject.toml.depr (95%) create mode 100644 tests/commands/test_sign.py diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml index 162458b..cbd2af5 100644 --- a/.idea/inspectionProfiles/Project_Default.xml +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -4,11 +4,13 @@ diff --git a/.idea/kecpkg-tools.iml b/.idea/kecpkg-tools.iml index b569552..688685a 100644 --- a/.idea/kecpkg-tools.iml +++ b/.idea/kecpkg-tools.iml @@ -2,7 +2,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 998812f..0b4d9e3 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,7 +1,7 @@ - + - \ No newline at end of file diff --git a/.idea/runConfigurations/kecpkg_new.xml b/.idea/runConfigurations/kecpkg_new.xml index 834b0a1..7962f3d 100644 --- a/.idea/runConfigurations/kecpkg_new.xml +++ b/.idea/runConfigurations/kecpkg_new.xml @@ -1,5 +1,6 @@ + \ No newline at end of file diff --git a/.idea/runConfigurations/kecpkg_sign.xml b/.idea/runConfigurations/kecpkg_sign.xml new file mode 100644 index 0000000..2c61007 --- /dev/null +++ b/.idea/runConfigurations/kecpkg_sign.xml @@ -0,0 +1,33 @@ + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/kecpkg_upload.xml b/.idea/runConfigurations/kecpkg_upload.xml index 92a55f8..75070a3 100644 --- a/.idea/runConfigurations/kecpkg_upload.xml +++ b/.idea/runConfigurations/kecpkg_upload.xml @@ -1,5 +1,6 @@ + \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 70790b0..b331123 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - "2.7" - "3.5" - "3.6" + - "3.7-dev" - "pypy" - "pypy3" diff --git a/CHANGELOG.md b/CHANGELOG.md index 5056192..f923c69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 1.0.0 (28MAY19) +Version 1.0.0 release of the `kecpkg-tools` as in the past year no updates were deemed necessary and it is heavily used internally by KE-works BV and at customers to manage ke-chain script packages (KECPKG's). Package signing is only available for Python 3. + + * Added the ability to manage signatures and keys. We built a Publick Key Infrastructure to sign packages and have the ability to trust packages signed with a developer key. The process of creating and submitting a key to be included in the trusted keyring of KE-chain will be on our [support portal](https://support.ke-chain.com) later when it is all available in KE-chain production. Please check out the documentation of the commandline interface using `kecpkg sign --help` for further information. + * The build process is does now provide a list of artifacts (ARTIFACTS) that are included in a kecpkg. The list of artifacts consist out of the (relative pathname), the hash of the file (normally sha256) and the filesize. KE-chain is able to check the contents of the kecpkgs after upload against this file and will determine of the kecpkgs is untempered on disk. + * The build process also now provides an optional `kecpkg build --sign` command flag to include a signature inside the keckpg. When package signing is enabled using the `--sign` flag, the list of artifacts (ARTIFACTS file) is signed with the cryptographic signature of the developer (ARTIFACTS.SIG). This signature can be checked by KE-chain after upload when the public key of the developer is known and trusted by KE-chain. This might enable running the contained scripts on higher than scope manager permissions. + * Adding dependent permission on GPG on linux or windows in order to enable the package signing features. + * Added dependent packages tabulate, appdirs and python-gnupg. + + ## 0.9.0 (16JAN18) * added the ability to add multiple configurations. You can use this to create multiple settings files and build for each setting file another kecpkg. Use `kecpkg build --settings ` to create a new kecpkg in the `dist` directory. The `package-info.json` will be recreated based on what is set in the `settings` and stored inside the kecpkg. Use `kecpkg upload --settings ` to upload this kecpkg to KE-chain. You can now use a cmd or batch script with multiple setting files to create a multitude of kecpkgs and automatically upload (and even replace) them in a KE-chain project. * added `--update` and `--no-update` flags to `kecpkg build`. The `package-info.json` file is needed for the KE-crunch server to understand what module and what function inside the kecpkg to execute. Normally this is re-rendered (updated) in each build sessions based on the contents of the settings file. If you have a custom `package-info.json`, you can use the `--no-update` flag on `kecpkg build --no-update` to prevent the updating the `package-info.json`. You might want to consider updating the settings file with the correct values for the `package-info.json` instead. diff --git a/MANIFEST.in b/MANIFEST.in index f5d73fc..e99fbf3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,16 +1,18 @@ -include README.rst +include README.md include LICENSE include CONTRIBUTED include CHANGELOG.md include requirements.txt +include Pipfile include tox.ini -include pyproject.toml include .coveragerc graft tests graft kecpkg prune .idea prune .env* +prune *.depr +exclude build_release.sh recursive-include kecpkg *.template global-exclude *.pyc diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..1871e24 --- /dev/null +++ b/Pipfile @@ -0,0 +1,24 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +twine = "*" +tox = "*" +pytest = "*" + + +[packages] +kecpkg-tools = {editable = true,file = "file:///Users/jochem/dev/kecpkg-tools"} +click = "*" +atomicwrites = "*" +pykechain = ">=2.0.0" +appdirs = "*" +python-gnupg = "*" +tabulate = "*" +toml = "*" +Jinja2 = "*" + +[requires] +python_version = "3" diff --git a/README.md b/README.md new file mode 100644 index 0000000..9d173fb --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +kecpkg-tools +============ + +[![PyPI](https://img.shields.io/pypi/v/kecpkg-tools.svg)](https://pypi.python.org/pypi/kecpkg-tools) +[![PyPI - Status](https://img.shields.io/pypi/status/kecpkg-tools.svg)](https://pypi.python.org/pypi/kecpkg-tools) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/kecpkg-tools.svg) +[![Travis Build](https://travis-ci.org/KE-works/kecpkg-tools.svg?branch=master)](https://travis-ci.org/KE-works/kecpkg-tools) +[![Join the chat at https://gitter.im/KE-works/pykechain](https://badges.gitter.im/KE-works/pykechain.svg)](https://gitter.im/KE-works/pykechain?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +Usage +----- + +kecpkg-tools provide a set of tools to easily create KE-chain packages. +These are executable python scripts aimed for execution on the KE-chain +SIM platform. + +It requires normal user access to a [KE-chain](http://www.ke-chain.com) +instance for it to work. KE-chain is the flexible engineering platform +of [KE-works](http://www.ke-works.com). + +See Also +-------- + +KE-chain packages for SIM are used in combination with +[pykechain](https://github.com/KE-works/pykechain), the open source +KE-chain python api. + +Installation +------------ + +kecpkg-tools is distributed on [PyPI](https://pypi.org) as a universal +wheel and is available on Linux/macOS and Windows and supports Python +2.7/3.4+ and PyPy. + +``` {.sourceCode .bash} +$ pip install --user --upgrade kecpkg-tools +``` + +or when pip is not installed on the system + +``` {.sourceCode .bash} +$ python3 -m pip install --user --upgrade kecpkg-tools +``` + +License +------- + +kecpkg-tools is distributed under the terms of the [Apache License, +Version 2.0](https://choosealicense.com/licenses/apache-2.0). diff --git a/README.rst b/README.rst index fbd38a9..d07a27f 100644 --- a/README.rst +++ b/README.rst @@ -44,7 +44,14 @@ Python 2.7/3.5+ and PyPy. .. code-block:: bash - $ pip install kecpkg-tools + $ pip install --user --upgrade kecpkg-tools + +or when pip is not installed on the system + +.. code-block:: bash + + $ python3 -m pip install --user --upgrade kecpkg-tools + License ------- diff --git a/build_release.sh b/build_release.sh new file mode 100755 index 0000000..6d79be3 --- /dev/null +++ b/build_release.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +rm -rf ./build ./dist +python setup.py bdist_wheel --universal +twine upload dist/kecpkg_tools-*.whl diff --git a/kecpkg/__init__.py b/kecpkg/__init__.py index e4e49b3..cd7ca49 100644 --- a/kecpkg/__init__.py +++ b/kecpkg/__init__.py @@ -1 +1 @@ -__version__ = '0.9.0' +__version__ = '1.0.1' diff --git a/kecpkg/cli.py b/kecpkg/cli.py index 9a07d8c..cb84ecc 100644 --- a/kecpkg/cli.py +++ b/kecpkg/cli.py @@ -4,6 +4,7 @@ from kecpkg.commands.new import new from kecpkg.commands.prune import prune from kecpkg.commands.purge import purge +from kecpkg.commands.sign import sign from kecpkg.commands.upload import upload from kecpkg.commands.config import config from kecpkg.commands.utils import CONTEXT_SETTINGS @@ -28,3 +29,4 @@ def kecpkg(): kecpkg.add_command(purge) kecpkg.add_command(prune) kecpkg.add_command(config) +kecpkg.add_command(sign) diff --git a/kecpkg/commands/build.py b/kecpkg/commands/build.py index aedf4f8..03ad5d9 100644 --- a/kecpkg/commands/build.py +++ b/kecpkg/commands/build.py @@ -1,11 +1,17 @@ +import hashlib import os +import sys +from pprint import pprint from zipfile import ZipFile import click as click -from kecpkg.commands.utils import CONTEXT_SETTINGS, echo_info -from kecpkg.settings import load_settings, SETTINGS_FILENAME -from kecpkg.utils import ensure_dir_exists, remove_path, get_package_dir, get_artifacts_on_disk, render_package_info +from kecpkg.commands.sign import verify_signature, verify_artifacts_hashes +from kecpkg.commands.utils import CONTEXT_SETTINGS +from kecpkg.gpg import hash_of_file, get_gpg, tabulate_keys +from kecpkg.settings import load_settings, SETTINGS_FILENAME, ARTIFACTS_SIG_FILENAME, ARTIFACTS_FILENAME +from kecpkg.utils import ensure_dir_exists, remove_path, get_package_dir, get_artifacts_on_disk, render_package_info, \ + create_file, echo_success, echo_failure, echo_info @click.command(context_settings=CONTEXT_SETTINGS, @@ -19,6 +25,14 @@ @click.option('--update/--no-update', 'update_package_info', is_flag=True, default=True, help="Update the `package-info.json` for the KE-crunch execution to point to correct entrypoint based on " "settings. This is okay to leave ON. Use `--no-update` if you have a custom `package-info.json`.") +@click.option('--sign/--no-sign', 'do_sign', is_flag=True, default=False, + help="Sign the contents of the package with a cryptographic key from the keyring. Defaults to not sign.") +@click.option('--keyid', '--key-id', '-k', 'sign_keyid', + help="ID of the cryptographic key to do the sign the contents of the built package. If not provided it " + "will use the default key from the KECPKG keystore. Use in combination with `--sign`") +@click.option('--passphrase', '-p', 'sign_passphrase', hide_input=True, + help="Passphrase of the cryptographic key to sing the contents of the built package. " + "Use in combination with `--sign` and `--keyid`") @click.option('-v', '--verbose', help="Be more verbose", is_flag=True) def build(package=None, **options): """Build the package and create a kecpkg file.""" @@ -40,18 +54,113 @@ def build(package=None, **options): ensure_dir_exists(build_path) # do package building - build_package(package_dir, build_path, settings, verbose=options.get('verbose')) + build_package(package_dir, build_path, settings, options=options, verbose=options.get('verbose')) + echo_success('Complete') -def build_package(package_dir, build_path, settings, verbose=False): + +def build_package(package_dir, build_path, settings, options=None, verbose=False): """Perform the actual building of the kecpkg zip.""" additional_exclude_paths = settings.get('exclude_paths') - artifacts = get_artifacts_on_disk(package_dir, verbose=verbose, additional_exclude_paths=additional_exclude_paths) + artifacts = get_artifacts_on_disk(package_dir, verbose=verbose, + additional_exclude_paths=additional_exclude_paths) # type: set dist_filename = '{}-{}-py{}.kecpkg'.format(settings.get('package_name'), settings.get('version'), settings.get('python_version')) echo_info('Creating package name `{}`'.format(dist_filename)) + if verbose: + echo_info("Creating 'ARTIFACTS' file with list of contents and their hashes") + generate_artifact_hashes(package_dir, artifacts, settings, verbose=verbose) + artifacts.add(settings.get('artifacts_filename', 'ARTIFACTS')) + + if options.get('do_sign'): + sign_package(package_dir, settings, options=options, verbose=verbose) + artifacts.add(settings.get('artifacts_filename', 'ARTIFACTS') + '.SIG') + with ZipFile(os.path.join(build_path, dist_filename), 'w') as dist_zip: for artifact in artifacts: dist_zip.write(os.path.join(package_dir, artifact), arcname=artifact) + + +def generate_artifact_hashes(package_dir, artifacts, settings, verbose=False): + """ + Generate artifact hashes and store it on disk in a ARTIFACTS file. + + using settings > artifacts_filename to retrieve the artifacts (default ARTIFACTS). + using settings > hash_algorithm to determine the right algoritm for hashing (default sha256) + + :param package_dir: package directory (fullpath) + :param artifacts: list of artifacts to store in kecpkg + :param settings: settings object + :param verbose: be verbose (or not) + :return: None + """ + artifacts_fn = settings.get('artifacts_filename', 'ARTIFACTS') + algorithm = settings.get('hash_algorithm', 'sha256') + if algorithm not in hashlib.algorithms_guaranteed: + raise + + # save content of the artifacts file + # A line is "README.md,sha256=d831....ccf79a,336" + # ^filename ^algo ^hash ^size in bytes + artifacts_content = [] + + for af in artifacts: + # we do not need to create a hash from the ARTIFACTS and ARTIFACTS.SIG file if they are present in the list + if af not in [artifacts_fn, artifacts_fn + '.SIG']: + af_fp = os.path.join(package_dir, af) + artifacts_content.append('{},{}={},{}\n'.format( + af, + algorithm, + hash_of_file(af_fp, algorithm=algorithm), + os.stat(af_fp).st_size + )) + + create_file(os.path.join(package_dir, artifacts_fn), + content=artifacts_content, + overwrite=True) + + +def sign_package(package_dir, settings, options=None, verbose=False): + """ + Sign the package with a GPG/PGP key. + + :param package_dir: directory fullpath of the package + :param settings: settings object + :param options: commandline options dictionary passed down. + :param verbose: be verbose (or not) + :return: None + """ + gpg = get_gpg() + + if options.get('sign_keyid') is None: + tabulate_keys(gpg, explain=True) + options['sign_keyid'] = click.prompt("Provide Key (Name, Comment, Email, Fingerprint) to sign package with", + default=settings.get('email')) + if options.get('sign_passphrase') is None: + options['sign_passphrase'] = click.prompt("Provide Passphrase", hide_input=True) + + echo_info('Signing package contents') + + with open(os.path.join(package_dir, settings.get('artifacts_filename', ARTIFACTS_FILENAME)), 'rb') as fd: + results = gpg.sign_file(fd, + keyid=options.get('sign_keyid'), + passphrase=options.get('sign_passphrase'), + detach=True, + output=settings.get('artifacts_sig_filename', ARTIFACTS_SIG_FILENAME) + ) + pprint(results.__dict__) + + if results and results.status is not None: + echo_info("Signed package contents: {}".format(results.status)) + else: + failure_text = results.stderr.split("\n")[-2] + echo_failure("Could not sign the package contents: '{}'".format(failure_text)) + sys.exit(1) + + if verbose: + echo_success('Successfully signed the package contents.') + + verify_signature(package_dir, ARTIFACTS_FILENAME, ARTIFACTS_SIG_FILENAME) + verify_artifacts_hashes(package_dir, ARTIFACTS_FILENAME) diff --git a/kecpkg/commands/config.py b/kecpkg/commands/config.py index 3cad738..075d0e6 100644 --- a/kecpkg/commands/config.py +++ b/kecpkg/commands/config.py @@ -1,10 +1,11 @@ import os import click +from tabulate import tabulate -from kecpkg.commands.utils import CONTEXT_SETTINGS, echo_info, echo_success +from kecpkg.commands.utils import CONTEXT_SETTINGS from kecpkg.settings import load_settings, copy_default_settings, save_settings, SETTINGS_FILENAME -from kecpkg.utils import get_package_dir, copy_path +from kecpkg.utils import get_package_dir, copy_path, echo_success, echo_info @click.command(context_settings=CONTEXT_SETTINGS, @@ -15,13 +16,15 @@ type=click.Path(exists=True), default=SETTINGS_FILENAME) @click.option('--init', is_flag=True, help="will init a settingsfile if not found") @click.option('--interactive', '-i', is_flag=True, help="interactive mode; guide me through the settings") +@click.option('--get', '-g', 'get_key', help="Key to get and display", required=False) +@click.option('--set', '-s', 'set_key', nargs=2, help="Key to set . Value is set as string.", + required=False) @click.option('--verbose', '-v', is_flag=True, help="be more verbose (print settings)") def config(package, **options): - r"""Manage the configuration (or settings) of the package. + """Manage the configuration (or settings) of the package. The various settings in the .kecpkg-settings.json file are: - \b package_name: name of the package version: version number of the package description: longer description of the package @@ -74,9 +77,20 @@ def config(package, **options): value_proc=process_additional_exclude_paths) save_settings(settings, package_dir=package_dir, settings_filename=options.get('settings_filename')) + if options.get('set_key'): + k, v = options.get('set_key') + if options.get('verbose'): + echo_info("Set the key '{}' to value '{}'".format(k, v)) + settings[k] = v + save_settings(settings, package_dir=package_dir, settings_filename=options.get('settings_filename')) + + if options.get('get_key'): + echo_info(tabulate([(options.get('get_key'), settings.get(options.get('get_key')))], + headers=("key", "value"))) + return + if options.get('verbose'): - for k, v in settings.items(): - echo_info(" {}: '{}'".format(k, v)) + echo_info(tabulate(settings.items(), headers=("key", "value"))) if not options.get('interactive'): echo_success('Settings file identified and correct') diff --git a/kecpkg/commands/new.py b/kecpkg/commands/new.py index c62268f..2ab9af8 100644 --- a/kecpkg/commands/new.py +++ b/kecpkg/commands/new.py @@ -4,10 +4,10 @@ import click from kecpkg.commands.config import process_additional_exclude_paths -from kecpkg.commands.utils import echo_failure, CONTEXT_SETTINGS, echo_info, echo_success +from kecpkg.commands.utils import CONTEXT_SETTINGS from kecpkg.create import create_package, create_venv, pip_install_venv from kecpkg.settings import load_settings, copy_default_settings, save_settings, SETTINGS_FILENAME -from kecpkg.utils import normalise_name +from kecpkg.utils import normalise_name, echo_success, echo_failure, echo_info @click.command(short_help="Create a new kecpkg SIM script package", diff --git a/kecpkg/commands/prune.py b/kecpkg/commands/prune.py index 6415a66..1160e01 100644 --- a/kecpkg/commands/prune.py +++ b/kecpkg/commands/prune.py @@ -2,9 +2,9 @@ import click -from kecpkg.commands.utils import CONTEXT_SETTINGS, echo_failure, echo_warning +from kecpkg.commands.utils import CONTEXT_SETTINGS from kecpkg.settings import load_settings -from kecpkg.utils import get_package_name, get_package_dir, remove_path +from kecpkg.utils import get_package_name, get_package_dir, remove_path, echo_failure, echo_warning @click.command(context_settings=CONTEXT_SETTINGS, diff --git a/kecpkg/commands/purge.py b/kecpkg/commands/purge.py index 6caa88a..714eefb 100644 --- a/kecpkg/commands/purge.py +++ b/kecpkg/commands/purge.py @@ -2,8 +2,8 @@ import click -from kecpkg.commands.utils import CONTEXT_SETTINGS, echo_warning, echo_failure, echo_success -from kecpkg.utils import remove_path, get_package_dir +from kecpkg.commands.utils import CONTEXT_SETTINGS +from kecpkg.utils import remove_path, get_package_dir, echo_success, echo_failure, echo_warning @click.command(context_settings=CONTEXT_SETTINGS, diff --git a/kecpkg/commands/sign.py b/kecpkg/commands/sign.py new file mode 100644 index 0000000..9e0bbeb --- /dev/null +++ b/kecpkg/commands/sign.py @@ -0,0 +1,286 @@ +import os +import sys +from pprint import pprint + +import click +from pykechain.utils import temp_chdir + +from kecpkg.commands.utils import CONTEXT_SETTINGS +from kecpkg.gpg import get_gpg, list_keys, hash_of_file +from kecpkg.settings import SETTINGS_FILENAME, GNUPG_KECPKG_HOME, load_settings, DEFAULT_SETTINGS, ARTIFACTS_FILENAME, \ + ARTIFACTS_SIG_FILENAME +from kecpkg.utils import remove_path, echo_info, echo_success, echo_failure, get_package_dir, unzip_package + + +@click.command(context_settings=CONTEXT_SETTINGS, + short_help="Perform package signing and key management.") +@click.argument('package', required=False) +@click.option('--settings', '--config', '-s', 'settings_filename', + help="path to the setting file (default `{}`".format(SETTINGS_FILENAME), + type=click.Path(), default=SETTINGS_FILENAME) +@click.option('--keyid', '--key-id', '-k', 'keyid', + help="ID (name, email, KeyID) of the cryptographic key to do the operation with. ") +# @click.option('--passphrase', '-p', 'sign_passphrase', hide_input=True, +# help="Passphrase of the cryptographic key to sign the contents of the package. " +# "Use in combination with `--sign` and `--keyid`") +@click.option('--import-key', '--import', '-i', 'do_import', type=click.Path(exists=True), + help="Import secret keyfile (in .asc) to the KECPKG keyring which will be used for signing. " + "You can export a created key in gpg with `gpg -a --export-secret-key [keyID] > secret_key.asc`.") +@click.option('--delete-key', '-d', 'do_delete_key', + help="Delete key by its fingerprint permanently from the KECPKG keyring. To retrieve the full " + "fingerprint of the key, use the `--list` option and look at the 'fingerprint' section.") +@click.option('--create-key', '-c', 'do_create_key', is_flag=True, + help="Create secret key and add it to the KECPKG keyring.") +@click.option('--export-key', '--export', '-e', 'do_export_key', type=click.Path(), + help="Export public key to filename with `--keyid KeyID` in .ASC format for public distribution.") +@click.option('--clear-keyring', 'do_clear', is_flag=True, default=False, + help="Clear all keys from the KECPKG keyring") +@click.option('--list', '-l', 'do_list', is_flag=True, + help="List all available keys in the KECPKG keyring") +@click.option('--verify-kecpkg', 'do_verify_kecpkg', type=click.Path(exists=True), + help="Verify contents and signature of an existing kecpkg.") +@click.option('--yes', '-y', 'do_yes', is_flag=True, + help="Don't ask questions, just do it.") +@click.option('-v', '--verbose', help="Be more verbose", is_flag=True) +def sign(package=None, **options): + """Sign the package.""" + # noinspection PyShadowingNames + def _do_clear(options): + echo_info("Clearing all keys from the KECPKG keyring") + if not options.get('do_yes'): + options['do_yes'] = click.confirm("Are you sure you want to clear the KECPKG keyring?", default=False) + if options.get('do_yes'): + remove_path(GNUPG_KECPKG_HOME) + echo_success("Completed") + sys.exit(0) + else: + echo_failure("Not removing the KECPKG keyring") + sys.exit(1) + + def _do_list(gpg, explain=False): + if explain: + echo_info("Listing all keys from the KECPKG keyring") + result = gpg.list_keys(secret=True) + if len(result): + from tabulate import tabulate + print(tabulate(list_keys(gpg=gpg), headers=("Name", "Comment", "E-mail", "Expires", "Fingerprint"))) + else: + if explain: + echo_info("No keys found in KECPKG keyring. Use `--import-key` or `--create-key` to add a " + "secret key to the KECPKG keyring in order to sign KECPKG's.") + sys.exit(1) + + # noinspection PyShadowingNames + def _do_import(gpg, options): + echo_info("Importing secret key into KECPKG keyring from '{}'".format(options.get('do_import'))) + result = gpg.import_keys(open(os.path.abspath(options.get('do_import')), 'rb').read()) + # pprint(result.__dict__) + if result and result.sec_imported: + echo_success("Succesfully imported secret key into the KECPKG keystore") + _do_list(gpg=gpg) + sys.exit(0) + elif result and result.unchanged: + echo_failure("Did not import the secret key into the KECPKG keystore. The key was already " + "in place and was unchanged") + _do_list(gpg=gpg) + sys.exit(1) + + echo_failure("Did not import a secret key into the KECPKG keystore. Is something wrong " + "with the file: '{}'? Are you sure it is a ASCII file containing a " + "private key block?".format(options.get('do_import'))) + sys.exit(1) + + # noinspection PyShadowingNames + def _do_delete_key(gpg, options): + echo_info("Deleting private key with ID '{}' from the KECPKG keyring".format(options.get('do_delete_key'))) + + # custom call to gpg using --delete-secret-and-public-key + result = gpg.result_map['delete'](gpg) + # noinspection PyProtectedMember + p = gpg._open_subprocess(['--yes', '--delete-secret-and-public-key', options.get('do_delete_key')]) + # noinspection PyProtectedMember + gpg._collect_output(p, result, stdin=p.stdin) + + # result = gpg.delete_keys(fingerprints=options.get('do_delete_key'), + # secret=True, + # passphrase=options.get('sign_passphrase')) + # pprint(result.__dict__) + if result and result.stderr.find("failed") < 0: + echo_success("Succesfully deleted key") + _do_list(gpg=gpg) + sys.exit(0) + + echo_failure("Could not delete key.") + sys.exit(1) + + # noinspection PyShadowingNames + def _do_create_key(gpg, options): + echo_info("Will create a secret key and store it into the KECPKG keyring.") + package_dir = get_package_dir(package_name=package, fail=False) + settings = DEFAULT_SETTINGS + if package_dir is not None: + package_name = os.path.basename(package_dir) + echo_info('Package `{}` has been selected'.format(package_name)) + settings = load_settings(package_dir=package_dir, settings_filename=options.get('settings_filename')) + + key_info = {'name_real': click.prompt("Name", default=settings.get('name')), + 'name_comment': click.prompt("Comment", default="KECPKG SIGNING KEY"), + 'name_email': click.prompt("Email", default=settings.get('email')), + 'expire_date': click.prompt("Expiration in months", default=12, + value_proc=lambda i: "{}m".format(i)), 'key_type': 'RSA', + 'key_length': 4096, + 'key_usage': '', + 'subkey_type': 'RSA', + 'subkey_length': 4096, + 'subkey_usage': 'encrypt,sign,auth', + 'passphrase': ''} + + passphrase = click.prompt("Passphrase", hide_input=True) + passphrase_confirmed = click.prompt("Confirm passphrase", hide_input=True) + if passphrase == passphrase_confirmed: + key_info['passphrase'] = passphrase + else: + raise ValueError("The passphrases did not match.") + + echo_info("Creating the secret key '{name_real} ({name_comment}) <{name_email}>'".format(**key_info)) + echo_info("Please move around mouse or generate other activity to introduce sufficient entropy. " + "This might take a minute...") + result = gpg.gen_key(gpg.gen_key_input(**key_info)) + pprint(result.__dict__) + if result and result.stderr.find('KEY_CREATED'): + echo_success("The key is succesfully created") + _do_list(gpg=gpg) + sys.exit(0) + + echo_failure("Could not generate the key due to an error: '{}'".format(result.stderr)) + sys.exit(1) + + # noinspection PyShadowingNames + def _do_export_key(gpg, options): + """Export public key.""" + echo_info("Exporting public key") + if options.get('keyid') is None: + _do_list(gpg=gpg) + options['keyid'] = click.prompt("Provide KeyId (name, comment, email, fingerprint) of the key to export") + result = gpg.export_keys(keyids=[options.get('keyid')], secret=False, armor=True) + + if result is not None: + with open(options.get('do_export_key'), 'w') as fd: + fd.write(result) + echo_success("Sucessfully written public key to '{}'".format(options.get('do_export_key'))) + sys.exit(0) + + echo_failure("Could not export key") + sys.exit(1) + + # noinspection PyShadowingNames + def _do_verify_kecpkg(gpg, options): + """Verify the kecpkg.""" + echo_info("Verify the contents of the KECPKG and if the KECPKG is signed with a valid signature.") + + current_working_directory = os.getcwd() + + with temp_chdir() as d: + unzip_package(package_path=os.path.join(current_working_directory, options.get('do_verify_kecpkg')), + target_path=d) + verify_signature(d, artifacts_filename=ARTIFACTS_FILENAME, artifacts_sig_filename=ARTIFACTS_SIG_FILENAME) + verify_artifacts_hashes(d, artifacts_filename=ARTIFACTS_FILENAME) + sys.exit(0) + + # + # Dispatcher to subfunctions + # + + if options.get('do_clear'): + _do_clear(options=options) + elif options.get('do_list'): + _do_list(gpg=get_gpg(), explain=True) + elif options.get('do_import'): + _do_import(gpg=get_gpg(), options=options) + elif options.get('do_delete_key'): + _do_delete_key(gpg=get_gpg(), options=options) + elif options.get('do_create_key'): + _do_create_key(gpg=get_gpg(), options=options) + elif options.get('do_export_key'): + _do_export_key(gpg=get_gpg(), options=options) + elif options.get('do_verify_kecpkg'): + _do_verify_kecpkg(gpg=get_gpg(), options=options) + else: + sys.exit(500) + sys.exit(0) + + +def verify_signature(package_dir, artifacts_filename, artifacts_sig_filename): + """ + Check signature of the package. + + :param package_dir: directory fullpath of the package + :param artifacts_filename: path of the artifacts file + :param artifacts_sig_filename: path of the artifacts signature file + :return: None + """ + gpg = get_gpg() + artifacts_fp = os.path.join(package_dir, artifacts_filename) + artifacts_sig_fp = os.path.join(package_dir, artifacts_sig_filename) + if not os.path.exists(artifacts_fp): + echo_failure("Artifacts file does not exist: '{}'".format(artifacts_filename)) + sys.exit(1) + if not os.path.exists(artifacts_sig_fp): + echo_failure("Artifacts signature file does not exist: '{}'. Is the package signed?". + format(artifacts_filename)) + sys.exit(1) + + with open(artifacts_sig_fp, 'rb') as sig_fd: + results = gpg.verify_file(sig_fd, data_filename=artifacts_fp) + + if results.valid: + echo_info("Verified the signature and the signature is valid") + echo_info("Signed with: '{}'".format(results.username)) + elif not results.valid: + echo_failure("Signature of the package is invalid") + echo_failure(pprint(results.__dict__)) + sys.exit(1) + + +def verify_artifacts_hashes(package_dir, artifacts_filename): + """ + Check the hashes of the artifacts in the package. + + :param package_dir: directory fullpath of the package + :param artifacts_filename: filename of the artifacts file + :return: + """ + artifacts_fp = os.path.join(package_dir, artifacts_filename) + if not os.path.exists(artifacts_fp): + echo_failure("Artifacts file does not exist: '{}'".format(artifacts_filename)) + sys.exit(1) + + with open(artifacts_fp, 'r') as fd: + artifacts = fd.readlines() + + # process the file contents + # A line is "README.md,sha256=d831....ccf79a,336" + # ^filename ^algo ^hash ^size in bytes + fails = [] + for af in artifacts: + # noinspection PyShadowingBuiltins,PyShadowingBuiltins + filename, hash, orig_size = af.split(',') + algorithm, orig_hash = hash.split('=') + fp = os.path.join(package_dir, filename) + if os.path.exists(fp): + found_hash = hash_of_file(fp, algorithm) + found_size = os.stat(fp).st_size + if found_hash != orig_hash.strip() or found_size != int(orig_size.strip()): + fails.append("File '{}' is changed in the package.".format(filename)) + fails.append("File '{}' original checksum: '{}', found: '{}'".format(filename, orig_hash, found_hash)) + fails.append("File '{}' original size: {}, found: {}".format(filename, orig_size, found_size)) + else: + fails.append("File '{}' does not exist".format(filename)) + + if fails: + echo_failure('The package has been changed after building the package.') + for fail in fails: + print(fail) + sys.exit(1) + else: + echo_info("Package contents succesfully verified.") diff --git a/kecpkg/commands/upload.py b/kecpkg/commands/upload.py index 971bccd..83bca17 100644 --- a/kecpkg/commands/upload.py +++ b/kecpkg/commands/upload.py @@ -4,9 +4,9 @@ import click as click from pykechain import Client, get_project -from kecpkg.commands.utils import CONTEXT_SETTINGS, echo_info, echo_success, echo_failure +from kecpkg.commands.utils import CONTEXT_SETTINGS from kecpkg.settings import load_settings, save_settings, SETTINGS_FILENAME -from kecpkg.utils import get_package_dir, get_package_name +from kecpkg.utils import get_package_dir, get_package_name, echo_success, echo_failure, echo_info @click.command(context_settings=CONTEXT_SETTINGS, @@ -176,6 +176,7 @@ def upload_package(scope, build_path=None, kecpkg_path=None, service_id=None, se # Wrap up party! echo_success("kecpkg `{}` successfully uploaded to KE-chain.".format(os.path.basename(kecpkg_path))) + # noinspection PyProtectedMember success_url = "{api_root}/#scopes/{scope_id}/scripts/{service_id}".format( api_root=scope._client.api_root, scope_id=scope.id, diff --git a/kecpkg/commands/utils.py b/kecpkg/commands/utils.py index 9df1565..48e89dd 100644 --- a/kecpkg/commands/utils.py +++ b/kecpkg/commands/utils.py @@ -1,58 +1,7 @@ -import click - CONTEXT_SETTINGS = { 'help_option_names': ['-h', '--help'], + 'max_content_width': 110 } UNKNOWN_OPTIONS = { 'ignore_unknown_options': True, }.update(CONTEXT_SETTINGS) - - -def echo_success(text, nl=True): - """ - Write to the console as a success (Cyan bold). - - :param text: string to write - :param nl: add newline - """ - click.secho(text, fg='cyan', bold=True, nl=nl) - - -def echo_failure(text, nl=True): - """ - Write to the console as a failure (Red bold). - - :param text: string to write - :param nl: add newline - """ - click.secho(text, fg='red', bold=True, nl=nl) - - -def echo_warning(text, nl=True): - """ - Write to the console as a warning (Yellow bold). - - :param text: string to write - :param nl: add newline - """ - click.secho(text, fg='yellow', bold=True, nl=nl) - - -def echo_waiting(text, nl=True): - """ - Write to the console as a waiting (Magenta bold). - - :param text: string to write - :param nl: add newline - """ - click.secho(text, fg='magenta', bold=True, nl=nl) - - -def echo_info(text, nl=True): - """ - Write to the console as a informational (bold). - - :param text: string to write - :param nl: add newline - """ - click.secho(text, bold=True, nl=nl) diff --git a/kecpkg/create.py b/kecpkg/create.py index c2ac88f..f728967 100644 --- a/kecpkg/create.py +++ b/kecpkg/create.py @@ -6,9 +6,9 @@ import six -from kecpkg.commands.utils import echo_failure, echo_info, echo_success from kecpkg.files.rendering import render_to_file -from kecpkg.utils import ensure_dir_exists, get_proper_python, NEED_SUBPROCESS_SHELL, venv +from kecpkg.utils import (ensure_dir_exists, get_proper_python, NEED_SUBPROCESS_SHELL, venv, + echo_success, echo_failure, echo_info) def create_package(package_dir, settings): @@ -57,6 +57,9 @@ def create_venv(package_dir, settings, pypath=None, use_global=False, verbose=Fa :param package_dir: the full path to the package directory :param settings: the settings dict (including the venv_dir name to create the right venv) + :param pypath: absolute path to the python binary interpreter to create the virtual environment with + :param use_global: Use global sysem site packages when creating virtual environment (default False) + :param verbose: Use verbosity (default False) """ venv_dir = os.path.join(package_dir, settings.get('venv_dir')) diff --git a/kecpkg/files/templates/.gitignore.template b/kecpkg/files/templates/.gitignore.template index 2ac6b54..b18344e 100644 --- a/kecpkg/files/templates/.gitignore.template +++ b/kecpkg/files/templates/.gitignore.template @@ -97,3 +97,12 @@ pip-selfcheck.json .idea/**/gradle.xml .idea/**/libraries +# VSCode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json + +# GnuPG +*.asc diff --git a/kecpkg/files/templates/script.py.template b/kecpkg/files/templates/script.py.template index 0b37beb..d9f184a 100644 --- a/kecpkg/files/templates/script.py.template +++ b/kecpkg/files/templates/script.py.template @@ -11,6 +11,7 @@ import sys __version__ = '{{ version }}' + def main(**kwargs): """ Main entry point of the script diff --git a/kecpkg/gpg.py b/kecpkg/gpg.py new file mode 100644 index 0000000..ed455e2 --- /dev/null +++ b/kecpkg/gpg.py @@ -0,0 +1,122 @@ +import hashlib +import logging +import os +import re +import subprocess +import sys +from datetime import datetime + +import gnupg +import six + +from kecpkg.settings import GNUPG_KECPKG_HOME +from kecpkg.utils import ON_LINUX, ON_WINDOWS, ON_MACOS, echo_failure, read_chunks, echo_info + +LOGLEVEL = logging.INFO + + +def hash_of_file(path, algorithm='sha256'): + """Return the hash digest of a file.""" + with open(path, 'rb') as archive: + hash = hashlib.new(algorithm) + for chunk in read_chunks(archive): + hash.update(chunk) + return hash.hexdigest() + + +__gpg = None # type: gnupg.GPG or None + + +def get_gpg(): + # type: () -> gnupg.GPG + """Return the GPG objects instantiated with custom KECPKG keyring in custom KECPKG GNUPG home.""" + global __gpg + if not __gpg: + if six.PY2: + echo_failure('Package signing capability is not available in python 2.7. Please use python 3 or greater.') + sys.exit(1) + + import gnupg + logging.basicConfig(level=LOGLEVEL) + logging.getLogger('gnupg') + gpg_bin = 'gpg' + if ON_LINUX: + gpg_bin = subprocess.getoutput('which gpg') + if ON_WINDOWS: + gpg_bin = 'C:\\Program Files\\GnuPG\\gpg' + elif ON_MACOS: + gpg_bin = '/usr/local/bin/gpg' + if not os.path.exists(gpg_bin): + echo_failure("Unable to detect installed GnuPG executable. Ensure you have it installed. " + "We checked: '{}'".format(gpg_bin)) + echo_failure("- For Linux please install GnuPG using your package manager. In Ubuntu/Debian this can be " + "achieved with `sudo apt install gnupg`.") + echo_failure("- For Mac OSX please install GnuPG using `brew install gpg`.") + echo_failure("- For Windows please install GnuPG using the downloads via: https://gnupg.org/download/") + + __gpg = gnupg.GPG(gpgbinary=gpg_bin, gnupghome=GNUPG_KECPKG_HOME) + + return __gpg + + +def list_keys(gpg): + """ + List all keys from the KECPKG keystore and return it as a list of list. + + :param gpg: GPG object + :return: list of [name, comment, email, expires(str), fingerprint] for each key in the keystore + """ + result = gpg.list_keys(secret=True) + key_list = [] + for r in result: + uids = parse_key_uids(r.get('uids')) + row = [ + uids.get('name'), + uids.get('comment'), + uids.get('email'), + str(datetime.fromtimestamp(int(r.get('expires')))), + r.get('fingerprint') + ] + key_list.append(row) + return key_list + + +def tabulate_keys(gpg, explain=False): + """ + List all keys in a table for printing on the CLI. + + Will print a nice table of keys with Name, Comment, E-mail, Expires and Fingerprint. + If explain = Truem, it will exit with returncode 1 when no keys are present. + + :param gpg: GPG objects + :param explain: With explain is True, more text is added and will exit(1) when no keys are present. + :return: None. + """ + result = gpg.list_keys(secret=True) + if len(result): + from tabulate import tabulate + print(tabulate(list_keys(gpg=gpg), headers=("Name", "Comment", "E-mail", "Expires", "Fingerprint"))) + else: + echo_info("No keys found in KECPKG keyring. Use `--import-key` or `--create-key` to add a " + "secret key to the KECPKG keyring in order to sign KECPKG's.") + if explain: + sys.exit(1) + + +def parse_key_uids(uids): + """ + Parse GPG key uids into a dictionary with Name, Comment and email. + + If the uids is a listof a (single) uids, the uids will be unpacked from the list. + example uids: `['KE-works BV (KECPKG SIGAUTH TEST KEY) ']` + + :param uids: the uids string of a GPG key. + :return: dict with the keys: {name=..., comment=..., email=...} + """ + uids_pattern = r"(?P.+) \((?P.+)\)( <(?P.+)>)?" + if isinstance(uids, list) and len(uids) == 1: + uids = uids[0] + + match = re.match(uids_pattern, uids) + + return match.groupdict() diff --git a/kecpkg/settings.py b/kecpkg/settings.py index 6318b52..ffba260 100644 --- a/kecpkg/settings.py +++ b/kecpkg/settings.py @@ -4,34 +4,44 @@ from copy import deepcopy import sys + +from appdirs import user_data_dir from atomicwrites import atomic_write -from kecpkg.commands.utils import echo_failure -from kecpkg.utils import ensure_dir_exists, create_file, get_package_dir +from kecpkg.utils import ensure_dir_exists, create_file, get_package_dir, echo_failure SETTINGS_FILENAME = '.kecpkg_settings.json' SETTINGS_FILE = os.path.join(os.getcwd(), SETTINGS_FILENAME) +ARTIFACTS_FILENAME = 'ARTIFACTS' +ARTIFACTS_SIG_FILENAME = 'ARTIFACTS.SIG' + +# using the appdirs.user_data_dir to manage user data on various platforms. +GNUPG_KECPKG_HOME = os.path.join(user_data_dir('kecpkg', 'KE-works BV'), '.gnupg') DEFAULT_SETTINGS = OrderedDict([ ('version', '0.0.1'), - ('pyversions', ['2.7', '3.5']), + ('pyversions', ['2.7', '3.5', '3.6']), ('python_version', '3.5'), ('venv_dir', 'venv'), ('entrypoint_script', 'script'), ('entrypoint_func', 'main'), ('build_dir', 'dist'), - ('requirements_filename', 'requirements.txt') + ('requirements_filename', 'requirements.txt'), + ('artifacts_filename', ARTIFACTS_FILENAME), + ('artifacts_sig_filename', ARTIFACTS_SIG_FILENAME), + ('hash_algorithm', 'sha256') ]) EXCLUDE_DIRS_IN_BUILD = [ 'venv', 'dist', '.idea', '.tox', '.cache', '.git', 'venv*', '_venv*', '.env', '__pycache__', 'develop-eggs', - 'downloads', 'eggs', 'lib', 'lib64', 'sdist', 'wheels', '.hypothesis', '.ipynb_checkpoints', '.mypy_cache' + 'downloads', 'eggs', 'lib', 'lib64', 'sdist', 'wheels', '.hypothesis', '.ipynb_checkpoints', '.mypy_cache', + '.vscode' ] EXCLUDE_PATHS_IN_BUILD = [ '.gitignore', '*.pyc', '*.pyo', '*.pyd', '*$py.class', '*.egg-info', '.installed.cfg', '.coveragerc', '*.egg', 'pip-log.txt', '*.log', 'pip-delete-this-directory.txt', '.coverage*', 'nosetests.xml', 'coverage.xml', '*.cover', - 'env.bak', 'venv.bak', 'pip-selfcheck.json', '*.so', '*-dist', '.*.swp' + 'env.bak', 'venv.bak', 'pip-selfcheck.json', '*.so', '*-dist', '.*.swp', '*.asc' ] EXCLUDE_IN_BUILD = EXCLUDE_DIRS_IN_BUILD + EXCLUDE_PATHS_IN_BUILD diff --git a/kecpkg/utils.py b/kecpkg/utils.py index abce76c..d6f9268 100644 --- a/kecpkg/utils.py +++ b/kecpkg/utils.py @@ -4,6 +4,7 @@ Parts are borrowed from hatch Those parts are are released under the MIT license """ import fnmatch +import io import os import platform import re @@ -11,10 +12,9 @@ import sys from contextlib import contextmanager +import click import six -from kecpkg.commands.utils import echo_failure, echo_info, echo_warning - def ensure_dir_exists(d): """Ensure that directory exists, otherwise make directory.""" @@ -23,13 +23,17 @@ def ensure_dir_exists(d): def create_file(filepath, content=None, overwrite=True): - """ + r""" Create file and optionally fill it with content. - Will overwrite file already in place if overwrite flag is set + Will overwrite file already in place if overwrite flag is set. + If a list is provided each line in the list is written on a new line in the file (`fp.writelines`) + otherwise the string will be written as such and newline characters (`\\\\n`) will be respected. :param filepath: full path to a file to create - :param content: + :param content: textual content. + :type content: list or string + :param overwrite: boolean if you want to overwrite :return: """ ensure_dir_exists(os.path.dirname(os.path.abspath(filepath))) @@ -38,7 +42,9 @@ def create_file(filepath, content=None, overwrite=True): if not os.path.exists(filepath) or (os.path.exists(filepath) and overwrite): with open(filepath, 'w') as fd: # os.utime(filepath, times=None) - if content: + if isinstance(content, list): + fd.writelines(content) + else: fd.write(content) else: echo_failure("File '{}' already exists.".format(filepath)) @@ -74,7 +80,7 @@ def remove_path(path): except (IOError, OSError): try: os.remove(path) - except (IOError): + except IOError: pass @@ -114,24 +120,22 @@ def _inner(d): return None package_dir = _inner(os.getcwd()) - if not package_dir: + if not package_dir and package_name is not None: package_dir = _inner(os.path.join(os.getcwd(), package_name)) - if not package_dir: + if not package_dir and package_name is not None: package_dir = _inner(package_name) - if not package_dir: + if not package_dir and package_name is not None: echo_failure('This does not seem to be a package in path `{}` - please check that there is a ' '`package_info.json` or a `{}`'.format(package_dir, SETTINGS_FILENAME)) if fail: sys.exit(1) - else: - return package_dir + return package_dir def get_package_name(): """ Provide the name of the package (in current dir). - :param fail: ensure that directory search does not fail in a exit. :return: package name or None """ package_dir = get_package_dir(fail=False) @@ -142,12 +146,18 @@ def get_package_name(): def get_artifacts_on_disk(root_path, additional_exclude_paths=None, default_exclude_paths=None, verbose=False): + # type: (str, list, list, bool) -> set """ Retrieve all artifacts on disk. + The artifacts are stripped from their rootpath. + :param root_path: root_path to collect all artifacts from - :param exclude_paths: (optional) directory names and filenames to exclude - :return: dictionary with {'property_id': ['attachment_path1', ...], ...} + :param additional_exclude_paths: (optional) directory names and filenames to exclude + :param default_exclude_paths: (optional) directory names and filenames to exclude + :param verbose: be verbose (or not) + :return: set with ['file_path1', ...] + :rtype: set """ from kecpkg.settings import EXCLUDE_IN_BUILD exclude_paths = default_exclude_paths or EXCLUDE_IN_BUILD @@ -187,7 +197,7 @@ def get_artifacts_on_disk(root_path, additional_exclude_paths=None, default_excl if verbose: echo_info('{}'.format(artifacts)) - return artifacts + return set(artifacts) def render_package_info(settings, package_dir, backup=True): @@ -215,10 +225,28 @@ def render_package_info(settings, package_dir, backup=True): target_dir=package_dir) +def unzip_package(package_path, target_path): + """ + Unzip package in the target_path. + + The package path has the full path of the zipped file. + + For example: package_path = /workspace/ops.zip + target_path = /workspace/target/ + + :param package_path: path of the package file + :param target_path: target path to unzip the package into + """ + import zipfile + with zipfile.ZipFile(package_path, 'r') as zip_file: + zip_file.extractall(target_path) + + # Python Operation regarding Virtual environments # Graceously borrowed From hatch package. __platform = platform.system() +ON_LINUX = os.name == 'posix' or __platform == 'Linux' ON_MACOS = os.name == 'mac' or __platform == 'Darwin' ON_WINDOWS = NEED_SUBPROCESS_SHELL = os.name == 'nt' or __platform == 'Windows' VENV_FLAGS = { @@ -348,3 +376,62 @@ def venv(venv_path, evars=None): with env_vars(evars, ignore={'__PYVENV_LAUNCHER__'}): yield venv_exe_dir + + +def echo_success(text, nl=True): + """ + Write to the console as a success (Cyan bold). + + :param text: string to write + :param nl: add newline + """ + click.secho(text, fg='cyan', bold=True, nl=nl) + + +def echo_failure(text, nl=True): + """ + Write to the console as a failure (Red bold). + + :param text: string to write + :param nl: add newline + """ + click.secho(text, fg='red', bold=True, nl=nl) + + +def echo_warning(text, nl=True): + """ + Write to the console as a warning (Yellow bold). + + :param text: string to write + :param nl: add newline + """ + click.secho(text, fg='yellow', bold=True, nl=nl) + + +def echo_waiting(text, nl=True): + """ + Write to the console as a waiting (Magenta bold). + + :param text: string to write + :param nl: add newline + """ + click.secho(text, fg='magenta', bold=True, nl=nl) + + +def echo_info(text, nl=True): + """ + Write to the console as a informational (bold). + + :param text: string to write + :param nl: add newline + """ + click.secho(text, bold=True, nl=nl) + + +def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): + """Yield pieces of data from a file-like object until EOF.""" + while True: + chunk = file.read(size) + if not chunk: + break + yield chunk diff --git a/pyproject.toml b/pyproject.toml.depr similarity index 95% rename from pyproject.toml rename to pyproject.toml.depr index d20bd31..c96a0ed 100644 --- a/pyproject.toml +++ b/pyproject.toml.depr @@ -9,7 +9,7 @@ url = 'https://github.com/_/kecpkg-tools' [requires] python_version = ['2.7', '3.5', '3.6', 'pypy', 'pypy3'] -requires = ['click', 'atomicwrites', 'jinja2', 'hatch', 'pykechain>=1.13'] +requires = ['click', 'atomicwrites', 'jinja2', 'hatch', 'pykechain>=1.13', 'python-gnupg'] testing_requires = ['coverage', 'pytest', 'toml', 'flake8', ] [build-system] diff --git a/requirements.txt b/requirements.txt index c93bc44..405be20 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,10 @@ click atomicwrites jinja2 -pykechain>=1.13 +pykechain>=2.0.0 +appdirs +python-gnupg +tabulate # testing @@ -11,4 +14,5 @@ pytest #hatch toml +twine diff --git a/setup.py b/setup.py index 4e42511..836621e 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ PACKAGE_NAME = 'kecpkg' HERE = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(HERE, 'README.rst'), 'r') as f: +with open(os.path.join(HERE, 'README.md'), encoding='utf-8') as f: long_description = f.read() ABOUT = {} @@ -18,6 +18,7 @@ version=ABOUT.get('__version__'), description='', long_description=long_description, + long_description_content_type='text/markdown', author='Jochem Berends', author_email='jochem.berends@ke-works.com', maintainer='Jochem Berends', @@ -31,11 +32,13 @@ 'pykechain', 'KE-chain', 'Services Integration Module', - 'SIM' + 'SIM', + 'KECPKG', + 'GPG' ), classifiers=( - 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Natural Language :: English', @@ -43,6 +46,7 @@ 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', ), @@ -51,7 +55,10 @@ 'click', 'atomicwrites', 'jinja2', - 'pykechain>=1.13', + 'pykechain>=2.0.0', + 'appdirs', + 'tabulate', + 'python-gnupg' ), tests_require=( diff --git a/tests/commands/test_build.py b/tests/commands/test_build.py index ba4d9db..65352fe 100644 --- a/tests/commands/test_build.py +++ b/tests/commands/test_build.py @@ -22,9 +22,10 @@ def test_build_non_interactive(self): os.chdir(package_dir) result = runner.invoke(kecpkg, ['build', pkgname]) - self.assertEqual(result.exit_code, 0) + self.assertEqual(result.exit_code, 0, "Results of the run were: \n---\n{}\n---".format(result.output)) self.assertExists(os.path.join(package_dir, 'dist')) + # check if dist is filled package_dir_contents = os.listdir(os.path.join(package_dir, 'dist')) self.assertTrue(len(package_dir_contents), 1) @@ -41,7 +42,7 @@ def test_build_with_prune(self): os.chdir(package_dir) result = runner.invoke(kecpkg, ['build', pkgname]) - self.assertEqual(result.exit_code, 0) + self.assertEqual(result.exit_code, 0, "Results of the run were: \n---\n{}\n---".format(result.output)) self.assertExists(os.path.join(package_dir, 'dist')) # check if dist is filled @@ -50,7 +51,7 @@ def test_build_with_prune(self): # restart the build, with prune and check if dist still has 1 result = runner.invoke(kecpkg, ['build', pkgname, '--prune']) - self.assertEqual(result.exit_code, 0) + self.assertEqual(result.exit_code, 0, "Results of the run were: \n---\n{}\n---".format(result.output)) self.assertExists(os.path.join(package_dir, 'dist')) # check if dist is filled @@ -87,7 +88,7 @@ def test_build_with_extra_ignores(self): # run the builder result = runner.invoke(kecpkg, ['build', pkgname, '--verbose']) - self.assertEqual(result.exit_code, 0) + self.assertEqual(result.exit_code, 0, "Results of the run were: \n---\n{}\n---".format(result.output)) self.assertExists(os.path.join(package_dir, 'dist')) # check the zip such that the extra files are not packaged @@ -116,11 +117,11 @@ def test_build_with_alternate_config(self): os.chdir(package_dir) result = runner.invoke(kecpkg, ['build', pkgname, '--config', alt_settings]) - self.assertEqual(result.exit_code, 0) + self.assertEqual(result.exit_code, 0, "Results of the run were: \n---\n{}\n---".format(result.output)) self.assertExists(os.path.join(package_dir, 'dist')) dist_dir_contents = os.listdir(os.path.join(package_dir, 'dist')) self.assertTrue(len(dist_dir_contents), 1) self.assertTrue(pkgname in dist_dir_contents[0], "the name of the pkg `{}` should be in the name of " - "the built kecpkg `{}`".format(pkgname, dist_dir_contents[0])) \ No newline at end of file + "the built kecpkg `{}`".format(pkgname, dist_dir_contents[0])) diff --git a/tests/commands/test_sign.py b/tests/commands/test_sign.py new file mode 100644 index 0000000..c1c5a6c --- /dev/null +++ b/tests/commands/test_sign.py @@ -0,0 +1,110 @@ +import os +import sys +from unittest import skipIf + +from kecpkg.cli import kecpkg +from kecpkg.gpg import list_keys, get_gpg +from kecpkg.utils import create_file +from tests.utils import BaseTestCase, temp_chdir + +TEST_SECRET_KEY = """ +-----BEGIN PGP PRIVATE KEY BLOCK----- + +lQIGBFzLAUQBBADZqa2AjOwDRb6D/lkuNKRFwTHF1x2SnhinTv5bosUZDRakZ6zd +dEeZiMBe6sSS9x+GTK8izkcZV6cEH8Xpcr7ngaH3bsfiiZvnI6ooyGTt/KFXP6jg +bPHEXi9zceZWIdZtZ3td4I7P52rwkPRSwEDHJcyyZYkG5/73s6Rf9xLkfQARAQAB +/gcDAoPb6jcCOLHi/RU1ynJZBz9PS+nrMUdzzRhRTKzgbhu+ijWhLUyWeXJeLtR2 +1OVlPUBPVUya7nII7Puk5a0YM4m7M4GDUdrOM9pGZ6t5ocfgQ/tb+fI5FkYP6d9/ +8eJaUn/OmnVl4QF2ZBBjMyWi9bJoUAa1Jq868f8qLN7YSUnJVEEx+CkMRAeP9oj/ +/O7KBYn+bIRitOEqTyQyxMbBj/YLRG6F3/2kZjPDJWR/rlbpGttYIxAA+TiQnBCN +0PuLg8vTLFEcmIgsrkT8Q3ODjunJq70GII72Hf3qSM6Po1lBqlyj0RxUydFJkkQa +WQZYOKZyUbnIcOMozxFuNWoNYl8KVBoWawEvS0pzTojzwQy8PEsiqOFYH59dy0VD +2fqnM9Lf6TzcFeHwMu2Ps/vDQEKUZkOPOydAFJZw0IwozZtdPfDcccMItAXjCxjr +x/1+keOSU8V1d3wuppYAO7qve2OTYf5CZAEEJmy1ovYNWqhjcNvZ+O20LnRlc3Rr +ZXkgKEtFQ1BLRyBUT09MUyB0ZXN0aW5nIG9ubHkga2V5IDNNQVkxOSmI1AQTAQoA +PhYhBI0JL8wGC8wel87EiYehd6qyNx5oBQJcywFEAhsDBQkAAVGABQsJCAcCBhUK +CQgLAgQWAgMBAh4BAheAAAoJEIehd6qyNx5o8ZsEAInYfs7EhwUknBDbFHuZt+AE +TI80SIj/VD528EZyrDzyz5p/eeg2HQd470HDSPgwnChUJdMOKSUR7oSTxoOyGJcP +p0M/ydnmSraCOhWI8srW8edtWp16OOK5y/t7CbJ97CVOenImkSwT5uzxHgZWM9Tu +qTdggGeiZjdzXYSbq0pW +=Xq/K +-----END PGP PRIVATE KEY BLOCK----- +""" +TEST_SECRET_KEY_PASSPHRASE = "test" +TEST_SECRET_KEY_FINGERPRINT = "8D092FCC060BCC1E97CEC48987A177AAB2371E68" + + +@skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", "Skipping this test on Travis CI.") +@skipIf("sys.version_info <= (2, 7)", + reason="Skipping tests for python 2.7, as PGP signing cannot be provided") +class TestCommandSign(BaseTestCase): + + def _import_test_key(self): + with self.runner.isolated_filesystem() as d: + create_file('TESTKEY.asc', TEST_SECRET_KEY) + self.runner.invoke(kecpkg, ['sign', '--import-key', 'TESTKEY.asc']) + + def tearDown(self): + super(TestCommandSign, self).tearDown() + self.runner.invoke(kecpkg, ['sign', '--delete-key', TEST_SECRET_KEY_FINGERPRINT]) + + def test_sign_list_keys(self): + self._import_test_key() + result = self.runner.invoke(kecpkg, ['sign', '--list']) + self.assertIn(result.exit_code, [0, 1], "Results of the run were: \n---\n{}\n---".format(result.output)) + + def test_import_key(self): + with temp_chdir() as d: + create_file('TESTKEY.asc', TEST_SECRET_KEY) + result = self.runner.invoke(kecpkg, ['sign', '--import-key', 'TESTKEY.asc']) + self.assertEqual(result.exit_code, 0, "Results of the run were: \n---\n{}\n---".format(result.output)) + + # teardown + result = self.runner.invoke(kecpkg, ['sign', '--delete-key', TEST_SECRET_KEY_FINGERPRINT]) + self.assertEqual(result.exit_code, 0, "Results of the run were: \n---\n{}\n---".format(result.output)) + + def test_delete_key(self): + self._import_test_key() + + result = self.runner.invoke(kecpkg, ['sign', '--delete-key', TEST_SECRET_KEY_FINGERPRINT]) + self.assertEqual(result.exit_code, 0, "Results of the run were: \n---\n{}\n---".format(result.output)) + + def test_delete_key_wrong_fingerprint(self): + self._import_test_key() + + result = self.runner.invoke(kecpkg, ['sign', '--delete-key', 'THISISAWRONGFINGERPRINT']) + self.assertEqual(result.exit_code, 1, "Results of the run were: \n---\n{}\n---".format(result.output)) + + def test_create_key(self): + result = self.runner.invoke(kecpkg, ['sign', '--create-key'], + input="Testing\n" + "KECPKG TESTING CREATE KEY\n" + "no-reply@ke-works.com\n" + "1\n" + "pass\n" + "pass\n") + self.assertEqual(result.exit_code, 0, "Results of the run were: \n---\n{}\n---".format(result.output)) + + keys = list_keys(get_gpg()) + last_key = keys[-1] + fingerprint = last_key[-1] + + result = self.runner.invoke(kecpkg, ['sign', '--delete-key', fingerprint]) + self.assertEqual(result.exit_code, 0, "Results of the run were: \n---\n{}\n---".format(result.output)) + + def test_export_key(self): + self._import_test_key() + + with self.runner.isolated_filesystem() as d: + result = self.runner.invoke(kecpkg, ['sign', + '--export-key', 'out.asc', + '--keyid', TEST_SECRET_KEY_FINGERPRINT]) + self.assertEqual(result.exit_code, 0, "Results of the run were: \n---\n{}\n---".format(result.output)) + self.assertExists('out.asc') + + +@skipIf("sys.version_info >= (3, 4)", reason="These tests are for python 2 only.") +class TestCommandSign27(BaseTestCase): + def test_sign_capability_unaivable(self): + result = self.runner.invoke(kecpkg, ['sign', '--list']) + self.assertEqual(result.exit_code, 1, "Results of the run were: \n---\n{}\n---".format(result.output)) diff --git a/tests/commands/test_upload.py b/tests/commands/test_upload.py index 5185088..2281f07 100644 --- a/tests/commands/test_upload.py +++ b/tests/commands/test_upload.py @@ -1,6 +1,6 @@ import os +from unittest import skipIf -import pytest from click.testing import CliRunner from envparse import Env @@ -9,13 +9,19 @@ from tests.utils import temp_chdir, BaseTestCase -@pytest.mark.skipif("os.getenv('TRAVIS', False)", - reason="Skipping tests when using Travis, as upload of services cannot be testing securely") +@skipIf("os.getenv('TRAVIS', False)", + reason="Skipping tests when using Travis, as upload of services cannot be testing securely") +@skipIf("os.getenv('KECHAIN_URL') is None", + reason="Skipping test as the KECHAIN_URL is not available as environment variable. Cannot upload kecpkg to " + "test this functionality. Provice a `.env` file locally to enable these tests.") class TestCommandUpload(BaseTestCase): def test_upload_non_interactive(self): pkgname = 'new_pkg' env = Env.read_envfile() + self.assertTrue(os.environ.get('KECHAIN_URL'), + "KECHAIN_URL is not set in environment, cannot perform this test") + with temp_chdir() as d: runner = CliRunner() result = runner.invoke(kecpkg, ['new', pkgname, '--no-venv']) diff --git a/tests/utils.py b/tests/utils.py index 9763791..ed475d9 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -10,17 +10,27 @@ class BaseTestCase(TestCase): - def SetUp(self): + def setUp(self): self.runner = CliRunner() def assertExists(self, path): self.assertTrue(os.path.exists(path), "Path `{}` does not exists".format(path)) +def is_travis(): + """Predicate to determine if the test is running in the context of Travis.""" + return "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true" + +def is_python27(): + """Predicate to determine if the runtime version of python is version 2.""" + import sys + return sys.version_info <= (2, 7) + + @contextmanager def temp_chdir(cwd=None): if six.PY3: from tempfile import TemporaryDirectory - with TemporaryDirectory() as tempwd: + with TemporaryDirectory(prefix="kecpkg_") as tempwd: origin = cwd or os.getcwd() os.chdir(tempwd) @@ -30,7 +40,7 @@ def temp_chdir(cwd=None): os.chdir(origin) else: from tempfile import mkdtemp - tempwd = mkdtemp() + tempwd = mkdtemp(prefix="kecpkg_") origin=cwd or os.getcwd() os.chdir(tempwd) try: @@ -63,4 +73,4 @@ def touch_file(path): requires_internet = pytest.mark.skipif( not connected_to_internet(), reason='Not connected to internet' -) \ No newline at end of file +) diff --git a/tox.ini b/tox.ini index 0e8ae7b..9d7b53b 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,7 @@ envlist = py27, py35, py36, + py37, pypy, pypy3, dist_and_docs @@ -41,7 +42,7 @@ commands = flake8 kecpkg pydocstyle kecpkg check-manifest - python setup.py check -m -r -s + python setup.py check -m -s # test settings @@ -62,4 +63,4 @@ ignore = D100,D104,D105,D203,D212,D213 [pytest] addopts = -l --color=yes -v -testpaths = tests \ No newline at end of file +testpaths = tests