From f9713c8e50b11be94d0bf973190de834cb39d11f Mon Sep 17 00:00:00 2001 From: Olivier Desenfans Date: Wed, 27 Sep 2023 17:07:40 +0200 Subject: [PATCH 1/3] Feature: documentation and deployment script Added documentation in the README and on all functions. --- .gitignore | 6 + .readthedocs.yml | 27 -- AUTHORS.rst | 6 - CHANGELOG.rst | 6 - Dockerfile | 3 +- README.md | 121 +++++++- README.rst | 49 --- deployment/deploy_vms.py | 190 ++++++++++++ docs/Makefile | 29 -- docs/_static/.gitignore | 1 - docs/authors.rst | 2 - docs/changelog.rst | 2 - docs/conf.py | 289 ------------------ docs/contributing.rst | 1 - docs/index.rst | 61 ---- docs/license.rst | 7 - docs/readme.rst | 2 - docs/requirements.txt | 5 - install.sh | 2 +- prepare_venv_volume.sh | 7 - run-in-docker.sh | 2 - setup.cfg | 11 +- .../coordinator/executor_selection.py | 51 +++- src/aleph_vrf/coordinator/main.py | 15 +- src/aleph_vrf/exceptions.py | 12 + src/aleph_vrf/executor/main.py | 16 +- src/aleph_vrf/models.py | 6 +- tests/conftest.py | 16 + tests/coordinator/test_executor_selection.py | 7 + tests/coordinator/test_integration_lib.py | 10 +- tests/malicious_executor.py | 5 + tests/mock_ccn.py | 4 + 32 files changed, 450 insertions(+), 521 deletions(-) delete mode 100644 .readthedocs.yml delete mode 100644 AUTHORS.rst delete mode 100644 CHANGELOG.rst delete mode 100644 README.rst create mode 100644 deployment/deploy_vms.py delete mode 100644 docs/Makefile delete mode 100644 docs/_static/.gitignore delete mode 100644 docs/authors.rst delete mode 100644 docs/changelog.rst delete mode 100644 docs/conf.py delete mode 100644 docs/contributing.rst delete mode 100644 docs/index.rst delete mode 100644 docs/license.rst delete mode 100644 docs/readme.rst delete mode 100644 docs/requirements.txt delete mode 100755 prepare_venv_volume.sh delete mode 100644 run-in-docker.sh diff --git a/.gitignore b/.gitignore index 5b0f7ee..06c5523 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,9 @@ MANIFEST venv/ .conda*/ .python-version + +# Volume files +*.squashfs + +# Key files +*.key diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index a2bcab3..0000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Read the Docs configuration file -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details - -# Required -version: 2 - -# Build documentation in the docs/ directory with Sphinx -sphinx: - configuration: docs/conf.py - -# Build documentation with MkDocs -#mkdocs: -# configuration: mkdocs.yml - -# Optionally build your docs in additional formats such as PDF -formats: - - pdf - -build: - os: ubuntu-22.04 - tools: - python: "3.11" - -python: - install: - - requirements: docs/requirements.txt - - {path: ., method: pip} diff --git a/AUTHORS.rst b/AUTHORS.rst deleted file mode 100644 index e33fb27..0000000 --- a/AUTHORS.rst +++ /dev/null @@ -1,6 +0,0 @@ -============ -Contributors -============ - -* Olivier Desenfans -* Andres Molins diff --git a/CHANGELOG.rst b/CHANGELOG.rst deleted file mode 100644 index 1396ed6..0000000 --- a/CHANGELOG.rst +++ /dev/null @@ -1,6 +0,0 @@ -========= -Changelog -========= - -Current version -=============== diff --git a/Dockerfile b/Dockerfile index 1ec0936..e5667b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM debian:bullseye RUN apt-get update && apt-get -y upgrade && apt-get install -y \ + git \ libsecp256k1-dev \ python3-pip \ python3-venv \ @@ -11,4 +12,4 @@ WORKDIR /usr/src/aleph_vrf COPY . . RUN mkdir /opt/packages -RUN pip install -t /opt/packages . \ No newline at end of file +RUN pip install -t /opt/packages . diff --git a/README.md b/README.md index 93dc202..0fb8396 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,119 @@ -# aleph-vrf -Aleph VRF +# Aleph.im Verifiable Random Functions + +## What is a Verifiable Random Function (VRF)? + +Verifiable Random Functions (VRF) are cryptographic primitives that generate random numbers that are both unpredictable +and verifiable. +This allows to create "trustless randomness", i.e. generate (pseudo-) random numbers in decentralized systems and +provide the assurance that the number was indeed generated randomly. + +## Aleph.im implementation + +Aleph.im uses a combination of virtual machines (VMs) and aleph.im network messages to implement VRFs. + +The implementation revolves around the following components: + +* The VRF coordinator +* N executors. + +The coordinator receives user requests to generate random numbers. +Upon receiving a request, it selects a set of compute resource nodes (CRNs) to act as executors. +Each of these executors generates a random number and computes its hash using SHA3–256. +These hashes are then posted to aleph.im using a POST message, which also includes a unique request identifier. +Once all the hashes are posted and confirmed, the coordinator requests the actual random numbers from each node. + +Finally, the coordinator performs a verification process to ensure that all random numbers correspond to their +previously posted hashes. The random numbers are then combined using an XOR operation to generate the final random +number. This final number, along with a summary of operations performed, is published on aleph.im for public +verification. + +## How to use aleph.im VRFs + +The VRF executors and coordinator are meant to be deployed as VM functions on the aleph.im network. +The coordinator can also be deployed in library mode (see below). + +We provide a script to deploy the VM functions. +Just run the following command to package the application and upload it to the aleph.im network. + +``` +python3 deployment/deploy_vms.py +``` + +If the deployment succeeds, the script will display links to the VMs on the aleph.im network. Example: + +> Executor VM: https://api2.aleph.im/api/v0/messages/558b0eeea54d80d2504b0287d047e0b78458d08022d3600bcf8478700dd0aac2 + Coordinator VM: https://api2.aleph.im/api/v0/messages/d9eef54544338685a9b4034cc16e285520eb3cf0c199eeade1d6b290365c95d0 + + + +### Use the coordinator in library mode + +The coordinator can also be used directly from Python code. +First, deploy the executors using the deployment script, without the coordinator VM: + +``` +python3 deployment/deploy_vms.py --no-coordinator +``` + +This will deploy an executor VM on the network and give you its ID. +Example: + +> Executor VM: https://api2.aleph.im/api/v0/messages/558b0eeea54d80d2504b0287d047e0b78458d08022d3600bcf8478700dd0aac2 + +Then, install the `aleph-vrf` module and call it from your code: + +```shell +pip install aleph-vrf +``` + +```python +from aleph_vrf.coordinator.vrf import generate_vrf +from aleph_message.models import ItemHash + + +async def main(): + aleph_account = ... # Specify your aleph.im account + vrf_response = await generate_vrf( + account=aleph_account, + vrf_function=ItemHash( + # The hash of the executor VM deployed above + "558b0eeea54d80d2504b0287d047e0b78458d08022d3600bcf8478700dd0aac2" + ), + ) + random_number = int(vrf_response.random_number) +``` + +## Contribute + +### Set up the development environment + +You can set up a development environment by configuring a Python virtual environment and installing the project in +development mode. + +```shell +python -m virtualenv venv +source venv/bin/activate +pip install -e .[build,testing] +``` + +### Run tests + +This project uses mypy for static type analysis and pytest for unit/integration tests. + +```shell +# Static analysis with mypy +mypy src/ tests/ +# Run unit/integration tests +pytest -v . +``` + +### Create a new release + +1. Deploy the VMs: `python3 deployment/deploy_vms.py` +2. Update the executor VM hash in the settings (Settings.FUNCTION) and create a Pull Request +3. Merge the Pull Request and create a new release on Github +4. Build and upload the package on PyPI: `python3 -m build && twine upload dist/*` + +## Other resources + +* [Article on Medium](https://medium.com/aleph-im/aleph-im-verifiable-random-function-vrf-b03544a7e904) diff --git a/README.rst b/README.rst deleted file mode 100644 index 27d44d9..0000000 --- a/README.rst +++ /dev/null @@ -1,49 +0,0 @@ -.. These are examples of badges you might want to add to your README: - please update the URLs accordingly - - .. image:: https://api.cirrus-ci.com/github//aleph-vrf.svg?branch=main - :alt: Built Status - :target: https://cirrus-ci.com/github//aleph-vrf - .. image:: https://readthedocs.org/projects/aleph-vrf/badge/?version=latest - :alt: ReadTheDocs - :target: https://aleph-vrf.readthedocs.io/en/stable/ - .. image:: https://img.shields.io/coveralls/github//aleph-vrf/main.svg - :alt: Coveralls - :target: https://coveralls.io/r//aleph-vrf - .. image:: https://img.shields.io/pypi/v/aleph-vrf.svg - :alt: PyPI-Server - :target: https://pypi.org/project/aleph-vrf/ - .. image:: https://img.shields.io/conda/vn/conda-forge/aleph-vrf.svg - :alt: Conda-Forge - :target: https://anaconda.org/conda-forge/aleph-vrf - .. image:: https://pepy.tech/badge/aleph-vrf/month - :alt: Monthly Downloads - :target: https://pepy.tech/project/aleph-vrf - .. image:: https://img.shields.io/twitter/url/http/shields.io.svg?style=social&label=Twitter - :alt: Twitter - :target: https://twitter.com/aleph-vrf - -.. image:: https://img.shields.io/badge/-PyScaffold-005CA0?logo=pyscaffold - :alt: Project generated with PyScaffold - :target: https://pyscaffold.org/ - -| - -========= -aleph-vrf -========= - - - Aleph.im Verifiable Random Function - - -A longer description of your project goes here... - - -.. _pyscaffold-notes: - -Note -==== - -This project has been set up using PyScaffold 4.5. For details and usage -information on PyScaffold see https://pyscaffold.org/. diff --git a/deployment/deploy_vms.py b/deployment/deploy_vms.py new file mode 100644 index 0000000..2cf5d7b --- /dev/null +++ b/deployment/deploy_vms.py @@ -0,0 +1,190 @@ +import argparse +import asyncio +import subprocess +import sys +from functools import partial +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Optional, Tuple, Dict + +from aleph.sdk import AuthenticatedAlephClient +from aleph.sdk.chains.common import get_fallback_private_key +from aleph.sdk.chains.ethereum import ETHAccount +from aleph_message.models import ItemHash, ProgramMessage +from aleph_message.models.execution.volume import ImmutableVolume +from aleph_message.status import MessageStatus + +from aleph_vrf.settings import settings + +DEBIAN12_RUNTIME = ItemHash( + "ed2c37ae857edaea1d36a43fdd0fb9fdb7a2c9394957e6b53d9c94bf67f32ac3" +) + + +def build_venv(package_path: Path, destination: Path) -> None: + subprocess.run(["pip", "install", "-t", str(destination), package_path], check=True) + + +def mksquashfs(path: Path, destination: Path) -> None: + subprocess.run(["mksquashfs", str(path), str(destination)], check=True) + + +async def upload_dir_as_volume( + aleph_client: AuthenticatedAlephClient, + dir_path: Path, + channel: str, + volume_path: Optional[Path] = None, +): + volume_path = volume_path or Path(f"{dir_path}.squashfs") + if volume_path.exists(): + print("Squashfs volume already exists, updating it...") + volume_path.unlink() + + mksquashfs(dir_path, volume_path) + + store_message, status = await aleph_client.create_store( + file_path=volume_path, sync=True, channel=channel + ) + if status not in (MessageStatus.PENDING, MessageStatus.PROCESSED): + raise RuntimeError(f"Could not upload venv volume: {status}") + + return store_message.item_hash + + +async def deploy_python_program( + aleph_client: AuthenticatedAlephClient, + code_volume_hash: ItemHash, + entrypoint: str, + venv_hash: ItemHash, + channel: str, + environment: Optional[Dict[str, str]] = None, +) -> ProgramMessage: + program_message, status = await aleph_client.create_program( + program_ref=code_volume_hash, + entrypoint=entrypoint, + runtime=DEBIAN12_RUNTIME, + volumes=[ + ImmutableVolume( + ref=venv_hash, + use_latest=True, + mount="/opt/packages", + comment="Aleph.im VRF virtualenv", + ).dict() + ], + memory=256, + sync=True, + environment_variables=environment, + channel=channel, + ) + + if status == MessageStatus.REJECTED: + raise RuntimeError("Could not upload program message") + + return program_message + + +deploy_executor_vm = partial( + deploy_python_program, + entrypoint="aleph_vrf.executor.main:app", +) + +deploy_coordinator_vm = partial( + deploy_python_program, + entrypoint="aleph_vrf.coordinator.main:app", +) + + +async def deploy_vrf( + source_dir: Path, venv_dir: Path, deploy_coordinator: bool = True +) -> Tuple[ProgramMessage, Optional[ProgramMessage]]: + private_key = get_fallback_private_key() + account = ETHAccount(private_key) + channel = "vrf-tests" + + async with AuthenticatedAlephClient( + account=account, api_server=settings.API_HOST + ) as aleph_client: + # Upload the code and venv volumes + print("Uploading code volume...") + code_volume_hash = await upload_dir_as_volume( + aleph_client=aleph_client, + dir_path=source_dir, + channel=channel, + volume_path=Path("aleph-vrf-code.squashfs"), + ) + print("Uploading virtualenv volume...") + venv_hash = await upload_dir_as_volume( + aleph_client=aleph_client, + dir_path=venv_dir, + channel=channel, + volume_path=Path("aleph-vrf-venv.squashfs"), + ) + + # Upload the executor and coordinator VMs + print("Creating executor VM...") + executor_program_message = await deploy_executor_vm( + aleph_client=aleph_client, + code_volume_hash=code_volume_hash, + venv_hash=venv_hash, + channel=channel, + environment={"PYTHONPATH": "/opt/packages"}, + ) + + if deploy_coordinator: + print("Creating coordinator VM...") + coordinator_program_message = await deploy_coordinator_vm( + aleph_client=aleph_client, + code_volume_hash=code_volume_hash, + venv_hash=venv_hash, + channel=channel, + environment={ + "PYTHONPATH": "/opt/packages", + "ALEPH_VRF_FUNCTION": executor_program_message.item_hash, + }, + ) + else: + coordinator_program_message = None + + return executor_program_message, coordinator_program_message + + +async def main(args: argparse.Namespace): + deploy_coordinator = args.deploy_coordinator + root_dir = Path(__file__).parent.parent + + with TemporaryDirectory() as venv_dir_str: + venv_dir = Path(venv_dir_str) + build_venv(package_path=root_dir, destination=venv_dir) + + executor_program_message, coordinator_program_message = await deploy_vrf( + source_dir=root_dir / "src", + venv_dir=venv_dir, + deploy_coordinator=deploy_coordinator, + ) + + print("Aleph.im VRF VMs were successfully deployed.") + print( + f"Executor VM: https://api2.aleph.im/api/v0/messages/{executor_program_message.item_hash}" + ) + if coordinator_program_message: + print( + f"Coordinator VM: https://api2.aleph.im/api/v0/messages/{coordinator_program_message.item_hash}" + ) + + +def parse_args(args) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="vm_supervisor", description="Aleph.im VM Supervisor" + ) + parser.add_argument( + "--no-coordinator", + dest="deploy_coordinator", + action="store_false", + default=True, + help="Deploy the coordinator as an aleph.im VM function", + ) + return parser.parse_args(args) + + +if __name__ == "__main__": + asyncio.run(main(args=parse_args(sys.argv[1:]))) diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 31655dd..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,29 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build -AUTODOCDIR = api - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $?), 1) -$(error "The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from https://sphinx-doc.org/") -endif - -.PHONY: help clean Makefile - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -clean: - rm -rf $(BUILDDIR)/* $(AUTODOCDIR) - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/.gitignore b/docs/_static/.gitignore deleted file mode 100644 index 3c96363..0000000 --- a/docs/_static/.gitignore +++ /dev/null @@ -1 +0,0 @@ -# Empty directory diff --git a/docs/authors.rst b/docs/authors.rst deleted file mode 100644 index cd8e091..0000000 --- a/docs/authors.rst +++ /dev/null @@ -1,2 +0,0 @@ -.. _authors: -.. include:: ../AUTHORS.rst diff --git a/docs/changelog.rst b/docs/changelog.rst deleted file mode 100644 index 871950d..0000000 --- a/docs/changelog.rst +++ /dev/null @@ -1,2 +0,0 @@ -.. _changes: -.. include:: ../CHANGELOG.rst diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index fa31c43..0000000 --- a/docs/conf.py +++ /dev/null @@ -1,289 +0,0 @@ -# This file is execfile()d with the current directory set to its containing dir. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import os -import shutil -import sys - -# -- Path setup -------------------------------------------------------------- - -__location__ = os.path.dirname(__file__) - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.join(__location__, "../src")) - -# -- Run sphinx-apidoc ------------------------------------------------------- -# This hack is necessary since RTD does not issue `sphinx-apidoc` before running -# `sphinx-build -b html . _build/html`. See Issue: -# https://github.com/readthedocs/readthedocs.org/issues/1139 -# DON'T FORGET: Check the box "Install your project inside a virtualenv using -# setup.py install" in the RTD Advanced Settings. -# Additionally it helps us to avoid running apidoc manually - -try: # for Sphinx >= 1.7 - from sphinx.ext import apidoc -except ImportError: - from sphinx import apidoc - -output_dir = os.path.join(__location__, "api") -module_dir = os.path.join(__location__, "../src/aleph_vrf") -try: - shutil.rmtree(output_dir) -except FileNotFoundError: - pass - -try: - import sphinx - - cmd_line = f"sphinx-apidoc --implicit-namespaces -f -o {output_dir} {module_dir}" - - args = cmd_line.split(" ") - if tuple(sphinx.__version__.split(".")) >= ("1", "7"): - # This is a rudimentary parse_version to avoid external dependencies - args = args[1:] - - apidoc.main(args) -except Exception as e: - print("Running `sphinx-apidoc` failed!\n{}".format(e)) - -# -- General configuration --------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.intersphinx", - "sphinx.ext.todo", - "sphinx.ext.autosummary", - "sphinx.ext.viewcode", - "sphinx.ext.coverage", - "sphinx.ext.doctest", - "sphinx.ext.ifconfig", - "sphinx.ext.mathjax", - "sphinx.ext.napoleon", -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = "aleph-vrf" -copyright = "2023, Olivier Desenfans" - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# version: The short X.Y version. -# release: The full version, including alpha/beta/rc tags. -# If you don’t need the separation provided between version and release, -# just set them both to the same value. -try: - from aleph_vrf import __version__ as version -except ImportError: - version = "" - -if not version or version.lower() == "unknown": - version = os.getenv("READTHEDOCS_VERSION", "unknown") # automatically set by RTD - -release = version - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ".venv"] - -# The reST default role (used for this markup: `text`) to use for all documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -# keep_warnings = False - -# If this is True, todo emits a warning for each TODO entries. The default is False. -todo_emit_warnings = True - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = "alabaster" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -html_theme_options = {"sidebar_width": "300px", "page_width": "1200px"} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = "" - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_domain_indices = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = "aleph-vrf-doc" - - -# -- Options for LaTeX output ------------------------------------------------ - -latex_elements = { - # The paper size ("letterpaper" or "a4paper"). - # "papersize": "letterpaper", - # The font size ("10pt", "11pt" or "12pt"). - # "pointsize": "10pt", - # Additional stuff for the LaTeX preamble. - # "preamble": "", -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ( - "index", - "user_guide.tex", - "aleph-vrf Documentation", - "Olivier Desenfans", - "manual", - ) -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = "" - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False - -# If true, show page references after internal links. -# latex_show_pagerefs = False - -# If true, show URL addresses after external links. -# latex_show_urls = False - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_domain_indices = True - -# -- External mapping -------------------------------------------------------- -python_version = ".".join(map(str, sys.version_info[0:2])) -intersphinx_mapping = { - "sphinx": ("https://www.sphinx-doc.org/en/master", None), - "python": ("https://docs.python.org/" + python_version, None), - "matplotlib": ("https://matplotlib.org", None), - "numpy": ("https://numpy.org/doc/stable", None), - "sklearn": ("https://scikit-learn.org/stable", None), - "pandas": ("https://pandas.pydata.org/pandas-docs/stable", None), - "scipy": ("https://docs.scipy.org/doc/scipy/reference", None), - "setuptools": ("https://setuptools.pypa.io/en/stable/", None), - "pyscaffold": ("https://pyscaffold.org/en/stable", None), -} - -print(f"loading configurations for {project} {version} ...", file=sys.stderr) diff --git a/docs/contributing.rst b/docs/contributing.rst deleted file mode 100644 index e582053..0000000 --- a/docs/contributing.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../CONTRIBUTING.rst diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 949cd3b..0000000 --- a/docs/index.rst +++ /dev/null @@ -1,61 +0,0 @@ -========= -aleph-vrf -========= - -This is the documentation of **aleph-vrf**. - -.. note:: - - This is the main page of your project's `Sphinx`_ documentation. - It is formatted in `reStructuredText`_. Add additional pages - by creating rst-files in ``docs`` and adding them to the `toctree`_ below. - Use then `references`_ in order to link them from this page, e.g. - :ref:`authors` and :ref:`changes`. - - It is also possible to refer to the documentation of other Python packages - with the `Python domain syntax`_. By default you can reference the - documentation of `Sphinx`_, `Python`_, `NumPy`_, `SciPy`_, `matplotlib`_, - `Pandas`_, `Scikit-Learn`_. You can add more by extending the - ``intersphinx_mapping`` in your Sphinx's ``conf.py``. - - The pretty useful extension `autodoc`_ is activated by default and lets - you include documentation from docstrings. Docstrings can be written in - `Google style`_ (recommended!), `NumPy style`_ and `classical style`_. - - -Contents -======== - -.. toctree:: - :maxdepth: 2 - - Overview - Contributions & Help - License - Authors - Changelog - Module Reference - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - -.. _toctree: https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html -.. _reStructuredText: https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html -.. _references: https://www.sphinx-doc.org/en/stable/markup/inline.html -.. _Python domain syntax: https://www.sphinx-doc.org/en/master/usage/restructuredtext/domains.html#the-python-domain -.. _Sphinx: https://www.sphinx-doc.org/ -.. _Python: https://docs.python.org/ -.. _Numpy: https://numpy.org/doc/stable -.. _SciPy: https://docs.scipy.org/doc/scipy/reference/ -.. _matplotlib: https://matplotlib.org/contents.html# -.. _Pandas: https://pandas.pydata.org/pandas-docs/stable -.. _Scikit-Learn: https://scikit-learn.org/stable -.. _autodoc: https://www.sphinx-doc.org/en/master/ext/autodoc.html -.. _Google style: https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings -.. _NumPy style: https://numpydoc.readthedocs.io/en/latest/format.html -.. _classical style: https://www.sphinx-doc.org/en/master/domains.html#info-field-lists diff --git a/docs/license.rst b/docs/license.rst deleted file mode 100644 index 3989c51..0000000 --- a/docs/license.rst +++ /dev/null @@ -1,7 +0,0 @@ -.. _license: - -======= -License -======= - -.. include:: ../LICENSE.txt diff --git a/docs/readme.rst b/docs/readme.rst deleted file mode 100644 index 81995ef..0000000 --- a/docs/readme.rst +++ /dev/null @@ -1,2 +0,0 @@ -.. _readme: -.. include:: ../README.rst diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 2ddf98a..0000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Requirements file for ReadTheDocs, check .readthedocs.yml. -# To build the module reference correctly, make sure every external package -# under `install_requires` in `setup.cfg` is also listed here! -sphinx>=3.2.1 -# sphinx_rtd_theme diff --git a/install.sh b/install.sh index 2232b92..f8102ce 100644 --- a/install.sh +++ b/install.sh @@ -3,4 +3,4 @@ python3 -m virtualenv venv source venv/bin/activate -pip install -e .[testing] \ No newline at end of file +pip install -e .[build,testing] diff --git a/prepare_venv_volume.sh b/prepare_venv_volume.sh deleted file mode 100755 index e590e9e..0000000 --- a/prepare_venv_volume.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -docker build -t aleph-vrf . -docker run --rm -ti -v "$(pwd)":/usr/src/aleph_vrf aleph-vrf \ - mksquashfs /opt/packages aleph-vrf-venv.squashfs \ No newline at end of file diff --git a/run-in-docker.sh b/run-in-docker.sh deleted file mode 100644 index c72f88a..0000000 --- a/run-in-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -docker build -t aleph-vrf . -docker run --rm -ti -v "$(pwd)":/usr/src/aleph_vrf aleph-vrf bash \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 5498224..9f42cfd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,11 +7,10 @@ name = aleph-vrf description = Aleph.im Verifiable Random Function author = Andres Molins -author_email = desenfans.olivier@gmail.com license = MIT license_files = LICENSE.txt -long_description = file: README.rst -long_description_content_type = text/x-rst; charset=UTF-8 +long_description = file: README.md +long_description_content_type = text/markdown; charset=UTF-8 url = https://github.com/pyscaffold/pyscaffold/ # Add here related links, for example: project_urls = @@ -50,7 +49,7 @@ package_dir = install_requires = aiohttp aleph-sdk-python==0.7.0 - fastapi + fastapi>=0.95.1 importlib-metadata; python_version<"3.8" utilitybelt @@ -75,6 +74,10 @@ testing = setuptools uvicorn +build = + build + twine + [options.entry_points] # Add here console scripts like: # console_scripts = diff --git a/src/aleph_vrf/coordinator/executor_selection.py b/src/aleph_vrf/coordinator/executor_selection.py index b788b25..e3aef64 100644 --- a/src/aleph_vrf/coordinator/executor_selection.py +++ b/src/aleph_vrf/coordinator/executor_selection.py @@ -13,12 +13,24 @@ class ExecutorSelectionPolicy(abc.ABC): + """ + How the coordinator selects executors. + """ + @abc.abstractmethod async def select_executors(self, nb_executors: int) -> List[Executor]: + """ + Returns nb_executors executor objects. + Raises NotEnoughExecutors if there are fewer than nb_executors available. + """ ... async def _get_corechannel_aggregate() -> Dict[str, Any]: + """ + Returns the "corechannel" aleph.im aggregate. + This aggregate contains an up-to-date list of staked nodes on the network. + """ async with aiohttp.ClientSession(settings.API_HOST) as session: url = ( f"/api/v0/aggregates/{settings.CORECHANNEL_AGGREGATE_ADDRESS}.json?" @@ -32,11 +44,20 @@ async def _get_corechannel_aggregate() -> Dict[str, Any]: class ExecuteOnAleph(ExecutorSelectionPolicy): - def __init__(self, vm_function: ItemHash): + """ + Select executors at random on the aleph.im network. + """ + + def __init__(self, vm_function: ItemHash, crn_score_threshold: float = 0.9): self.vm_function = vm_function + self.crn_score_threshold = crn_score_threshold + + async def _list_compute_nodes(self) -> AsyncIterator[ComputeResourceNode]: + """ + Returns a list of all compute resource nodes that are linked to a core channel node + and have a score above the required threshold. + """ - @staticmethod - async def _list_compute_nodes() -> AsyncIterator[ComputeResourceNode]: content = await _get_corechannel_aggregate() if ( @@ -49,7 +70,10 @@ async def _list_compute_nodes() -> AsyncIterator[ComputeResourceNode]: for resource_node in resource_nodes: # Filter nodes by score, with linked status - if resource_node["status"] == "linked" and resource_node["score"] > 0.9: + if ( + resource_node["status"] == "linked" + and resource_node["score"] > self.crn_score_threshold + ): node_address = resource_node["address"].strip("/") node = ComputeResourceNode( hash=resource_node["hash"], @@ -60,6 +84,11 @@ async def _list_compute_nodes() -> AsyncIterator[ComputeResourceNode]: @staticmethod def _get_unauthorized_nodes() -> List[str]: + """ + Returns a list of unauthorized nodes. + The caller may provide a blacklist of nodes by specifying a list of URLs in a file + named `unauthorized_node_list.json` in the working directory. + """ unauthorized_nodes_list_path = Path(__file__).with_name( "unauthorized_node_list.json" ) @@ -70,6 +99,10 @@ def _get_unauthorized_nodes() -> List[str]: return [] async def select_executors(self, nb_executors: int) -> List[Executor]: + """ + Selects nb_executors compute resource nodes at random from the aleph.im network. + """ + compute_nodes = self._list_compute_nodes() blacklisted_nodes = self._get_unauthorized_nodes() whitelisted_nodes = ( @@ -88,10 +121,20 @@ async def select_executors(self, nb_executors: int) -> List[Executor]: class UsePredeterminedExecutors(ExecutorSelectionPolicy): + """ + Use a hardcoded list of executors. + """ + def __init__(self, executors: List[Executor]): self.executors = executors async def select_executors(self, nb_executors: int) -> List[Executor]: + """ + Returns nb_executors from the hardcoded list of executors. + If nb_executors is lower than the total number of executors, this method + will always return the nb_executors first executors in the list. + """ + if len(self.executors) < nb_executors: raise NotEnoughExecutors( requested=nb_executors, available=len(self.executors) diff --git a/src/aleph_vrf/coordinator/main.py b/src/aleph_vrf/coordinator/main.py index daeb210..bcb0e29 100644 --- a/src/aleph_vrf/coordinator/main.py +++ b/src/aleph_vrf/coordinator/main.py @@ -6,8 +6,6 @@ logger = logging.getLogger(__name__) logger.debug("import aleph_client") -from aleph.sdk.chains.common import get_fallback_private_key -from aleph.sdk.chains.ethereum import ETHAccount from aleph.sdk.vm.app import AlephApp from aleph.sdk.vm.cache import VmCache @@ -16,7 +14,7 @@ logger.debug("local imports") from aleph_vrf.coordinator.vrf import generate_vrf -from aleph_vrf.models import APIResponse, PublishedVRFResponse +from aleph_vrf.models import APIResponse, PublishedVRFResponse, APIError logger.debug("imports done") @@ -36,14 +34,19 @@ async def index(): @app.post("/vrf") -async def receive_vrf() -> APIResponse: +async def receive_vrf() -> APIResponse[Union[PublishedVRFResponse, APIError]]: + """ + Goes through the VRF random number generation process and returns a random number + along with details on how the number was generated. + """ + account = settings.aleph_account() - response: Union[PublishedVRFResponse, Dict[str, str]] + response: Union[PublishedVRFResponse, APIError] try: response = await generate_vrf(account) except Exception as err: - response = {"error": str(err)} + response = APIError(error=str(err)) return APIResponse(data=response) diff --git a/src/aleph_vrf/exceptions.py b/src/aleph_vrf/exceptions.py index 1ce6131..14a4ab5 100644 --- a/src/aleph_vrf/exceptions.py +++ b/src/aleph_vrf/exceptions.py @@ -29,6 +29,10 @@ def __str__(self): class ExecutorError(Exception): + """ + An error occurred while communicating with an executor. + """ + def __init__(self, executor: Executor): self.executor = executor @@ -69,6 +73,10 @@ def __str__(self): class HashValidationFailed(VrfException): + """ + A random number does not match the SHA3 hash sent by the executor. + """ + def __init__( self, random_bytes: PublishedVRFRandomBytes, @@ -88,6 +96,10 @@ def __str__(self): class NotEnoughExecutors(VrfException): + """ + There are not enough executors available to satisfy the user requirements. + """ + def __init__(self, requested: int, available: int): self.requested = requested self.available = available diff --git a/src/aleph_vrf/executor/main.py b/src/aleph_vrf/executor/main.py index 37c0975..7bbbdc6 100644 --- a/src/aleph_vrf/executor/main.py +++ b/src/aleph_vrf/executor/main.py @@ -19,8 +19,6 @@ logger = logging.getLogger(__name__) logger.debug("import aleph_client") -from aleph.sdk.chains.common import get_fallback_private_key -from aleph.sdk.chains.ethereum import ETHAccount from aleph.sdk.client import AlephClient, AuthenticatedAlephClient from aleph.sdk.vm.app import AlephApp from aleph_message.models import ItemHash, PostMessage @@ -94,6 +92,10 @@ async def receive_generate( AuthenticatedAlephClient, Depends(authenticated_aleph_client) ], ) -> APIResponse[PublishedVRFResponseHash]: + """ + Generates a random number and returns its SHA3 hash. + """ + global SAVED_GENERATED_BYTES, ANSWERED_REQUESTS message = await _get_message(client=aleph_client, item_hash=vrf_request) @@ -145,6 +147,12 @@ async def receive_publish( AuthenticatedAlephClient, Depends(authenticated_aleph_client) ], ) -> APIResponse[PublishedVRFRandomBytes]: + """ + Publishes the random number associated with the specified message hash. + If a user attempts to call this endpoint several times for the same message hash, + data will only be returned on the first call. + """ + global SAVED_GENERATED_BYTES message = await _get_message(client=aleph_client, item_hash=hash_message) @@ -183,6 +191,10 @@ async def publish_data( data: Union[VRFResponseHash, VRFRandomBytes], ref: str, ) -> ItemHash: + """ + Publishes the generation/publication artefacts on the aleph.im network as POST messages. + """ + channel = f"vrf_{data.request_id}" message, status = await aleph_client.create_post( diff --git a/src/aleph_vrf/models.py b/src/aleph_vrf/models.py index b463c4d..d31fd10 100644 --- a/src/aleph_vrf/models.py +++ b/src/aleph_vrf/models.py @@ -1,4 +1,4 @@ -from typing import List, Optional, overload +from typing import List from typing import TypeVar, Generic from uuid import uuid4 @@ -184,5 +184,9 @@ def from_vrf_response( M = TypeVar("M", bound=BaseModel) +class APIError(BaseModel): + error: str + + class APIResponse(GenericModel, Generic[M]): data: M diff --git a/tests/conftest.py b/tests/conftest.py index d2eeae8..67f549f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -113,6 +113,10 @@ def _executor_servers( @pytest.fixture def executor_server(mock_ccn: str) -> str: + """ + Spawns one executor server, configured to use a fake CCN to read/post aleph messages. + """ + assert mock_ccn, "The mock CCN server must be running" with _executor_servers(nb_executors=1) as executor_urls: @@ -121,6 +125,10 @@ def executor_server(mock_ccn: str) -> str: @pytest_asyncio.fixture def executor_servers(mock_ccn: str, request) -> Tuple[str]: + """ + Spawns N executor servers, using the port range [start_port, start_port + N - 1]. + """ + assert mock_ccn, "The mock CCN server must be running" nb_executors = request.param @@ -130,6 +138,10 @@ def executor_servers(mock_ccn: str, request) -> Tuple[str]: @pytest_asyncio.fixture async def executor_client(executor_server: str) -> aiohttp.ClientSession: + """ + Spawns an executor server and provides an aiohttp client to communicate with it. + """ + async with aiohttp.ClientSession(executor_server) as client: yield client @@ -138,6 +150,10 @@ async def executor_client(executor_server: str) -> aiohttp.ClientSession: async def executor_clients( executor_servers: Tuple[str], ) -> Tuple[aiohttp.ClientSession]: + """ + Spawns N executor servers and provides N aiohttp clients to communicate with them. + """ + async with AsyncExitStack() as cm: clients = [ cm.enter_async_context(aiohttp.ClientSession(executor_server)) diff --git a/tests/coordinator/test_executor_selection.py b/tests/coordinator/test_executor_selection.py index 5912255..1d4722d 100644 --- a/tests/coordinator/test_executor_selection.py +++ b/tests/coordinator/test_executor_selection.py @@ -137,6 +137,9 @@ def fixture_nodes_aggregate() -> Dict[str, Any]: @pytest.mark.asyncio async def test_select_random_nodes(fixture_nodes_aggregate: Dict[str, Any], mocker): + """ + Checks that the ExecuteOnAleph policy is able to select CRNs at random. + """ network_fixture = mocker.patch( "aleph_vrf.coordinator.executor_selection._get_corechannel_aggregate", return_value=fixture_nodes_aggregate, @@ -161,6 +164,10 @@ async def test_select_random_nodes(fixture_nodes_aggregate: Dict[str, Any], mock async def test_select_random_nodes_with_unauthorized( fixture_nodes_aggregate: Dict[str, Any], mocker ): + """ + Checks that the blacklisting feature works during node selection. + """ + network_fixture = mocker.patch( "aleph_vrf.coordinator.executor_selection._get_corechannel_aggregate", return_value=fixture_nodes_aggregate, diff --git a/tests/coordinator/test_integration_lib.py b/tests/coordinator/test_integration_lib.py index f5cfdcd..458a8e4 100644 --- a/tests/coordinator/test_integration_lib.py +++ b/tests/coordinator/test_integration_lib.py @@ -93,6 +93,10 @@ async def test_normal_flow( nb_executors: int, nb_bytes: int, ): + """ + Test that the coordinator can call executors to generate a random number. + """ + executors = [Executor(node=Node(address=address)) for address in executor_servers] vrf_response = await generate_vrf( @@ -105,8 +109,6 @@ async def test_normal_flow( assert vrf_response.nb_executors == nb_executors assert len(vrf_response.nodes) == nb_executors assert vrf_response.nb_bytes == nb_bytes - # TODO: determine if this check makes sense as leading zeroes get removed. - # assert int(vrf_response.random_number).bit_length() == nb_bytes * 8 for executor_response in vrf_response.nodes: assert verify( @@ -219,8 +221,8 @@ async def test_call_publish_before_coordinator( This should result in the coordinator call failing and an exception to be raised. """ - # TODO - ... + # To simulate an attack, the simplest solution is to patch `send_generate_requests` + # with a function that does exactly the same and then calls /publish right after generation. mocker.patch( "aleph_vrf.coordinator.vrf.send_generate_requests", send_generate_requests_and_call_publish, diff --git a/tests/malicious_executor.py b/tests/malicious_executor.py index f5a6ca4..789a6c5 100644 --- a/tests/malicious_executor.py +++ b/tests/malicious_executor.py @@ -1,3 +1,8 @@ +""" +A malicious executor that returns an incorrect random number willingly. +""" + + import logging import sys diff --git a/tests/mock_ccn.py b/tests/mock_ccn.py index 726ed5a..df24073 100644 --- a/tests/mock_ccn.py +++ b/tests/mock_ccn.py @@ -1,3 +1,7 @@ +""" +A simplified core channel node used for testing to avoid relying on the aleph.im network. +""" + import json import logging from enum import Enum From 2d33cacfa778e538756876382e70bf2ab6aa10d5 Mon Sep 17 00:00:00 2001 From: Olivier Desenfans Date: Fri, 29 Sep 2023 10:40:57 +0200 Subject: [PATCH 2/3] add back run-in-docker.sh --- run-in-docker.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 run-in-docker.sh diff --git a/run-in-docker.sh b/run-in-docker.sh new file mode 100644 index 0000000..c72f88a --- /dev/null +++ b/run-in-docker.sh @@ -0,0 +1,2 @@ +docker build -t aleph-vrf . +docker run --rm -ti -v "$(pwd)":/usr/src/aleph_vrf aleph-vrf bash \ No newline at end of file From 2d7c80249e9306cdc28bfc158f64395d9a08839d Mon Sep 17 00:00:00 2001 From: Olivier Desenfans Date: Fri, 29 Sep 2023 10:49:18 +0200 Subject: [PATCH 3/3] fix deploy script --- README.md | 6 +++--- deployment/{deploy_vms.py => deploy_vrf_vms.py} | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename deployment/{deploy_vms.py => deploy_vrf_vms.py} (98%) diff --git a/README.md b/README.md index 0fb8396..2763e3a 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ We provide a script to deploy the VM functions. Just run the following command to package the application and upload it to the aleph.im network. ``` -python3 deployment/deploy_vms.py +python3 deployment/deploy_vrf_vms.py ``` If the deployment succeeds, the script will display links to the VMs on the aleph.im network. Example: @@ -52,7 +52,7 @@ The coordinator can also be used directly from Python code. First, deploy the executors using the deployment script, without the coordinator VM: ``` -python3 deployment/deploy_vms.py --no-coordinator +python3 deployment/deploy_vrf_vms.py --no-coordinator ``` This will deploy an executor VM on the network and give you its ID. @@ -109,7 +109,7 @@ pytest -v . ### Create a new release -1. Deploy the VMs: `python3 deployment/deploy_vms.py` +1. Deploy the VMs: `python3 deployment/deploy_vrf_vms.py` 2. Update the executor VM hash in the settings (Settings.FUNCTION) and create a Pull Request 3. Merge the Pull Request and create a new release on Github 4. Build and upload the package on PyPI: `python3 -m build && twine upload dist/*` diff --git a/deployment/deploy_vms.py b/deployment/deploy_vrf_vms.py similarity index 98% rename from deployment/deploy_vms.py rename to deployment/deploy_vrf_vms.py index 2cf5d7b..14e9b33 100644 --- a/deployment/deploy_vms.py +++ b/deployment/deploy_vrf_vms.py @@ -174,7 +174,7 @@ async def main(args: argparse.Namespace): def parse_args(args) -> argparse.Namespace: parser = argparse.ArgumentParser( - prog="vm_supervisor", description="Aleph.im VM Supervisor" + prog="deploy_vrf_vms", description="Deploys the VRF VMs on the aleph.im network." ) parser.add_argument( "--no-coordinator",