S2E support

CMakeLists.txt

@@ -114,20 +114,62 @@ if (BUILD_LIBFUZZER)
        src/lib/Option.c
        src/lib/Stream.c
     )
-    
+
     target_compile_options(${PROJECT_NAME}_LF PUBLIC -DLIBFUZZER -mno-avx -fsanitize=fuzzer-no-link,undefined)
-    
+
     target_include_directories(${PROJECT_NAME}_LF
        PUBLIC SYSTEM "${CMAKE_SOURCE_DIR}/src/include"
     )
-    
+
     install(
        TARGETS ${PROJECT_NAME} ${PROJECT_NAME}_LF
        LIBRARY DESTINATION lib
        ARCHIVE DESTINATION lib
     )
 endif()
 
+if (NOT DEFINED BUILD_S2E AND DEFINED ENV{BUILD_S2E})
+  set(BUILD_S2E "$ENV{BUILD_S2E}")
+endif()
+
+if (BUILD_S2E)
+  if (NOT DEFINED S2E_ENV_PATH AND DEFINED ENV{S2EDIR})
+    SET(S2E_ENV_PATH "$ENV{S2EDIR}")
+  endif()
+
+  GET_FILENAME_COMPONENT(S2E_ENV_PATH ${S2E_ENV_PATH} ABSOLUTE)
+  if ("${S2E_ENV_PATH}" STREQUAL "")
+    message(FATAL_ERROR "S2E support enabled but S2E environment path not provided."
+                        " Please activate your S2E environment or set S2E_ENV_PATH")
+  endif ()
+
+  set(S2E_INCLUDE "${S2E_ENV_PATH}/source/s2e/guest/common/include")
+  if (EXISTS "${S2E_INCLUDE}/s2e/s2e.h")
+    message(STATUS "Valid S2E environment found at ${S2E_ENV_PATH}")
+  else ()
+    message(FATAL_ERROR "s2e.h not found in ${S2E_INCLUDE}/s2e. Are you sure that this is a valid S2E environment?")
+  endif ()
+
+  add_library(${PROJECT_NAME}_S2E STATIC
+    src/lib/DeepState.c
+    src/lib/Log.c
+    src/lib/Option.c
+    src/lib/Stream.c
+  )
+
+  target_compile_options(${PROJECT_NAME}_S2E PUBLIC -DBUILD_S2E)
+
+  target_include_directories(${PROJECT_NAME}_S2E
+    PUBLIC SYSTEM "${CMAKE_SOURCE_DIR}/src/include"
+    PUBLIC SYSTEM "${S2E_INCLUDE}"
+  )
+
+  install(
+    TARGETS ${PROJECT_NAME} ${PROJECT_NAME}_S2E
+    LIBRARY DESTINATION lib
+    ARCHIVE DESTINATION lib
+  )
+endif()
 
 set(SETUP_PY_IN "${CMAKE_SOURCE_DIR}/bin/setup.py.in")
 set(SETUP_PY "${CMAKE_CURRENT_BINARY_DIR}/setup.py")

README.md

@@ -18,7 +18,7 @@ There are two blog posts on using DeepState: [Part 1](https://blog.trailofbits.c
 
 * Tests look like Google Test, but can use symbolic execution/fuzzing to generate data (parameterized unit testing)
   * Easier to learn than binary analysis tools/fuzzers, but provides similar functionality
-* Already supports Manticore, Angr, libFuzzer, file-based fuzzing with e.g., AFL; more back-ends likely in future
+* Already supports Manticore, Angr, libFuzzer, S2E, file-based fuzzing with e.g., AFL; more back-ends likely in future
   * Switch test generation tool without re-writing test harness
     * Work around show-stopper bugs
     * Find out which tool works best for your code under test
@@ -477,6 +477,21 @@ running inside a VM, due to AFL (unless in persistent mode) relying
 extensively on
 forks, which are very slow on macOS.
 
+## Test-case Generation with S2E
+
+To enable S2E support, run CMake with the following options:
+
+```shell
+cmake -DBUILD_S2E=On -DS2E_ENV_PATH=/path/to/s2e/environment ../
+```
+
+Where `S2E_ENV_PATH` is a path to an S2E environment created with [s2e-env](https://github.com/S2E/s2e-env).
+
+The `deepstate-s2e` command will create analysis projects in your S2E
+environment. You can then start the S2E analysis by running `launch-s2e.sh`.
+Once the analysis completes tests will be available in the `s2e-last/tests/`
+directory.
+
 ## Contributing
 
 All accepted PRs are awarded bounties by Trail of Bits. Join the #deepstate channel on the [Empire Hacking Slack](https://empireslacking.herokuapp.com/) to discuss ongoing development and claim bounties. Check the [good first issue](https://github.com/trailofbits/deepstate/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) label for suggested contributions.

bin/deepstate/main_s2e.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# Copyright (c) 2018 Adrian Herrera
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+
+from s2e_env.manage import call_command
+from s2e_env.commands.new_project import Command as NewProjectCommand
+from s2e_env.utils import log as s2e_log
+
+from deepstate.common import LOG_LEVEL_DEBUG, LOG_LEVEL_TRACE, LOG_LEVEL_INFO, \
+        LOG_LEVEL_WARNING, LOG_LEVEL_ERROR, LOG_LEVEL_FATAL
+from .common import DeepState
+from .s2e.project import DeepStateProject
+
+
+L = logging.getLogger("deepstate.s2e")
+L.setLevel(logging.INFO)
+
+LOG_LEVEL_TO_LOGGING_LEVEL = {
+    LOG_LEVEL_DEBUG: logging.DEBUG,
+    LOG_LEVEL_TRACE: 15,
+    LOG_LEVEL_INFO: logging.INFO,
+    LOG_LEVEL_WARNING: logging.WARNING,
+    LOG_LEVEL_ERROR: logging.ERROR,
+    LOG_LEVEL_FATAL: logging.CRITICAL,
+}
+
+
+def get_s2e_env():
+    s2e_env_dir = os.getenv("S2EDIR")
+    if not s2e_env_dir:
+        raise Exception("S2EDIR environment variable not specified. Ensure "
+                        "that s2e_activate.sh has been sourced")
+    if not os.path.isdir(s2e_env_dir):
+        raise Exception("S2EDIR {} is invalid".format(s2e_env_dir))
+
+    return s2e_env_dir
+
+
+def main():
+    """
+    Create an s2e-env project that is suitable for analyzing a DeepState test.
+    """
+    args = DeepState.parse_args()
+
+    # Sync S2E and DeepState logging levels
+    s2e_log.configure_logging(level=LOG_LEVEL_TO_LOGGING_LEVEL[args.verbosity])
+
+    try:
+        s2e_env_path = get_s2e_env()
+        proj_name = "{}-deepstate".format(os.path.basename(args.binary))
+
+        call_command(NewProjectCommand(), args.binary, env=s2e_env_path,
+                     name=proj_name, project_class=DeepStateProject,
+                     **vars(args))
+    except Exception as e:
+        L.critical("Cannot create an S2E project for %s: %s", args.binary, e)
+        return 1
+
+    return 0
+
+
+if __name__ == '__main__':
+    exit(main())

bin/deepstate/s2e/project.py

@@ -0,0 +1,203 @@
+# Copyright (c) 2018 Adrian Herrera
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import datetime
+import logging
+import os
+import shutil
+
+from s2e_env.command import CommandError
+from s2e_env.commands.project_creation.abstract_project import AbstractProject
+from s2e_env.utils.templates import render_template
+
+
+L = logging.getLogger('deepstate.s2e')
+L.setLevel(logging.INFO)
+
+# Only Linux targets are supported
+ARCH_TO_IMAGE = {
+    'i386': 'debian-9.2.1-i386',
+    'x86_64': 'debian-9.2.1-x86_64',
+}
+
+DEEPSTATE_TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), 'templates')
+
+INSTRUCTIONS = """
+Your DeepState project is available in {project_dir}.
+
+This is a simplified version of a regular S2E analysis project. To start the
+analysis:
+
+    * cd {project_dir} && ./launch-s2e.sh
+
+This will run the DeepState-enabled program in an S2E guest VM. The generated
+tests will appear in the s2e-last/tests directory. The tests can be run using
+the `--input_test_files_dir` option on a **NON** S2E-compiled program (running
+an S2E-compiled program **outside** of S2E will result in an illegal
+instruction error).
+
+Customization
+=============
+
+* By default, your analysis will run with {num_workers} worker(s). This can be
+  changed by modifying the S2E_MAX_PROCESSES variable in launch-s2e.sh
+* If your target program depends on other files (e.g., shared libraries, etc.),
+  then these should be retrieved in bootstrap.sh by adding a call to ${{S2EGET}}
+* Only the minimum plugins required to generate tests have been enabled. To
+  enable more, edit s2e-config.lua
+"""
+
+
+class DeepStateProject(AbstractProject):
+    """A simplified S2E analysis project for DeepState-compiled programs."""
+
+    def _configure(self, target, *args, **options):
+        """
+        Generate the S2E analysis project configuration.
+        """
+        if target.is_empty():
+            raise CommandError('Cannot use an empty target for a DeepState '
+                               'project')
+
+        # Decide on the image to use
+        image = ARCH_TO_IMAGE.get(target.arch)
+        if not image:
+            raise CommandError('Unable to find a suitable image for %s' %
+                               target.path)
+        img_desc = self._select_image(target, image, download_image=False)
+        L.info('Using %s', img_desc['name'])
+
+        # Determine if guestfs is available for this image
+        guestfs_path = self._select_guestfs(img_desc)
+        if not guestfs_path:
+            L.warn('No guestfs available. The VMI plugin may not run optimally')
+
+        # Return the project config dict
+        return {
+            'creation_time': str(datetime.datetime.now()),
+            'project_dir': self.env_path('projects', options['name']),
+            'image': img_desc,
+            'has_guestfs': guestfs_path is not None,
+            'guestfs_path': guestfs_path,
+            'target_path': target.path,
+            'target_arch': target.arch,
+            'num_workers': options['num_workers'],
+        }
+
+    def _create(self, config, force=False):
+        """
+        Create the S2E analysis project, based on the given configuration.
+        """
+        project_dir = config['project_dir']
+
+        # The force option is not exposed on the command-line for a DeepState
+        # project, so fail if the project directory already exists
+        if os.path.isdir(project_dir):
+            raise CommandError('Project directory %s already exists' %
+                               project_dir)
+
+        os.mkdir(project_dir)
+
+        try:
+            # Create a symlink to the analysis target
+            self._symlink_project_files(project_dir, config['target_path'])
+
+            # Create a symlink to the guest tools directory
+            self._symlink_guest_tools(project_dir, config['image'])
+
+            # Create a symlink to the guestfs (if it exists)
+            if config['guestfs_path']:
+                self._symlink_guestfs(project_dir, config['guestfs_path'])
+
+            # Render the templates
+            self._create_launch_script(project_dir, config)
+            self._create_lua_config(project_dir, config)
+            self._create_bootstrap(project_dir, config)
+        except Exception:
+            # If anything goes wrong during project creation, remove anything
+            # incomplete
+            shutil.rmtree(project_dir)
+            raise
+
+        return project_dir
+
+    def _get_instructions(self, config):
+        """
+        Generate instructions.
+        """
+        return INSTRUCTIONS.format(**config)
+
+    def _create_launch_script(self, project_dir, config):
+        """
+        Create the S2E launch script.
+        """
+        L.info('Creating launch script')
+
+        img_desc = config['image']
+        context = {
+            'creation_time': config['creation_time'],
+            'env_dir': self.env_path(),
+            'rel_image_path': os.path.relpath(img_desc['path'], self.env_path()),
+            'max_processes': config['num_workers'],
+            'qemu_arch': img_desc['qemu_build'],
+            'qemu_memory': img_desc['memory'],
+            'qemu_snapshot': img_desc['snapshot'],
+            'qemu_extra_flags': img_desc['qemu_extra_flags'],
+        }
+
+        output_file = 'launch-s2e.sh'
+        output_path = os.path.join(project_dir, output_file)
+
+        render_template(context, '%s.j2' % output_file, output_path,
+                        templates_dir=DEEPSTATE_TEMPLATES_DIR, executable=True)
+
+    def _create_lua_config(self, project_dir, config):
+        """
+        Create the S2E Lua config.
+        """
+        L.info('Creating S2E configuration')
+
+        self._copy_lua_library(project_dir)
+
+        context = {
+            'creation_time': config['creation_time'],
+            'project_dir': config['project_dir'],
+            'target_process': os.path.basename(config['target_path']),
+            'has_guestfs': config['has_guestfs'],
+            'guestfs_path': config['guestfs_path'],
+        }
+
+        output_file = 's2e-config.lua'
+        output_path = os.path.join(project_dir, output_file)
+
+        render_template(context, '%s.j2' % output_file, output_path,
+                        templates_dir=DEEPSTATE_TEMPLATES_DIR)
+
+    def _create_bootstrap(self, project_dir, config):
+        """
+        Create the S2E bootstrap script.
+        """
+        L.info('Creating S2E bootstrap script')
+
+        context = {
+            'creation_time': config['creation_time'],
+            'target': os.path.basename(config['target_path']),
+        }
+
+        output_file = 'bootstrap.sh'
+        output_path = os.path.join(project_dir, output_file)
+
+        render_template(context, '%s.j2' % output_file, output_path,
+                        templates_dir=DEEPSTATE_TEMPLATES_DIR)