diff --git a/CMakeLists.txt b/CMakeLists.txt index 119701f47a..5b9593a22d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,9 +70,6 @@ include(autopas) # ----- include mamico include(mamico) -# ----- resilience plugin dependencies are set here -include(compression) - # ----- add adios2 dependency include(adios2) diff --git a/cmake/modules/compression.cmake b/cmake/modules/compression.cmake deleted file mode 100644 index 4fd1cdd023..0000000000 --- a/cmake/modules/compression.cmake +++ /dev/null @@ -1,9 +0,0 @@ -# compression wrapper -option(ENABLE_COMPRESSION "Use compression wrapper" ${ENABLE_COMPRESSION_WRAPPER}) -if(ENABLE_COMPRESSION) - message(STATUS "Using Compression wrapper.") - option(ENABLE_LZ4 "Use LZ4 in compression wrapper" ON) - include(lz4) -else() - message(STATUS "Compression wrapper disabled.") -endif() diff --git a/cmake/modules/lz4.cmake b/cmake/modules/lz4.cmake deleted file mode 100644 index 484b777794..0000000000 --- a/cmake/modules/lz4.cmake +++ /dev/null @@ -1,39 +0,0 @@ -# lz4 library -if(ENABLE_LZ4) - message(STATUS "Using LZ4.") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_LZ4") - - # Enable ExternalProject CMake module - include(ExternalProject) - - set(LZ4_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/lz4) - set(LZ4_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/libs/lz4) - - # Select https (default) or ssh path. - set(lz4RepoPath https://github.com/lz4/lz4.git) - if(GIT_SUBMODULES_SSH) - set(lz4RepoPath git@github.com:lz4/lz4.git) - endif() - - # Download and install lz4 - ExternalProject_Add( - lz4 - GIT_REPOSITORY ${lz4RepoPath} - GIT_TAG dev - SOURCE_DIR ${LZ4_SOURCE_DIR} - BINARY_DIR ${LZ4_BINARY_DIR} - CONFIGURE_COMMAND cmake ${LZ4_SOURCE_DIR}/contrib/cmake_unofficial -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DBUILD_STATIC_LIBS=ON - BUILD_COMMAND make - INSTALL_COMMAND "" - ) - # Create a liblz4 target to be used as a dependency by the program - add_library(liblz4 IMPORTED STATIC) - add_dependencies(liblz4 lz4) - - # set include directory associated with this target - include_directories(${LZ4_SOURCE_DIR}/lib) - set(LZ4_LIB ${LZ4_BINARY_DIR}/liblz4.a) -else() - message(STATUS "Not using LZ4.") - set(LZ4_LIB "") -endif() diff --git a/doc/compression.dox b/doc/compression.dox deleted file mode 100644 index 987e23766c..0000000000 --- a/doc/compression.dox +++ /dev/null @@ -1,50 +0,0 @@ -/** \page compression Compression algorithm wrapper - -This wrapper provides some convenience when you want to use some sort of compression -somewhere in the project. The syntax is simple and LZ4 is already included as one -compression algorithm.\n -To use the functionality, turn on the option `ENABLE_COMPRESSION` in the CMake options. -New options to turn on individual encoding algorithms will be available.\n -LZ4 is used by default when the wrapper was enabled.\n -In the code, instantiate a `Compression` object, passing the requested compression algorithm as a std::string tag (see \ref tags below), which will offer the necessary functionality. -Consider the following snippet: - - #include // for std::cout - #include // for std::unique_ptr - #include "compression.h" // compression stuff - - std::vector data; //your data - std::vector compressed; //object to store the compressed result - std::vector decompressed; //object to store the decompressed result - - std::unique_ptr compression_instance; - try { - compression_instance = Compression::create("LZ4"); - } - catch (std::invalid_argument ia) { - std::cout << ia.what() << std::endl; - } - - compression_instance.compress(data.begin(), data.end(), compressed); //run compression algorithm - -The encoded data is now stored in `compressed`. The first size_t bytes (actually decltype(_uncompressedSize), see Compression class) contain the uncompressed size.\n -If the instance creation fails, create() will throw a std::invalid_argument (strong guarantee). The string in the error message is the string the factory received to select an encoding, which should be one of the tags in \ref tags. -If the compression fails, compress() will throw a std::exception. The error code from the message can be looked up in the errors of the documentation matching the encoding, i.e. it will show the error code returned by the library function. E.g. for lz4, you would need to go the lz4 github page and read the appropriate docs. - -Decompression works similarly: - - compression_instance.decompress(compressed.begin(), compressed.end(), decompressed); - -Decompressed now contains exactly the same values as `data`.\n -The decompression will throw on failure. The error code provided is again library specific. -The compress and decompress methods for "LZ4" and "None" tags provide strong guarantee. - -If you are having issues with any library, try using "None" as encoding tag. This will only do a copy, and therefore is less error prone. - - - -
Algorithm Tags
Tag Algorithm -
None No compression -
LZ4 lz4 lz4.github.io/lz4/ -
-*/ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f204718945..93ecd78d13 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -61,11 +61,6 @@ else() ) endif() -# dependencies for lz4 -if (ENABLE_LZ4) - add_dependencies(MarDyn liblz4) -endif() - # find adios if (NOT ENABLE_ADIOS2) list(FILTER MY_SRC EXCLUDE REGEX "adios2") diff --git a/src/plugins/compression.cpp b/src/plugins/compression.cpp deleted file mode 100644 index 676ba1dc33..0000000000 --- a/src/plugins/compression.cpp +++ /dev/null @@ -1,117 +0,0 @@ -#include "compression.h" - -std::unique_ptr Compression::create(std::string encoding) { -#ifdef ENABLE_LZ4 - if (encoding.compare("LZ4") == 0) { - return std::unique_ptr(new Lz4Compression()); - } - else -#endif /* ENABLE_LZ4 */ - if (encoding.compare("None") == 0) { - return std::unique_ptr(new NoCompression()); - } - else { - std::string const errormsg("CompressionWrapper error > Invalid encoding: "+encoding); - throw std::invalid_argument(errormsg); - } -} - -#ifdef ENABLE_LZ4 -void Lz4Compression::compress(ByteIterator uncompressedStart, ByteIterator uncompressedEnd, std::vector& compressed) { - decltype(_uncompressedSize) uncompressedSize = uncompressedEnd-uncompressedStart; - decltype(_compressedSize) compressedSize; - // create a temp vector to store the compressed data (may throw) - std::vector temp; - temp.resize(LZ4_compressBound(uncompressedSize)); - // compress the uncompressed source (may throw) - auto actualCompressedSize = LZ4_compress_default( - &(*uncompressedStart), //input data begin of type char* - temp.data()+sizeof(_compressedSize), //output data pointer of type char* - uncompressedSize, //input data size (decltype of _uncompressedSize, usually size_t), imp. cast to int - temp.size()); //output data pointer of decltype(temp.size()), most probably size_t, imp. cast to int - //following may throw (duh) - if (actualCompressedSize == 0) { - std::string const errormsg( - "CompressionWrapper error > LZ4_compress_default failed with error code: " - +std::to_string(actualCompressedSize)); - throw std::runtime_error(errormsg); - } - else { - compressedSize = actualCompressedSize; - } - // add the uncompressed size to the beginning of the result array as meta data (may throw) - std::copy(reinterpret_cast(&uncompressedSize), - reinterpret_cast(&uncompressedSize)+sizeof(_uncompressedSize), - temp.data()); - // add these extra bytes for the meta data to the tracked size - compressedSize += sizeof(_uncompressedSize); - temp.resize(compressedSize); - // If vector allocators match (and they have to as method is not templated), - // swap can't throw -> method has strong guarantee - _uncompressedSize = uncompressedSize; - _compressedSize = compressedSize; - compressed.swap(temp); -} - -void Lz4Compression::decompress(ByteIterator compressedStart, ByteIterator compressedEnd, std::vector& decompressed) { - decltype(_compressedSize) compressedSize = compressedEnd-compressedStart; - decltype(_uncompressedSize) uncompressedSize; - // get the decompressed size, may throw - std::copy(&(*compressedStart), - &(*compressedStart)+sizeof(_uncompressedSize), - reinterpret_cast(&uncompressedSize)); - // resize may throw - std::vector temp; - temp.resize(uncompressedSize); - auto actualDecompressedSize = LZ4_decompress_safe(&(*compressedStart)+sizeof(_uncompressedSize), - temp.data(), - compressedSize-sizeof(_uncompressedSize), - uncompressedSize); - // following may throw (duh) - if (actualDecompressedSize != uncompressedSize) { - std::string const errormsg( - "CompressionWrapper error > LZ4_decompress_default failed with error code: " - +std::to_string(actualDecompressedSize)); - throw std::runtime_error(errormsg); - } - // If vector allocators match (and they have to as method is not templated), - // swap can't throw -> method has strong guarantee - _compressedSize = compressedSize; - _uncompressedSize = uncompressedSize; - decompressed.swap(temp); -} -#endif /* ENABLE_LZ4 */ - -void NoCompression::compress(ByteIterator uncompressedStart, ByteIterator uncompressedEnd, std::vector& compressed) { - // Everything is either const on operating on temporaries, until the swap -> strong guarantee - std::vector temp; - auto uncompressedSize = uncompressedEnd-uncompressedStart; - auto compressedSize = uncompressedSize; - temp.resize(compressedSize); - auto curPosCompressed = temp.begin(); - while (uncompressedStart != uncompressedEnd) { - *curPosCompressed = *uncompressedStart; - ++curPosCompressed; - ++uncompressedStart; - } - _compressedSize = compressedSize; - _uncompressedSize = uncompressedSize; - compressed.swap(temp); -} - -void NoCompression::decompress(ByteIterator compressedStart, ByteIterator compressedEnd, std::vector& decompressed) { - // Everything is either const on operating on temporaries, until the swap -> strong guarantee - std::vector temp; - auto compressedSize = compressedEnd-compressedStart; - auto uncompressedSize = compressedSize; - temp.resize(uncompressedSize); - auto curPosDecompressed = temp.begin(); - while (compressedStart != compressedEnd) { - *curPosDecompressed = *compressedStart; - ++curPosDecompressed; - ++compressedStart; - } - _uncompressedSize = uncompressedSize; - _compressedSize = compressedSize; - decompressed.swap(temp); -} diff --git a/src/plugins/compression.h b/src/plugins/compression.h deleted file mode 100644 index f60f290035..0000000000 --- a/src/plugins/compression.h +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Compression Library Wrapper - * - * Created on: 08 Nov 2018 - * Author: Oliver Fernandes - */ - -#ifndef SRC_PLUGINS_COMPRESSION_H_ -#define SRC_PLUGINS_COMPRESSION_H_ - -#include -#include -#include -#include -#include - -#ifdef ENABLE_LZ4 -#include "lz4.h" -#endif - -/** @brief The Compression class provides easy to use compression methods. - * - * The idea of this class is to provide a "drop-in" possibility to (de-)compress a std::vector, without worrying - * about to much boiler plate code. The usage is simple, consider the following example: - * - * \code{.cpp} - * #include "compression.h" - * std::unique_ptr comp = Compression::create("LZ4"); // Possible tags are "LZ4" and "None" atm - * std::vector myData = {'1','2','3','a','b','c'}; - * std::vector compressedData; - * std::vector decompressedData; - * comp.compress(someData.begin(), someData.end(), compressedData); - * comp.decompress(compressedData.begin(), compressedData.end(), decompressedData); - * for (size_t i = 0; i::iterator; - /** - * Compresses a series of bytes. - * - * Call to compress data. - * @param[in] uncompressedStart Iterator pointing to the start of the array to be compressed - * @param[in] uncompressedEnd Iterator pointing to the element past the last element of the array to be compressed - * @param[in] compressed A std::vector holding the compressed result. The vector will be appropriately resized. - * Any previous contents will be destroyed. - * @return Error codes. Returns 0 on success. Error codes are specific to the employed algorithm. - */ - virtual void compress(ByteIterator uncompressedStart, ByteIterator uncompressedEnd, std::vector& compressed) = 0; - /** - * Decompresses a series of bytes. - * - * Call to decompress data. - * @param[in] compressedStart Iterator pointing to the start of the array to be decompressed - * @param[in] compressedEnd Iterator pointing to the element past the last element of the array to be decompressed - * @param[in] decompressed A std::vector holding the decompressed result. The vector will be appropriately resized. - * Any previous contents will be destroyed. The first sizeof(size_t) bytes hold the uncompressed - * size. - * @return Error codes. Returns 0 on success. Error codes are specific to the employed algorithm. - */ - virtual void decompress(ByteIterator compressedStart, ByteIterator compressedEnd, std::vector& decompressed) = 0; - /** - * Returns the uncompressed size of the data. - * - * Returns the uncompressed size of the data. This will only give sane results if either compression or decompression has - * been attempted. Zero means that there is no data, or something went wrong during the processing calls. - * @return Uncompressed size of data. Depending on if the instance was used for compression or decompression, this is the - * input or output size, respectively. - */ - size_t getUncompressedSize(void) const { - return _uncompressedSize; - }; - /** - * Returns the compressed size of the data. - * - * Returns the compressed size of the data. This will only give sane results if either compression or decompression has - * been attempted. Zero means that there is no data, or something went wrong during the processing calls. - * @return Uncompressed size of data. Depending on if the instance was used for compression or decompression, this is the - * output or input size, respectively. - */ - size_t getCompressedSize(void) const { - return _compressedSize; - }; - /** - * Create an instance of the Compression class. - * - * Use this to instantiate an object which is able to do compression on arrays. The argument passed is a std::string - * containing the tag of the compression algorithm. At the moment, this is just "LZ4" and "None". - * If the tag is not recognized a nullptr will be returned. - * @param[in] encoding A tag from a list of tags associated with various compression algorithms. - * @return A std::unique_ptr to the Compression object. - */ - static std::unique_ptr create(std::string encoding); -protected: - size_t _uncompressedSize = 0; - size_t _compressedSize = 0; -}; - -class NoCompression : public Compression { -public: - /** - * Copies a series of bytes. - * As the class name says, it just does a copy. - * @param[in] compressedStart Iterator pointing to the start of the array to be copied. - * @param[in] compressedEnd Iterator pointing to the element past the last element of the array to be compressed - * @param[in] decompressed A std::vector holding the copied result. The vector will be appropriately resized. - * Any previous contents will be destroyed. The first sizeof(size_t) bytes hold the uncompressed - * size. - * @return Error codes. Returns 0 on success. May provide additional information when getError() is not COMP_SUCCESS. - */ - void compress(ByteIterator uncompressedStart, ByteIterator uncompressedEnd, std::vector& compressed) override; - /** - * Copies a series of bytes back. - * Call this to decompress data. - * @param[in] compressedStart Iterator pointing to the start of the array to be decompressed - * @param[in] compressedEnd Iterator pointing to the element past the last element of the array to be decompressed - * @param[in] decompressed A std::vector holding the copied result. The vector will be appropriately resized. - * Any previous contents will be destroyed. The first sizeof(size_t) bytes hold the uncompressed - * size. - * @return Error codes. Returns 0 on success. May provide additional information when getError() is not COMP_SUCCESS. - */ - void decompress(ByteIterator uncompressedStart, ByteIterator uncompressedEnd, std::vector& compressed) override; -}; - -#ifdef ENABLE_LZ4 -class Lz4Compression : public Compression { -public: - /** - * Copies a series of bytes. - * As the class name says, it just does a copy. - * @param[in] compressedStart Iterator pointing to the start of the array to be copied. - * @param[in] compressedEnd Iterator pointing to the element past the last element of the array to be compressed - * @param[in] decompressed A std::vector holding the compressed result. The vector will be appropriately resized. - * Any previous contents will be destroyed. The first sizeof(size_t) bytes hold the uncompressed - * size. - * @return Error codes. Returns 0 on success. Error codes match the LZ4 documentation. May provide additional information - * when getError() is not COMP_SUCCESS. - */ - void compress(ByteIterator uncompressedStart, ByteIterator uncompressedEnd, std::vector& compressed) override; - /** - * Decompresses a series of bytes. - * Call this to decompress data. - * @param[in] compressedStart Iterator pointing to the start of the array to be decompressed - * @param[in] compressedEnd Iterator pointing to the element past the last element of the array to be decompressed - * @param[in] decompressed A std::vector holding the decompressed result. The vector will be appropriately resized. - * Any previous contents will be destroyed. The first sizeof(size_t) bytes hold the uncompressed - * size. - * @return Error codes. Returns 0 on success. Error codes match the LZ4 documentation. May provide additional information - * when getError() is not COMP_SUCCESS. - */ - void decompress(ByteIterator uncompressedStart, ByteIterator uncompressedEnd, std::vector& compressed) override; -}; -#endif /* ENABLE_LZ4 */ - -#endif /* SRC_PLUGINS_COMPRESSION_H_ */ diff --git a/src/plugins/tests/compressionTest.cpp b/src/plugins/tests/compressionTest.cpp deleted file mode 100644 index 44e3eead64..0000000000 --- a/src/plugins/tests/compressionTest.cpp +++ /dev/null @@ -1,122 +0,0 @@ -// -// Created by fernanor on 2018-12-13 -// - -#include "compressionTest.h" - - -TEST_SUITE_REGISTRATION(compressionTest); - -compressionTest::compressionTest() {} - -compressionTest::~compressionTest() {} - -void compressionTest::testNone() { - std::unique_ptr compInstance = Compression::create("None"); - ASSERT_TRUE(compInstance.get()); - - constexpr size_t datasize = 1024*1024; - std::vector someData(datasize); - std::mt19937 gen(0); - std::uniform_int_distribution<> charsonly(0, 255); - for (auto& elem : someData) { - elem = charsonly(gen); - } - std::vector compressedData; - std::vector decompressedData; - - //compression - compInstance->compress(someData.begin(), someData.end(), compressedData); - size_t correctUncompressedSize = datasize; - size_t correctCompressedSize = datasize; - ASSERT_EQUAL(someData.size(), compInstance->getUncompressedSize()); - ASSERT_EQUAL(correctUncompressedSize, compInstance->getUncompressedSize()); - ASSERT_EQUAL(compressedData.size(), compInstance->getCompressedSize()); - ASSERT_EQUAL(correctCompressedSize, compInstance->getCompressedSize()); - - //decompression - compInstance->decompress(compressedData.begin(), compressedData.end(), decompressedData); - ASSERT_EQUAL(correctUncompressedSize, decompressedData.size()); - - //assert symmetry - for (auto i=0; i compInstance = Compression::create("LZ4"); - ASSERT_TRUE(compInstance.get()); - - constexpr size_t datasize = 1024*1024; - std::vector someData(datasize); - //fill input with random data - std::mt19937 gen(0); - std::uniform_int_distribution<> charsonly(0, 255); - for (auto& elem : someData) { - elem = charsonly(gen); - } - std::vector compressedData; - std::vector decompressedData; - - //compression - compInstance->compress(someData.begin(), someData.end(), compressedData); - size_t correctUncompressedSize = datasize; - ASSERT_EQUAL(someData.size(), compInstance->getUncompressedSize()); - ASSERT_EQUAL(correctUncompressedSize, compInstance->getUncompressedSize()); - ASSERT_EQUAL(compressedData.size(), compInstance->getCompressedSize()); //this is actually larger than uncompressed, Kudos to mt19937 entropy :P - - //decompression - compInstance->decompress(compressedData.begin(), compressedData.end(), decompressedData); - ASSERT_EQUAL(correctUncompressedSize, decompressedData.size()); - - // assert symmetry - for (auto i=0; i compInstance = Compression::create("LZ4"); - ASSERT_TRUE(compInstance.get()); - - constexpr size_t datasize = 1024*1024; - std::vector someData(datasize); - // fill with 4 full sine periods - constexpr double dphi = 8.*M_PI/datasize; - int t = 0; - for (auto& elem : someData) { - elem = static_cast(sin(dphi*t++/datasize)*127.); - } - std::vector compressedData; - std::vector decompressedData; - - //compression - compInstance->compress(someData.begin(), someData.end(), compressedData); - size_t correctUncompressedSize = datasize; - ASSERT_EQUAL(someData.size(), compInstance->getUncompressedSize()); - ASSERT_EQUAL(correctUncompressedSize, compInstance->getUncompressedSize()); - ASSERT_EQUAL(compressedData.size(), compInstance->getCompressedSize()); - - //decompression - compInstance->decompress(compressedData.begin(), compressedData.end(), decompressedData); - ASSERT_EQUAL(correctUncompressedSize, decompressedData.size()); - - // assert symmetry - for (auto i=0; i compInstance = Compression::create("FailMe"); - } - catch (const std::invalid_argument& ia) { - std::string const correctMsg("CompressionWrapper error > Invalid encoding: FailMe"); - std::string const exceptionMsg(ia.what()); - ASSERT_EQUAL(correctMsg, exceptionMsg); - } -} diff --git a/src/plugins/tests/compressionTest.h b/src/plugins/tests/compressionTest.h deleted file mode 100644 index e376bb8b06..0000000000 --- a/src/plugins/tests/compressionTest.h +++ /dev/null @@ -1,43 +0,0 @@ -// -// Created by fernanor on 2018-12-13. -// - - -#ifndef COMPRESSIONTEST_H -#define COMPRESSIONTEST_H - -#include -#include -#include -#include -#include -#include - -#include "utils/TestWithSimulationSetup.h" -#include "plugins/compression.h" - -class compressionTest : public utils::TestWithSimulationSetup { - - TEST_SUITE(compressionTest); - TEST_METHOD(testNone); -#ifdef ENABLE_LZ4 - TEST_METHOD(testLz4Random); - TEST_METHOD(testLz4Sine); -#endif - TEST_METHOD(testThrowOnFailedCreate); - TEST_SUITE_END; - -public: - - compressionTest(); - - virtual ~compressionTest(); - void testNone(); -#ifdef ENABLE_LZ4 - void testLz4Random(); - void testLz4Sine(); -#endif - void testThrowOnFailedCreate(); -}; - -#endif //COMPRESSIONTEST_H