diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acece783..043eb1bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,8 +96,8 @@ jobs: ls dist - name: Install from source package - run: - pip install --pre dist/hdf5plugin* + run: | + pip install --pre "$(ls dist/hdf5plugin-*)[test]" --only-binary blosc2 || pip install --pre dist/hdf5plugin-* - name: Print python info run: | @@ -129,5 +129,7 @@ jobs: CIBW_BUILD: cp39-macosx_* CIBW_ARCHS_MACOS: arm64 HDF5PLUGIN_SSE2: False + HDF5PLUGIN_SSSE3: False HDF5PLUGIN_AVX2: False + HDF5PLUGIN_AVX512: False HDF5PLUGIN_NATIVE: False diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 45e1a701..9acc0fa8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,7 +45,7 @@ jobs: name: cibw-sdist path: dist - name: Install sdist - run: pip install --pre dist/hdf5plugin*.tar.gz + run: pip install --pre "$(ls dist/hdf5plugin*.tar.gz)[test]" - name: Run tests run: python test/test.py @@ -85,13 +85,14 @@ jobs: HDF5PLUGIN_OPENMP: "False" HDF5PLUGIN_NATIVE: "False" HDF5PLUGIN_SSE2: ${{ matrix.with_sse2 && 'True' || 'False' }} + HDF5PLUGIN_SSSE3: "False" HDF5PLUGIN_AVX2: "False" HDF5PLUGIN_AVX512: "False" HDF5PLUGIN_BMI2: "False" HDF5PLUGIN_CPP11: "True" HDF5PLUGIN_CPP14: "True" - CIBW_ENVIRONMENT_PASS_LINUX: HDF5PLUGIN_OPENMP HDF5PLUGIN_NATIVE HDF5PLUGIN_SSE2 HDF5PLUGIN_AVX2 HDF5PLUGIN_AVX512 HDF5PLUGIN_BMI2 HDF5PLUGIN_CPP11 HDF5PLUGIN_CPP14 + CIBW_ENVIRONMENT_PASS_LINUX: HDF5PLUGIN_OPENMP HDF5PLUGIN_NATIVE HDF5PLUGIN_SSE2 HDF5PLUGIN_SSSE3 HDF5PLUGIN_AVX2 HDF5PLUGIN_AVX512 HDF5PLUGIN_BMI2 HDF5PLUGIN_CPP11 HDF5PLUGIN_CPP14 # Use Python3.11 to build wheels that are compatible with all supported version of Python CIBW_BUILD: cp311-* @@ -132,10 +133,11 @@ jobs: pattern: cibw-wheels-* path: dist merge-multiple: true - - name: Install h5py and hdf5plugin + - name: Install hdf5plugin + # First select the right wheel from dist/ with pip download, then install it run: | - pip install h5py - pip install --no-index --no-cache --find-links=./dist hdf5plugin --only-binary hdf5plugin + pip download --no-index --no-cache --no-deps --find-links=./dist --only-binary :all: hdf5plugin + pip install "$(ls ./hdf5plugin-*.whl)[test]" --only-binary blosc2 || pip install "$(ls ./hdf5plugin-*.whl)" - name: Run test with latest h5py run: python test/test.py - name: Run test with oldest h5py diff --git a/doc/information.rst b/doc/information.rst index 4218efb6..e8825195 100644 --- a/doc/information.rst +++ b/doc/information.rst @@ -76,7 +76,7 @@ HDF5 compression filters and compression libraries sources were obtained from: * `hdf5-blosc plugin `_ (v1.0.0) using `c-blosc `_ (v1.21.5), LZ4, Snappy, ZLib and ZStd. * hdf5-blosc2 plugin (from `PyTables `_ v3.9.2) - using `c-blosc2 `_ (v2.13.1), LZ4, ZLib and ZStd. + using `c-blosc2 `_ (v2.13.2), LZ4, ZLib and ZStd. * `FCIDECOMP plugin `_ (v1.0.2) using `CharLS `_ (1.x branch, commit `25160a4 `_). @@ -93,9 +93,9 @@ HDF5 compression filters and compression libraries sources were obtained from: Sources of compression libraries shared accross multiple filters were obtained from: -* `LZ4 v1.9.4 `_ +* `LZ4 v1.9.4 `_ * `Snappy v1.1.10 `_ -* `ZStd v1.5.5 `_ +* `ZStd v1.5.5 `_ * `ZLib v1.2.13 `_ When compiled with Intel IPP, the LZ4 compression library is replaced with `LZ4 v1.9.3 `_ patched with a patch from Intel IPP 2021.7.0. diff --git a/doc/install.rst b/doc/install.rst index 22e74548..3435f08b 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -50,6 +50,9 @@ Available options * - ``HDF5PLUGIN_SSE2`` - Whether or not to compile with `SSE2`_ support. Default: True on ppc64le and when probed on x86, False otherwise + * - ``HDF5PLUGIN_SSSE3`` + - Whether or not to compile with `SSSE3`_ support. + Default: True when probed on x86, False otherwise * - ``HDF5PLUGIN_AVX2`` - Whether or not to compile with `AVX2`_ support. It requires enabling `SSE2`_ support. @@ -80,4 +83,5 @@ Note: Boolean options are passed as ``True`` or ``False``. .. _BMI2: https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set .. _IPP: https://en.wikipedia.org/wiki/Integrated_Performance_Primitives .. _SSE2: https://en.wikipedia.org/wiki/SSE2 +.. _SSSE3: https://en.wikipedia.org/wiki/SSSE3 .. _OpenMP: https://www.openmp.org/ diff --git a/doc/usage.rst b/doc/usage.rst index 7fc1c1e9..8adae727 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -18,6 +18,11 @@ In order to read compressed dataset with `h5py`_, use: It registers ``hdf5plugin`` supported compression filters with the HDF5 library used by `h5py`_. Hence, HDF5 compressed datasets can be read as any other dataset (see `h5py documentation `_). +.. note:: + + HDF5 datasets compressed with `Blosc2`_ can require additional plugins to enable decompression, such as `blosc2-grok `_ or `blosc2-openhtj2k `_. + See list of Blosc2 `filters `_ and `codecs `_. + Write compressed datasets +++++++++++++++++++++++++ diff --git a/package/debian11/rules b/package/debian11/rules index 6f74de2c..153134a8 100755 --- a/package/debian11/rules +++ b/package/debian11/rules @@ -6,7 +6,9 @@ export PYBUILD_NAME=hdf5plugin # Build options export HDF5PLUGIN_NATIVE=False export HDF5PLUGIN_SSE2=True +export HDF5PLUGIN_SSSE3=False export HDF5PLUGIN_AVX2=False +export HDF5PLUGIN_AVX512=False export HDF5PLUGIN_OPENMP=True export HDF5PLUGIN_CPP11=True diff --git a/package/debian12/rules b/package/debian12/rules index ff8cb157..6aec2c65 100755 --- a/package/debian12/rules +++ b/package/debian12/rules @@ -6,7 +6,9 @@ export PYBUILD_NAME=hdf5plugin # Build options export HDF5PLUGIN_NATIVE=False export HDF5PLUGIN_SSE2=True +export HDF5PLUGIN_SSSE3=False export HDF5PLUGIN_AVX2=False +export HDF5PLUGIN_AVX512=False export HDF5PLUGIN_OPENMP=True export HDF5PLUGIN_CPP11=True diff --git a/setup.py b/setup.py index a5a320ee..0e6110a6 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ # coding: utf-8 # /*########################################################################## # -# Copyright (c) 2016-2022 European Synchrotron Radiation Facility +# Copyright (c) 2016-2024 European Synchrotron Radiation Facility # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -158,6 +158,11 @@ def __init__(self, compiler=None): else: self.sse2_compile_args = () + if self.ARCH in ('X86_32', 'X86_64'): + self.ssse3_compile_args = ('-mssse3',) # There is no /arch:SSSE3 + else: + self.ssse3_compile_args = () + if self.ARCH in ('X86_32', 'X86_64'): self.avx2_compile_args = ('-mavx2', '/arch:AVX2') else: @@ -201,6 +206,16 @@ def has_sse2(self) -> bool: return check_compile_flags(self.__compiler, "-msse2") return False # Disabled by default + def has_ssse3(self) -> bool: + """Check SSSE3 availability on host""" + if self.ARCH in ('X86_32', 'X86_64'): + if not has_cpu_flag('ssse3'): + return False # SSSE3 not available on host + if self.__compiler.compiler_type == "msvc": + return True + return check_compile_flags(self.__compiler, "-mssse3") + return False # Disabled by default + def has_avx2(self) -> bool: """Check AVX2 availability on host""" if self.ARCH in ('X86_32', 'X86_64'): @@ -273,20 +288,24 @@ def __init__( use_sse2 = host_config.has_sse2() if env_sse2 is None else env_sse2 == "True" self.__use_sse2 = bool(use_sse2) + env_ssse3 = os.environ.get("HDF5PLUGIN_SSSE3", None) + use_ssse3 = host_config.has_ssse3() if env_ssse3 is None else env_ssse3 == "True" + self.__use_ssse3 = bool(use_ssse3) + if use_avx2 is None: env_avx2 = os.environ.get("HDF5PLUGIN_AVX2", None) use_avx2 = host_config.has_avx2() if env_avx2 is None else env_avx2 == "True" - if use_avx2 and not use_sse2: + if use_avx2 and not (use_sse2 and use_ssse3): logger.error( - "use_avx2=True disabled: incompatible with use_sse2=False") + "use_avx2=True disabled: incompatible with use_sse2=False and use_ssse3=False") use_avx2 = False self.__use_avx2 = bool(use_avx2) env_avx512 = os.environ.get("HDF5PLUGIN_AVX512", None) use_avx512 = host_config.has_avx512() if env_avx512 is None else env_avx512 == "True" - if use_avx512 and not (use_sse2 and use_avx2): + if use_avx512 and not (use_sse2 and use_ssse3 and use_avx2): logger.error( - "use_avx512=True disabled: incompatible with use_sse2=False or use_avx2=False") + "use_avx512=True disabled: incompatible with use_sse2=False, use_ssse3=False and use_avx2=False") use_avx512 = False self.__use_avx512 = bool(use_avx512) @@ -304,6 +323,8 @@ def __init__( compile_args = [] if self.__use_sse2: compile_args.extend(host_config.sse2_compile_args) + if self.__use_ssse3: + compile_args.extend(host_config.ssse3_compile_args) if self.__use_avx2: compile_args.extend(host_config.avx2_compile_args) if self.__use_avx512: @@ -319,6 +340,7 @@ def __init__( use_cpp11 = property(lambda self: self.__use_cpp11) use_cpp14 = property(lambda self: self.__use_cpp14) use_sse2 = property(lambda self: self.__use_sse2) + use_ssse3 = property(lambda self: self.__use_ssse3) use_avx2 = property(lambda self: self.__use_avx2) use_avx512 = property(lambda self: self.__use_avx512) use_openmp = property(lambda self: self.__use_openmp) @@ -346,6 +368,7 @@ def get_config_string(self): 'native': self.use_native, 'bmi2': self.USE_BMI2, 'sse2': self.use_sse2, + 'ssse3': self.use_ssse3, 'avx2': self.use_avx2, 'avx512': self.use_avx512, 'cpp11': self.use_cpp11, @@ -865,11 +888,28 @@ def get_blosc2_plugin(): """ hdf5_blosc2_dir = 'src/PyTables/hdf5-blosc2/src' blosc2_dir = 'src/c-blosc2' + plugins_dir = f'{blosc2_dir}/plugins' # blosc sources sources = glob(f'{blosc2_dir}/blosc/*.c') - include_dirs = [blosc2_dir, f'{blosc2_dir}/blosc', f'{blosc2_dir}/include'] - define_macros = [('SHUFFLE_AVX512_ENABLED', 1), ('SHUFFLE_NEON_ENABLED', 1)] + sources += [ # Add embedded codecs, filters and tuners + src_file + for src_file in glob(f'{plugins_dir}/*.c') + glob(f'{plugins_dir}/*/*.c') + glob(f'{plugins_dir}/*/*/*.c') + if not os.path.basename(src_file).startswith("test") + ] + sources += glob(f'{plugins_dir}/codecs/zfp/src/*.c') # Add ZFP embedded sources + + include_dirs = [ + blosc2_dir, + f'{blosc2_dir}/blosc', + f'{blosc2_dir}/include', + f'{blosc2_dir}/plugins/codecs/zfp/include', + ] + define_macros = [ + ('HAVE_PLUGINS', 1), + ('SHUFFLE_AVX512_ENABLED', 1), + ('SHUFFLE_NEON_ENABLED', 1), + ] extra_compile_args = [] extra_link_args = [] libraries = [] @@ -902,9 +942,8 @@ def get_blosc2_plugin(): include_dirs += get_zstd_clib('include_dirs') define_macros.append(('HAVE_ZSTD', 1)) - extra_compile_args += ['-std=gnu99'] # Needed to build manylinux1 wheels - extra_compile_args += ['-O3', '-ffast-math'] - extra_compile_args += ['/Ox', '/fp:fast'] + extra_compile_args += ['-O3', '-std=gnu99'] + extra_compile_args += ['/Ox'] extra_compile_args += ['-pthread'] extra_link_args += ['-pthread'] @@ -1289,7 +1328,10 @@ def get_version(): ext_modules=extensions, install_requires=['h5py'], setup_requires=['setuptools', 'wheel'], - extras_require={'dev': ['sphinx', 'sphinx_rtd_theme']}, + extras_require={ + 'dev': ['sphinx', 'sphinx_rtd_theme'], + 'test': ['blosc2>=2.5.1', 'blosc2-grok>=0.2.2'], + }, cmdclass=cmdclass, libraries=libraries, zip_safe=False, diff --git a/src/c-blosc2/ANNOUNCE.md b/src/c-blosc2/ANNOUNCE.md index 634a3aa3..f0a271ee 100644 --- a/src/c-blosc2/ANNOUNCE.md +++ b/src/c-blosc2/ANNOUNCE.md @@ -1,9 +1,13 @@ -# Announcing C-Blosc2 2.13.1 +# Announcing C-Blosc2 2.13.2 A fast, compressed and persistent binary data store library for C. ## What is new? -This is a patch release for fixing a bug regarding the included files in `b2nd.h`. +This is a patch release for improving of SSSE3 detection on Visual Studio. +Also, documentation for the globally registered filters and codecs has been +added: +https://www.blosc.org/c-blosc2/reference/utility_variables.html#codes-for-filters +https://www.blosc.org/c-blosc2/reference/utility_variables.html#compressor-codecs For more info, please see the release notes in: diff --git a/src/c-blosc2/RELEASE_NOTES.md b/src/c-blosc2/RELEASE_NOTES.md index 36f8441d..137cf357 100644 --- a/src/c-blosc2/RELEASE_NOTES.md +++ b/src/c-blosc2/RELEASE_NOTES.md @@ -1,6 +1,17 @@ Release notes for C-Blosc2 ========================== +Changes from 2.13.1 to 2.13.2 +============================= + +* Better checking for `SSSE3` availability in Visual Studio. Probably fixes #546 too. + Thanks to @t20100 (Thomas Vincent) for the PR (#586). + +* Documented the globally registered filters and codecs. See: + https://www.blosc.org/c-blosc2/reference/utility_variables.html#codes-for-filters + https://www.blosc.org/c-blosc2/reference/utility_variables.html#compressor-codecs + + Changes from 2.13.0 to 2.13.1 ============================= diff --git a/src/c-blosc2/blosc/frame.c b/src/c-blosc2/blosc/frame.c index 1c5290b3..eada94fd 100644 --- a/src/c-blosc2/blosc/frame.c +++ b/src/c-blosc2/blosc/frame.c @@ -1358,7 +1358,7 @@ static int get_meta_from_header(blosc2_frame_s* frame, blosc2_schunk* schunk, ui schunk->metalayers[nmetalayer] = metalayer; // Populate the metalayer string - int8_t nslen = *idxp & (uint8_t)0x1F; + uint8_t nslen = *idxp & (uint8_t)0x1F; idxp += 1; header_pos += nslen; if (header_len < header_pos) { @@ -1538,7 +1538,7 @@ static int get_vlmeta_from_trailer(blosc2_frame_s* frame, blosc2_schunk* schunk, schunk->vlmetalayers[nmetalayer] = metalayer; // Populate the metalayer string - int8_t nslen = *idxp & (uint8_t)0x1F; + uint8_t nslen = *idxp & (uint8_t)0x1F; idxp += 1; trailer_pos += nslen; if (trailer_len < trailer_pos) { @@ -1885,6 +1885,11 @@ blosc2_schunk* frame_to_schunk(blosc2_frame_s* frame, bool copy, const blosc2_io } if (chunk_cbytes > (int32_t)prev_alloc) { data_chunk = realloc(data_chunk, chunk_cbytes); + if (data_chunk == NULL) { + BLOSC_TRACE_ERROR("Cannot realloc space for the data_chunk."); + rc = BLOSC2_ERROR_MEMORY_ALLOC; + break; + } prev_alloc = chunk_cbytes; } if (!frame->sframe) { @@ -3119,7 +3124,7 @@ void* frame_update_chunk(blosc2_frame_s* frame, int64_t nchunk, void* chunk, blo } // Add the new offset - int64_t sframe_chunk_id; + int64_t sframe_chunk_id = -1; if (frame->sframe) { if (offsets[nchunk] < 0) { sframe_chunk_id = -1; diff --git a/src/c-blosc2/doc/Doxyfile b/src/c-blosc2/doc/Doxyfile index df232aa6..3b56e428 100644 --- a/src/c-blosc2/doc/Doxyfile +++ b/src/c-blosc2/doc/Doxyfile @@ -7,7 +7,7 @@ GENERATE_RTF = NO CASE_SENSE_NAMES = NO GENERATE_HTML = NO GENERATE_XML = YES -RECURSIVE = NO +RECURSIVE = YES QUIET = YES JAVADOC_AUTOBRIEF = YES WARN_IF_UNDOCUMENTED = NO diff --git a/src/c-blosc2/doc/reference/utility_variables.rst b/src/c-blosc2/doc/reference/utility_variables.rst index 767a27a7..c0383315 100644 --- a/src/c-blosc2/doc/reference/utility_variables.rst +++ b/src/c-blosc2/doc/reference/utility_variables.rst @@ -33,6 +33,14 @@ Codes for filters .. doxygenenumvalue:: BLOSC_TRUNC_PREC +.. doxygenenumvalue:: BLOSC_FILTER_NDCELL + +.. doxygenenumvalue:: BLOSC_FILTER_NDMEAN + +.. doxygenenumvalue:: BLOSC_FILTER_BYTEDELTA + +.. doxygenenumvalue:: BLOSC_FILTER_INT_TRUNC + Compressor codecs ----------------- @@ -46,6 +54,18 @@ Compressor codecs .. doxygenenumvalue:: BLOSC_ZSTD +.. doxygenenumvalue:: BLOSC_CODEC_NDLZ + +.. doxygenenumvalue:: BLOSC_CODEC_ZFP_FIXED_ACCURACY + +.. doxygenenumvalue:: BLOSC_CODEC_ZFP_FIXED_PRECISION + +.. doxygenenumvalue:: BLOSC_CODEC_ZFP_FIXED_RATE + +.. doxygenenumvalue:: BLOSC_CODEC_OPENHTJ2K + +.. doxygenenumvalue:: BLOSC_CODEC_GROK + Compressor names ---------------- diff --git a/src/c-blosc2/examples/CMakeLists.txt b/src/c-blosc2/examples/CMakeLists.txt index 385030f7..3d5c5479 100644 --- a/src/c-blosc2/examples/CMakeLists.txt +++ b/src/c-blosc2/examples/CMakeLists.txt @@ -1,7 +1,7 @@ # Examples with correspondingly named source files set(EXAMPLES contexts instrument_codec delta_schunk_ex multithread simple frame_metalayers noinit find_roots schunk_simple frame_simple schunk_postfilter urcodecs urfilters frame_vlmetalayers - sframe_simple frame_backed_schunk compress_file frame_offset frame_roundtrip get_set_slice get_blocksize) + sframe_simple frame_backed_schunk compress_file decompress_file frame_offset frame_roundtrip get_set_slice get_blocksize) add_subdirectory(b2nd) @@ -44,8 +44,12 @@ if(BUILD_TESTS) foreach(example ${EXAMPLES}) if(example STREQUAL compress_file) add_test(NAME test_example_${example} - COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ - "${PROJECT_BINARY_DIR}/CMakeCache.txt" CMakeCache.b2frame) + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ + "${PROJECT_BINARY_DIR}/CMakeCache.txt" CMakeCache.b2frame) + elseif(example STREQUAL decompress_file) + add_test(NAME test_example_${example} + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ + CMakeCache.b2frame CMakeCache-2.txt) else() add_test(NAME test_example_${example} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) diff --git a/src/c-blosc2/examples/decompress_file.c b/src/c-blosc2/examples/decompress_file.c new file mode 100644 index 00000000..fd443988 --- /dev/null +++ b/src/c-blosc2/examples/decompress_file.c @@ -0,0 +1,89 @@ +/* + Copyright (c) 2024 The Blosc Development Team + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + Example program demonstrating use of the Blosc filter from C code. + + To compile this program: + + $ gcc decompress_file.c -o decompress_file -lblosc2 + + Example usage for compression/decompression verification: + + $ sha512sum compress_file + 385c93c..feaf38dbec compress_file + $ ./compress_file compress_file compress_file.bl2 + Blosc version info: 2.13.2.dev ($Date:: 2023-01-25 #$) + Compression ratio: 5.1 MB -> 2.0 MB (2.5x) + Compression time: 0.07 s, 72.8 MB/s + $ ./decompress_file compress_file.bl2 compress_file.1 + Blosc version info: 2.13.2.dev ($Date:: 2023-01-25 #$) + Decompression ratio: 2.0 MB -> 5.1 MB (0.4x) + Decompression time: 0.0343 s, 148.5 MB/s + $ sha512sum compress_file.1 + 385c93c..feaf38dbec compress_file.1 + + */ + +#include +#include + +#define KB 1024. +#define MB (1024*KB) +#define GB (1024*MB) + +int main(int argc, char* argv[]) { + blosc2_init(); + static char* data; + int32_t dsize; + int64_t nbytes, cbytes; + blosc_timestamp_t last, current; + double ttotal; + + if (argc != 3) { + fprintf(stderr, "Usage: decompress_file input_file.b2frame output_file\n"); + return -1; + } + + printf("Blosc version info: %s (%s)\n", + BLOSC2_VERSION_STRING, BLOSC2_VERSION_DATE); + + /* Open an existing super-chunk that is on-disk (frame). */ + blosc2_schunk* schunk = blosc2_schunk_open(argv[1]); + + data = (char*)malloc(schunk->chunksize); + + // Decompress the file + blosc_set_timestamp(&last); + FILE* foutput = fopen(argv[2], "wb"); + if (foutput == NULL) { + printf("Output file cannot be open."); + exit(1); + } + for (int nchunk = 0; nchunk < schunk->nchunks; nchunk++) { + dsize = blosc2_schunk_decompress_chunk(schunk, nchunk, data, schunk->chunksize); + if (dsize < 0) { + fprintf(stderr, "Decompression error. Error code: %d\n", dsize); + return dsize; + } + fwrite(data, dsize, 1, foutput); + } + fclose(foutput); + + /* Gather some info */ + nbytes = schunk->nbytes; + cbytes = schunk->cbytes; + blosc_set_timestamp(¤t); + ttotal = blosc_elapsed_secs(last, current); + printf("Decompression ratio: %.1f MB -> %.1f MB (%.1fx)\n", + (float)cbytes / MB, (float)nbytes / MB, (1. * (float)cbytes) / (float)nbytes); + printf("Decompression time: %.3g s, %.1f MB/s\n", + ttotal, (float)nbytes / (ttotal * MB)); + + /* Free resources */ + free(data); + blosc2_schunk_free(schunk); + blosc2_destroy(); + return 0; +} diff --git a/src/c-blosc2/include/blosc2.h b/src/c-blosc2/include/blosc2.h index a8372cdb..73559441 100644 --- a/src/c-blosc2/include/blosc2.h +++ b/src/c-blosc2/include/blosc2.h @@ -83,10 +83,10 @@ extern "C" { /* Version numbers */ #define BLOSC2_VERSION_MAJOR 2 /* for major interface/format changes */ #define BLOSC2_VERSION_MINOR 13 /* for minor interface/format changes */ -#define BLOSC2_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ +#define BLOSC2_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ -#define BLOSC2_VERSION_STRING "2.13.1" /* string version. Sync with above! */ -#define BLOSC2_VERSION_DATE "$Date:: 2023-01-25 #$" /* date version */ +#define BLOSC2_VERSION_STRING "2.13.2" /* string version. Sync with above! */ +#define BLOSC2_VERSION_DATE "$Date:: 2023-02-07 #$" /* date version */ /* The maximum number of dimensions for Blosc2 NDim arrays */ @@ -251,7 +251,7 @@ enum { BLOSC_BITSHUFFLE = 2, //!< Bit-wise shuffle. #endif // BLOSC_H BLOSC_DELTA = 3, //!< Delta filter. - BLOSC_TRUNC_PREC = 4, //!< Truncate mantissa precision; positive values in cparams.filters_meta will keep bits; negative values will reduce bits. + BLOSC_TRUNC_PREC = 4, //!< Truncate mantissa precision; positive values in `filters_meta` will keep bits; negative values will zero bits. BLOSC_LAST_FILTER = 5, //!< sentinel BLOSC_LAST_REGISTERED_FILTER = BLOSC2_GLOBAL_REGISTERED_FILTERS_START + BLOSC2_GLOBAL_REGISTERED_FILTERS - 1, //!< Determine the last registered filter. It is used to check if a filter is registered or not. @@ -2505,6 +2505,11 @@ BLOSC_EXPORT void blosc2_multidim_to_unidim(const int64_t *index, int8_t ndim, c BLOSC_EXPORT int blosc2_get_slice_nchunks(blosc2_schunk* schunk, int64_t *start, int64_t *stop, int64_t **chunks_idx); +/********************************************************************* + Private functions, these are here for convenience, + and are not meant to be included in public docs +*********************************************************************/ + // Private function needed in b2nd.h for deserializing meta static inline void swap_store(void *dest, const void *pa, int size) { uint8_t *pa_ = (uint8_t *) pa; diff --git a/src/c-blosc2/include/blosc2/codecs-registry.h b/src/c-blosc2/include/blosc2/codecs-registry.h index fdc25d8c..4c8ea34e 100644 --- a/src/c-blosc2/include/blosc2/codecs-registry.h +++ b/src/c-blosc2/include/blosc2/codecs-registry.h @@ -19,11 +19,22 @@ extern "C" { enum { BLOSC_CODEC_NDLZ = 32, + //!< Simple Lempel-Ziv compressor for NDim data. Experimental, mainly for teaching purposes. BLOSC_CODEC_ZFP_FIXED_ACCURACY = 33, + //!< ZFP compressor for fixed accuracy mode. The desired accuracy is set in `compcode_meta`. + //!< See https://github.com/Blosc/c-blosc2/blob/main/plugins/codecs/zfp/README.md BLOSC_CODEC_ZFP_FIXED_PRECISION = 34, + //!< ZFP compressor for fixed precision. The desired precision is set in `compcode_meta`. + //!< See https://github.com/Blosc/c-blosc2/blob/main/plugins/codecs/zfp/README.md BLOSC_CODEC_ZFP_FIXED_RATE = 35, + //!< ZFP compressor for fixed precision. The desired rate is set in `compcode_meta`. + //!< See https://github.com/Blosc/c-blosc2/blob/main/plugins/codecs/zfp/README.md BLOSC_CODEC_OPENHTJ2K = 36, + //!< OpenHTJ2K compressor for JPEG 2000 HT. + //!< See https://github.com/Blosc/blosc2_openhtj2k BLOSC_CODEC_GROK = 37, + //!< Grok compressor for JPEG 2000. + //!< See https://github.com/Blosc/blosc2_grok }; void register_codecs(void); diff --git a/src/c-blosc2/include/blosc2/filters-registry.h b/src/c-blosc2/include/blosc2/filters-registry.h index efcd7cfd..0dd3fb8a 100644 --- a/src/c-blosc2/include/blosc2/filters-registry.h +++ b/src/c-blosc2/include/blosc2/filters-registry.h @@ -17,10 +17,20 @@ extern "C" { enum { BLOSC_FILTER_NDCELL = 32, + //!< Simple filter for grouping NDim cell data together. + //!< See https://github.com/Blosc/c-blosc2/blob/main/plugins/filters/ndcell/README.md BLOSC_FILTER_NDMEAN = 33, - BLOSC_FILTER_BYTEDELTA_BUGGY = 34, // buggy version. See #524 - BLOSC_FILTER_BYTEDELTA = 35, // fixed version - BLOSC_FILTER_INT_TRUNC = 36, // truncate int precision; positive values in cparams.filters_meta will keep bits; negative values will reduce bits. + //!< Simple filter for replacing content of a NDim cell with its mean value. + //!< See https://github.com/Blosc/c-blosc2/blob/main/plugins/filters/ndmean/README.md + BLOSC_FILTER_BYTEDELTA_BUGGY = 34, + // buggy version. See #524 + BLOSC_FILTER_BYTEDELTA = 35, + //!< Byteshuffle + delta. Sometimes this can represent an advantage over + //!< @ref BLOSC_SHUFFLE or @ref BLOSC_BITSHUFFLE. + //!< See https://www.blosc.org/posts/bytedelta-enhance-compression-toolset/ + BLOSC_FILTER_INT_TRUNC = 36, + //!< Truncate int precision; positive values in `filter_meta` will keep bits; negative values will zero bits. + //!< This is similar to @ref BLOSC_TRUNC_PREC, but for integers instead of floating point data. }; void register_filters(void); diff --git a/src/c-blosc2/plugins/filters/bytedelta/bytedelta.c b/src/c-blosc2/plugins/filters/bytedelta/bytedelta.c index 87d4ed16..ba78120c 100644 --- a/src/c-blosc2/plugins/filters/bytedelta/bytedelta.c +++ b/src/c-blosc2/plugins/filters/bytedelta/bytedelta.c @@ -18,7 +18,15 @@ #include #include -#if defined __i386__ || defined _M_IX86 || defined __x86_64__ || defined _M_X64 +/* Define the __SSSE3__ symbol if compiling with Visual C++ and + targeting the minimum architecture level. +*/ +#if !defined(__SSSE3__) && defined(_MSC_VER) && \ + (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2)) + #define __SSSE3__ +#endif + +#if defined(__SSSE3__) // SSSE3 code path for x64/x64 #define CPU_HAS_SIMD 1 #include diff --git a/src/hdf5plugin/_filters.py b/src/hdf5plugin/_filters.py index 63a269a1..9bff685c 100644 --- a/src/hdf5plugin/_filters.py +++ b/src/hdf5plugin/_filters.py @@ -221,8 +221,6 @@ def __init__(self, cname='lz4', clevel=5, shuffle=SHUFFLE): class Blosc2(_FilterRefClass): """``h5py.Group.create_dataset``'s compression arguments for using blosc2 filter. - WARNING: This is a pre-release version of the HDF5 filter, only for testing purpose. - It can be passed as keyword arguments: .. code-block:: python diff --git a/src/hdf5plugin/test.py b/src/hdf5plugin/test.py index f9ed5b03..4e3f8809 100644 --- a/src/hdf5plugin/test.py +++ b/src/hdf5plugin/test.py @@ -1,7 +1,7 @@ # coding: utf-8 # /*########################################################################## # -# Copyright (c) 2019-2023 European Synchrotron Radiation Facility +# Copyright (c) 2019-2024 European Synchrotron Radiation Facility # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -23,6 +23,10 @@ # # ###########################################################################*/ """Provides tests """ +from __future__ import annotations + +import importlib.util +import io import os import shutil import tempfile @@ -31,6 +35,10 @@ import h5py import hdf5plugin +try: + import blosc2 +except ImportError: + blosc2 = None from hdf5plugin import _filters @@ -465,9 +473,100 @@ def testAbsoluteMode(self): ) +class TestBlosc2Plugins(unittest.TestCase): + """Specific tests for Blosc2 compression with Blosc2 plugins""" + + def setUp(self): + if not should_test("blosc2"): + self.skipTest("Blosc2 filter not available") + if blosc2 is None: + self.skipTest("Blosc2 package not available") + + def _readback_hdf5_blosc2_dataset( + self, + data: numpy.ndarray, + blocks: tuple[int, ...] | None = None, + **cparams + ) -> numpy.ndarray: + """Compress data with blosc2, write it as HDF5 file with direct chunk write and read it back with h5py + + :param data: data array to compress + :param blocks: Blosc2 block shape + :param cparams: Blosc2 compression parameters + """ + # Convert data to a blosc2 array: This is where compression happens + blosc_array = blosc2.asarray( + data, + chunks=data.shape, + blocks=blocks, + cparams=cparams, + ) + + # Write blosc2 array as a hdf5 dataset + with io.BytesIO() as buffer: + with h5py.File(buffer, 'w') as f: + dataset = f.create_dataset( + 'data', + shape=data.shape, + dtype=data.dtype, + chunks=data.shape, + compression=hdf5plugin.Blosc2(), + ) + dataset.id.write_direct_chunk( + (0,) * data.ndim, + blosc_array.schunk.to_cframe(), + ) + f.flush() + + return dataset[()] + + def test_blosc2_filter_int_trunc(self): + """Read blosc2 dataset written with int truncate filter plugin""" + data = numpy.arange(2**16, dtype=numpy.int16) + + removed_bits = 2 + read_data = self._readback_hdf5_blosc2_dataset( + data, + codec=blosc2.Codec.ZSTD, + filters=[blosc2.Filter.INT_TRUNC], + filters_meta=[-removed_bits], + ) + assert numpy.allclose(read_data, data, rtol=0.0, atol=2**removed_bits) + + def test_blosc2_codec_zfp(self): + """Read blosc2 dataset written with zfp codec plugin""" + data = numpy.outer(numpy.arange(128), numpy.arange(128)).astype(numpy.float32) + + read_data = self._readback_hdf5_blosc2_dataset( + data, + codec=blosc2.Codec.ZFP_PREC, + codec_meta=8, + filters=[], + filters_meta=[], + splitmode=blosc2.SplitMode.NEVER_SPLIT, + ) + assert numpy.allclose(read_data, data, rtol=1e-3, atol=0) + + @unittest.skipIf(importlib.util.find_spec("blosc2_grok") is None, "blosc2_grok package is not available") + def test_blosc2_codec_grok(self): + """Read blosc2 dataset written with blosc2-grok external codec plugin""" + shape = 10, 128, 128 + data = numpy.arange(numpy.prod(shape), dtype=numpy.uint16).reshape(shape) + + read_data = self._readback_hdf5_blosc2_dataset( + data, + blocks=(1,) + data.shape[1:], # 1 block per slice + codec=blosc2.Codec.GROK, + # Disable the filters and the splitmode, because these don't work with grok. + filters=[], + splitmode=blosc2.SplitMode.NEVER_SPLIT, + ) + assert numpy.array_equal(read_data, data) + + def suite(): test_suite = unittest.TestSuite() - for cls in (TestHDF5PluginRW, TestPackage, TestRegisterFilter, TestGetFilters, TestSZ): + for cls in (TestHDF5PluginRW, TestPackage, TestRegisterFilter, TestGetFilters, TestSZ, TestBlosc2Plugins): test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(cls)) return test_suite