Skip to content

Commit

Permalink
Merge pull request #320 from t20100/updates
Browse files Browse the repository at this point in the history
Updated filters: `c-blosc` v1.21.6, `c-blosc2` v2.15.1, `fcidecomp` v2.1.1; Updated libraries: `lz4` v1.10.0, `zlib` v1.3.1
  • Loading branch information
t20100 authored Aug 28, 2024
2 parents ad7622d + 9db17b9 commit 8252d67
Show file tree
Hide file tree
Showing 419 changed files with 27,128 additions and 21,103 deletions.
16 changes: 8 additions & 8 deletions doc/information.rst
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,14 @@ HDF5 compression filters and compression libraries sources were obtained from:
(commit `49e3b65 <https://github.com/nexusformat/HDF5-External-Filter-Plugins/tree/49e3b65eca772bca77af13ba047d8b577673afba>`_)
using LZ4.
* `bitshuffle plugin <https://github.com/kiyo-masui/bitshuffle>`_ (v0.5.1) using LZ4 and ZStd.
* bzip2 plugin (from `PyTables <https://github.com/PyTables/PyTables/>`_ v3.9.2)
* bzip2 plugin (from `PyTables <https://github.com/PyTables/PyTables/>`_ v3.10.1)
using `BZip2 <https://sourceware.org/git/bzip2.git>`_ (v1.0.8).
* `hdf5-blosc plugin <https://github.com/Blosc/hdf5-blosc>`_ (v1.0.1)
using `c-blosc <https://github.com/Blosc/c-blosc>`_ (v1.21.5), LZ4, Snappy, ZLib and ZStd.
* hdf5-blosc2 plugin (from `PyTables <https://github.com/PyTables/PyTables/>`_ v3.9.2)
using `c-blosc2 <https://github.com/Blosc/c-blosc2>`_ (v2.15.0), LZ4, ZLib and ZStd.
using `c-blosc <https://github.com/Blosc/c-blosc>`_ (v1.21.6), LZ4, Snappy, ZLib and ZStd.
* hdf5-blosc2 plugin (from `PyTables <https://github.com/PyTables/PyTables/>`_ v3.10.1)
using `c-blosc2 <https://github.com/Blosc/c-blosc2>`_ (v2.15.1), LZ4, ZLib and ZStd.
* `FCIDECOMP plugin <https://gitlab.eumetsat.int/open-source/data-tailor-plugins/fcidecomp>`_
(`v2.0.1 <https://gitlab.eumetsat.int/open-source/data-tailor-plugins/fcidecomp/-/tree/e88f83c03bafcd0769c167dca14aa7aabf728e1b>`_)
(`v2.1.1 <https://gitlab.eumetsat.int/open-source/data-tailor-plugins/fcidecomp/-/tree/2.1.1>`_)
using `CharLS <https://github.com/team-charls/charls>`_ (v2.1.0).
* `SZ plugin <https://github.com/szcompressor/SZ>`_
(commit `f466775 <https://github.com/szcompressor/SZ/tree/f4667759ead6a902110e80ff838ccdfddbc8dcd7>`_)
Expand All @@ -88,10 +88,10 @@ HDF5 compression filters and compression libraries sources were obtained from:

Sources of compression libraries shared accross multiple filters were obtained from:

* `LZ4 v1.9.4 <https://github.com/Blosc/c-blosc2/tree/v2.15.0/internal-complibs/lz4-1.9.4>`_
* `LZ4 v1.10.0 <https://github.com/Blosc/c-blosc2/tree/v2.15.1/internal-complibs/lz4-1.10.0>`_
* `Snappy v1.2.1 <https://github.com/google/snappy>`_
* `ZStd v1.5.6 <https://github.com/Blosc/c-blosc2/tree/v2.15.0/internal-complibs/zstd-1.5.6>`_
* `ZLib v1.2.13 <https://github.com/Blosc/c-blosc/tree/v1.21.5/internal-complibs/zlib-1.2.13>`_
* `ZStd v1.5.6 <https://github.com/Blosc/c-blosc2/tree/v2.15.1/internal-complibs/zstd-1.5.6>`_
* `ZLib v1.3.1 <https://github.com/Blosc/c-blosc/tree/v1.21.6/internal-complibs/zlib-1.3.1>`_

When compiled with Intel IPP, the LZ4 compression library is replaced with `LZ4 v1.9.3 <https://github.com/lz4/lz4/releases/tag/v1.9.3>`_ patched with a patch from Intel IPP 2021.7.0.

Expand Down
4 changes: 3 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -942,15 +942,17 @@ def get_blosc2_plugin():
]
define_macros = [
('HAVE_PLUGINS', 1),
('SHUFFLE_SSE2_ENABLED', 1),
('SHUFFLE_AVX2_ENABLED', 1),
('SHUFFLE_AVX512_ENABLED', 1),
('SHUFFLE_NEON_ENABLED', 1),
('SHUFFLE_ALTIVEC_ENABLED', 1),
]
extra_compile_args = []
extra_link_args = []
libraries = []

if platform.machine() == 'ppc64le':
define_macros.append(('SHUFFLE_ALTIVEC_ENABLED', 1))
define_macros.append(('NO_WARN_X86_INTRINSICS', None))
if HostConfig.ARCH == 'ARM_8':
extra_compile_args += ['-flax-vector-conversions']
Expand Down
20 changes: 20 additions & 0 deletions src/PyTables/hdf5-blosc2/src/blosc2_filter.c
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,7 @@ size_t blosc2_filter_function(unsigned flags, size_t cd_nelmts,
/* Filter params that are only set for B2ND */
int ndim = -1;
int32_t chunkshape[BLOSC2_MAX_DIM];
size_t chunksize = typesize;
if (cd_nelmts >= 8) {
/* Get chunk shape for B2ND */
ndim = cd_values[7];
Expand All @@ -321,6 +322,7 @@ size_t blosc2_filter_function(unsigned flags, size_t cd_nelmts,
}
for (int i = 0; i < ndim; i++) {
chunkshape[i] = cd_values[8 + i];
chunksize *= (size_t) cd_values[8 + i];
}
}

Expand Down Expand Up @@ -367,6 +369,14 @@ size_t blosc2_filter_function(unsigned flags, size_t cd_nelmts,

blosc2_storage storage = {.cparams=&cparams, .contiguous=false};

if (ndim > 1 && nbytes != chunksize) {
BLOSC_TRACE_INFO("Filter input size %lu does not match chunk data size %lu "
"(e.g. Fletcher32 checksum added before compression step), "
"using plain Blosc2 instead of B2ND",
nbytes, chunksize);
ndim = -1;
}

if (ndim > 1) {

b2nd_context_t *ctx = NULL;
Expand Down Expand Up @@ -453,6 +463,15 @@ size_t blosc2_filter_function(unsigned flags, size_t cd_nelmts,
/* declare dummy variables */
int32_t cbytes;

/* It would be cool to have a check to detect whether the buffer contains
* an unexpected amout of bytes because of the application of a previous
* filter (e.g. a Fletcher32 checksum), thus disabling B2ND optimizations.
* However, we cannot know before parsing the super-chunk, and that
* operation accepts trailing bytes without reporting their presence.
* Thus such a test can only happen outside of the filter, with more
* information available about the whole pipeline setup.
*/

blosc2_schunk* schunk = blosc2_schunk_from_buffer(*buf, (int64_t)nbytes, false);
if (schunk == NULL) {
PUSH_ERR("blosc2_filter", H5E_CALLBACK, "Cannot get super-chunk from buffer");
Expand Down Expand Up @@ -549,6 +568,7 @@ size_t blosc2_filter_function(unsigned flags, size_t cd_nelmts,
}

blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS;
dparams.schunk = schunk;
dctx = blosc2_create_dctx(dparams);
status = blosc2_decompress_ctx(dctx, chunk, cbytes, outbuf, (int32_t) outbuf_size);
if (status <= 0) {
Expand Down
2 changes: 1 addition & 1 deletion src/c-blosc/.github/workflows/cmake.yml
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ jobs:
compiler: gcc

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4

- name: Install packages (Ubuntu)
if: runner.os == 'Linux' && matrix.packages
Expand Down
2 changes: 1 addition & 1 deletion src/c-blosc/.github/workflows/fuzz.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
fuzz-seconds: 600
dry-run: false
- name: Upload Crash
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
if: failure()
with:
name: artifacts
Expand Down
7 changes: 4 additions & 3 deletions src/c-blosc/ANNOUNCE.rst
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
===============================================================
Announcing C-Blosc 1.21.5
Announcing C-Blosc 1.21.6
A blocking, shuffling and lossless compression library for C
===============================================================

What is new?
============

This is a maintenance release. Here, we have provided a fix for a SSE2/AVX2
build issue. Thanks to Thomas VINCENT and Mark Kittisopikul.
This is a maintenance release. Here, we are providing new versions of
internal Zlib and Zstd compressors, which are now 1.3.1 and 1.5.6
respectively.

For more info, please see the release notes in:

Expand Down
9 changes: 9 additions & 0 deletions src/c-blosc/RELEASE_NOTES.rst
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,18 @@
===========================


Changes from 1.21.5 to 1.21.6
=============================

* Zlib updated to 1.3.1. Thanks to Lachlan Deakin.
* Zstd updated to 1.5.6
* Fixed many typos. Thanks to Dimitri Papadopoulos.


Changes from 1.21.4 to 1.21.5
=============================


* Fix SSE2/AVX2 build issue. Fixes #352. Thanks to Thomas VINCENT
and Mark Kittisopikul.

Expand Down
2 changes: 1 addition & 1 deletion src/c-blosc/bench/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ LDFLAGS += -DHAVE_SNAPPY -lsnappy
LDFLAGS += -DHAVE_ZLIB -lz

# Support for internal Zlib
#ZLIB_DIR = ../internal-complibs/zlib-1.2.13
#ZLIB_DIR = ../internal-complibs/zlib-1.3.1
#CFLAGS += -DHAVE_ZLIB -I$(ZLIB_DIR)
#SOURCES += $(wildcard $(ZLIB_DIR)/*.c)

Expand Down
2 changes: 1 addition & 1 deletion src/c-blosc/bench/Makefile.mingw
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ LDFLAGS=-DHAVE_ZLIB -I$(ZLIB_ROOT)/include -lzdll -L$(ZLIB_ROOT)/lib
#LDFLAGS=-DHAVE_ZLIB -I$(ZLIB_ROOT)/include -lz -L$(ZLIB_ROOT)/lib

# Support for internal Zlib
#ZLIB_DIR = ../internal-complibs/zlib-1.2.13
#ZLIB_DIR = ../internal-complibs/zlib-1.3.1
#CFLAGS += -DHAVE_ZLIB -I$(ZLIB_DIR)
#SOURCES += $(wildcard $(ZLIB_DIR)/*.c)

Expand Down
4 changes: 2 additions & 2 deletions src/c-blosc/blosc/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ if(NOT DEACTIVATE_ZLIB)
if (ZLIB_FOUND)
set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIR})
else(ZLIB_FOUND)
set(ZLIB_LOCAL_DIR ${INTERNAL_LIBS}/zlib-1.2.13)
set(ZLIB_LOCAL_DIR ${INTERNAL_LIBS}/zlib-1.3.1)
set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZLIB_LOCAL_DIR})
endif(ZLIB_FOUND)
endif(NOT DEACTIVATE_ZLIB)
Expand All @@ -40,7 +40,7 @@ if (NOT DEACTIVATE_ZSTD)
if (ZSTD_FOUND)
set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZSTD_INCLUDE_DIR})
else (ZSTD_FOUND)
set(ZSTD_LOCAL_DIR ${INTERNAL_LIBS}/zstd-1.5.5)
set(ZSTD_LOCAL_DIR ${INTERNAL_LIBS}/zstd-1.5.6)
set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZSTD_LOCAL_DIR} ${ZSTD_LOCAL_DIR}/common)
endif (ZSTD_FOUND)
endif (NOT DEACTIVATE_ZSTD)
Expand Down
6 changes: 3 additions & 3 deletions src/c-blosc/blosc/blosc.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@ extern "C" {
/* Version numbers */
#define BLOSC_VERSION_MAJOR 1 /* for major interface/format changes */
#define BLOSC_VERSION_MINOR 21 /* for minor interface/format changes */
#define BLOSC_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */
#define BLOSC_VERSION_RELEASE 6 /* for tweaks, bug-fixes, or development */

#define BLOSC_VERSION_STRING "1.21.5" /* string version. Sync with above! */
#define BLOSC_VERSION_STRING "1.21.6" /* string version. Sync with above! */
#define BLOSC_VERSION_REVISION "$Rev$" /* revision version */
#define BLOSC_VERSION_DATE "$Date:: 2023-05-16 #$" /* date version */
#define BLOSC_VERSION_DATE "$Date:: 2024-06-24 #$" /* date version */

/* The *_FORMAT symbols should be just 1-byte long */
#define BLOSC_VERSION_FORMAT 2 /* Blosc format version, starting at 1 */
Expand Down

This file was deleted.

Loading

0 comments on commit 8252d67

Please sign in to comment.