Merge pull request #283 from t20100/update-blosc2

Updated `c-blosc2` to v2.11.1; Enabled AVX512 support in `c-blosc2`
silx-kit · Nov 7, 2023 · f203a28 · f203a28
2 parents e1dfedf + d77c9df
commit f203a28
Show file tree

Hide file tree

Showing 53 changed files with 1,887 additions and 1,542 deletions.
diff --git a/doc/information.rst b/doc/information.rst
@@ -58,7 +58,7 @@ HDF5 compression filters and compression libraries sources were obtained from:
 * `hdf5-blosc plugin <https://github.com/Blosc/hdf5-blosc>`_ (v1.0.0)
   using `c-blosc <https://github.com/Blosc/c-blosc>`_ (v1.21.5), LZ4, Snappy, ZLib and ZStd.
 * hdf5-blosc2 plugin (from `PyTables <https://github.com/PyTables/PyTables/>`_ v3.9.2.dev0, commit `3ba4e78 <https://github.com/PyTables/PyTables/tree/3ba4e78336f21f5e32ddf49ced4560f610de70dd/hdf5-blosc2/src>`_)
-  using `c-blosc2 <https://github.com/Blosc/c-blosc2>`_ (v2.10.2), LZ4, ZLib and ZStd.
+  using `c-blosc2 <https://github.com/Blosc/c-blosc2>`_ (v2.11.1), LZ4, ZLib and ZStd.
 * `FCIDECOMP plugin <ftp://ftp.eumetsat.int/pub/OPS/out/test-data/Test-data-for-External-Users/MTG_FCI_Test-Data/FCI_Decompression_Software_V1.0.2>`_ (v1.0.2)
   using `CharLS <https://github.com/team-charls/charls>`_
   (1.x branch, commit `25160a4 <https://github.com/team-charls/charls/tree/25160a42fb62e71e4b0ce081f5cb3f8bb73938b5>`_).

diff --git a/setup.py b/setup.py
@@ -869,7 +869,7 @@ def get_blosc2_plugin():
     # blosc sources
     sources = glob(f'{blosc2_dir}/blosc/*.c')
     include_dirs = [blosc2_dir, f'{blosc2_dir}/blosc', f'{blosc2_dir}/include']
-    define_macros = [('SHUFFLE_NEON_ENABLED', 1)]
+    define_macros = [('SHUFFLE_AVX512_ENABLED', 1), ('SHUFFLE_NEON_ENABLED', 1)]
     extra_compile_args = []
     extra_link_args = []
     libraries = []

diff --git a/src/c-blosc2/.github/workflows/cmake.yml b/src/c-blosc2/.github/workflows/cmake.yml
@@ -71,6 +71,11 @@ jobs:
             compiler: clang
             cmake-args: -D DEACTIVATE_AVX2=ON
 
+          - name: Ubuntu Clang No AVX512
+            os: ubuntu-latest
+            compiler: clang
+            cmake-args: -D DEACTIVATE_AVX512=ON
+
           - name: Ubuntu Clang No ZLIB
             os: ubuntu-latest
             compiler: clang
@@ -105,18 +110,17 @@ jobs:
             compiler: gcc
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Install packages (Ubuntu)
       if: runner.os == 'Linux' && matrix.packages
       run: |
         sudo apt-get update
         sudo apt-get install -y ${{ matrix.packages }}
 
-    # Ninja should be not necessary anymore (see note on Win / GCC above)
-    # - name: Install packages (Windows)
-    #   if: runner.os == 'Windows'
-    #   run: choco install ninja ${{ matrix.packages }}
+    - name: Install packages (Windows)
+      if: runner.os == 'Windows'
+      run: choco install ninja ${{ matrix.packages }}
 
     - name: Install packages (macOS)
       if: runner.os == 'macOS'

diff --git a/src/c-blosc2/.guix-channel b/src/c-blosc2/.guix-channel
@@ -0,0 +1,3 @@
+(channel
+  (version 0)
+  (directory ".guix/modules"))
diff --git a/src/c-blosc2/.guix/modules/c-blosc2-package.scm b/src/c-blosc2/.guix/modules/c-blosc2-package.scm
@@ -0,0 +1,99 @@
+;;; This file follows the suggestions in the article "From development
+;;; environments to continuous integration—the ultimate guide to software
+;;; development with Guix" by Ludovic Courtès at the Guix blog:
+;;; <https://guix.gnu.org/es/blog/2023/from-development-environments-to-continuous-integrationthe-ultimate-guide-to-software-development-with-guix/>.
+
+(define-module (c-blosc2-package)
+  #:use-module (guix)
+  #:use-module (guix build-system cmake)
+  #:use-module (guix git-download)
+  #:use-module ((guix licenses)
+                #:prefix license:)
+  #:use-module (gnu packages compression)
+  #:use-module (ice-9 regex)
+  #:use-module (ice-9 textual-ports))
+
+(define (current-source-root)
+  (dirname (dirname (current-source-directory))))
+
+(define (get-c-blosc2-version)
+  (let ((version-path (string-append (current-source-root) "/include/blosc2.h"))
+        (version-rx (make-regexp
+                     "^\\s*#\\s*define\\s*BLOSC2_VERSION_STRING\\s*\"([^\"]*)\".*"
+                     regexp/newline)))
+    (call-with-input-file version-path
+      (lambda (port)
+        (let* ((version-body (get-string-all port))
+               (version-match (regexp-exec version-rx version-body)))
+          (and version-match
+               (match:substring version-match 1)))))))
+
+(define vcs-file?
+  ;; Return true if the given file is under version control.
+  (or (git-predicate (current-source-root))
+      (const #t)))
+
+(define-public c-blosc2
+  (package
+    (name "c-blosc2")
+    (version (get-c-blosc2-version))
+    (source (local-file "../.."
+                        "c-blosc2-checkout"
+                        #:recursive? #t
+                        #:select? (lambda (path stat)
+                                    (and (vcs-file? path stat)
+                                         (not (string-contains path
+                                               "/internal-complibs"))))))
+    (build-system cmake-build-system)
+    (arguments
+     ;; Disable AVX2 by default as in Guix' c-blosc package.
+     `(#:configure-flags '("-DBUILD_STATIC=OFF"
+                           "-DDEACTIVATE_AVX2=ON"
+                           "-DDEACTIVATE_AVX512=ON"
+                           "-DPREFER_EXTERNAL_LZ4=ON"
+                           "-DPREFER_EXTERNAL_ZLIB=ON"
+                           "-DPREFER_EXTERNAL_ZSTD=ON")))
+    (inputs (list lz4 zlib
+                  ;; The only input with a separate libs-only output.
+                  `(,zstd "lib")))
+    (home-page "https://blosc.org")
+    (synopsis "Blocking, shuffling and lossless compression library")
+    (description
+     "Blosc is a high performance compressor optimized for binary
+data (i.e. floating point numbers, integers and booleans, although it can
+handle string data too).  It has been designed to transmit data to the
+processor cache faster than the traditional, non-compressed, direct memory
+fetch approach via a @code{memcpy()} system call.  Blosc main goal is not just
+to reduce the size of large datasets on-disk or in-memory, but also to
+accelerate memory-bound computations.
+
+C-Blosc2 is the new major version of C-Blosc, and is backward compatible with
+both the C-Blosc1 API and its in-memory format.  However, the reverse thing is
+generally not true for the format; buffers generated with C-Blosc2 are not
+format-compatible with C-Blosc1 (i.e. forward compatibility is not
+supported).")
+    (license license:bsd-3)))
+
+(define (package-with-configure-flags p flags)
+  "Return P with FLAGS as additional 'configure' flags."
+  (package/inherit p
+    (arguments (substitute-keyword-arguments (package-arguments p)
+                 ((#:configure-flags original-flags
+                   #~(list))
+                  #~(append #$original-flags
+                            #$flags))))))
+
+(define-public c-blosc2-with-avx2
+  (package
+    (inherit (package-with-configure-flags c-blosc2
+                                           #~(list "-DDEACTIVATE_AVX2=OFF")))
+    (name "c-blosc2-with-avx2")))
+
+(define-public c-blosc2-with-avx512
+  (package
+    (inherit (package-with-configure-flags c-blosc2
+                                           #~(list "-DDEACTIVATE_AVX2=OFF"
+                                                   "-DDEACTIVATE_AVX512=OFF")))
+    (name "c-blosc2-with-avx512")))
+
+c-blosc2
diff --git a/src/c-blosc2/ANNOUNCE.md b/src/c-blosc2/ANNOUNCE.md
@@ -1,11 +1,12 @@
-# Announcing C-Blosc2 2.10.2
+# Announcing C-Blosc2 2.11.1
 A fast, compressed and persistent binary data store library for C.
 
 ## What is new?
 
-This is a maintenance release with also several improvements for helping
-integration of C-Blosc2 in other projects (thanks to Alex Huebl).  Also,
-some fixes for MinGW platform are in (thanks to Biswapriyo Nath).
+This is a maintenance release for fixing the ALTIVEC header.
+Only affects to IBM POWER builds.
+
+Also, some other fixes and improvements are in.
 
 For more info, please see the release notes in:
 

diff --git a/src/c-blosc2/Blosc2Config.cmake.in b/src/c-blosc2/Blosc2Config.cmake.in
@@ -12,7 +12,7 @@ endif()
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/Modules")
 
 # this section stores which configuration options were set
-set(HAVE_THREADS @Threads_FOUND@)
+set(HAVE_THREADS @HAVE_THREADS@)
 set(HAVE_IPP @HAVE_IPP@)
 set(HAVE_ZLIB_NG @HAVE_ZLIB_NG@)
 set(DEACTIVATE_IPP @DEACTIVATE_IPP@)
@@ -26,16 +26,13 @@ set(PREFER_EXTERNAL_ZSTD @PREFER_EXTERNAL_ZSTD@)
 #   additionally, the Blosc2_..._FOUND variables are used to support
 #   find_package(Blosc2 ... COMPONENTS ... ...)
 #   this enables downstream projects to express the need for specific features.
-if(WIN32)
-    if(HAVE_THREADS)
-        find_dependency(Threads)
-        set(Blosc2_THREADS_FOUND TRUE)
-    else()
-        set(Blosc2_THREADS_FOUND FALSE)
-    endif()
-else()
+set(CMAKE_THREAD_PREFER_PTHREAD TRUE) # pre 3.1
+set(THREADS_PREFER_PTHREAD_FLAG TRUE) # CMake 3.1+
+if(HAVE_THREADS)
     find_dependency(Threads)
     set(Blosc2_THREADS_FOUND TRUE)
+else()
+    set(Blosc2_THREADS_FOUND FALSE)
 endif()
 
 if(NOT DEACTIVATE_IPP AND HAVE_IPP)

diff --git a/src/c-blosc2/CMakeLists.txt b/src/c-blosc2/CMakeLists.txt
@@ -29,6 +29,8 @@
 #       build a lite version (only with BloscLZ and LZ4/LZ4HC) of the blosc library
 #   DEACTIVATE_AVX2: default OFF
 #       do not attempt to build with AVX2 instructions
+#   DEACTIVATE_AVX512: default OFF
+#       do not attempt to build with AVX512 instructions
 #   DEACTIVATE_ZLIB: default OFF
 #       do not include support for the Zlib library
 #   DEACTIVATE_ZSTD: default OFF
@@ -115,6 +117,8 @@ option(BUILD_LITE
     "Build a lite version (only with BloscLZ and LZ4/LZ4HC) of the blosc library." OFF)
 option(DEACTIVATE_AVX2
     "Do not attempt to build with AVX2 instructions" OFF)
+option(DEACTIVATE_AVX512
+    "Do not attempt to build with AVX512 instructions" OFF)
 option(DEACTIVATE_ZLIB
     "Do not include support for the Zlib library." OFF)
 option(DEACTIVATE_ZSTD
@@ -151,6 +155,21 @@ if(BUILD_LITE)
     set(DEACTIVATE_ZSTD ON)
 endif()
 
+# Threads
+set(CMAKE_THREAD_PREFER_PTHREAD TRUE) # pre 3.1
+set(THREADS_PREFER_PTHREAD_FLAG TRUE) # CMake 3.1+
+if(WIN32)
+    # try to use the system library
+    find_package(Threads)
+else()
+    find_package(Threads REQUIRED)
+endif()
+if(Threads_FOUND)
+    set(HAVE_THREADS ON)
+else()
+    set(HAVE_THREADS OFF)
+endif()
+
 if(PREFER_EXTERNAL_LZ4)
     find_package(LZ4)
 else()
@@ -266,30 +285,42 @@ if(CMAKE_SYSTEM_PROCESSOR STREQUAL i386 OR
         else()
             set(COMPILER_SUPPORT_AVX2 FALSE)
         endif()
+        if(CMAKE_C_COMPILER_VERSION VERSION_GREATER 4.9 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 4.9)
+            set(COMPILER_SUPPORT_AVX512 TRUE)
+        else()
+            set(COMPILER_SUPPORT_AVX512 FALSE)
+        endif()
     elseif(CMAKE_C_COMPILER_ID STREQUAL Clang OR CMAKE_C_COMPILER_ID STREQUAL AppleClang)
         set(COMPILER_SUPPORT_SSE2 TRUE)
         if(CMAKE_C_COMPILER_VERSION VERSION_GREATER 3.2 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 3.2)
             set(COMPILER_SUPPORT_AVX2 TRUE)
         else()
             set(COMPILER_SUPPORT_AVX2 FALSE)
         endif()
-    elseif(CMAKE_C_COMPILER_ID STREQUAL Intel)
-        set(COMPILER_SUPPORT_SSE2 TRUE)
-        if(CMAKE_C_COMPILER_VERSION VERSION_GREATER 14.0 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 14.0)
-            set(COMPILER_SUPPORT_AVX2 TRUE)
+        if(CMAKE_C_COMPILER_VERSION VERSION_GREATER 10.0 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 10.0)
+            set(COMPILER_SUPPORT_AVX512 TRUE)
         else()
-            set(COMPILER_SUPPORT_AVX2 FALSE)
+            set(COMPILER_SUPPORT_AVX512 FALSE)
         endif()
+    elseif(CMAKE_C_COMPILER_ID STREQUAL Intel)
+        # All Intel compilers since the introduction of AVX512 in 2016 should support it, so activate all SIMD flavors
+        set(COMPILER_SUPPORT_SSE2 TRUE)
+        set(COMPILER_SUPPORT_AVX2 TRUE)
+        set(COMPILER_SUPPORT_AVX512 TRUE)
     elseif(MSVC)
         set(COMPILER_SUPPORT_SSE2 TRUE)
         if(CMAKE_C_COMPILER_VERSION VERSION_GREATER 18.00.30501 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 18.00.30501)
             set(COMPILER_SUPPORT_AVX2 TRUE)
+        # AVX512 starts to be supported since Visual Studio 17 15.0
+        elseif(CMAKE_C_COMPILER_VERSION VERSION_GREATER 19.10.25017 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 19.10.25017)
+                set(COMPILER_SUPPORT_AVX512 TRUE)
         else()
             set(COMPILER_SUPPORT_AVX2 FALSE)
         endif()
     else()
         set(COMPILER_SUPPORT_SSE2 FALSE)
         set(COMPILER_SUPPORT_AVX2 FALSE)
+        set(COMPILER_SUPPORT_AVX512 FALSE)
         # Unrecognized compiler. Emit a warning message to let the user know hardware-acceleration won't be available.
         message(WARNING "Unable to determine which ${CMAKE_SYSTEM_PROCESSOR} hardware features are supported by the C compiler (${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}).")
     endif()
@@ -330,6 +361,13 @@ endif()
 # disable AVX2 if specified
 if(DEACTIVATE_AVX2)
     set(COMPILER_SUPPORT_AVX2 FALSE)
+    # AVX512 functions in bitshuffle depend on AVX2 too
+    set(COMPILER_SUPPORT_AVX512 FALSE)
+endif()
+
+# disable AVX512 if specified
+if(DEACTIVATE_AVX512)
+    set(COMPILER_SUPPORT_AVX512 FALSE)
 endif()
 
 # flags

diff --git a/src/c-blosc2/CONTRIBUTING.rst b/src/c-blosc2/CONTRIBUTING.rst
@@ -36,5 +36,5 @@ Coding Style
 License
 -------
 By contributing to C-Blosc2, you agree that your contributions will be licensed
-under the `LICENSE <https://github.com/Blosc/c-blosc2/blob/main/LICENSES/BLOSC.txt>`_ 
+under the `LICENSE <https://github.com/Blosc/c-blosc2/blob/main/LICENSE.txt>`_ 
 file of the project.
diff --git a/src/c-blosc2/RELEASE_NOTES.md b/src/c-blosc2/RELEASE_NOTES.md
@@ -1,12 +1,73 @@
 Release notes for C-Blosc2
 ==========================
 
+Changes from 2.11.0 to 2.11.1
+=============================
+
+* Fix ALTIVEC header.  Only affects to IBM POWER builds. Thanks to
+  Michael Kuhn for providing a patch.
+
+
+Changes from 2.10.5 to 2.11.0
+=============================
+
+* New AVX512 support for the bitshuffle filter.  This is a backport of the upstream
+  bitshuffle project (https://github.com/kiyo-masui/bitshuffle).  Expect up to [20%
+  better compression speed](https://github.com/Blosc/c-blosc2/pull/567#issuecomment-1789239842)
+  on AMD Zen4 architecture (7950X3D CPU).
+
+* Add c-blosc2 package definition for Guix.  Thanks to Ivan Vilata.
+
+* Properly check calls to `strtol`.  Fixes #558.
+
+* Export the `b2nd_copy_buffer` function. This may be useful for other projects
+  dealing with multidimensional arrays in memory. Thanks to Ivan Vilata.
+
+* Better check that nthreads must be >= 1 and <= INT16_MAX.  Fixes #559.
+
+* Fix compile arguments for armv7l. Thanks to Ben Greiner.
+
+
+Changes from 2.10.4 to 2.10.5
+=============================
+
+* Fix a variable name in a test that was causing a segfault in some platforms.
+
+* Change tuner's functions signature to return always an error code.  This allows
+  for better error checking when using pluggable tuners in Blosc2.
+
+* Do checks when creating contexts.
+
+
+Changes from 2.10.3 to 2.10.4
+=============================
+
+* Remove duplicated tune initialization since it is already done in blosc2_create_cctx.
+  Thanks to Marta Iborra
+
+* Typos fixed.  Thanks to Dimitri Papadopoulos.
+
+
+Changes from 2.10.2 to 2.10.3
+=============================
+
+* Globally registered new codec `openhtj2k`. This will be loaded dynamically. See PR #557.
+
+* Added a `BLOSC_INFO` macro for details on compression params.
+
+* Added `get_blocksize.c` example on automatic blocksizes.
+
+* Warning fixes.
+
+* Fixes for mingw.
+
+
 Changes from 2.10.1 to 2.10.2
 =============================
 
 * Several fixes for the CMake system.  Thanks to Axel Huebl. See PR #541 and #542.
 
-* Several fixes for mingw plaform.  Thanks to Biswapriyo Nath.  See PR #540 and #543.
+* Several fixes for mingw platform.  Thanks to Biswapriyo Nath.  See PR #540 and #543.
 
 
 Changes from 2.10.0 to 2.10.1