From a5aa301ac06bd5b5a049b5f072a9764528eb0fc7 Mon Sep 17 00:00:00 2001 From: Dmitriy Musatkin <63878209+DmitriyMusatkin@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:25:06 -0700 Subject: [PATCH] Bind out CRC64 (#831) --- .github/workflows/ci.yml | 30 +++--- .github/workflows/docs.yml | 2 +- .../awssdk/crt/checksums/CRC64NVME.java | 98 +++++++++++++++++++ src/native/checksums.c | 34 +++++-- .../amazon/awssdk/crt/test/CrcTest.java | 46 +++++++++ 5 files changed, 184 insertions(+), 26 deletions(-) create mode 100644 src/main/java/software/amazon/awssdk/crt/checksums/CRC64NVME.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d296f22a5..8f8e33164 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: - 'docs' env: - BUILDER_VERSION: v0.9.62 + BUILDER_VERSION: v0.9.67 BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-java @@ -34,7 +34,7 @@ jobs: #- manylinux2014-x86 until we find 32-bit linux binaries we can use steps: - name: Checkout Sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Build ${{ env.PACKAGE_NAME }} @@ -60,7 +60,7 @@ jobs: - gcc-8 steps: - name: Checkout Sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Build ${{ env.PACKAGE_NAME }} @@ -167,7 +167,7 @@ jobs: runs-on: windows-2022 # latest steps: - name: Checkout Sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Build ${{ env.PACKAGE_NAME }} + consumers @@ -182,7 +182,7 @@ jobs: arch: [x86, x64] steps: - name: Checkout Sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Build ${{ env.PACKAGE_NAME }} + consumers @@ -196,7 +196,7 @@ jobs: runs-on: macos-14 #latest steps: - name: Checkout Sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Build ${{ env.PACKAGE_NAME }} + consumers @@ -210,7 +210,7 @@ jobs: runs-on: macos-14-large #latest steps: - name: Checkout Sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Build ${{ env.PACKAGE_NAME }} + consumers @@ -227,12 +227,12 @@ jobs: runs-on: ubuntu-20.04 # latest steps: - name: Checkout Sources - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: true # Setup JDK 11 - name: set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: '11' distribution: 'temurin' @@ -276,7 +276,7 @@ jobs: check-docs: runs-on: ubuntu-22.04 # use same version as docs.yml steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Check docs @@ -287,7 +287,7 @@ jobs: runs-on: ubuntu-22.04 # latest steps: - name: Checkout Source - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true fetch-depth: 0 @@ -301,7 +301,7 @@ jobs: runs-on: ubuntu-22.04 # latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Configure local host @@ -319,7 +319,7 @@ jobs: runs-on: macos-14 # latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Configure local host @@ -340,7 +340,7 @@ jobs: runs-on: windows-2022 # latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Configure local host @@ -364,7 +364,7 @@ jobs: java-version: ['22', '21', '17'] steps: - name: Checkout Sources - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: true - name: Setup GraalVM diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index fc99ab2c7..73a20eb76 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -14,7 +14,7 @@ jobs: contents: write # allow push steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Update docs branch diff --git a/src/main/java/software/amazon/awssdk/crt/checksums/CRC64NVME.java b/src/main/java/software/amazon/awssdk/crt/checksums/CRC64NVME.java new file mode 100644 index 000000000..aaeef281e --- /dev/null +++ b/src/main/java/software/amazon/awssdk/crt/checksums/CRC64NVME.java @@ -0,0 +1,98 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +package software.amazon.awssdk.crt.checksums; + +import software.amazon.awssdk.crt.CRT; +import java.util.zip.Checksum; + +/** + * CRT implementation of the Java Checksum interface for making CRC64NVME checksum calculations + */ +public class CRC64NVME implements Checksum, Cloneable { + static { + new CRT(); + }; + + private long value = 0; + + /** + * Default constructor + */ + public CRC64NVME() { + } + + private CRC64NVME(long value) { + this.value = value; + } + + @Override + public Object clone() { + return new CRC64NVME(value); + } + + /** + * Returns the current checksum value. + * + * @return the current checksum value. + */ + @Override + public long getValue() { + return value; + } + + /** + * Resets the checksum to its initial value. + */ + @Override + public void reset() { + value = 0; + } + + /** + * Updates the current checksum with the specified array of bytes. + * + * @param b the byte array to update the checksum with + * @param off the starting offset within b of the data to use + * @param len the number of bytes to use in the update + */ + @Override + public void update(byte[] b, int off, int len) { + if (b == null) { + throw new NullPointerException(); + } + if (off < 0 || len < 0 || off > b.length - len) { + throw new ArrayIndexOutOfBoundsException(); + } + value = crc64nvme(b, value, off, len); + } + + /** + * Updates the current checksum with the specified array of bytes. + * + * @param b the byte array to update the checksum with + */ + public void update(byte[] b) { + update(b, 0, b.length); + } + + /** + * Updates the current checksum with the specified byte. + * + * @param b the byte to update the checksum with + */ + @Override + public void update(int b) { + if (b < 0 || b > 0xff) { + throw new IllegalArgumentException(); + } + byte[] buf = { (byte) (b & 0x000000ff) }; + this.update(buf); + } + + /******************************************************************************* + * native methods + ******************************************************************************/ + private static native long crc64nvme(byte[] input, long previous, int offset, int length);; +} diff --git a/src/native/checksums.c b/src/native/checksums.c index 7da8c2fc1..d8e61a6b2 100644 --- a/src/native/checksums.c +++ b/src/native/checksums.c @@ -9,23 +9,18 @@ #include "crt.h" #include "java_class_ids.h" -jint crc_common( +jint crc32_common( JNIEnv *env, jbyteArray input, jint previous, const size_t start, size_t length, - uint32_t (*checksum_fn)(const uint8_t *, int, uint32_t)) { + uint32_t (*checksum_fn)(const uint8_t *, size_t, uint32_t)) { struct aws_byte_cursor c_byte_array = aws_jni_byte_cursor_from_jbyteArray_acquire(env, input); struct aws_byte_cursor cursor = c_byte_array; aws_byte_cursor_advance(&cursor, start); cursor.len = aws_min_size(length, cursor.len); - uint32_t res = (uint32_t)previous; - while (cursor.len > INT_MAX) { - res = checksum_fn(cursor.ptr, INT_MAX, res); - aws_byte_cursor_advance(&cursor, INT_MAX); - } - jint res_signed = (jint)checksum_fn(cursor.ptr, (int)cursor.len, res); + jint res_signed = (jint)checksum_fn(cursor.ptr, cursor.len, previous); aws_jni_byte_cursor_from_jbyteArray_release(env, input, c_byte_array); return res_signed; } @@ -40,7 +35,7 @@ JNIEXPORT jint JNICALL Java_software_amazon_awssdk_crt_checksums_CRC32_crc32( (void)jni_class; aws_cache_jni_ids(env); - return crc_common(env, input, previous, offset, length, aws_checksums_crc32); + return crc32_common(env, input, previous, offset, length, aws_checksums_crc32_ex); } JNIEXPORT jint JNICALL Java_software_amazon_awssdk_crt_checksums_CRC32C_crc32c( @@ -53,5 +48,24 @@ JNIEXPORT jint JNICALL Java_software_amazon_awssdk_crt_checksums_CRC32C_crc32c( (void)jni_class; aws_cache_jni_ids(env); - return crc_common(env, input, previous, offset, length, aws_checksums_crc32c); + return crc32_common(env, input, previous, offset, length, aws_checksums_crc32c_ex); +} + +JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_checksums_CRC64NVME_crc64nvme( + JNIEnv *env, + jclass jni_class, + jbyteArray input, + jlong previous, + jint offset, + jint length) { + (void)jni_class; + aws_cache_jni_ids(env); + + struct aws_byte_cursor c_byte_array = aws_jni_byte_cursor_from_jbyteArray_acquire(env, input); + struct aws_byte_cursor cursor = c_byte_array; + aws_byte_cursor_advance(&cursor, offset); + cursor.len = aws_min_size(length, cursor.len); + jlong res_signed = (jlong)aws_checksums_crc64nvme_ex(cursor.ptr, cursor.len, previous); + aws_jni_byte_cursor_from_jbyteArray_release(env, input, c_byte_array); + return res_signed; } diff --git a/src/test/java/software/amazon/awssdk/crt/test/CrcTest.java b/src/test/java/software/amazon/awssdk/crt/test/CrcTest.java index 19afbbcf9..9b1fbcc25 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/CrcTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/CrcTest.java @@ -167,4 +167,50 @@ public void testCrc32CLargeBuffer() { int expected = 0xfb5b991d; assertEquals(expected, (int) crcc.getValue()); } + + @Test + public void testCrc64NVMEZeroes() { + byte[] zeroes = new byte[32]; + software.amazon.awssdk.crt.checksums.CRC64NVME crc64 = new software.amazon.awssdk.crt.checksums.CRC64NVME(); + crc64.update(zeroes); + long expected = 0xCF3473434D4ECF3BL; + assertEquals(expected, crc64.getValue()); + } + + @Test + public void testCrc64NVMEZeroesIterated() { + byte[] zeroes = new byte[32]; + software.amazon.awssdk.crt.checksums.CRC64NVME crc64 = new software.amazon.awssdk.crt.checksums.CRC64NVME(); + for (int i = 0; i < 32; i++) { + crc64.update(zeroes, i, 1); + } + long expected = 0xCF3473434D4ECF3BL; + assertEquals(expected, crc64.getValue()); + } + + @Test + public void testCrc64NVMEValues() { + byte[] values = new byte[32]; + for (byte i = 0; i < 32; i++) { + values[i] = i; + } + software.amazon.awssdk.crt.checksums.CRC64NVME crc64 = new software.amazon.awssdk.crt.checksums.CRC64NVME(); + crc64.update(values); + long expected = 0xB9D9D4A8492CBD7FL; + assertEquals(expected, crc64.getValue()); + } + + @Test + public void testCrc64NVMEValuesIterated() { + byte[] values = new byte[32]; + for (byte i = 0; i < 32; i++) { + values[i] = i; + } + software.amazon.awssdk.crt.checksums.CRC64NVME crc64 = new software.amazon.awssdk.crt.checksums.CRC64NVME(); + for (int i = 0; i < 32; i++) { + crc64.update(values, i, 1); + } + long expected = 0xB9D9D4A8492CBD7FL; + assertEquals(expected, crc64.getValue()); + } }