Skip to content

Commit

Permalink
Bind out CRC64 (#831)
Browse files Browse the repository at this point in the history
  • Loading branch information
DmitriyMusatkin authored Sep 25, 2024
1 parent 7c48ca0 commit a5aa301
Show file tree
Hide file tree
Showing 5 changed files with 184 additions and 26 deletions.
30 changes: 15 additions & 15 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ on:
- 'docs'

env:
BUILDER_VERSION: v0.9.62
BUILDER_VERSION: v0.9.67
BUILDER_SOURCE: releases
BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
PACKAGE_NAME: aws-crt-java
Expand All @@ -34,7 +34,7 @@ jobs:
#- manylinux2014-x86 until we find 32-bit linux binaries we can use
steps:
- name: Checkout Sources
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: true
- name: Build ${{ env.PACKAGE_NAME }}
Expand All @@ -60,7 +60,7 @@ jobs:
- gcc-8
steps:
- name: Checkout Sources
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: true
- name: Build ${{ env.PACKAGE_NAME }}
Expand Down Expand Up @@ -167,7 +167,7 @@ jobs:
runs-on: windows-2022 # latest
steps:
- name: Checkout Sources
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: true
- name: Build ${{ env.PACKAGE_NAME }} + consumers
Expand All @@ -182,7 +182,7 @@ jobs:
arch: [x86, x64]
steps:
- name: Checkout Sources
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: true
- name: Build ${{ env.PACKAGE_NAME }} + consumers
Expand All @@ -196,7 +196,7 @@ jobs:
runs-on: macos-14 #latest
steps:
- name: Checkout Sources
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: true
- name: Build ${{ env.PACKAGE_NAME }} + consumers
Expand All @@ -210,7 +210,7 @@ jobs:
runs-on: macos-14-large #latest
steps:
- name: Checkout Sources
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: true
- name: Build ${{ env.PACKAGE_NAME }} + consumers
Expand All @@ -227,12 +227,12 @@ jobs:
runs-on: ubuntu-20.04 # latest
steps:
- name: Checkout Sources
uses: actions/checkout@v2
uses: actions/checkout@v4
with:
submodules: true
# Setup JDK 11
- name: set up JDK 11
uses: actions/setup-java@v3
uses: actions/setup-java@v4
with:
java-version: '11'
distribution: 'temurin'
Expand Down Expand Up @@ -276,7 +276,7 @@ jobs:
check-docs:
runs-on: ubuntu-22.04 # use same version as docs.yml
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
submodules: true
- name: Check docs
Expand All @@ -287,7 +287,7 @@ jobs:
runs-on: ubuntu-22.04 # latest
steps:
- name: Checkout Source
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: true
fetch-depth: 0
Expand All @@ -301,7 +301,7 @@ jobs:
runs-on: ubuntu-22.04 # latest
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: true
- name: Configure local host
Expand All @@ -319,7 +319,7 @@ jobs:
runs-on: macos-14 # latest
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: true
- name: Configure local host
Expand All @@ -340,7 +340,7 @@ jobs:
runs-on: windows-2022 # latest
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: true
- name: Configure local host
Expand All @@ -364,7 +364,7 @@ jobs:
java-version: ['22', '21', '17']
steps:
- name: Checkout Sources
uses: actions/checkout@v2
uses: actions/checkout@v4
with:
submodules: true
- name: Setup GraalVM
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
contents: write # allow push
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: true
- name: Update docs branch
Expand Down
98 changes: 98 additions & 0 deletions src/main/java/software/amazon/awssdk/crt/checksums/CRC64NVME.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
/**
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0.
*/
package software.amazon.awssdk.crt.checksums;

import software.amazon.awssdk.crt.CRT;
import java.util.zip.Checksum;

/**
* CRT implementation of the Java Checksum interface for making CRC64NVME checksum calculations
*/
public class CRC64NVME implements Checksum, Cloneable {
static {
new CRT();
};

private long value = 0;

/**
* Default constructor
*/
public CRC64NVME() {
}

private CRC64NVME(long value) {
this.value = value;
}

@Override
public Object clone() {
return new CRC64NVME(value);
}

/**
* Returns the current checksum value.
*
* @return the current checksum value.
*/
@Override
public long getValue() {
return value;
}

/**
* Resets the checksum to its initial value.
*/
@Override
public void reset() {
value = 0;
}

/**
* Updates the current checksum with the specified array of bytes.
*
* @param b the byte array to update the checksum with
* @param off the starting offset within b of the data to use
* @param len the number of bytes to use in the update
*/
@Override
public void update(byte[] b, int off, int len) {
if (b == null) {
throw new NullPointerException();
}
if (off < 0 || len < 0 || off > b.length - len) {
throw new ArrayIndexOutOfBoundsException();
}
value = crc64nvme(b, value, off, len);
}

/**
* Updates the current checksum with the specified array of bytes.
*
* @param b the byte array to update the checksum with
*/
public void update(byte[] b) {
update(b, 0, b.length);
}

/**
* Updates the current checksum with the specified byte.
*
* @param b the byte to update the checksum with
*/
@Override
public void update(int b) {
if (b < 0 || b > 0xff) {
throw new IllegalArgumentException();
}
byte[] buf = { (byte) (b & 0x000000ff) };
this.update(buf);
}

/*******************************************************************************
* native methods
******************************************************************************/
private static native long crc64nvme(byte[] input, long previous, int offset, int length);;
}
34 changes: 24 additions & 10 deletions src/native/checksums.c
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,18 @@
#include "crt.h"
#include "java_class_ids.h"

jint crc_common(
jint crc32_common(
JNIEnv *env,
jbyteArray input,
jint previous,
const size_t start,
size_t length,
uint32_t (*checksum_fn)(const uint8_t *, int, uint32_t)) {
uint32_t (*checksum_fn)(const uint8_t *, size_t, uint32_t)) {
struct aws_byte_cursor c_byte_array = aws_jni_byte_cursor_from_jbyteArray_acquire(env, input);
struct aws_byte_cursor cursor = c_byte_array;
aws_byte_cursor_advance(&cursor, start);
cursor.len = aws_min_size(length, cursor.len);
uint32_t res = (uint32_t)previous;
while (cursor.len > INT_MAX) {
res = checksum_fn(cursor.ptr, INT_MAX, res);
aws_byte_cursor_advance(&cursor, INT_MAX);
}
jint res_signed = (jint)checksum_fn(cursor.ptr, (int)cursor.len, res);
jint res_signed = (jint)checksum_fn(cursor.ptr, cursor.len, previous);
aws_jni_byte_cursor_from_jbyteArray_release(env, input, c_byte_array);
return res_signed;
}
Expand All @@ -40,7 +35,7 @@ JNIEXPORT jint JNICALL Java_software_amazon_awssdk_crt_checksums_CRC32_crc32(
(void)jni_class;
aws_cache_jni_ids(env);

return crc_common(env, input, previous, offset, length, aws_checksums_crc32);
return crc32_common(env, input, previous, offset, length, aws_checksums_crc32_ex);
}

JNIEXPORT jint JNICALL Java_software_amazon_awssdk_crt_checksums_CRC32C_crc32c(
Expand All @@ -53,5 +48,24 @@ JNIEXPORT jint JNICALL Java_software_amazon_awssdk_crt_checksums_CRC32C_crc32c(
(void)jni_class;
aws_cache_jni_ids(env);

return crc_common(env, input, previous, offset, length, aws_checksums_crc32c);
return crc32_common(env, input, previous, offset, length, aws_checksums_crc32c_ex);
}

JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_checksums_CRC64NVME_crc64nvme(
JNIEnv *env,
jclass jni_class,
jbyteArray input,
jlong previous,
jint offset,
jint length) {
(void)jni_class;
aws_cache_jni_ids(env);

struct aws_byte_cursor c_byte_array = aws_jni_byte_cursor_from_jbyteArray_acquire(env, input);
struct aws_byte_cursor cursor = c_byte_array;
aws_byte_cursor_advance(&cursor, offset);
cursor.len = aws_min_size(length, cursor.len);
jlong res_signed = (jlong)aws_checksums_crc64nvme_ex(cursor.ptr, cursor.len, previous);
aws_jni_byte_cursor_from_jbyteArray_release(env, input, c_byte_array);
return res_signed;
}
46 changes: 46 additions & 0 deletions src/test/java/software/amazon/awssdk/crt/test/CrcTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -167,4 +167,50 @@ public void testCrc32CLargeBuffer() {
int expected = 0xfb5b991d;
assertEquals(expected, (int) crcc.getValue());
}

@Test
public void testCrc64NVMEZeroes() {
byte[] zeroes = new byte[32];
software.amazon.awssdk.crt.checksums.CRC64NVME crc64 = new software.amazon.awssdk.crt.checksums.CRC64NVME();
crc64.update(zeroes);
long expected = 0xCF3473434D4ECF3BL;
assertEquals(expected, crc64.getValue());
}

@Test
public void testCrc64NVMEZeroesIterated() {
byte[] zeroes = new byte[32];
software.amazon.awssdk.crt.checksums.CRC64NVME crc64 = new software.amazon.awssdk.crt.checksums.CRC64NVME();
for (int i = 0; i < 32; i++) {
crc64.update(zeroes, i, 1);
}
long expected = 0xCF3473434D4ECF3BL;
assertEquals(expected, crc64.getValue());
}

@Test
public void testCrc64NVMEValues() {
byte[] values = new byte[32];
for (byte i = 0; i < 32; i++) {
values[i] = i;
}
software.amazon.awssdk.crt.checksums.CRC64NVME crc64 = new software.amazon.awssdk.crt.checksums.CRC64NVME();
crc64.update(values);
long expected = 0xB9D9D4A8492CBD7FL;
assertEquals(expected, crc64.getValue());
}

@Test
public void testCrc64NVMEValuesIterated() {
byte[] values = new byte[32];
for (byte i = 0; i < 32; i++) {
values[i] = i;
}
software.amazon.awssdk.crt.checksums.CRC64NVME crc64 = new software.amazon.awssdk.crt.checksums.CRC64NVME();
for (int i = 0; i < 32; i++) {
crc64.update(values, i, 1);
}
long expected = 0xB9D9D4A8492CBD7FL;
assertEquals(expected, crc64.getValue());
}
}

0 comments on commit a5aa301

Please sign in to comment.