Skip to content

Commit 8f2c7ba

Browse files
zoewanggzoewangg
committed
Add CRC64 checksum via CRT
1 parent f614efe commit 8f2c7ba

File tree

12 files changed

+196
-103
lines changed

12 files changed

+196
-103
lines changed

‎core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ public final class DefaultChecksumAlgorithm {
2929
public static final ChecksumAlgorithm MD5 = of("MD5");
3030
public static final ChecksumAlgorithm SHA256 = of("SHA256");
3131
public static final ChecksumAlgorithm SHA1 = of("SHA1");
32+
public static final ChecksumAlgorithm CRC64NVME = of("CRC64NVME");
3233

3334
private DefaultChecksumAlgorithm() {
3435
}

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/BaseCrcChecksum.java

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.http.auth.aws.internal.signer.checksums;
17+
18+
import java.util.zip.Checksum;
19+
import software.amazon.awssdk.annotations.SdkInternalApi;
20+
21+
/**
22+
* Base class for CRC related checksums
23+
*/
24+
@SdkInternalApi
25+
public abstract class BaseCrcChecksum implements SdkChecksum {
26+
27+
private Checksum checksum;
28+
private Checksum lastMarkedChecksum;
29+
30+
public BaseCrcChecksum(Checksum checksum) {
31+
this.checksum = checksum;
32+
}
33+
34+
public Checksum getChecksum() {
35+
return checksum;
36+
}
37+
38+
@Override
39+
public void mark(int readLimit) {
40+
this.lastMarkedChecksum = cloneChecksum(checksum);
41+
}
42+
43+
@Override
44+
public void update(int b) {
45+
checksum.update(b);
46+
}
47+
48+
@Override
49+
public void update(byte[] b, int off, int len) {
50+
checksum.update(b, off, len);
51+
}
52+
53+
@Override
54+
public long getValue() {
55+
return checksum.getValue();
56+
}
57+
58+
@Override
59+
public void reset() {
60+
if (lastMarkedChecksum == null) {
61+
checksum.reset();
62+
} else {
63+
checksum = cloneChecksum(lastMarkedChecksum);
64+
}
65+
}
66+
67+
abstract Checksum cloneChecksum(Checksum checksum);
68+
}

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Crc32CChecksum.java

Lines changed: 12 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@
1515

1616
package software.amazon.awssdk.http.auth.aws.internal.signer.checksums;
1717

18-
import java.nio.ByteBuffer;
18+
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.ChecksumUtil.longToByte;
19+
1920
import java.util.Arrays;
2021
import java.util.zip.Checksum;
2122
import software.amazon.awssdk.annotations.SdkInternalApi;
@@ -26,23 +27,24 @@
2627
* Implementation of {@link SdkChecksum} to calculate an CRC32C checksum.
2728
*/
2829
@SdkInternalApi
29-
public class Crc32CChecksum implements SdkChecksum {
30+
public class Crc32CChecksum extends BaseCrcChecksum {
3031

3132
private static final String CRT_CLASSPATH_FOR_CRC32C = "software.amazon.awssdk.crt.checksums.CRC32C";
3233
private static final ThreadLocal<Boolean> IS_CRT_AVAILABLE = ThreadLocal.withInitial(Crc32CChecksum::isCrtAvailable);
3334

34-
private Checksum crc32c;
35-
private Checksum lastMarkedCrc32C;
36-
3735
/**
3836
* Creates CRT Based Crc32C checksum if Crt classpath for Crc32c is loaded, else create Sdk Implemented Crc32c
3937
*/
4038
public Crc32CChecksum() {
39+
super(createChecksum());
40+
}
41+
42+
private static Checksum createChecksum() {
4143
if (IS_CRT_AVAILABLE.get()) {
42-
crc32c = new CRC32C();
43-
} else {
44-
crc32c = SdkCrc32CChecksum.create();
44+
return new CRC32C();
4545
}
46+
// TODO: use Java implementation if it's Java 9+
47+
return SdkCrc32CChecksum.create();
4648
}
4749

4850
private static boolean isCrtAvailable() {
@@ -55,47 +57,13 @@ private static boolean isCrtAvailable() {
5557
return true;
5658
}
5759

58-
private static byte[] longToByte(Long input) {
59-
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
60-
buffer.putLong(input);
61-
return buffer.array();
62-
}
63-
6460
@Override
6561
public byte[] getChecksumBytes() {
66-
return Arrays.copyOfRange(longToByte(crc32c.getValue()), 4, 8);
62+
return Arrays.copyOfRange(longToByte(getChecksum().getValue()), 4, 8);
6763
}
6864

6965
@Override
70-
public void mark(int readLimit) {
71-
this.lastMarkedCrc32C = cloneChecksum(crc32c);
72-
}
73-
74-
@Override
75-
public void update(int b) {
76-
crc32c.update(b);
77-
}
78-
79-
@Override
80-
public void update(byte[] b, int off, int len) {
81-
crc32c.update(b, off, len);
82-
}
83-
84-
@Override
85-
public long getValue() {
86-
return crc32c.getValue();
87-
}
88-
89-
@Override
90-
public void reset() {
91-
if (lastMarkedCrc32C == null) {
92-
crc32c.reset();
93-
} else {
94-
crc32c = cloneChecksum(lastMarkedCrc32C);
95-
}
96-
}
97-
98-
private Checksum cloneChecksum(Checksum checksum) {
66+
public Checksum cloneChecksum(Checksum checksum) {
9967
if (checksum instanceof CRC32C) {
10068
return (Checksum) ((CRC32C) checksum).clone();
10169
}

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Crc32Checksum.java

Lines changed: 13 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@
1515

1616
package software.amazon.awssdk.http.auth.aws.internal.signer.checksums;
1717

18-
import java.nio.ByteBuffer;
18+
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.ChecksumUtil.longToByte;
19+
1920
import java.util.Arrays;
2021
import java.util.zip.Checksum;
2122
import software.amazon.awssdk.annotations.SdkInternalApi;
@@ -26,23 +27,24 @@
2627
* Implementation of {@link SdkChecksum} to calculate an CRC32 checksum.
2728
*/
2829
@SdkInternalApi
29-
public class Crc32Checksum implements SdkChecksum {
30+
public class Crc32Checksum extends BaseCrcChecksum {
3031
private static final String CRT_CLASSPATH_FOR_CRC32 = "software.amazon.awssdk.crt.checksums.CRC32";
3132
private static final ThreadLocal<Boolean> IS_CRT_AVAILABLE = ThreadLocal.withInitial(Crc32Checksum::isCrtAvailable);
3233

33-
private Checksum crc32;
34-
private Checksum lastMarkedCrc32;
35-
36-
3734
/**
3835
* Creates CRT Based Crc32 checksum if Crt classpath for Crc32 is loaded, else create Sdk Implemented Crc32.
3936
*/
4037
public Crc32Checksum() {
38+
super(createChecksum());
39+
}
40+
41+
private static Checksum createChecksum() {
4142
if (IS_CRT_AVAILABLE.get()) {
42-
crc32 = new CRC32();
43-
} else {
44-
crc32 = SdkCrc32Checksum.create();
43+
return new CRC32();
4544
}
45+
46+
// TODO: use Java implementation
47+
return SdkCrc32Checksum.create();
4648
}
4749

4850
private static boolean isCrtAvailable() {
@@ -55,47 +57,13 @@ private static boolean isCrtAvailable() {
5557
return true;
5658
}
5759

58-
private static byte[] longToByte(Long input) {
59-
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
60-
buffer.putLong(input);
61-
return buffer.array();
62-
}
63-
6460
@Override
6561
public byte[] getChecksumBytes() {
66-
return Arrays.copyOfRange(longToByte(crc32.getValue()), 4, 8);
67-
}
68-
69-
@Override
70-
public void mark(int readLimit) {
71-
this.lastMarkedCrc32 = cloneChecksum(crc32);
62+
return Arrays.copyOfRange(longToByte(getChecksum().getValue()), 4, 8);
7263
}
7364

7465
@Override
75-
public void update(int b) {
76-
crc32.update(b);
77-
}
78-
79-
@Override
80-
public void update(byte[] b, int off, int len) {
81-
crc32.update(b, off, len);
82-
}
83-
84-
@Override
85-
public long getValue() {
86-
return crc32.getValue();
87-
}
88-
89-
@Override
90-
public void reset() {
91-
if (lastMarkedCrc32 == null) {
92-
crc32.reset();
93-
} else {
94-
crc32 = cloneChecksum(lastMarkedCrc32);
95-
}
96-
}
97-
98-
private Checksum cloneChecksum(Checksum checksum) {
66+
public Checksum cloneChecksum(Checksum checksum) {
9967
if (checksum instanceof CRC32) {
10068
return (Checksum) ((CRC32) checksum).clone();
10169
}

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Crc64NvmeChecksum.java

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.http.auth.aws.internal.signer.checksums;
17+
18+
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.ChecksumUtil.longToByte;
19+
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.OptionalDependencyLoaderUtil.getCrc64Nvme;
20+
21+
import java.util.zip.Checksum;
22+
import software.amazon.awssdk.annotations.SdkInternalApi;
23+
import software.amazon.awssdk.crt.checksums.CRC64NVME;
24+
25+
/**
26+
* Implementation of {@link SdkChecksum} to calculate an CRC64NVME checksum.
27+
*/
28+
@SdkInternalApi
29+
public final class Crc64NvmeChecksum extends BaseCrcChecksum {
30+
31+
public Crc64NvmeChecksum() {
32+
super(getCrc64Nvme());
33+
}
34+
35+
@Override
36+
public Checksum cloneChecksum(Checksum checksum) {
37+
if (checksum instanceof CRC64NVME) {
38+
return (Checksum) ((CRC64NVME) checksum).clone();
39+
}
40+
41+
throw new IllegalStateException("Unsupported checksum");
42+
}
43+
44+
@Override
45+
public byte[] getChecksumBytes() {
46+
return longToByte(getChecksum().getValue());
47+
}
48+
}

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtil.java

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,13 @@
1717

1818
import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32;
1919
import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32C;
20+
import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC64NVME;
2021
import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.MD5;
2122
import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA1;
2223
import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA256;
2324

2425
import java.io.InputStream;
26+
import java.nio.ByteBuffer;
2527
import java.util.Locale;
2628
import java.util.Map;
2729
import java.util.function.Supplier;
@@ -30,6 +32,7 @@
3032
import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.ConstantChecksum;
3133
import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Crc32CChecksum;
3234
import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Crc32Checksum;
35+
import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Crc64NvmeChecksum;
3336
import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Md5Checksum;
3437
import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.SdkChecksum;
3538
import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Sha1Checksum;
@@ -42,13 +45,14 @@ public final class ChecksumUtil {
4245

4346
private static final String CONSTANT_CHECKSUM = "CONSTANT";
4447

45-
private static final Map<String, Supplier<SdkChecksum>> CHECKSUM_MAP = ImmutableMap.of(
46-
SHA256.algorithmId(), Sha256Checksum::new,
47-
SHA1.algorithmId(), Sha1Checksum::new,
48-
CRC32.algorithmId(), Crc32Checksum::new,
49-
CRC32C.algorithmId(), Crc32CChecksum::new,
50-
MD5.algorithmId(), Md5Checksum::new
51-
);
48+
private static final Map<String, Supplier<SdkChecksum>> CHECKSUM_MAP =
49+
ImmutableMap.<String, Supplier<SdkChecksum>>builder()
50+
.put(SHA256.algorithmId(), Sha256Checksum::new)
51+
.put(SHA1.algorithmId(), Sha1Checksum::new)
52+
.put(CRC32.algorithmId(), Crc32Checksum::new)
53+
.put(CRC32C.algorithmId(), Crc32CChecksum::new)
54+
.put(MD5.algorithmId(), Md5Checksum::new)
55+
.put(CRC64NVME.algorithmId(), Crc64NvmeChecksum::new).build();
5256

5357
private ChecksumUtil() {
5458
}
@@ -93,6 +97,12 @@ public static void readAll(InputStream inputStream) {
9397
});
9498
}
9599

100+
public static byte[] longToByte(Long input) {
101+
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
102+
buffer.putLong(input);
103+
return buffer.array();
104+
}
105+
96106
/**
97107
* An implementation of a {@link ChecksumAlgorithm} that will map to {@link ConstantChecksum}, which provides a constant
98108
* checksum. This isn't super useful, but is needed in cases such as signing, where the content-hash (a

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/OptionalDependencyLoaderUtil.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import java.time.Clock;
1919
import software.amazon.awssdk.annotations.SdkInternalApi;
20+
import software.amazon.awssdk.crt.checksums.CRC64NVME;
2021
import software.amazon.awssdk.http.auth.aws.crt.internal.signer.DefaultAwsCrtV4aHttpSigner;
2122
import software.amazon.awssdk.http.auth.aws.eventstream.internal.signer.EventStreamV4PayloadSigner;
2223
import software.amazon.awssdk.http.auth.aws.internal.signer.CredentialScope;
@@ -38,6 +39,10 @@ public final class OptionalDependencyLoaderUtil {
3839
private static final String HTTP_AUTH_AWS_EVENT_STREAM_PATH =
3940
"software.amazon.awssdk.http.auth.aws.eventstream.HttpAuthAwsEventStream";
4041
private static final String HTTP_AUTH_AWS_EVENT_STREAM_MODULE = "software.amazon.awssdk:http-auth-aws-eventstream";
42+
private static final String CRT_CRC64NVME_PATH =
43+
"software.amazon.awssdk.crt.checksums.CRC64NVME";
44+
private static final String CRT_MODULE =
45+
"software.amazon.awssdk.crt:aws-crt";
4146

4247
private OptionalDependencyLoaderUtil() {
4348
}
@@ -65,6 +70,11 @@ public static DefaultAwsCrtV4aHttpSigner getDefaultAwsCrtV4aHttpSigner() {
6570
return new DefaultAwsCrtV4aHttpSigner();
6671
}
6772

73+
public static CRC64NVME getCrc64Nvme() {
74+
requireClass(CRT_CRC64NVME_PATH, CRT_MODULE, "CRC64NVME");
75+
return new CRC64NVME();
76+
}
77+
6878
public static EventStreamV4PayloadSigner getEventStreamV4PayloadSigner(
6979
AwsCredentialsIdentity credentials,
7080
CredentialScope credentialScope,

0 commit comments

Comments
 (0)