Merge pull request #3371 from aws/feature/master/flexiblechecksums-v2-phase1

Releasing CRC64 and checksum refactoring change

Author: zoewangg

.changes/next-release/deprecation-AWSSDKforJavav2-eb641b5.json

@@ -0,0 +1,6 @@
+{
+    "type": "deprecation",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "Deprecate internal checksum algorithm classes."
+}

.changes/next-release/feature-AWSSDKforJavav2-3676574.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "The SDK now defaults to Java built-in CRC32 and CRC32C(if it's Java 9+) implementations, resulting in improved performance."
+}

build-tools/src/main/resources/software/amazon/awssdk/spotbugs-suppressions.xml

@@ -303,8 +303,8 @@
     <!-- Retrieves and updates crc value in update() -->
      <Match>
          <Or>
-             <Class name="software.amazon.awssdk.http.auth.aws.internal.signer.checksums.SdkCrc32CChecksum"/>
-             <Class name="software.amazon.awssdk.http.auth.aws.internal.signer.checksums.SdkCrc32Checksum"/>
+             <Class name="software.amazon.awssdk.checksums.internal.SdkCrc32CChecksum"/>
+             <Class name="software.amazon.awssdk.checksums.internal.SdkCrc32Checksum"/>
              <Class name="software.amazon.awssdk.core.internal.checksums.factory.SdkCrc32C"/>
              <Class name="software.amazon.awssdk.core.internal.checksums.factory.SdkCrc32"/>
          </Or>

core/checksums/pom.xml

@@ -45,6 +45,19 @@
             <version>${awsjavasdk.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>utils</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>software.amazon.awssdk.crt</groupId>
+            <artifactId>aws-crt</artifactId>
+            <version>${awscrt.version}</version>
+            <optional>true</optional>
+        </dependency>
+
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>

core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java

@@ -29,6 +29,7 @@ public final class DefaultChecksumAlgorithm {
     public static final ChecksumAlgorithm MD5 = of("MD5");
     public static final ChecksumAlgorithm SHA256 = of("SHA256");
     public static final ChecksumAlgorithm SHA1 = of("SHA1");
+    public static final ChecksumAlgorithm CRC64NVME = of("CRC64NVME");
 
     private DefaultChecksumAlgorithm() {
     }

core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/SdkChecksum.java renamed to core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java

@@ -13,18 +13,48 @@
  * permissions and limitations under the License.
  */
 
-package software.amazon.awssdk.http.auth.aws.internal.signer.checksums;
+package software.amazon.awssdk.checksums;
 
 import java.nio.ByteBuffer;
 import java.util.zip.Checksum;
-import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.annotations.SdkProtectedApi;
+import software.amazon.awssdk.checksums.internal.Crc32Checksum;
+import software.amazon.awssdk.checksums.internal.Crc32cProvider;
+import software.amazon.awssdk.checksums.internal.Crc64NvmeChecksum;
+import software.amazon.awssdk.checksums.internal.Md5Checksum;
+import software.amazon.awssdk.checksums.internal.Sha1Checksum;
+import software.amazon.awssdk.checksums.internal.Sha256Checksum;
+import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
 
 /**
  * Extension of {@link Checksum} to support checksums and checksum validations used by the SDK that are not provided by the JDK.
  */
-@SdkInternalApi
+@SdkProtectedApi
 public interface SdkChecksum extends Checksum {
 
+    /**
+     * Returns an {@link SdkChecksum} based on the {@link ChecksumAlgorithm} provided.
+     * UnsupportedOperationException will be thrown for unsupported algorithm.
+     */
+    static SdkChecksum forAlgorithm(ChecksumAlgorithm algorithm) {
+        switch (algorithm.algorithmId()) {
+            case "CRC32C":
+                return Crc32cProvider.create();
+            case "CRC32":
+                return new Crc32Checksum();
+            case "SHA1":
+                return new Sha1Checksum();
+            case "SHA256":
+                return new Sha256Checksum();
+            case "MD5":
+                return new Md5Checksum();
+            case "CRC64NVME":
+                return new Crc64NvmeChecksum();
+            default:
+                throw new UnsupportedOperationException("Unsupported checksum algorithm: " + algorithm);
+        }
+    }
+
     /**
      * Returns the computed checksum in a byte array rather than the long provided by {@link #getValue()}.
      *
@@ -49,7 +79,6 @@ default void update(byte[] b) {
         update(b, 0, b.length);
     }
 
-
     /**
      * Updates the current checksum with the bytes from the specified buffer.
      * <p>

core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/BaseCrcChecksum.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.checksums.internal;
+
+import java.util.zip.Checksum;
+import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.checksums.SdkChecksum;
+
+/**
+ * Base class for CRC related checksums
+ */
+@SdkInternalApi
+public abstract class BaseCrcChecksum implements SdkChecksum {
+
+    private Checksum checksum;
+    private Checksum lastMarkedChecksum;
+
+    public BaseCrcChecksum(Checksum checksum) {
+        this.checksum = checksum;
+    }
+
+    public Checksum getChecksum() {
+        return checksum;
+    }
+
+    @Override
+    public void mark(int readLimit) {
+        this.lastMarkedChecksum = cloneChecksum(checksum);
+    }
+
+    @Override
+    public void update(int b) {
+        checksum.update(b);
+    }
+
+    @Override
+    public void update(byte[] b, int off, int len) {
+        checksum.update(b, off, len);
+    }
+
+    @Override
+    public long getValue() {
+        return checksum.getValue();
+    }
+
+    @Override
+    public void reset() {
+        if (lastMarkedChecksum == null) {
+            checksum.reset();
+        } else {
+            checksum = cloneChecksum(lastMarkedChecksum);
+        }
+    }
+
+    abstract Checksum cloneChecksum(Checksum checksum);
+}

core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/ConstructorCache.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.checksums.internal;
+
+import java.lang.ref.WeakReference;
+import java.lang.reflect.Constructor;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Optional;
+import java.util.WeakHashMap;
+import java.util.concurrent.ConcurrentHashMap;
+import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.utils.ClassLoaderHelper;
+import software.amazon.awssdk.utils.Logger;
+
+/**
+ * A cache that stores classes and their constructors by class name and class loader.
+ * <p>
+ * This cache uses weak references to both class loaders and classes, allowing them to be garbage collected
+ * when no longer needed. It provides methods to retrieve the zero-argument constructor for a class,
+ * based on the current thread's context class loader or the system class loader.
+ * <p>
+ * If a class or its zero-argument constructor cannot be found, an empty result is returned.
+ */
+@SdkInternalApi
+public final class ConstructorCache {
+    private static final Logger log = Logger.loggerFor(ConstructorCache.class);
+
+    /**
+     * Cache storing classes by class name and class loader.
+     * Uses weak references to allow garbage collection when not needed.
+     */
+    private final Map<String, Map<ClassLoader, Optional<WeakReference<Class<?>>>>> classesByClassName =
+        new ConcurrentHashMap<>();
+
+    /**
+     * Retrieve the class for the given class name from the context or system class loader.
+     * Returns an empty result if the class is not found.
+     */
+    private Optional<Class<?>> getClass(String className) {
+        Map<ClassLoader, Optional<WeakReference<Class<?>>>> classesByClassLoader =
+            classesByClassName.computeIfAbsent(className, k -> Collections.synchronizedMap(new WeakHashMap<>()));
+
+        ClassLoader classLoader = ClassLoaderHelper.contextClassLoader();
+        Optional<WeakReference<Class<?>>> classRef = classesByClassLoader.computeIfAbsent(classLoader, k -> {
+            try {
+                Class<?> clazz = classLoader.loadClass(className);
+                return Optional.of(new WeakReference<>(clazz));
+            } catch (ClassNotFoundException e) {
+                return Optional.empty();
+            }
+        });
+        return classRef.map(WeakReference::get);
+    }
+
+    /**
+     * Retrieve the zero-argument constructor for the given class name.
+     * Returns an empty result if no such constructor is found.
+     */
+    public Optional<Constructor<?>> getConstructor(String className) {
+        return getClass(className).flatMap(clazz -> {
+            try {
+                return Optional.of(clazz.getConstructor());
+            } catch (NoSuchMethodException e) {
+                log.debug(() -> "Classloader contains " + className + ", but without a zero-arg constructor.", e);
+                return Optional.empty();
+            }
+        });
+    }
+}

core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32Checksum.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.checksums.internal;
+
+
+import java.util.zip.CRC32;
+import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.checksums.SdkChecksum;
+
+@SdkInternalApi
+public final class Crc32Checksum implements SdkChecksum {
+    private final CrcCombineOnMarkChecksum crc32;
+
+    public Crc32Checksum() {
+        // Delegates to CrcCombineOnMarkChecksum with CRC32
+        this.crc32 = new CrcCombineOnMarkChecksum(
+            new CRC32(),
+            SdkCrc32Checksum::combine
+        );
+    }
+
+    @Override
+    public byte[] getChecksumBytes() {
+        return crc32.getChecksumBytes();
+    }
+
+    @Override
+    public void mark(int readLimit) {
+        crc32.mark(readLimit);
+    }
+
+    @Override
+    public void update(int b) {
+        crc32.update(b);
+    }
+
+    @Override
+    public void update(byte[] b, int off, int len) {
+        crc32.update(b, off, len);
+    }
+
+    @Override
+    public long getValue() {
+        return crc32.getValue();
+    }
+
+    @Override
+    public void reset() {
+        crc32.reset();
+    }
+}