Use system property proxy settings for netty and crt http client

Author: Erin Yang

http-clients/aws-crt-client/src/main/java/software/amazon/awssdk/http/crt/ProxyConfiguration.java

@@ -18,6 +18,7 @@
 import java.util.Objects;
 import software.amazon.awssdk.annotations.SdkPreviewApi;
 import software.amazon.awssdk.annotations.SdkPublicApi;
+import software.amazon.awssdk.utils.ProxySystemSetting;
 import software.amazon.awssdk.utils.builder.CopyableBuilder;
 import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
 
@@ -39,11 +40,13 @@ public final class ProxyConfiguration implements ToCopyableBuilder<ProxyConfigur
 
     private final String username;
     private final String password;
+    private final Boolean useSystemPropertyValues;
 
     private ProxyConfiguration(BuilderImpl builder) {
+        this.useSystemPropertyValues = builder.useSystemPropertyValues;
         this.scheme = builder.scheme;
-        this.host = builder.host;
-        this.port = builder.port;
+        this.host = resolveHost(builder.host);
+        this.port = resolvePort(builder.port);
         this.username = builder.username;
         this.password = builder.password;
     }
@@ -56,31 +59,35 @@ public String scheme() {
     }
 
     /**
-     * @return The proxy host.
+     * @return The proxy host from the configuration if set, or from the "http.proxyHost" system property if
+     * {@link Builder#useSystemPropertyValues(Boolean)} is set to true
      */
     public String host() {
         return host;
     }
 
     /**
-     * @return The proxy port.
+     * @return The proxy port from the configuration if set, or from the "http.proxyPort" system property if
+     * {@link Builder#useSystemPropertyValues(Boolean)} is set to true
      */
     public int port() {
         return port;
     }
 
     /**
-     * @return Basic authentication username
-     */
+     * @return The proxy username from the configuration if set, or from the "http.proxyUser" system property if
+     * {@link Builder#useSystemPropertyValues(Boolean)} is set to true
+     * */
     public String username() {
-        return username;
+        return resolveValue(username, ProxySystemSetting.PROXY_USERNAME);
     }
 
     /**
-     * @return Basic authentication password
-     */
+     * @return The proxy password from the configuration if set, or from the "http.proxyPassword" system property if
+     * {@link Builder#useSystemPropertyValues(Boolean)} is set to true
+     * */
     public String password() {
-        return password;
+        return resolveValue(password, ProxySystemSetting.PROXY_PASSWORD);
     }
 
     @Override
@@ -182,6 +189,34 @@ public interface Builder extends CopyableBuilder<Builder, ProxyConfiguration> {
          * @return This object for method chaining.
          */
         Builder password(String password);
+
+        /**
+         * The option whether to use system property values from {@link ProxySystemSetting} if any of the config options
+         * are missing. The value is set to "true" by default which means SDK will automatically use system property values if
+         * options are not provided during building the {@link ProxyConfiguration} object. To disable this behaviour, set this
+         * value to false.
+         *
+         * @param useSystemPropertyValues The option whether to use system proerpty values
+         * @return This object for method chaining.
+         */
+        Builder useSystemPropertyValues(Boolean useSystemPropertyValues);
+    }
+
+    private String resolveHost(String host) {
+        return resolveValue(host, ProxySystemSetting.PROXY_HOST);
+    }
+
+    private int resolvePort(int port) {
+        return useSystemPropertyValues ? ProxySystemSetting.PROXY_PORT.getStringValue().map(Integer::parseInt).orElse(0)
+                                       : port;
+    }
+
+    /**
+     * Uses the configuration options, system setting property and returns the final value of the given member.
+     */
+    private String resolveValue(String value, ProxySystemSetting systemSetting) {
+        return value == null && useSystemPropertyValues ? systemSetting.getStringValue().orElse(null)
+                                                        : value;
     }
 
     private static final class BuilderImpl implements Builder {
@@ -190,11 +225,13 @@ private static final class BuilderImpl implements Builder {
         private int port;
         private String username;
         private String password;
+        private Boolean useSystemPropertyValues = Boolean.TRUE;
 
         private BuilderImpl() {
         }
 
         private BuilderImpl(ProxyConfiguration proxyConfiguration) {
+            this.useSystemPropertyValues = proxyConfiguration.useSystemPropertyValues;
             this.scheme = proxyConfiguration.scheme;
             this.host = proxyConfiguration.host;
             this.port = proxyConfiguration.port;
@@ -232,6 +269,16 @@ public Builder password(String password) {
             return this;
         }
 
+        @Override
+        public Builder useSystemPropertyValues(Boolean useSystemPropertyValues) {
+            this.useSystemPropertyValues = useSystemPropertyValues;
+            return this;
+        }
+
+        public void setUseSystemPropertyValues(Boolean useSystemPropertyValues) {
+            useSystemPropertyValues(useSystemPropertyValues);
+        }
+
         @Override
         public ProxyConfiguration build() {
             return new ProxyConfiguration(this);

http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/ProxyConfigurationTest.java

@@ -27,12 +27,45 @@
  */
 public class ProxyConfigurationTest {
     private static final Random RNG = new Random();
+    private static final String TEST_HOST = "foo.com";
+    private static final int TEST_PORT = 7777;
+    private static final String TEST_USER = "testuser";
+    private static final String TEST_PASSWORD = "123";
 
     @Test
     public void build_setsAllProperties() {
         verifyAllPropertiesSet(allPropertiesSetConfig());
     }
 
+    @Test
+    public void build_systemPropertyEnabled() {
+        setSystemProperty();
+        ProxyConfiguration config = ProxyConfiguration.builder().useSystemPropertyValues(true).build();
+
+        assertThat(config.host()).isEqualTo(TEST_HOST);
+        assertThat(config.port()).isEqualTo(TEST_PORT);
+        assertThat(config.username()).isEqualTo(TEST_USER);
+        assertThat(config.password()).isEqualTo(TEST_PASSWORD);
+        assertThat(config.scheme()).isNull();
+    }
+
+    @Test
+    public void build_systemPropertyDisabled() {
+        setSystemProperty();
+        ProxyConfiguration config = ProxyConfiguration.builder()
+                                                      .host("localhost")
+                                                      .port(8888)
+                                                      .username("username")
+                                                      .password("password")
+                                                      .useSystemPropertyValues(false).build();
+
+        assertThat(config.host()).isEqualTo("localhost");
+        assertThat(config.port()).isEqualTo(8888);
+        assertThat(config.username()).isEqualTo("username");
+        assertThat(config.password()).isEqualTo("password");
+        assertThat(config.scheme()).isNull();
+    }
+
     @Test
     public void toBuilder_roundTrip_producesExactCopy() {
         ProxyConfiguration original = allPropertiesSetConfig();
@@ -76,6 +109,8 @@ private void setRandomValue(Object o, Method setter) throws InvocationTargetExce
             setter.invoke(o, randomString());
         } else if (int.class.equals(paramClass)) {
             setter.invoke(o, RNG.nextInt());
+        } else if (Boolean.class.equals(paramClass)) {
+            setter.invoke(o, Boolean.FALSE);
         } else {
             throw new RuntimeException("Don't know how create random value for type " + paramClass);
         }
@@ -108,4 +143,11 @@ private String randomString() {
 
         return sb.toString();
     }
+
+    private void setSystemProperty() {
+        System.setProperty("http.proxyHost", TEST_HOST);
+        System.setProperty("http.proxyPort", Integer.toString(TEST_PORT));
+        System.setProperty("http.proxyUser", TEST_USER);
+        System.setProperty("http.proxyPassword", TEST_PASSWORD);
+    }
 }

http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/ProxyConfiguration.java

@@ -15,10 +15,14 @@
 
 package software.amazon.awssdk.http.nio.netty;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.stream.Collectors;
 import software.amazon.awssdk.annotations.SdkPublicApi;
+import software.amazon.awssdk.utils.ProxySystemSetting;
+import software.amazon.awssdk.utils.StringUtils;
 import software.amazon.awssdk.utils.builder.CopyableBuilder;
 import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
 
@@ -30,6 +34,7 @@
  */
 @SdkPublicApi
 public final class ProxyConfiguration implements ToCopyableBuilder<ProxyConfiguration.Builder, ProxyConfiguration> {
+    private final Boolean useSystemPropertyValues;
     private final String scheme;
     private final String host;
     private final int port;
@@ -38,12 +43,13 @@ public final class ProxyConfiguration implements ToCopyableBuilder<ProxyConfigur
     private final Set<String> nonProxyHosts;
 
     private ProxyConfiguration(BuilderImpl builder) {
+        this.useSystemPropertyValues = builder.useSystemPropertyValues;
         this.scheme = builder.scheme;
-        this.host = builder.host;
-        this.port = builder.port;
+        this.host = resolveHost(builder.host);
+        this.port = resolvePort(builder.port);
         this.username = builder.username;
         this.password = builder.password;
-        this.nonProxyHosts = Collections.unmodifiableSet(builder.nonProxyHosts);
+        this.nonProxyHosts = builder.nonProxyHosts;
     }
 
     public static Builder builder() {
@@ -58,38 +64,45 @@ public String scheme() {
     }
 
     /**
-     * @return The proxy host.
+     * @return The proxy host from the configuration if set, or from the "http.proxyHost" system property if
+     * {@link Builder#useSystemPropertyValues(Boolean)} is set to true
      */
     public String host() {
         return host;
     }
 
     /**
-     * @return The proxy port.
+     * @return The proxy port from the configuration if set, or from the "http.proxyPort" system property if
+     * {@link Builder#useSystemPropertyValues(Boolean)} is set to true
      */
     public int port() {
         return port;
     }
 
     /**
-     * @return The proxy username.
-     */
+     * @return The proxy username from the configuration if set, or from the "http.proxyUser" system property if
+     * {@link Builder#useSystemPropertyValues(Boolean)} is set to true
+     * */
     public String username() {
-        return username;
+        return resolveValue(username, ProxySystemSetting.PROXY_USERNAME);
     }
 
     /**
-     * @return The proxy password.
-     */
+     * @return The proxy password from the configuration if set, or from the "http.proxyPassword" system property if
+     * {@link Builder#useSystemPropertyValues(Boolean)} is set to true
+     * */
     public String password() {
-        return password;
+        return resolveValue(password, ProxySystemSetting.PROXY_PASSWORD);
     }
 
     /**
-     * @return The set of hosts that should not be proxied.
+     * @return The set of hosts that should not be proxied. If the value is not set, the value present by "http.nonProxyHost
+     * system property os returned. If system property is also not set, an unmodifiable empty set is returned.
      */
     public Set<String> nonProxyHosts() {
-        return nonProxyHosts;
+        Set<String> hosts = nonProxyHosts == null && useSystemPropertyValues ? parseNonProxyHostProperty()
+                                                                : nonProxyHosts;
+        return Collections.unmodifiableSet(hosts != null ? hosts : Collections.emptySet());
     }
 
     @Override
@@ -199,6 +212,47 @@ public interface Builder extends CopyableBuilder<Builder, ProxyConfiguration> {
          * @return This object for method chaining.
          */
         Builder password(String password);
+
+        /**
+         * Set the option whether to use system property values from {@link ProxySystemSetting} if any of the config options
+         * are missing. The value is set to "true" by default which means SDK will automatically use system property values if
+         * options are not provided during building the {@link ProxyConfiguration} object. To disable this behaviour, set this
+         * value to false.
+         *
+         * @param useSystemPropertyValues The option whether to use system proerpty values
+         * @return This object for method chaining.
+         */
+        Builder useSystemPropertyValues(Boolean useSystemPropertyValues);
+
+    }
+
+    private String resolveHost(String host) {
+        return resolveValue(host, ProxySystemSetting.PROXY_HOST);
+    }
+
+    private int resolvePort(int port) {
+        return useSystemPropertyValues ? ProxySystemSetting.PROXY_PORT.getStringValue().map(Integer::parseInt).orElse(0)
+                                       : port;
+    }
+
+    /**
+     * Uses the configuration options, system setting property and returns the final value of the given member.
+     */
+    private String resolveValue(String value, ProxySystemSetting systemSetting) {
+        return value == null && useSystemPropertyValues ? systemSetting.getStringValue().orElse(null)
+                                                        : value;
+    }
+
+    private Set<String> parseNonProxyHostProperty() {
+        String nonProxyHosts = ProxySystemSetting.NON_PROXY_HOSTS.getStringValue().orElse(null);
+
+        if (!StringUtils.isEmpty(nonProxyHosts)) {
+            return Arrays.stream(nonProxyHosts.split("\\|"))
+                         .map(String::toLowerCase)
+                         .map(s -> s.replace("*", ".*?"))
+                         .collect(Collectors.toSet());
+        }
+        return Collections.emptySet();
     }
 
     private static final class BuilderImpl implements Builder {
@@ -208,11 +262,13 @@ private static final class BuilderImpl implements Builder {
         private String username;
         private String password;
         private Set<String> nonProxyHosts = Collections.emptySet();
+        private Boolean useSystemPropertyValues = Boolean.TRUE;
 
         private BuilderImpl() {
         }
 
         private BuilderImpl(ProxyConfiguration proxyConfiguration) {
+            this.useSystemPropertyValues = proxyConfiguration.useSystemPropertyValues;
             this.scheme = proxyConfiguration.scheme;
             this.host = proxyConfiguration.host;
             this.port = proxyConfiguration.port;
@@ -262,6 +318,16 @@ public Builder password(String password) {
             return this;
         }
 
+        @Override
+        public Builder useSystemPropertyValues(Boolean useSystemPropertyValues) {
+            this.useSystemPropertyValues = useSystemPropertyValues;
+            return this;
+        }
+
+        public void setUseSystemPropertyValues(Boolean useSystemPropertyValues) {
+            useSystemPropertyValues(useSystemPropertyValues);
+        }
+
         @Override
         public ProxyConfiguration build() {
             return new ProxyConfiguration(this);