GH-1251: Jackson2JsonMessageConverter Improvements

Resolves #1420

- detect and use `charset` in `contentType` when present
- allow Jackson to determine the decode `charset` via `ByteSourceJsonBootstrapper.detectEncoding()`
- allow configuration of the `MimeType` to use, which can include a `charset` parameter

**cherry-pick to main - will require what's new fix**

* Fix typo in doc.
# Conflicts:
#	src/reference/asciidoc/whats-new.adoc

Author: garyrussell

spring-amqp/src/main/java/org/springframework/amqp/support/converter/AbstractJackson2MessageConverter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2020 the original author or authors.
+ * Copyright 2018-2022 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -33,6 +33,7 @@
 import org.springframework.util.Assert;
 import org.springframework.util.ClassUtils;
 import org.springframework.util.MimeType;
+import org.springframework.util.MimeTypeUtils;
 
 import com.fasterxml.jackson.databind.JavaType;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -61,13 +62,18 @@ public abstract class AbstractJackson2MessageConverter extends AbstractMessageCo
 	 */
 	public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
 
+	protected final ObjectMapper objectMapper; // NOSONAR protected
+
 	/**
-	 * The supported content type; only the subtype is checked, e.g. */json,
-	 * */xml.
+	 * The supported content type; only the subtype is checked when decoding, e.g.
+	 * */json, */xml. If this contains a charset parameter, when encoding, the
+	 * contentType header will not be set, when decoding, the raw bytes are passed to
+	 * Jackson which can dynamically determine the encoding; otherwise the contentEncoding
+	 * or default charset is used.
 	 */
-	private final MimeType supportedContentType;
+	private MimeType supportedContentType;
 
-	protected final ObjectMapper objectMapper; // NOSONAR protected
+	private String supportedCTCharset;
 
 	@Nullable
 	private ClassMapper classMapper = null;
@@ -93,8 +99,11 @@ public abstract class AbstractJackson2MessageConverter extends AbstractMessageCo
 	/**
 	 * Construct with the provided {@link ObjectMapper} instance.
 	 * @param objectMapper the {@link ObjectMapper} to use.
-	 * @param contentType supported content type when decoding messages, only the subtype
-	 * is checked, e.g. */json, */xml.
+	 * @param contentType the supported content type; only the subtype is checked when
+	 * decoding, e.g. */json, */xml. If this contains a charset parameter, when
+	 * encoding, the contentType header will not be set, when decoding, the raw bytes are
+	 * passed to Jackson which can dynamically determine the encoding; otherwise the
+	 * contentEncoding or default charset is used.
 	 * @param trustedPackages the trusted Java packages for deserialization
 	 * @see DefaultJackson2JavaTypeMapper#setTrustedPackages(String...)
 	 */
@@ -105,9 +114,41 @@ protected AbstractJackson2MessageConverter(ObjectMapper objectMapper, MimeType c
 		Assert.notNull(contentType, "'contentType' must not be null");
 		this.objectMapper = objectMapper;
 		this.supportedContentType = contentType;
+		this.supportedCTCharset = this.supportedContentType.getParameter("charset");
 		((DefaultJackson2JavaTypeMapper) this.javaTypeMapper).setTrustedPackages(trustedPackages);
 	}
 
+
+	/**
+	 * Get the supported content type; only the subtype is checked when decoding, e.g.
+	 * */json, */xml. If this contains a charset parameter, when encoding, the
+	 * contentType header will not be set, when decoding, the raw bytes are passed to
+	 * Jackson which can dynamically determine the encoding; otherwise the contentEncoding
+	 * or default charset is used.
+	 * @return the supportedContentType
+	 * @since 2.4.3
+	 */
+	protected MimeType getSupportedContentType() {
+		return this.supportedContentType;
+	}
+
+
+	/**
+	 * Set the supported content type; only the subtype is checked when decoding, e.g.
+	 * */json, */xml. If this contains a charset parameter, when encoding, the
+	 * contentType header will not be set, when decoding, the raw bytes are passed to
+	 * Jackson which can dynamically determine the encoding; otherwise the contentEncoding
+	 * or default charset is used.
+	 * @param supportedContentType the supportedContentType to set.
+	 * @since 2.4.3
+	 */
+	public void setSupportedContentType(MimeType supportedContentType) {
+		Assert.notNull(supportedContentType, "'supportedContentType' cannot be null");
+		this.supportedContentType = supportedContentType;
+		this.supportedCTCharset = this.supportedContentType.getParameter("charset");
+	}
+
+
 	@Nullable
 	public ClassMapper getClassMapper() {
 		return this.classMapper;
@@ -264,10 +305,7 @@ public Object fromMessage(Message message, @Nullable Object conversionHint) thro
 			if ((this.assumeSupportedContentType // NOSONAR Boolean complexity
 					&& (contentType == null || contentType.equals(MessageProperties.DEFAULT_CONTENT_TYPE)))
 					|| (contentType != null && contentType.contains(this.supportedContentType.getSubtype()))) {
-				String encoding = properties.getContentEncoding();
-				if (encoding == null) {
-					encoding = getDefaultCharset();
-				}
+				String encoding = determineEncoding(properties, contentType);
 				content = doFromMessage(message, conversionHint, properties, encoding);
 			}
 			else {
@@ -283,6 +321,24 @@ public Object fromMessage(Message message, @Nullable Object conversionHint) thro
 		return content;
 	}
 
+	private String determineEncoding(MessageProperties properties, @Nullable String contentType) {
+		String encoding = properties.getContentEncoding();
+		if (encoding == null && contentType != null) {
+			try {
+				MimeType mimeType = MimeTypeUtils.parseMimeType(contentType);
+				if (mimeType != null) {
+					encoding = mimeType.getParameter("charset");
+				}
+			}
+			catch (RuntimeException e) {
+			}
+		}
+		if (encoding == null) {
+			encoding = this.supportedCTCharset != null ? this.supportedCTCharset : getDefaultCharset();
+		}
+		return encoding;
+	}
+
 	private Object doFromMessage(Message message, Object conversionHint, MessageProperties properties,
 			String encoding) {
 
@@ -348,11 +404,17 @@ private Object tryConverType(Message message, String encoding, JavaType inferred
 	}
 
 	private Object convertBytesToObject(byte[] body, String encoding, JavaType targetJavaType) throws IOException {
+		if (this.supportedCTCharset != null) { // Jackson will determine encoding
+			return this.objectMapper.readValue(body, targetJavaType);
+		}
 		String contentAsString = new String(body, encoding);
 		return this.objectMapper.readValue(contentAsString, targetJavaType);
 	}
 
 	private Object convertBytesToObject(byte[] body, String encoding, Class<?> targetClass) throws IOException {
+		if (this.supportedCTCharset != null) { // Jackson will determine encoding
+			return this.objectMapper.readValue(body, this.objectMapper.constructType(targetClass));
+		}
 		String contentAsString = new String(body, encoding);
 		return this.objectMapper.readValue(contentAsString, this.objectMapper.constructType(targetClass));
 	}
@@ -370,20 +432,23 @@ protected Message createMessage(Object objectToConvert, MessageProperties messag
 
 		byte[] bytes;
 		try {
-			if (this.charsetIsUtf8) {
+			if (this.charsetIsUtf8 && this.supportedCTCharset == null) {
 				bytes = this.objectMapper.writeValueAsBytes(objectToConvert);
 			}
 			else {
 				String jsonString = this.objectMapper
 						.writeValueAsString(objectToConvert);
-				bytes = jsonString.getBytes(getDefaultCharset());
+				String encoding = this.supportedCTCharset != null ? this.supportedCTCharset : getDefaultCharset();
+				bytes = jsonString.getBytes(encoding);
 			}
 		}
 		catch (IOException e) {
 			throw new MessageConversionException("Failed to convert Message content", e);
 		}
 		messageProperties.setContentType(this.supportedContentType.toString());
-		messageProperties.setContentEncoding(getDefaultCharset());
+		if (this.supportedCTCharset == null) {
+			messageProperties.setContentEncoding(getDefaultCharset());
+		}
 		messageProperties.setContentLength(bytes.length);
 
 		if (getClassMapper() == null) {

spring-amqp/src/test/java/org/springframework/amqp/support/converter/Jackson2JsonMessageConverterTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2020 the original author or authors.
+ * Copyright 2002-2022 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 package org.springframework.amqp.support.converter;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 
 import java.io.IOException;
 import java.math.BigDecimal;
@@ -34,6 +35,7 @@
 import org.springframework.core.ParameterizedTypeReference;
 import org.springframework.data.web.JsonPath;
 import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
+import org.springframework.util.MimeTypeUtils;
 
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -399,6 +401,67 @@ void concreteInMapRegression() throws Exception {
 		assertThat(foos.values().iterator().next().getField()).isEqualTo("baz");
 	}
 
+	@Test
+	void charsetInContentType() {
+		trade.setUserName("John Doe ∫");
+		Jackson2JsonMessageConverter converter = new Jackson2JsonMessageConverter();
+		String utf8 = "application/json;charset=utf-8";
+		converter.setSupportedContentType(MimeTypeUtils.parseMimeType(utf8));
+		Message message = converter.toMessage(trade, new MessageProperties());
+		int bodyLength8 = message.getBody().length;
+		assertThat(message.getMessageProperties().getContentEncoding()).isNull();
+		assertThat(message.getMessageProperties().getContentType()).isEqualTo(utf8);
+		SimpleTrade marshalledTrade = (SimpleTrade) converter.fromMessage(message);
+		assertThat(marshalledTrade).isEqualTo(trade);
+
+		// use content type property
+		String utf16 = "application/json;charset=utf-16";
+		converter.setSupportedContentType(MimeTypeUtils.parseMimeType(utf16));
+		message = converter.toMessage(trade, new MessageProperties());
+		assertThat(message.getBody().length).isNotEqualTo(bodyLength8);
+		assertThat(message.getMessageProperties().getContentEncoding()).isNull();
+		assertThat(message.getMessageProperties().getContentType()).isEqualTo(utf16);
+		marshalledTrade = (SimpleTrade) converter.fromMessage(message);
+		assertThat(marshalledTrade).isEqualTo(trade);
+
+		// no encoding in message, use configured default
+		converter.setSupportedContentType(MimeTypeUtils.parseMimeType("application/json"));
+		converter.setDefaultCharset("UTF-16");
+		message = converter.toMessage(trade, new MessageProperties());
+		assertThat(message.getBody().length).isNotEqualTo(bodyLength8);
+		assertThat(message.getMessageProperties().getContentEncoding()).isNotNull();
+		message.getMessageProperties().setContentEncoding(null);
+		marshalledTrade = (SimpleTrade) converter.fromMessage(message);
+		assertThat(marshalledTrade).isEqualTo(trade);
+
+	}
+
+	@Test
+	void noConfigForCharsetInContentType() {
+		trade.setUserName("John Doe ∫");
+		Jackson2JsonMessageConverter converter = new Jackson2JsonMessageConverter();
+		Message message = converter.toMessage(trade, new MessageProperties());
+		int bodyLength8 = message.getBody().length;
+		SimpleTrade marshalledTrade = (SimpleTrade) converter.fromMessage(message);
+		assertThat(marshalledTrade).isEqualTo(trade);
+
+		// no encoding in message; use configured default
+		message = converter.toMessage(trade, new MessageProperties());
+		assertThat(message.getMessageProperties().getContentEncoding()).isNotNull();
+		message.getMessageProperties().setContentEncoding(null);
+		marshalledTrade = (SimpleTrade) converter.fromMessage(message);
+		assertThat(marshalledTrade).isEqualTo(trade);
+
+		converter.setDefaultCharset("UTF-16");
+		Message message2 = converter.toMessage(trade, new MessageProperties());
+		message2.getMessageProperties().setContentEncoding(null);
+		assertThat(message2.getBody().length).isNotEqualTo(bodyLength8);
+		converter.setDefaultCharset("UTF-8");
+
+		assertThatExceptionOfType(MessageConversionException.class).isThrownBy(
+				() -> converter.fromMessage(message2));
+	}
+
 	public List<Foo> fooLister() {
 		return null;
 	}

src/reference/asciidoc/amqp.adoc

@@ -3938,11 +3938,34 @@ public DefaultClassMapper classMapper() {
 Now, if the sending system sets the header to `thing1`, the converter creates a `Thing1` object, and so on.
 See the <<spring-rabbit-json>> sample application for a complete discussion about converting messages from non-Spring applications.
 
+Starting with version 2.4.3, the converter will not add a `contentEncoding` message property if the `supportedMediaType` has a `charset` parameter; this is also used for the encoding.
+A new method `setSupportedMediaType` has been added:
+
+====
+[source, java]
+----
+String utf16 = "application/json; charset=utf-16";
+converter.setSupportedContentType(MimeTypeUtils.parseMimeType(utf16));
+----
+====
+
 [[Jackson2JsonMessageConverter-from-message]]
 ====== Converting from a `Message`
 
 Inbound messages are converted to objects according to the type information added to headers by the sending system.
 
+Starting with version 2.4.3, if there is no `contentEncoding` message property, the converter will attempt to detect a `charset` parameter in the `contentType` message property and use that.
+If neither exist, if the `supportedMediaType` has a `charset` parameter, it will be used for decoding, with a final fallback to the `defaultCharset` property.
+A new method `setSupportedMediaType` has been added:
+
+====
+[source, java]
+----
+String utf16 = "application/json; charset=utf-16";
+converter.setSupportedContentType(MimeTypeUtils.parseMimeType(utf16));
+----
+====
+
 In versions prior to 1.6, if type information is not present, conversion would fail.
 Starting with version 1.6, if type information is missing, the converter converts the JSON by using Jackson defaults (usually a map).
 

src/reference/asciidoc/whats-new.adoc

@@ -11,4 +11,7 @@ This version requires Spring Framework 6.0 and Java 17
 
 The remoting feature (using RMI) is no longer supported.
 
-See <<remoting>> for alternatives.
+==== Message Converter Changes
+
+The `Jackson2JsonMessageConverter` can now determine the charset from the `contentEncoding` header.
+See <<json-message-converter>> for more information.