Skip to content

Commit 17abb4d

Browse files
bclozelbclozel
committed
Add codecs for JSON support with Protobuf
Prior to this commit, WebFlux had Protobuf codecs for managing the `Message`/`"application/x-protobuf"` encoding and decoding. The `com.google.protobuf:protobuf-java-util` library has additional support for JSON (de)serialization, but this is not supported by existing codecs. This commit adds the new `ProtobufJsonEncode` and `ProtobufJsonDecoder` classes that support this use case. Note, the `ProtobufJsonDecoder` has a significant limitation: it cannot decode JSON arrays as `Flux<Message>` because there is no available non-blocking parser able to tokenize JSON arrays into streams of `Databuffer`. Instead, applications should decode to `Mono<List<Message>>` which causes additional buffering but is properly supported. Closes gh-25457
1 parent 24bbc6d commit 17abb4d

File tree

6 files changed

+560
-5
lines changed

6 files changed

+560
-5
lines changed

‎spring-web/src/main/java/org/springframework/http/codec/protobuf/ProtobufEncoder.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2002-2022 the original author or authors.
2+
* Copyright 2002-2024 the original author or authors.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -47,7 +47,7 @@
4747
*
4848
* <p>To generate {@code Message} Java classes, you need to install the {@code protoc} binary.
4949
*
50-
* <p>This encoder requires Protobuf 3 or higher, and supports
50+
* <p>This encoder requires Protobuf 3.29 or higher, and supports
5151
* {@code "application/x-protobuf"} and {@code "application/octet-stream"} with the official
5252
* {@code "com.google.protobuf:protobuf-java"} library.
5353
*

‎spring-web/src/main/java/org/springframework/http/codec/protobuf/ProtobufHttpMessageWriter.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2002-2018 the original author or authors.
2+
* Copyright 2002-2024 the original author or authors.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -28,8 +28,8 @@
2828
import reactor.core.publisher.Mono;
2929

3030
import org.springframework.core.ResolvableType;
31-
import org.springframework.core.codec.DecodingException;
3231
import org.springframework.core.codec.Encoder;
32+
import org.springframework.core.codec.EncodingException;
3333
import org.springframework.http.MediaType;
3434
import org.springframework.http.ReactiveHttpOutputMessage;
3535
import org.springframework.http.codec.EncoderHttpMessageWriter;
@@ -97,7 +97,7 @@ else if (!ProtobufEncoder.DELIMITED_VALUE.equals(mediaType.getParameters().get(P
9797
return super.write(inputStream, elementType, mediaType, message, hints);
9898
}
9999
catch (Exception ex) {
100-
return Mono.error(new DecodingException("Could not read Protobuf message: " + ex.getMessage(), ex));
100+
return Mono.error(new EncodingException("Could not write Protobuf message: " + ex.getMessage(), ex));
101101
}
102102
}
103103

‎spring-web/src/main/java/org/springframework/http/codec/protobuf/ProtobufJsonDecoder.java

+173
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
/*
2+
* Copyright 2002-2024 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.http.codec.protobuf;
18+
19+
import java.io.InputStreamReader;
20+
import java.lang.reflect.Method;
21+
import java.util.List;
22+
import java.util.Map;
23+
import java.util.concurrent.ConcurrentMap;
24+
25+
import com.google.protobuf.Message;
26+
import com.google.protobuf.util.JsonFormat;
27+
import org.reactivestreams.Publisher;
28+
import reactor.core.publisher.Flux;
29+
import reactor.core.publisher.Mono;
30+
31+
import org.springframework.core.ResolvableType;
32+
import org.springframework.core.codec.Decoder;
33+
import org.springframework.core.codec.DecodingException;
34+
import org.springframework.core.io.buffer.DataBuffer;
35+
import org.springframework.core.io.buffer.DataBufferLimitException;
36+
import org.springframework.core.io.buffer.DataBufferUtils;
37+
import org.springframework.http.MediaType;
38+
import org.springframework.lang.Nullable;
39+
import org.springframework.util.ConcurrentReferenceHashMap;
40+
import org.springframework.util.MimeType;
41+
42+
/**
43+
* A {@code Decoder} that reads a JSON byte stream and converts it to
44+
* <a href="https://developers.google.com/protocol-buffers/">Google Protocol Buffers</a>
45+
* {@link com.google.protobuf.Message}s.
46+
*
47+
* <p>Flux deserialized via
48+
* {@link #decode(Publisher, ResolvableType, MimeType, Map)} are not supported because
49+
* the Protobuf Java Util library does not provide a non-blocking parser
50+
* that splits a JSON stream into tokens.
51+
* Applications should consider decoding to {@code Mono<Message>} or
52+
* {@code Mono<List<Message>>}, which will use the supported
53+
* {@link #decodeToMono(Publisher, ResolvableType, MimeType, Map)}.
54+
*
55+
* <p>To generate {@code Message} Java classes, you need to install the
56+
* {@code protoc} binary.
57+
*
58+
* <p>This decoder requires Protobuf 3.29 or higher, and supports
59+
* {@code "application/json"} and {@code "application/*+json"} with
60+
* the official {@code "com.google.protobuf:protobuf-java-util"} library.
61+
*
62+
* @author Brian Clozel
63+
* @since 6.2
64+
* @see ProtobufJsonEncoder
65+
*/
66+
public class ProtobufJsonDecoder implements Decoder<Message> {
67+
68+
/** The default max size for aggregating messages. */
69+
protected static final int DEFAULT_MESSAGE_MAX_SIZE = 256 * 1024;
70+
71+
private static final List<MimeType> defaultMimeTypes = List.of(MediaType.APPLICATION_JSON,
72+
new MediaType("application", "*+json"));
73+
74+
private static final ConcurrentMap<Class<?>, Method> methodCache = new ConcurrentReferenceHashMap<>();
75+
76+
private final JsonFormat.Parser parser;
77+
78+
private int maxMessageSize = DEFAULT_MESSAGE_MAX_SIZE;
79+
80+
/**
81+
* Construct a new {@link ProtobufJsonDecoder} using a default {@link JsonFormat.Parser} instance.
82+
*/
83+
public ProtobufJsonDecoder() {
84+
this(JsonFormat.parser());
85+
}
86+
87+
/**
88+
* Construct a new {@link ProtobufJsonDecoder} using the given {@link JsonFormat.Parser} instance.
89+
*/
90+
public ProtobufJsonDecoder(JsonFormat.Parser parser) {
91+
this.parser = parser;
92+
}
93+
94+
/**
95+
* Return the {@link #setMaxMessageSize configured} message size limit.
96+
*/
97+
public int getMaxMessageSize() {
98+
return this.maxMessageSize;
99+
}
100+
101+
/**
102+
* The max size allowed per message.
103+
* <p>By default, this is set to 256K.
104+
* @param maxMessageSize the max size per message, or -1 for unlimited
105+
*/
106+
public void setMaxMessageSize(int maxMessageSize) {
107+
this.maxMessageSize = maxMessageSize;
108+
}
109+
110+
@Override
111+
public boolean canDecode(ResolvableType elementType, @Nullable MimeType mimeType) {
112+
return Message.class.isAssignableFrom(elementType.toClass()) && supportsMimeType(mimeType);
113+
}
114+
115+
private static boolean supportsMimeType(@Nullable MimeType mimeType) {
116+
if (mimeType == null) {
117+
return false;
118+
}
119+
for (MimeType m : defaultMimeTypes) {
120+
if (m.isCompatibleWith(mimeType)) {
121+
return true;
122+
}
123+
}
124+
return false;
125+
}
126+
127+
128+
@Override
129+
public List<MimeType> getDecodableMimeTypes() {
130+
return defaultMimeTypes;
131+
}
132+
133+
@Override
134+
public Flux<Message> decode(Publisher<DataBuffer> inputStream, ResolvableType targetType, @Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
135+
return Flux.error(new UnsupportedOperationException("Protobuf decoder does not support Flux, use Mono<List<...>> instead."));
136+
}
137+
138+
@Override
139+
public Message decode(DataBuffer dataBuffer, ResolvableType targetType, @Nullable MimeType mimeType, @Nullable Map<String, Object> hints) throws DecodingException {
140+
try {
141+
Message.Builder builder = getMessageBuilder(targetType.toClass());
142+
this.parser.merge(new InputStreamReader(dataBuffer.asInputStream()), builder);
143+
return builder.build();
144+
}
145+
catch (Exception ex) {
146+
throw new DecodingException("Could not read Protobuf message: " + ex.getMessage(), ex);
147+
}
148+
finally {
149+
DataBufferUtils.release(dataBuffer);
150+
}
151+
}
152+
153+
/**
154+
* Create a new {@code Message.Builder} instance for the given class.
155+
* <p>This method uses a ConcurrentHashMap for caching method lookups.
156+
*/
157+
private static Message.Builder getMessageBuilder(Class<?> clazz) throws Exception {
158+
Method method = methodCache.get(clazz);
159+
if (method == null) {
160+
method = clazz.getMethod("newBuilder");
161+
methodCache.put(clazz, method);
162+
}
163+
return (Message.Builder) method.invoke(clazz);
164+
}
165+
166+
@Override
167+
public Mono<Message> decodeToMono(Publisher<DataBuffer> inputStream, ResolvableType elementType, @Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
168+
return DataBufferUtils.join(inputStream, this.maxMessageSize)
169+
.map(dataBuffer -> decode(dataBuffer, elementType, mimeType, hints))
170+
.onErrorMap(DataBufferLimitException.class, exc -> new DecodingException("Could not decode JSON as Protobuf message", exc));
171+
}
172+
173+
}

‎spring-web/src/main/java/org/springframework/http/codec/protobuf/ProtobufJsonEncoder.java

+173
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
/*
2+
* Copyright 2002-2024 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.http.codec.protobuf;
18+
19+
import java.io.IOException;
20+
import java.io.OutputStreamWriter;
21+
import java.nio.charset.StandardCharsets;
22+
import java.util.List;
23+
import java.util.Map;
24+
25+
import com.google.protobuf.Message;
26+
import com.google.protobuf.util.JsonFormat;
27+
import org.reactivestreams.Publisher;
28+
import reactor.core.publisher.Flux;
29+
import reactor.core.publisher.Mono;
30+
31+
import org.springframework.core.ResolvableType;
32+
import org.springframework.core.io.buffer.DataBuffer;
33+
import org.springframework.core.io.buffer.DataBufferFactory;
34+
import org.springframework.http.MediaType;
35+
import org.springframework.http.codec.HttpMessageEncoder;
36+
import org.springframework.lang.Nullable;
37+
import org.springframework.util.FastByteArrayOutputStream;
38+
import org.springframework.util.MimeType;
39+
40+
/**
41+
* A {@code Encoder} that writes {@link com.google.protobuf.Message}s as JSON.
42+
*
43+
* <p>To generate {@code Message} Java classes, you need to install the
44+
* {@code protoc} binary.
45+
*
46+
* <p>This encoder requires Protobuf 3.29 or higher, and supports
47+
* {@code "application/json"} and {@code "application/*+json"} with
48+
* the official {@code "com.google.protobuf:protobuf-java-util"} library.
49+
*
50+
* @author Brian Clozel
51+
* @since 6.2
52+
* @see ProtobufJsonDecoder
53+
*/
54+
public class ProtobufJsonEncoder implements HttpMessageEncoder<Message> {
55+
56+
private static final byte[] EMPTY_BYTES = new byte[0];
57+
58+
private static final ResolvableType MESSAGE_TYPE = ResolvableType.forClass(Message.class);
59+
60+
private static final List<MimeType> defaultMimeTypes = List.of(
61+
MediaType.APPLICATION_JSON,
62+
new MediaType("application", "*+json"));
63+
64+
private final JsonFormat.Printer printer;
65+
66+
67+
/**
68+
* Construct a new {@link ProtobufJsonEncoder} using a default {@link JsonFormat.Printer} instance.
69+
*/
70+
public ProtobufJsonEncoder() {
71+
this(JsonFormat.printer());
72+
}
73+
74+
/**
75+
* Construct a new {@link ProtobufJsonEncoder} using the given {@link JsonFormat.Printer} instance.
76+
*/
77+
public ProtobufJsonEncoder(JsonFormat.Printer printer) {
78+
this.printer = printer;
79+
}
80+
81+
@Override
82+
public List<MediaType> getStreamingMediaTypes() {
83+
return List.of(MediaType.APPLICATION_NDJSON);
84+
}
85+
86+
@Override
87+
public List<MimeType> getEncodableMimeTypes() {
88+
return defaultMimeTypes;
89+
}
90+
91+
@Override
92+
public boolean canEncode(ResolvableType elementType, @Nullable MimeType mimeType) {
93+
return Message.class.isAssignableFrom(elementType.toClass()) && supportsMimeType(mimeType);
94+
}
95+
96+
private static boolean supportsMimeType(@Nullable MimeType mimeType) {
97+
if (mimeType == null) {
98+
return false;
99+
}
100+
for (MimeType m : defaultMimeTypes) {
101+
if (m.isCompatibleWith(mimeType)) {
102+
return true;
103+
}
104+
}
105+
return false;
106+
}
107+
108+
@Override
109+
public Flux<DataBuffer> encode(Publisher<? extends Message> inputStream, DataBufferFactory bufferFactory, ResolvableType elementType, @Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
110+
if (inputStream instanceof Mono) {
111+
return Mono.from(inputStream)
112+
.map(value -> encodeValue(value, bufferFactory, elementType, mimeType, hints))
113+
.flux();
114+
}
115+
JsonArrayJoinHelper helper = new JsonArrayJoinHelper();
116+
117+
// Do not prepend JSON array prefix until first signal is known, onNext vs onError
118+
// Keeps response not committed for error handling
119+
return Flux.from(inputStream)
120+
.map(value -> {
121+
byte[] prefix = helper.getPrefix();
122+
byte[] delimiter = helper.getDelimiter();
123+
DataBuffer dataBuffer = encodeValue(value, bufferFactory, MESSAGE_TYPE, mimeType, hints);
124+
return (prefix.length > 0 ?
125+
bufferFactory.join(List.of(bufferFactory.wrap(prefix), bufferFactory.wrap(delimiter), dataBuffer)) :
126+
bufferFactory.join(List.of(bufferFactory.wrap(delimiter), dataBuffer)));
127+
})
128+
.switchIfEmpty(Mono.fromCallable(() -> bufferFactory.wrap(helper.getPrefix())))
129+
.concatWith(Mono.fromCallable(() -> bufferFactory.wrap(helper.getSuffix())));
130+
}
131+
132+
@Override
133+
public DataBuffer encodeValue(Message message, DataBufferFactory bufferFactory, ResolvableType valueType, @Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
134+
FastByteArrayOutputStream bos = new FastByteArrayOutputStream();
135+
OutputStreamWriter writer = new OutputStreamWriter(bos, StandardCharsets.UTF_8);
136+
try {
137+
this.printer.appendTo(message, writer);
138+
writer.flush();
139+
byte[] bytes = bos.toByteArrayUnsafe();
140+
return bufferFactory.wrap(bytes);
141+
}
142+
catch (IOException ex) {
143+
throw new IllegalStateException("Unexpected I/O error while writing to data buffer", ex);
144+
}
145+
}
146+
147+
private static class JsonArrayJoinHelper {
148+
149+
private static final byte[] COMMA_SEPARATOR = {','};
150+
151+
private static final byte[] OPEN_BRACKET = {'['};
152+
153+
private static final byte[] CLOSE_BRACKET = {']'};
154+
155+
private boolean firstItemEmitted;
156+
157+
public byte[] getDelimiter() {
158+
if (this.firstItemEmitted) {
159+
return COMMA_SEPARATOR;
160+
}
161+
this.firstItemEmitted = true;
162+
return EMPTY_BYTES;
163+
}
164+
165+
public byte[] getPrefix() {
166+
return (this.firstItemEmitted ? EMPTY_BYTES : OPEN_BRACKET);
167+
}
168+
169+
public byte[] getSuffix() {
170+
return CLOSE_BRACKET;
171+
}
172+
}
173+
}

0 commit comments

Comments
 (0)