From 806cd13574b9ccbbfba6552deba6db2ce490bb0e Mon Sep 17 00:00:00 2001 From: Sylvain Wallez Date: Thu, 25 May 2023 17:23:30 +0200 Subject: [PATCH 1/5] Extract base abstract behavior from RestClientTransport --- .../clients/transport/TransportBase.java | 396 ++++++++++++++++ .../endpoints/BinaryDataResponse.java | 53 +++ .../RestClientMonolithTransport.java | 442 ++++++++++++++++++ .../rest_client/RestClientOptions.java | 2 +- .../rest_client/RestClientTransport.java | 397 ++++------------ .../co/elastic/clients/util/BinaryData.java | 23 +- .../clients/util/ByteArrayBinaryData.java | 24 + .../util/NoCopyByteArrayOutputStream.java | 8 + .../rest_client/RestClientOptionsTest.java | 23 +- 9 files changed, 1060 insertions(+), 308 deletions(-) create mode 100644 java-client/src/main/java/co/elastic/clients/transport/TransportBase.java create mode 100644 java-client/src/main/java/co/elastic/clients/transport/endpoints/BinaryDataResponse.java create mode 100644 java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientMonolithTransport.java diff --git a/java-client/src/main/java/co/elastic/clients/transport/TransportBase.java b/java-client/src/main/java/co/elastic/clients/transport/TransportBase.java new file mode 100644 index 000000000..941ae4edc --- /dev/null +++ b/java-client/src/main/java/co/elastic/clients/transport/TransportBase.java @@ -0,0 +1,396 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport; + +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch._types.ErrorResponse; +import co.elastic.clients.json.JsonpDeserializer; +import co.elastic.clients.json.JsonpMapper; +import co.elastic.clients.json.NdJsonpSerializable; +import co.elastic.clients.transport.endpoints.BinaryDataResponse; +import co.elastic.clients.transport.endpoints.BinaryEndpoint; +import co.elastic.clients.transport.endpoints.BooleanEndpoint; +import co.elastic.clients.transport.endpoints.BooleanResponse; +import co.elastic.clients.util.ApiTypeHelper; +import co.elastic.clients.util.BinaryData; +import co.elastic.clients.util.ByteArrayBinaryData; +import co.elastic.clients.util.ContentType; +import co.elastic.clients.util.MissingRequiredPropertyException; +import co.elastic.clients.util.NoCopyByteArrayOutputStream; +import jakarta.json.JsonException; +import jakarta.json.stream.JsonGenerator; +import jakarta.json.stream.JsonParser; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +public abstract class TransportBase< + Options extends TransportOptions + > implements Transport { + + public static class TransportRequest { + public final String method; + public final String path; + public final Map queryParams; + @Nullable public final String contentType; + @Nullable public final Iterable body; + + public TransportRequest(String method, String path, Map queryParams, @Nullable String contentType, @Nullable Iterable body) { + this.method = method; + this.path = path; + this.queryParams = queryParams; + this.contentType = contentType; + this.body = body; + } + } + + public static final String JsonContentType; + + static { + if (Version.VERSION == null) { + JsonContentType = ContentType.APPLICATION_JSON; + } else { + JsonContentType = + "application/vnd.elasticsearch+json; compatible-with=" + + Version.VERSION.major(); + } + } + + /** + * Create implementation-specific options from optional initial options. + */ + protected abstract Options createOptions(@Nullable TransportOptions options); + protected abstract TransportResponse performRequest(TransportRequest request, Options options) throws IOException; + protected abstract CompletableFuture performRequestAsync(TransportRequest request, Options options); + + private final JsonpMapper mapper; + protected final Options transportOptions; + + protected TransportBase(JsonpMapper jsonpMapper, Options options) { + this.mapper = jsonpMapper; + this.transportOptions = options; + } + + @Override + public final JsonpMapper jsonpMapper() { + return mapper; + } + + @Override + public final TransportOptions options() { + return transportOptions; + } + + @Override + public final ResponseT performRequest( + RequestT request, + Endpoint endpoint, + @Nullable TransportOptions options + ) throws IOException { + TransportRequest req = prepareTransportRequest(request, endpoint, options); + Options opts = createOptions(options); + TransportResponse resp = performRequest(req, opts); + return getApiResponse(resp, endpoint); + } + + @Override + public final CompletableFuture performRequestAsync(RequestT request, Endpoint endpoint, @Nullable TransportOptions options) { + TransportRequest clientReq; + try { + clientReq = prepareTransportRequest(request, endpoint, options); + } catch (Exception e) { + // Terminate early + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(e); + return future; + } + + // Propagate required property checks to the thread that will decode the response + boolean disableRequiredChecks = ApiTypeHelper.requiredPropertiesCheckDisabled(); + + CompletableFuture clientFuture = performRequestAsync(clientReq, createOptions(options)); + + // Cancelling the result will cancel the upstream future created by the http client, allowing to stop in-flight requests + CompletableFuture future = new CompletableFuture() { + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + boolean cancelled = super.cancel(mayInterruptIfRunning); + if (cancelled) { + clientFuture.cancel(mayInterruptIfRunning); + } + return cancelled; + } + }; + + clientFuture.handle((clientResp, thr) -> { + if (thr != null) { + future.completeExceptionally(thr); + } else { + try (ApiTypeHelper.DisabledChecksHandle h = + ApiTypeHelper.DANGEROUS_disableRequiredPropertiesCheck(disableRequiredChecks)) { + + ResponseT response = getApiResponse(clientResp, endpoint); + future.complete(response); + + } catch (Throwable e) { + future.completeExceptionally(e); + } + } + return null; + }); + + return future; + } + + private TransportRequest prepareTransportRequest( + RequestT request, + Endpoint endpoint, + @Nullable TransportOptions options + ) throws IOException { + String method = endpoint.method(request); + String path = endpoint.requestUrl(request); + Map params = endpoint.queryParameters(request); + + List bodyBuffers = null; + String contentType = null; + + Object body = endpoint.body(request); + if (body != null) { + // Request has a body + if (body instanceof NdJsonpSerializable) { + bodyBuffers = new ArrayList<>(); + collectNdJsonLines(bodyBuffers, (NdJsonpSerializable) request); + contentType = JsonContentType; + + } else if (body instanceof BinaryData) { + BinaryData data = (BinaryData)body; + + // ES expects the Accept and Content-Type headers to be consistent. + String dataContentType = data.contentType(); + if (ContentType.APPLICATION_JSON.equals(dataContentType)) { + // Fast path + contentType = JsonContentType; + } else { + contentType = dataContentType; + } + bodyBuffers = Collections.singletonList(data.asByteBuffer()); + + } else { + NoCopyByteArrayOutputStream baos = new NoCopyByteArrayOutputStream(); + JsonGenerator generator = mapper.jsonProvider().createGenerator(baos); + mapper.serialize(body, generator); + generator.close(); + bodyBuffers = Collections.singletonList(baos.asByteBuffer()); + contentType = JsonContentType; + } + } + + return new TransportRequest(method, path, params, contentType, bodyBuffers); + } + + private static final ByteBuffer NdJsonSeparator = ByteBuffer.wrap("\n".getBytes(StandardCharsets.UTF_8)); + + private void collectNdJsonLines(List lines, NdJsonpSerializable value) throws IOException { + Iterator values = value._serializables(); + while(values.hasNext()) { + Object item = values.next(); + if (item == null) { + // Skip + } else if (item instanceof NdJsonpSerializable && item != value) { // do not recurse on the item itself + collectNdJsonLines(lines, (NdJsonpSerializable)item); + } else { + // TODO: items that aren't already BinaryData could be serialized to ByteBuffers lazily + // to reduce the number of buffers to keep in memory + lines.add(BinaryData.of(item, this.mapper).asByteBuffer()); + lines.add(NdJsonSeparator); + } + } + } + + protected interface TransportResponse { + int statusCode(); + String getHeader(String name); + @Nullable BinaryData getBody() throws IOException; + Throwable createException() throws IOException; + void close() throws IOException; + } + + private ResponseT getApiResponse( + TransportResponse clientResp, + Endpoint endpoint + ) throws IOException { + + int statusCode = clientResp.statusCode(); + + try { + if (statusCode == 200) { + checkProductHeader(clientResp, endpoint); + } + + if (endpoint.isError(statusCode)) { + JsonpDeserializer errorDeserializer = endpoint.errorDeserializer(statusCode); + if (errorDeserializer == null) { + throw new TransportException( + statusCode, + "Request failed with status code '" + statusCode + "'", + endpoint.id(), clientResp.createException() + ); + } + + BinaryData entity = clientResp.getBody(); + if (entity == null) { + throw new TransportException( + statusCode, + "Expecting a response body, but none was sent", + endpoint.id(), clientResp.createException() + ); + } + + // We may have to replay it. + if (!entity.isRepeatable()) { + entity = new ByteArrayBinaryData(entity); + } + + try { + InputStream content = entity.asInputStream(); + try (JsonParser parser = mapper.jsonProvider().createParser(content)) { + ErrorT error = errorDeserializer.deserialize(parser, mapper); + // TODO: have the endpoint provide the exception constructor + throw new ElasticsearchException(endpoint.id(), (ErrorResponse) error); + } + } catch(JsonException | MissingRequiredPropertyException errorEx) { + // Could not decode exception, try the response type + try { + ResponseT response = decodeTransportResponse(statusCode, entity, clientResp, endpoint); + return response; + } catch(Exception respEx) { + // No better luck: throw the original error decoding exception + throw new TransportException(statusCode, + "Failed to decode error response, check exception cause for additional details", endpoint.id(), + clientResp.createException() + ); + } + } + } else { + return decodeTransportResponse(statusCode, clientResp.getBody(), clientResp, endpoint); + } + + + } finally { + // Consume the entity unless this is a successful binary endpoint, where the user must consume the entity + if (!(endpoint instanceof BinaryEndpoint && !endpoint.isError(statusCode))) { + clientResp.close(); + } + } + } + + private ResponseT decodeTransportResponse( + int statusCode, @Nullable BinaryData entity, TransportResponse clientResp, Endpoint endpoint + ) throws IOException { + + if (endpoint instanceof JsonEndpoint) { + @SuppressWarnings("unchecked") + JsonEndpoint jsonEndpoint = (JsonEndpoint) endpoint; + // Successful response + ResponseT response = null; + JsonpDeserializer responseParser = jsonEndpoint.responseDeserializer(); + if (responseParser != null) { + // Expecting a body + if (entity == null) { + throw new TransportException( + statusCode, + "Expecting a response body, but none was sent", + endpoint.id(), clientResp.createException() + ); + } + InputStream content = entity.asInputStream(); + try (JsonParser parser = mapper.jsonProvider().createParser(content)) { + response = responseParser.deserialize(parser, mapper); + } catch (Exception e) { + throw new TransportException( + statusCode, + "Failed to decode response", + endpoint.id(), + e + ); + } + } + return response; + + } else if(endpoint instanceof BooleanEndpoint) { + BooleanEndpoint bep = (BooleanEndpoint) endpoint; + + @SuppressWarnings("unchecked") + ResponseT response = (ResponseT) new BooleanResponse(bep.getResult(statusCode)); + return response; + + + } else if (endpoint instanceof BinaryEndpoint) { + @SuppressWarnings("unchecked") + ResponseT response = (ResponseT) new BinaryDataResponse(entity); + return response; + + } else { + throw new TransportException(statusCode, "Unhandled endpoint type: '" + endpoint.getClass().getName() + "'", endpoint.id()); + } + } + + // Endpoints that (incorrectly) do not return the Elastic product header + private static final Set endpointsMissingProductHeader = new HashSet<>(Arrays.asList( + "es/snapshot.create" // #74 / elastic/elasticsearch#82358 + )); + + private void checkProductHeader(TransportResponse clientResp, Endpoint endpoint) throws IOException { + String header = clientResp.getHeader("X-Elastic-Product"); + if (header == null) { + if (endpointsMissingProductHeader.contains(endpoint.id())) { + return; + } + throw new TransportException( + clientResp.statusCode(), + "Missing [X-Elastic-Product] header. Please check that you are connecting to an Elasticsearch " + + "instance, and that any networking filters are preserving that header.", + endpoint.id(), + clientResp.createException() + ); + } + + if (!"Elasticsearch".equals(header)) { + throw new TransportException( + clientResp.statusCode(), + "Invalid value '" + header + "' for 'X-Elastic-Product' header.", + endpoint.id(), + clientResp.createException() + ); + } + } +} diff --git a/java-client/src/main/java/co/elastic/clients/transport/endpoints/BinaryDataResponse.java b/java-client/src/main/java/co/elastic/clients/transport/endpoints/BinaryDataResponse.java new file mode 100644 index 000000000..937edc10f --- /dev/null +++ b/java-client/src/main/java/co/elastic/clients/transport/endpoints/BinaryDataResponse.java @@ -0,0 +1,53 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport.endpoints; + +import co.elastic.clients.util.BinaryData; + +import java.io.IOException; +import java.io.InputStream; + +public class BinaryDataResponse implements BinaryResponse { + + private final BinaryData data; + + public BinaryDataResponse(BinaryData data) { + this.data = data; + } + + @Override + public String contentType() { + return data.contentType(); + } + + @Override + public long contentLength() { + return data.size(); + } + + @Override + public InputStream content() throws IOException { + return data.asInputStream(); + } + + @Override + public void close() throws IOException { + } +} diff --git a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientMonolithTransport.java b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientMonolithTransport.java new file mode 100644 index 000000000..85b530f18 --- /dev/null +++ b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientMonolithTransport.java @@ -0,0 +1,442 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport.rest_client; + +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch._types.ErrorResponse; +import co.elastic.clients.json.JsonpDeserializer; +import co.elastic.clients.json.JsonpMapper; +import co.elastic.clients.json.NdJsonpSerializable; +import co.elastic.clients.transport.JsonEndpoint; +import co.elastic.clients.transport.TransportException; +import co.elastic.clients.transport.Version; +import co.elastic.clients.transport.endpoints.BinaryEndpoint; +import co.elastic.clients.transport.endpoints.BooleanEndpoint; +import co.elastic.clients.transport.endpoints.BooleanResponse; +import co.elastic.clients.transport.ElasticsearchTransport; +import co.elastic.clients.transport.Endpoint; +import co.elastic.clients.transport.TransportOptions; +import co.elastic.clients.util.ApiTypeHelper; +import co.elastic.clients.util.BinaryData; +import co.elastic.clients.util.MissingRequiredPropertyException; +import jakarta.json.JsonException; +import jakarta.json.stream.JsonGenerator; +import jakarta.json.stream.JsonParser; +import org.apache.http.HttpEntity; +import org.apache.http.entity.BufferedHttpEntity; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Cancellable; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.ResponseListener; +import org.elasticsearch.client.RestClient; + +import javax.annotation.Nullable; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +public class RestClientMonolithTransport implements ElasticsearchTransport { + + static final ContentType JsonContentType; + + static { + + if (Version.VERSION == null) { + JsonContentType = ContentType.APPLICATION_JSON; + } else { + JsonContentType = ContentType.create( + "application/vnd.elasticsearch+json", + new BasicNameValuePair("compatible-with", String.valueOf(Version.VERSION.major())) + ); + } + } + + /** + * The {@code Future} implementation returned by async requests. + * It wraps the RestClient's cancellable and propagates cancellation. + */ + private static class RequestFuture extends CompletableFuture { + private volatile Cancellable cancellable; + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + boolean cancelled = super.cancel(mayInterruptIfRunning); + if (cancelled && cancellable != null) { + cancellable.cancel(); + } + return cancelled; + } + } + + private final RestClient restClient; + private final JsonpMapper mapper; + private final RestClientOptions transportOptions; + + public RestClientMonolithTransport(RestClient restClient, JsonpMapper mapper, @Nullable TransportOptions options) { + this.restClient = restClient; + this.mapper = mapper; + this.transportOptions = createOptions(options); + } + + protected RestClientOptions createOptions(@Nullable TransportOptions options) { + return options == null ? RestClientOptions.initialOptions() : RestClientOptions.of(options); + } + + public RestClientMonolithTransport(RestClient restClient, JsonpMapper mapper) { + this(restClient, mapper, null); + } + + /** + * Returns the underlying low level Rest Client used by this transport. + */ + public RestClient restClient() { + return this.restClient; + } + + /** + * Copies this {@link #RestClientTransport} with specific request options. + */ + public RestClientTransport withRequestOptions(@Nullable TransportOptions options) { + return new RestClientTransport(this.restClient, this.mapper, options); + } + + @Override + public JsonpMapper jsonpMapper() { + return mapper; + } + + @Override + public TransportOptions options() { + return transportOptions; + } + + @Override + public void close() throws IOException { + this.restClient.close(); + } + + public ResponseT performRequest( + RequestT request, + Endpoint endpoint, + @Nullable TransportOptions options + ) throws IOException { + + org.elasticsearch.client.Request clientReq = prepareLowLevelRequest(request, endpoint, options); + org.elasticsearch.client.Response clientResp = restClient.performRequest(clientReq); + return getHighLevelResponse(clientResp, endpoint); + } + + public CompletableFuture performRequestAsync( + RequestT request, + Endpoint endpoint, + @Nullable TransportOptions options + ) { + RequestFuture future = new RequestFuture<>(); + org.elasticsearch.client.Request clientReq; + try { + clientReq = prepareLowLevelRequest(request, endpoint, options); + } catch (Exception e) { + // Terminate early + future.completeExceptionally(e); + return future; + } + + // Propagate required property checks to the thread that will decode the response + boolean disableRequiredChecks = ApiTypeHelper.requiredPropertiesCheckDisabled(); + + future.cancellable = restClient.performRequestAsync(clientReq, new ResponseListener() { + @Override + public void onSuccess(Response clientResp) { + try (ApiTypeHelper.DisabledChecksHandle h = + ApiTypeHelper.DANGEROUS_disableRequiredPropertiesCheck(disableRequiredChecks)) { + + ResponseT response = getHighLevelResponse(clientResp, endpoint); + future.complete(response); + + } catch (Exception e) { + future.completeExceptionally(e); + } + } + + @Override + public void onFailure(Exception e) { + future.completeExceptionally(e); + } + }); + + return future; + } + + private org.elasticsearch.client.Request prepareLowLevelRequest( + RequestT request, + Endpoint endpoint, + @Nullable TransportOptions options + ) throws IOException { + String method = endpoint.method(request); + String path = endpoint.requestUrl(request); + Map params = endpoint.queryParameters(request); + + org.elasticsearch.client.Request clientReq = new org.elasticsearch.client.Request(method, path); + + RequestOptions restOptions = options == null ? + transportOptions.restClientRequestOptions() : + RestClientOptions.of(options).restClientRequestOptions(); + + if (restOptions != null) { + clientReq.setOptions(restOptions); + } + + clientReq.addParameters(params); + + Object body = endpoint.body(request); + if (body != null) { + // Request has a body + if (body instanceof NdJsonpSerializable) { + List lines = new ArrayList<>(); + collectNdJsonLines(lines, (NdJsonpSerializable) request); + clientReq.setEntity(new MultiBufferEntity(lines, JsonContentType)); + + } else if (body instanceof BinaryData) { + BinaryData data = (BinaryData)body; + + // ES expects the Accept and Content-Type headers to be consistent. + ContentType contentType; + String dataContentType = data.contentType(); + if (co.elastic.clients.util.ContentType.APPLICATION_JSON.equals(dataContentType)) { + // Fast path + contentType = JsonContentType; + } else { + contentType = ContentType.parse(dataContentType); + } + + clientReq.setEntity(new MultiBufferEntity( + Collections.singletonList(data.asByteBuffer()), + contentType + )); + + } else { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator generator = mapper.jsonProvider().createGenerator(baos); + mapper.serialize(body, generator); + generator.close(); + clientReq.setEntity(new ByteArrayEntity(baos.toByteArray(), JsonContentType)); + } + } + + // Request parameter intercepted by LLRC + clientReq.addParameter("ignore", "400,401,403,404,405"); + return clientReq; + } + + private static final ByteBuffer NdJsonSeparator = ByteBuffer.wrap("\n".getBytes(StandardCharsets.UTF_8)); + + private void collectNdJsonLines(List lines, NdJsonpSerializable value) throws IOException { + Iterator values = value._serializables(); + while(values.hasNext()) { + Object item = values.next(); + if (item == null) { + // Skip + } else if (item instanceof NdJsonpSerializable && item != value) { // do not recurse on the item itself + collectNdJsonLines(lines, (NdJsonpSerializable)item); + } else { + // TODO: items that aren't already BinaryData could be serialized to ByteBuffers lazily + // to reduce the number of buffers to keep in memory + lines.add(BinaryData.of(item, this.mapper).asByteBuffer()); + lines.add(NdJsonSeparator); + } + } + } + +// /** +// * Write an nd-json value by serializing each of its items on a separate line, recursing if its items themselves implement +// * {@link NdJsonpSerializable} to flattening nested structures. +// */ +// private void writeNdJson(NdJsonpSerializable value, ByteArrayOutputStream baos) throws IOException { +// Iterator values = value._serializables(); +// while(values.hasNext()) { +// Object item = values.next(); +// if (item instanceof NdJsonpSerializable && item != value) { // do not recurse on the item itself +// writeNdJson((NdJsonpSerializable) item, baos); +// } else { +// JsonGenerator generator = mapper.jsonProvider().createGenerator(baos); +// mapper.serialize(item, generator); +// generator.close(); +// baos.write('\n'); +// } +// } +// } + + private ResponseT getHighLevelResponse( + org.elasticsearch.client.Response clientResp, + Endpoint endpoint + ) throws IOException { + + int statusCode = clientResp.getStatusLine().getStatusCode(); + try { + + if (statusCode == 200) { + checkProductHeader(clientResp, endpoint); + } + + if (endpoint.isError(statusCode)) { + JsonpDeserializer errorDeserializer = endpoint.errorDeserializer(statusCode); + if (errorDeserializer == null) { + throw new TransportException( + statusCode, + "Request failed with status code '" + statusCode + "'", + endpoint.id(), new ResponseException(clientResp) + ); + } + + HttpEntity entity = clientResp.getEntity(); + if (entity == null) { + throw new TransportException( + statusCode, + "Expecting a response body, but none was sent", + endpoint.id(), new ResponseException(clientResp) + ); + } + + // We may have to replay it. + entity = new BufferedHttpEntity(entity); + + try { + InputStream content = entity.getContent(); + try (JsonParser parser = mapper.jsonProvider().createParser(content)) { + ErrorT error = errorDeserializer.deserialize(parser, mapper); + // TODO: have the endpoint provide the exception constructor + throw new ElasticsearchException(endpoint.id(), (ErrorResponse) error); + } + } catch(JsonException | MissingRequiredPropertyException errorEx) { + // Could not decode exception, try the response type + try { + ResponseT response = decodeResponse(statusCode, entity, clientResp, endpoint); + return response; + } catch(Exception respEx) { + // No better luck: throw the original error decoding exception + throw new TransportException(statusCode, + "Failed to decode error response, check exception cause for additional details", endpoint.id(), + new ResponseException(clientResp) + ); + } + } + } else { + return decodeResponse(statusCode, clientResp.getEntity(), clientResp, endpoint); + } + } finally { + // Consume the entity unless this is a successful binary endpoint, where the user must consume the entity + if (!(endpoint instanceof BinaryEndpoint && !endpoint.isError(statusCode))) { + EntityUtils.consume(clientResp.getEntity()); + } + } + } + + private ResponseT decodeResponse( + int statusCode, @Nullable HttpEntity entity, Response clientResp, Endpoint endpoint + ) throws IOException { + + if (endpoint instanceof JsonEndpoint) { + @SuppressWarnings("unchecked") + JsonEndpoint jsonEndpoint = (JsonEndpoint) endpoint; + // Successful response + ResponseT response = null; + JsonpDeserializer responseParser = jsonEndpoint.responseDeserializer(); + if (responseParser != null) { + // Expecting a body + if (entity == null) { + throw new TransportException( + statusCode, + "Expecting a response body, but none was sent", + endpoint.id(), new ResponseException(clientResp) + ); + } + InputStream content = entity.getContent(); + try (JsonParser parser = mapper.jsonProvider().createParser(content)) { + response = responseParser.deserialize(parser, mapper); + } + } + return response; + + } else if(endpoint instanceof BooleanEndpoint) { + BooleanEndpoint bep = (BooleanEndpoint) endpoint; + + @SuppressWarnings("unchecked") + ResponseT response = (ResponseT) new BooleanResponse(bep.getResult(statusCode)); + return response; + + + } else if (endpoint instanceof BinaryEndpoint) { + BinaryEndpoint bep = (BinaryEndpoint) endpoint; + + @SuppressWarnings("unchecked") + ResponseT response = (ResponseT) new HttpClientBinaryResponse(entity); + return response; + + } else { + throw new TransportException(statusCode, "Unhandled endpoint type: '" + endpoint.getClass().getName() + "'", endpoint.id()); + } + } + + // Endpoints that (incorrectly) do not return the Elastic product header + private static final Set endpointsMissingProductHeader = new HashSet<>(Arrays.asList( + "es/snapshot.create" // #74 / elastic/elasticsearch#82358 + )); + + private void checkProductHeader(Response clientResp, Endpoint endpoint) throws IOException { + String header = clientResp.getHeader("X-Elastic-Product"); + if (header == null) { + if (endpointsMissingProductHeader.contains(endpoint.id())) { + return; + } + throw new TransportException( + clientResp.getStatusLine().getStatusCode(), + "Missing [X-Elastic-Product] header. Please check that you are connecting to an Elasticsearch " + + "instance, and that any networking filters are preserving that header.", + endpoint.id(), + new ResponseException(clientResp) + ); + } + + if (!"Elasticsearch".equals(header)) { + throw new TransportException( + clientResp.getStatusLine().getStatusCode(), + "Invalid value '" + header + "' for 'X-Elastic-Product' header.", + endpoint.id(), + new ResponseException(clientResp) + ); + } + } +} diff --git a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientOptions.java b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientOptions.java index 26943dc82..627d8fa11 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientOptions.java +++ b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientOptions.java @@ -173,7 +173,7 @@ private static RequestOptions.Builder addBuiltinHeaders(RequestOptions.Builder b builder.addHeader(USER_AGENT_HEADER, USER_AGENT_VALUE); } if (builder.getHeaders().stream().noneMatch(h -> h.getName().equalsIgnoreCase("Accept"))) { - builder.addHeader("Accept", RestClientTransport.JsonContentType.toString()); + builder.addHeader("Accept", RestClientTransport.JsonContentType); } return builder; diff --git a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientTransport.java b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientTransport.java index 7e4bfa3cc..b0ee3e284 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientTransport.java +++ b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientTransport.java @@ -19,33 +19,18 @@ package co.elastic.clients.transport.rest_client; -import co.elastic.clients.elasticsearch._types.ElasticsearchException; -import co.elastic.clients.elasticsearch._types.ErrorResponse; -import co.elastic.clients.json.JsonpDeserializer; import co.elastic.clients.json.JsonpMapper; -import co.elastic.clients.json.NdJsonpSerializable; -import co.elastic.clients.transport.JsonEndpoint; -import co.elastic.clients.transport.TransportException; -import co.elastic.clients.transport.Version; -import co.elastic.clients.transport.endpoints.BinaryEndpoint; -import co.elastic.clients.transport.endpoints.BooleanEndpoint; -import co.elastic.clients.transport.endpoints.BooleanResponse; +import co.elastic.clients.transport.TransportBase; import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.Endpoint; import co.elastic.clients.transport.TransportOptions; -import co.elastic.clients.util.ApiTypeHelper; import co.elastic.clients.util.BinaryData; -import co.elastic.clients.util.MissingRequiredPropertyException; -import jakarta.json.JsonException; -import jakarta.json.stream.JsonGenerator; -import jakarta.json.stream.JsonParser; +import co.elastic.clients.util.NoCopyByteArrayOutputStream; +import org.apache.http.Header; import org.apache.http.HttpEntity; -import org.apache.http.entity.BufferedHttpEntity; -import org.apache.http.entity.ByteArrayEntity; import org.apache.http.entity.ContentType; -import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Cancellable; +import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; @@ -53,36 +38,16 @@ import org.elasticsearch.client.RestClient; import javax.annotation.Nullable; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; -public class RestClientTransport implements ElasticsearchTransport { +public class RestClientTransport extends TransportBase implements ElasticsearchTransport { - static final ContentType JsonContentType; - - static { - - if (Version.VERSION == null) { - JsonContentType = ContentType.APPLICATION_JSON; - } else { - JsonContentType = ContentType.create( - "application/vnd.elasticsearch+json", - new BasicNameValuePair("compatible-with", String.valueOf(Version.VERSION.major())) - ); - } - } + private static final ConcurrentHashMap ContentTypeCache = new ConcurrentHashMap<>(); /** * The {@code Future} implementation returned by async requests. @@ -102,19 +67,16 @@ public boolean cancel(boolean mayInterruptIfRunning) { } private final RestClient restClient; - private final JsonpMapper mapper; - private final RestClientOptions transportOptions; - - public RestClientTransport(RestClient restClient, JsonpMapper mapper, @Nullable TransportOptions options) { - this.restClient = restClient; - this.mapper = mapper; - this.transportOptions = options == null ? RestClientOptions.initialOptions() : RestClientOptions.of(options); - } public RestClientTransport(RestClient restClient, JsonpMapper mapper) { this(restClient, mapper, null); } + public RestClientTransport(RestClient restClient, JsonpMapper mapper, @Nullable TransportOptions options) { + super(mapper, options == null ? RestClientOptions.initialOptions() : RestClientOptions.of(options)); + this.restClient = restClient; + } + /** * Returns the underlying low level Rest Client used by this transport. */ @@ -122,90 +84,56 @@ public RestClient restClient() { return this.restClient; } - /** - * Copies this {@link #RestClientTransport} with specific request options. - */ - public RestClientTransport withRequestOptions(@Nullable TransportOptions options) { - return new RestClientTransport(this.restClient, this.mapper, options); - } - @Override - public JsonpMapper jsonpMapper() { - return mapper; + protected RestClientOptions createOptions(@Nullable TransportOptions options) { + return options == null ? transportOptions : RestClientOptions.of(options); } @Override - public TransportOptions options() { - return transportOptions; + protected TransportResponse performRequest(TransportRequest request, RestClientOptions options) throws IOException { + Request restRequest = createRestRequest(request, options); + Response restResponse = restClient.performRequest(restRequest); + return new RestResponse(restResponse); } @Override - public void close() throws IOException { - this.restClient.close(); - } - - public ResponseT performRequest( - RequestT request, - Endpoint endpoint, - @Nullable TransportOptions options - ) throws IOException { + protected CompletableFuture performRequestAsync(TransportRequest request, RestClientOptions options) { - org.elasticsearch.client.Request clientReq = prepareLowLevelRequest(request, endpoint, options); - org.elasticsearch.client.Response clientResp = restClient.performRequest(clientReq); - return getHighLevelResponse(clientResp, endpoint); - } + RequestFuture future = new RequestFuture<>(); + org.elasticsearch.client.Request restRequest; - public CompletableFuture performRequestAsync( - RequestT request, - Endpoint endpoint, - @Nullable TransportOptions options - ) { - RequestFuture future = new RequestFuture<>(); - org.elasticsearch.client.Request clientReq; try { - clientReq = prepareLowLevelRequest(request, endpoint, options); - } catch (Exception e) { + restRequest = createRestRequest(request, options); + } catch(Throwable thr) { // Terminate early - future.completeExceptionally(e); + future.completeExceptionally(thr); return future; } - // Propagate required property checks to the thread that will decode the response - boolean disableRequiredChecks = ApiTypeHelper.requiredPropertiesCheckDisabled(); - - future.cancellable = restClient.performRequestAsync(clientReq, new ResponseListener() { + future.cancellable = restClient.performRequestAsync(restRequest, new ResponseListener() { @Override - public void onSuccess(Response clientResp) { - try (ApiTypeHelper.DisabledChecksHandle h = - ApiTypeHelper.DANGEROUS_disableRequiredPropertiesCheck(disableRequiredChecks)) { - - ResponseT response = getHighLevelResponse(clientResp, endpoint); - future.complete(response); - - } catch (Exception e) { - future.completeExceptionally(e); - } + public void onSuccess(org.elasticsearch.client.Response response) { + future.complete(new RestResponse(response)); } @Override - public void onFailure(Exception e) { - future.completeExceptionally(e); + public void onFailure(Exception exception) { + future.completeExceptionally(exception); } }); return future; } - private org.elasticsearch.client.Request prepareLowLevelRequest( - RequestT request, - Endpoint endpoint, - @Nullable TransportOptions options - ) throws IOException { - String method = endpoint.method(request); - String path = endpoint.requestUrl(request); - Map params = endpoint.queryParameters(request); + @Override + public void close() throws IOException { + this.restClient.close(); + } - org.elasticsearch.client.Request clientReq = new org.elasticsearch.client.Request(method, path); + private org.elasticsearch.client.Request createRestRequest(TransportRequest request, RestClientOptions options) { + org.elasticsearch.client.Request clientReq = new org.elasticsearch.client.Request( + request.method, request.path + ); RequestOptions restOptions = options == null ? transportOptions.restClientRequestOptions() : @@ -215,41 +143,12 @@ private org.elasticsearch.client.Request prepareLowLevelRequest( clientReq.setOptions(restOptions); } - clientReq.addParameters(params); + clientReq.addParameters(request.queryParams); - Object body = endpoint.body(request); + Iterable body = request.body; if (body != null) { - // Request has a body - if (body instanceof NdJsonpSerializable) { - List lines = new ArrayList<>(); - collectNdJsonLines(lines, (NdJsonpSerializable) request); - clientReq.setEntity(new MultiBufferEntity(lines, JsonContentType)); - - } else if (body instanceof BinaryData) { - BinaryData data = (BinaryData)body; - - // ES expects the Accept and Content-Type headers to be consistent. - ContentType contentType; - String dataContentType = data.contentType(); - if (co.elastic.clients.util.ContentType.APPLICATION_JSON.equals(dataContentType)) { - // Fast path - contentType = JsonContentType; - } else { - contentType = ContentType.parse(dataContentType); - } - - clientReq.setEntity(new MultiBufferEntity( - Collections.singletonList(data.asByteBuffer()), - contentType - )); - - } else { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - JsonGenerator generator = mapper.jsonProvider().createGenerator(baos); - mapper.serialize(body, generator); - generator.close(); - clientReq.setEntity(new ByteArrayEntity(baos.toByteArray(), JsonContentType)); - } + ContentType ct = ContentTypeCache.computeIfAbsent(request.contentType, ContentType::parse); + clientReq.setEntity(new MultiBufferEntity(body, ct)); } // Request parameter intercepted by LLRC @@ -257,182 +156,80 @@ private org.elasticsearch.client.Request prepareLowLevelRequest( return clientReq; } - private static final ByteBuffer NdJsonSeparator = ByteBuffer.wrap("\n".getBytes(StandardCharsets.UTF_8)); - - private void collectNdJsonLines(List lines, NdJsonpSerializable value) { - Iterator values = value._serializables(); - while(values.hasNext()) { - Object item = values.next(); - if (item == null) { - // Skip - } else if (item instanceof NdJsonpSerializable && item != value) { // do not recurse on the item itself - collectNdJsonLines(lines, (NdJsonpSerializable)item); - } else { - // TODO: items that aren't already BinaryData could be serialized to ByteBuffers lazily - // to reduce the number of buffers to keep in memory - lines.add(BinaryData.of(item, this.mapper).asByteBuffer()); - lines.add(NdJsonSeparator); - } + private static class RestResponse implements TransportResponse { + private final org.elasticsearch.client.Response restResponse; + + public RestResponse(org.elasticsearch.client.Response restResponse) { + this.restResponse = restResponse; } - } - /** - * Write an nd-json value by serializing each of its items on a separate line, recursing if its items themselves implement - * {@link NdJsonpSerializable} to flattening nested structures. - */ - private void writeNdJson(NdJsonpSerializable value, ByteArrayOutputStream baos) throws IOException { - Iterator values = value._serializables(); - while(values.hasNext()) { - Object item = values.next(); - if (item instanceof NdJsonpSerializable && item != value) { // do not recurse on the item itself - writeNdJson((NdJsonpSerializable) item, baos); - } else { - JsonGenerator generator = mapper.jsonProvider().createGenerator(baos); - mapper.serialize(item, generator); - generator.close(); - baos.write('\n'); - } + @Override + public int statusCode() { + return restResponse.getStatusLine().getStatusCode(); } - } - private ResponseT getHighLevelResponse( - org.elasticsearch.client.Response clientResp, - Endpoint endpoint - ) throws IOException { + @Override + public String getHeader(String name) { + return restResponse.getHeader(name); + } - int statusCode = clientResp.getStatusLine().getStatusCode(); - try { + @Nullable + @Override + public BinaryData getBody() throws IOException { + HttpEntity entity = restResponse.getEntity(); + return entity == null ? null : new HttpEntityBinaryData(restResponse.getEntity()); + } - if (statusCode == 200) { - checkProductHeader(clientResp, endpoint); - } + @Override + public Throwable createException() throws IOException { + return new ResponseException(this.restResponse); + } - if (endpoint.isError(statusCode)) { - JsonpDeserializer errorDeserializer = endpoint.errorDeserializer(statusCode); - if (errorDeserializer == null) { - throw new TransportException( - statusCode, - "Request failed with status code '" + statusCode + "'", - endpoint.id(), new ResponseException(clientResp) - ); - } - - HttpEntity entity = clientResp.getEntity(); - if (entity == null) { - throw new TransportException( - statusCode, - "Expecting a response body, but none was sent", - endpoint.id(), new ResponseException(clientResp) - ); - } - - // We may have to replay it. - entity = new BufferedHttpEntity(entity); - - try { - InputStream content = entity.getContent(); - try (JsonParser parser = mapper.jsonProvider().createParser(content)) { - ErrorT error = errorDeserializer.deserialize(parser, mapper); - // TODO: have the endpoint provide the exception constructor - throw new ElasticsearchException(endpoint.id(), (ErrorResponse) error); - } - } catch(JsonException | MissingRequiredPropertyException errorEx) { - // Could not decode exception, try the response type - try { - ResponseT response = decodeResponse(statusCode, entity, clientResp, endpoint); - return response; - } catch(Exception respEx) { - // No better luck: throw the original error decoding exception - throw new TransportException(statusCode, - "Failed to decode error response, check exception cause for additional details", endpoint.id(), - new ResponseException(clientResp) - ); - } - } - } else { - return decodeResponse(statusCode, clientResp.getEntity(), clientResp, endpoint); - } - } finally { - // Consume the entity unless this is a successful binary endpoint, where the user must consume the entity - if (!(endpoint instanceof BinaryEndpoint && !endpoint.isError(statusCode))) { - EntityUtils.consume(clientResp.getEntity()); - } + @Override + public void close() throws IOException { + EntityUtils.consume(restResponse.getEntity()); } } - private ResponseT decodeResponse( - int statusCode, @Nullable HttpEntity entity, Response clientResp, Endpoint endpoint - ) throws IOException { - - if (endpoint instanceof JsonEndpoint) { - @SuppressWarnings("unchecked") - JsonEndpoint jsonEndpoint = (JsonEndpoint) endpoint; - // Successful response - ResponseT response = null; - JsonpDeserializer responseParser = jsonEndpoint.responseDeserializer(); - if (responseParser != null) { - // Expecting a body - if (entity == null) { - throw new TransportException( - statusCode, - "Expecting a response body, but none was sent", - endpoint.id(), new ResponseException(clientResp) - ); - } - InputStream content = entity.getContent(); - try (JsonParser parser = mapper.jsonProvider().createParser(content)) { - response = responseParser.deserialize(parser, mapper); - } - } - return response; - - } else if(endpoint instanceof BooleanEndpoint) { - BooleanEndpoint bep = (BooleanEndpoint) endpoint; - - @SuppressWarnings("unchecked") - ResponseT response = (ResponseT) new BooleanResponse(bep.getResult(statusCode)); - return response; + private static class HttpEntityBinaryData implements BinaryData { + private final HttpEntity entity; + public HttpEntityBinaryData(HttpEntity entity) { + this.entity = entity; + } - } else if (endpoint instanceof BinaryEndpoint) { - BinaryEndpoint bep = (BinaryEndpoint) endpoint; + @Override + public String contentType() { + Header h = entity.getContentType(); + return h == null ? "application/octet-stream" : h.getValue(); + } - @SuppressWarnings("unchecked") - ResponseT response = (ResponseT) new HttpClientBinaryResponse(entity); - return response; + @Override + public void writeTo(OutputStream out) throws IOException { + entity.writeTo(out); + } - } else { - throw new TransportException(statusCode, "Unhandled endpoint type: '" + endpoint.getClass().getName() + "'", endpoint.id()); + @Override + public ByteBuffer asByteBuffer() throws IOException { + NoCopyByteArrayOutputStream out = new NoCopyByteArrayOutputStream(); + entity.writeTo(out); + return out.asByteBuffer(); } - } - // Endpoints that (incorrectly) do not return the Elastic product header - private static final Set endpointsMissingProductHeader = new HashSet<>(Arrays.asList( - "es/snapshot.create" // #74 / elastic/elasticsearch#82358 - )); + @Override + public InputStream asInputStream() throws IOException { + return entity.getContent(); + } - private void checkProductHeader(Response clientResp, Endpoint endpoint) throws IOException { - String header = clientResp.getHeader("X-Elastic-Product"); - if (header == null) { - if (endpointsMissingProductHeader.contains(endpoint.id())) { - return; - } - throw new TransportException( - clientResp.getStatusLine().getStatusCode(), - "Missing [X-Elastic-Product] header. Please check that you are connecting to an Elasticsearch " - + "instance, and that any networking filters are preserving that header.", - endpoint.id(), - new ResponseException(clientResp) - ); + @Override + public boolean isRepeatable() { + return entity.isRepeatable(); } - if (!"Elasticsearch".equals(header)) { - throw new TransportException( - clientResp.getStatusLine().getStatusCode(), - "Invalid value '" + header + "' for 'X-Elastic-Product' header.", - endpoint.id(), - new ResponseException(clientResp) - ); + @Override + public long size() { + long len = entity.getContentLength(); + return len < 0 ? -1 : entity.getContentLength(); } } } diff --git a/java-client/src/main/java/co/elastic/clients/util/BinaryData.java b/java-client/src/main/java/co/elastic/clients/util/BinaryData.java index 02ec646b1..ddd9123b7 100644 --- a/java-client/src/main/java/co/elastic/clients/util/BinaryData.java +++ b/java-client/src/main/java/co/elastic/clients/util/BinaryData.java @@ -27,6 +27,7 @@ import jakarta.json.stream.JsonParser; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; @@ -43,13 +44,31 @@ public interface BinaryData { /** * Write this data to an output stream. + * @throws IllegalStateException if the content has already been consumed and the object + * isn't replayable. */ void writeTo(OutputStream out) throws IOException; /** - * Return this data as a {@code ByteBuffer} + * Return this data as a {@code ByteBuffer}. + * + * @throws IllegalStateException if the content has already been consumed and the object + * isn't replayable. + */ + ByteBuffer asByteBuffer() throws IOException; + + /** + * Return this data as an {@code InputStream}. + * + * @throws IllegalStateException if the content has already been consumed and the object + * isn't replayable. + */ + InputStream asInputStream() throws IOException; + + /** + * Can this object be consumed several times? */ - ByteBuffer asByteBuffer(); + boolean isRepeatable(); /** * Get the estimated size in bytes of the data. diff --git a/java-client/src/main/java/co/elastic/clients/util/ByteArrayBinaryData.java b/java-client/src/main/java/co/elastic/clients/util/ByteArrayBinaryData.java index 4b128a2e3..6509e77ab 100644 --- a/java-client/src/main/java/co/elastic/clients/util/ByteArrayBinaryData.java +++ b/java-client/src/main/java/co/elastic/clients/util/ByteArrayBinaryData.java @@ -27,8 +27,10 @@ import jakarta.json.stream.JsonGenerator; import jakarta.json.stream.JsonParser; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.EnumSet; @@ -55,6 +57,18 @@ public class ByteArrayBinaryData implements BinaryData { this.length = bytes.length; } + /** + * Copy another {@link BinaryData}. Typically used to make a replayable {@link BinaryData} + * from a non-replayable one. + */ + public ByteArrayBinaryData(BinaryData data) throws IOException { + NoCopyByteArrayOutputStream out = new NoCopyByteArrayOutputStream(); + data.writeTo(out); + this.contentType = data.contentType(); + this.bytes = out.array(); + this.offset = 0; + this.length = out.size(); + } @Override public String contentType() { @@ -76,6 +90,16 @@ public ByteBuffer asByteBuffer() { return ByteBuffer.wrap(bytes, offset, length); } + @Override + public InputStream asInputStream() { + return new ByteArrayInputStream(bytes, offset, length); + } + + @Override + public boolean isRepeatable() { + return true; + } + private static class Deserializer extends JsonpDeserializerBase { Deserializer() { diff --git a/java-client/src/main/java/co/elastic/clients/util/NoCopyByteArrayOutputStream.java b/java-client/src/main/java/co/elastic/clients/util/NoCopyByteArrayOutputStream.java index 5d91204a6..2021a50bc 100644 --- a/java-client/src/main/java/co/elastic/clients/util/NoCopyByteArrayOutputStream.java +++ b/java-client/src/main/java/co/elastic/clients/util/NoCopyByteArrayOutputStream.java @@ -21,6 +21,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; /** * A {@code ByteArrayOutputStream} that reduces copy operations of its underlying buffer. @@ -48,4 +49,11 @@ public byte[] array() { public ByteArrayInputStream asInputStream() { return new ByteArrayInputStream(this.buf, 0, this.count); } + + /** + * Get a {@code ByteBuffer} view on this object, based on the current buffer and size. + */ + public ByteBuffer asByteBuffer() { + return ByteBuffer.wrap(this.buf, 0, this.count); + } } diff --git a/java-client/src/test/java/co/elastic/clients/transport/rest_client/RestClientOptionsTest.java b/java-client/src/test/java/co/elastic/clients/transport/rest_client/RestClientOptionsTest.java index 488e55ff1..cd6558a4f 100644 --- a/java-client/src/test/java/co/elastic/clients/transport/rest_client/RestClientOptionsTest.java +++ b/java-client/src/test/java/co/elastic/clients/transport/rest_client/RestClientOptionsTest.java @@ -20,7 +20,9 @@ package co.elastic.clients.transport.rest_client; import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.json.JsonpMapper; import co.elastic.clients.json.SimpleJsonpMapper; +import co.elastic.clients.transport.ElasticsearchTransport; import co.elastic.clients.transport.Version; import co.elastic.clients.transport.endpoints.BooleanResponse; import com.sun.net.httpserver.Headers; @@ -58,6 +60,8 @@ public static void setup() throws IOException { // Register a handler on the core.exists("capture-handler/{name}") endpoint that will capture request headers. httpServer.createContext("/capture-headers/_doc/", exchange -> { String testName = exchange.getRequestURI().getPath().substring("/capture-headers/_doc/".length()); + System.out.println(exchange.getResponseHeaders()); + System.out.println(); collectedHeaders.put(testName, exchange.getRequestHeaders()); // Reply with an empty 200 response @@ -75,6 +79,15 @@ public static void cleanup() { httpServer = null; collectedHeaders = null; } + + private ElasticsearchTransport newRestClientTransport(RestClient restClient, JsonpMapper mapper) { + return newRestClientTransport(restClient, mapper, null); + } + + private ElasticsearchTransport newRestClientTransport(RestClient restClient, JsonpMapper mapper, RestClientOptions options) { + return new RestClientTransport(restClient, mapper, options); + //return new RestClientMonolithTransport(restClient, mapper, options); + } /** * Make a server call, capture request headers and check their consistency. @@ -114,7 +127,7 @@ void testNoRequestOptions() throws Exception { new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort(), "http") ).build(); - RestClientTransport transport = new RestClientTransport(llrc, new SimpleJsonpMapper()); + ElasticsearchTransport transport = newRestClientTransport(llrc, new SimpleJsonpMapper()); ElasticsearchClient esClient = new ElasticsearchClient(transport); String id = checkHeaders(esClient); @@ -127,7 +140,7 @@ void testTransportRequestOptions() throws Exception { new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort(), "http") ).build(); - RestClientTransport transport = new RestClientTransport(llrc, new SimpleJsonpMapper(), + ElasticsearchTransport transport = newRestClientTransport(llrc, new SimpleJsonpMapper(), new RestClientOptions.Builder(RequestOptions.DEFAULT.toBuilder()).build() ); ElasticsearchClient esClient = new ElasticsearchClient(transport); @@ -142,7 +155,7 @@ void testClientRequestOptions() throws Exception { new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort(), "http") ).build(); - RestClientTransport transport = new RestClientTransport(llrc, new SimpleJsonpMapper()); + ElasticsearchTransport transport = newRestClientTransport(llrc, new SimpleJsonpMapper()); ElasticsearchClient esClient = new ElasticsearchClient(transport).withTransportOptions( new RestClientOptions.Builder(RequestOptions.DEFAULT.toBuilder()).build() ); @@ -157,7 +170,7 @@ void testLambdaOptionsBuilder() throws Exception { new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort(), "http") ).build(); - RestClientTransport transport = new RestClientTransport(llrc, new SimpleJsonpMapper()); + ElasticsearchTransport transport = newRestClientTransport(llrc, new SimpleJsonpMapper()); ElasticsearchClient esClient = new ElasticsearchClient(transport) .withTransportOptions(o -> o .addHeader("Foo", "bar") @@ -179,7 +192,7 @@ void testRequestOptionsOverridingBuiltin() throws Exception { new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort(), "http") ).build(); - RestClientTransport transport = new RestClientTransport(llrc, new SimpleJsonpMapper(), new RestClientOptions(options)); + ElasticsearchTransport transport = newRestClientTransport(llrc, new SimpleJsonpMapper(), new RestClientOptions(options)); ElasticsearchClient esClient = new ElasticsearchClient(transport); // Should not override client meta String id = checkHeaders(esClient); From 6417f4ac9b2240465bcd2ef7487b55844d28ef7f Mon Sep 17 00:00:00 2001 From: Sylvain Wallez Date: Tue, 30 May 2023 19:10:14 +0200 Subject: [PATCH 2/5] Split transport and http client --- ...e.java => ElasticsearchTransportBase.java} | 74 +++--- .../transport/http/TransportHttpClient.java | 144 ++++++++++++ .../rest_client/RestClientHttpClient.java | 221 ++++++++++++++++++ .../RestClientMonolithTransport.java | 6 +- .../rest_client/RestClientTransport.java | 204 +--------------- .../ElasticsearchTestServer.java | 4 +- 6 files changed, 406 insertions(+), 247 deletions(-) rename java-client/src/main/java/co/elastic/clients/transport/{TransportBase.java => ElasticsearchTransportBase.java} (85%) create mode 100644 java-client/src/main/java/co/elastic/clients/transport/http/TransportHttpClient.java create mode 100644 java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientHttpClient.java diff --git a/java-client/src/main/java/co/elastic/clients/transport/TransportBase.java b/java-client/src/main/java/co/elastic/clients/transport/ElasticsearchTransportBase.java similarity index 85% rename from java-client/src/main/java/co/elastic/clients/transport/TransportBase.java rename to java-client/src/main/java/co/elastic/clients/transport/ElasticsearchTransportBase.java index 941ae4edc..99e589039 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/TransportBase.java +++ b/java-client/src/main/java/co/elastic/clients/transport/ElasticsearchTransportBase.java @@ -28,6 +28,7 @@ import co.elastic.clients.transport.endpoints.BinaryEndpoint; import co.elastic.clients.transport.endpoints.BooleanEndpoint; import co.elastic.clients.transport.endpoints.BooleanResponse; +import co.elastic.clients.transport.http.TransportHttpClient; import co.elastic.clients.util.ApiTypeHelper; import co.elastic.clients.util.BinaryData; import co.elastic.clients.util.ByteArrayBinaryData; @@ -53,25 +54,9 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; -public abstract class TransportBase< +public abstract class ElasticsearchTransportBase< Options extends TransportOptions - > implements Transport { - - public static class TransportRequest { - public final String method; - public final String path; - public final Map queryParams; - @Nullable public final String contentType; - @Nullable public final Iterable body; - - public TransportRequest(String method, String path, Map queryParams, @Nullable String contentType, @Nullable Iterable body) { - this.method = method; - this.path = path; - this.queryParams = queryParams; - this.contentType = contentType; - this.body = body; - } - } + > implements ElasticsearchTransport { public static final String JsonContentType; @@ -85,19 +70,20 @@ public TransportRequest(String method, String path, Map queryPar } } - /** - * Create implementation-specific options from optional initial options. - */ - protected abstract Options createOptions(@Nullable TransportOptions options); - protected abstract TransportResponse performRequest(TransportRequest request, Options options) throws IOException; - protected abstract CompletableFuture performRequestAsync(TransportRequest request, Options options); + private final TransportHttpClient httpClient; + + @Override + public void close() throws IOException { + httpClient.close(); + } private final JsonpMapper mapper; protected final Options transportOptions; - protected TransportBase(JsonpMapper jsonpMapper, Options options) { + public ElasticsearchTransportBase(JsonpMapper jsonpMapper, TransportHttpClient httpClient, Options options) { this.mapper = jsonpMapper; - this.transportOptions = options; + this.httpClient = httpClient; + this.transportOptions = options == null ? httpClient.createOptions(null) : options; } @Override @@ -116,15 +102,19 @@ public final ResponseT performRequest( Endpoint endpoint, @Nullable TransportOptions options ) throws IOException { - TransportRequest req = prepareTransportRequest(request, endpoint, options); - Options opts = createOptions(options); - TransportResponse resp = performRequest(req, opts); + TransportHttpClient.Request req = prepareTransportRequest(request, endpoint, options); + Options opts = options == null ? transportOptions : httpClient.createOptions(options); + TransportHttpClient.Response resp = httpClient.performRequest(endpoint.id(), req, opts); return getApiResponse(resp, endpoint); } @Override - public final CompletableFuture performRequestAsync(RequestT request, Endpoint endpoint, @Nullable TransportOptions options) { - TransportRequest clientReq; + public final CompletableFuture performRequestAsync( + RequestT request, + Endpoint endpoint, + @Nullable TransportOptions options + ) { + TransportHttpClient.Request clientReq; try { clientReq = prepareTransportRequest(request, endpoint, options); } catch (Exception e) { @@ -137,7 +127,9 @@ public final CompletableFuture performR // Propagate required property checks to the thread that will decode the response boolean disableRequiredChecks = ApiTypeHelper.requiredPropertiesCheckDisabled(); - CompletableFuture clientFuture = performRequestAsync(clientReq, createOptions(options)); + CompletableFuture clientFuture = httpClient.performRequestAsync( + endpoint.id(), clientReq, httpClient.createOptions(options) + ); // Cancelling the result will cancel the upstream future created by the http client, allowing to stop in-flight requests CompletableFuture future = new CompletableFuture() { @@ -171,7 +163,7 @@ public boolean cancel(boolean mayInterruptIfRunning) { return future; } - private TransportRequest prepareTransportRequest( + private TransportHttpClient.Request prepareTransportRequest( RequestT request, Endpoint endpoint, @Nullable TransportOptions options @@ -214,7 +206,7 @@ private TransportRequest prepareTransportRequest( } } - return new TransportRequest(method, path, params, contentType, bodyBuffers); + return new TransportHttpClient.Request(method, path, params, contentType, bodyBuffers); } private static final ByteBuffer NdJsonSeparator = ByteBuffer.wrap("\n".getBytes(StandardCharsets.UTF_8)); @@ -236,16 +228,8 @@ private void collectNdJsonLines(List lines, NdJsonpSerializable valu } } - protected interface TransportResponse { - int statusCode(); - String getHeader(String name); - @Nullable BinaryData getBody() throws IOException; - Throwable createException() throws IOException; - void close() throws IOException; - } - private ResponseT getApiResponse( - TransportResponse clientResp, + TransportHttpClient.Response clientResp, Endpoint endpoint ) throws IOException { @@ -314,7 +298,7 @@ private ResponseT getApiResponse( } private ResponseT decodeTransportResponse( - int statusCode, @Nullable BinaryData entity, TransportResponse clientResp, Endpoint endpoint + int statusCode, @Nullable BinaryData entity, TransportHttpClient.Response clientResp, Endpoint endpoint ) throws IOException { if (endpoint instanceof JsonEndpoint) { @@ -369,7 +353,7 @@ private ResponseT decodeTransportResponse( "es/snapshot.create" // #74 / elastic/elasticsearch#82358 )); - private void checkProductHeader(TransportResponse clientResp, Endpoint endpoint) throws IOException { + private void checkProductHeader(TransportHttpClient.Response clientResp, Endpoint endpoint) throws IOException { String header = clientResp.getHeader("X-Elastic-Product"); if (header == null) { if (endpointsMissingProductHeader.contains(endpoint.id())) { diff --git a/java-client/src/main/java/co/elastic/clients/transport/http/TransportHttpClient.java b/java-client/src/main/java/co/elastic/clients/transport/http/TransportHttpClient.java new file mode 100644 index 000000000..8e6af31b2 --- /dev/null +++ b/java-client/src/main/java/co/elastic/clients/transport/http/TransportHttpClient.java @@ -0,0 +1,144 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport.http; + +import co.elastic.clients.transport.TransportOptions; +import co.elastic.clients.util.BinaryData; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +/** + * Minimal http client interface needed to implement an Elasticsearch transport. + * + * @param the client's options type + */ +public interface TransportHttpClient { + + /** + * Create a client-specific options value from an existing option object. If {@code null}, this must + * create the default options to which additional options can be added. + */ + Options createOptions(@Nullable TransportOptions options); + + /** + * Perform a blocking request. + * + * @param endpointId the endpoint identifier. Can be used to have specific strategies depending on the endpoint. + * @param request the request + * @param options additional options for the http client + * + * @return the request response + */ + Response performRequest(String endpointId, Request request, Options options) throws IOException; + + /** + * Perform an asynchronous request. + * + * @param endpointId the endpoint identifier. Can be used to have specific strategies depending on the endpoint. + * @param request the request + * @param options additional options for the http client + * + * @return the request response + */ + CompletableFuture performRequestAsync(String endpointId, Request request, Options options); + + /** + * Close this client, freeing associated resources. + */ + void close() throws IOException; + + /** + * An http request. + */ + class Request { + private final String method; + private final String path; + private final Map queryParams; + @Nullable + private final String contentType; + @Nullable + private final Iterable body; + + public Request(String method, String path, Map queryParams, @Nullable String contentType, + @Nullable Iterable body) { + this.method = method; + this.path = path; + this.queryParams = queryParams; + this.contentType = contentType; + this.body = body; + } + + public String method() { + return method; + } + + public String path() { + return path; + } + + public Map queryParams() { + return queryParams; + } + + @Nullable + public String contentType() { + return contentType; + } + + @Nullable + public Iterable body() { + return body; + } + } + + /** + * An http response. + */ + interface Response { + + /** + * The response status code. + */ + int statusCode(); + + /** + * Get a header value, or the first value if the header has multiple values. + */ + String getHeader(String name); + + /** + * The response body, if any. + */ + @Nullable + BinaryData getBody() throws IOException; + + + Throwable createException() throws IOException; + + /** + * Close this response, freeing its associated resource, consuming the remaining body, if needed. + */ + void close() throws IOException; + } +} diff --git a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientHttpClient.java b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientHttpClient.java new file mode 100644 index 000000000..a887fcae2 --- /dev/null +++ b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientHttpClient.java @@ -0,0 +1,221 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport.rest_client; + +import co.elastic.clients.transport.TransportOptions; +import co.elastic.clients.transport.http.TransportHttpClient; +import co.elastic.clients.util.BinaryData; +import co.elastic.clients.util.NoCopyByteArrayOutputStream; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Cancellable; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.ResponseListener; +import org.elasticsearch.client.RestClient; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; + +public class RestClientHttpClient implements TransportHttpClient { + + private static final ConcurrentHashMap ContentTypeCache = new ConcurrentHashMap<>(); + + /** + * The {@code Future} implementation returned by async requests. + * It wraps the RestClient's cancellable and propagates cancellation. + */ + private static class RequestFuture extends CompletableFuture { + private volatile Cancellable cancellable; + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + boolean cancelled = super.cancel(mayInterruptIfRunning); + if (cancelled && cancellable != null) { + cancellable.cancel(); + } + return cancelled; + } + } + + private final RestClient restClient; + + public RestClientHttpClient(RestClient restClient) { + this.restClient = restClient; + } + + /** + * Returns the underlying low level Rest Client used by this transport. + */ + public RestClient restClient() { + return this.restClient; + } + + @Override + public RestClientOptions createOptions(@Nullable TransportOptions options) { + return options == null ? RestClientOptions.initialOptions() : RestClientOptions.of(options); + } + + @Override + public Response performRequest(String endpointId, Request request, RestClientOptions options) throws IOException { + org.elasticsearch.client.Request restRequest = createRestRequest(request, options); + org.elasticsearch.client.Response restResponse = restClient.performRequest(restRequest); + return new RestResponse(restResponse); + } + + @Override + public CompletableFuture performRequestAsync(String endpointId, Request request, RestClientOptions options) { + + RequestFuture future = new RequestFuture<>(); + org.elasticsearch.client.Request restRequest; + + try { + restRequest = createRestRequest(request, options); + } catch(Throwable thr) { + // Terminate early + future.completeExceptionally(thr); + return future; + } + + future.cancellable = restClient.performRequestAsync(restRequest, new ResponseListener() { + @Override + public void onSuccess(org.elasticsearch.client.Response response) { + future.complete(new RestResponse(response)); + } + + @Override + public void onFailure(Exception exception) { + future.completeExceptionally(exception); + } + }); + + return future; + } + + @Override + public void close() throws IOException { + this.restClient.close(); + } + + private org.elasticsearch.client.Request createRestRequest(Request request, RestClientOptions options) { + org.elasticsearch.client.Request clientReq = new org.elasticsearch.client.Request( + request.method(), request.path() + ); + + if (options != null) { + clientReq.setOptions(options.restClientRequestOptions()); + } + + clientReq.addParameters(request.queryParams()); + + Iterable body = request.body(); + if (body != null) { + ContentType ct = ContentTypeCache.computeIfAbsent(request.contentType(), ContentType::parse); + clientReq.setEntity(new MultiBufferEntity(body, ct)); + } + + // Request parameter intercepted by LLRC + clientReq.addParameter("ignore", "400,401,403,404,405"); + return clientReq; + } + + private static class RestResponse implements Response { + private final org.elasticsearch.client.Response restResponse; + + RestResponse(org.elasticsearch.client.Response restResponse) { + this.restResponse = restResponse; + } + + @Override + public int statusCode() { + return restResponse.getStatusLine().getStatusCode(); + } + + @Override + public String getHeader(String name) { + return restResponse.getHeader(name); + } + + @Nullable + @Override + public BinaryData getBody() throws IOException { + HttpEntity entity = restResponse.getEntity(); + return entity == null ? null : new HttpEntityBinaryData(restResponse.getEntity()); + } + + @Override + public Throwable createException() throws IOException { + return new ResponseException(this.restResponse); + } + + @Override + public void close() throws IOException { + EntityUtils.consume(restResponse.getEntity()); + } + } + + private static class HttpEntityBinaryData implements BinaryData { + private final HttpEntity entity; + + HttpEntityBinaryData(HttpEntity entity) { + this.entity = entity; + } + + @Override + public String contentType() { + Header h = entity.getContentType(); + return h == null ? "application/octet-stream" : h.getValue(); + } + + @Override + public void writeTo(OutputStream out) throws IOException { + entity.writeTo(out); + } + + @Override + public ByteBuffer asByteBuffer() throws IOException { + NoCopyByteArrayOutputStream out = new NoCopyByteArrayOutputStream(); + entity.writeTo(out); + return out.asByteBuffer(); + } + + @Override + public InputStream asInputStream() throws IOException { + return entity.getContent(); + } + + @Override + public boolean isRepeatable() { + return entity.isRepeatable(); + } + + @Override + public long size() { + long len = entity.getContentLength(); + return len < 0 ? -1 : entity.getContentLength(); + } + } +} diff --git a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientMonolithTransport.java b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientMonolithTransport.java index 85b530f18..7877df636 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientMonolithTransport.java +++ b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientMonolithTransport.java @@ -127,10 +127,10 @@ public RestClient restClient() { } /** - * Copies this {@link #RestClientTransport} with specific request options. + * Copies this {@link #RestClientMonolithTransport} with specific request options. */ - public RestClientTransport withRequestOptions(@Nullable TransportOptions options) { - return new RestClientTransport(this.restClient, this.mapper, options); + public RestClientMonolithTransport withRequestOptions(@Nullable TransportOptions options) { + return new RestClientMonolithTransport(this.restClient, this.mapper, options); } @Override diff --git a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientTransport.java b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientTransport.java index b0ee3e284..0aa9021f2 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientTransport.java +++ b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientTransport.java @@ -20,216 +20,24 @@ package co.elastic.clients.transport.rest_client; import co.elastic.clients.json.JsonpMapper; -import co.elastic.clients.transport.TransportBase; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.TransportOptions; -import co.elastic.clients.util.BinaryData; -import co.elastic.clients.util.NoCopyByteArrayOutputStream; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.entity.ContentType; -import org.apache.http.util.EntityUtils; -import org.elasticsearch.client.Cancellable; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.ResponseListener; +import co.elastic.clients.transport.ElasticsearchTransportBase; import org.elasticsearch.client.RestClient; -import javax.annotation.Nullable; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -public class RestClientTransport extends TransportBase implements ElasticsearchTransport { - - private static final ConcurrentHashMap ContentTypeCache = new ConcurrentHashMap<>(); - - /** - * The {@code Future} implementation returned by async requests. - * It wraps the RestClient's cancellable and propagates cancellation. - */ - private static class RequestFuture extends CompletableFuture { - private volatile Cancellable cancellable; - - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - boolean cancelled = super.cancel(mayInterruptIfRunning); - if (cancelled && cancellable != null) { - cancellable.cancel(); - } - return cancelled; - } - } +public class RestClientTransport extends ElasticsearchTransportBase { private final RestClient restClient; - public RestClientTransport(RestClient restClient, JsonpMapper mapper) { - this(restClient, mapper, null); + public RestClientTransport(RestClient restClient, JsonpMapper jsonpMapper) { + this(restClient, jsonpMapper, null); } - public RestClientTransport(RestClient restClient, JsonpMapper mapper, @Nullable TransportOptions options) { - super(mapper, options == null ? RestClientOptions.initialOptions() : RestClientOptions.of(options)); + public RestClientTransport(RestClient restClient, JsonpMapper jsonpMapper, RestClientOptions options) { + super(jsonpMapper, new RestClientHttpClient(restClient), options); this.restClient = restClient; } - /** - * Returns the underlying low level Rest Client used by this transport. - */ public RestClient restClient() { return this.restClient; } - - @Override - protected RestClientOptions createOptions(@Nullable TransportOptions options) { - return options == null ? transportOptions : RestClientOptions.of(options); - } - - @Override - protected TransportResponse performRequest(TransportRequest request, RestClientOptions options) throws IOException { - Request restRequest = createRestRequest(request, options); - Response restResponse = restClient.performRequest(restRequest); - return new RestResponse(restResponse); - } - - @Override - protected CompletableFuture performRequestAsync(TransportRequest request, RestClientOptions options) { - - RequestFuture future = new RequestFuture<>(); - org.elasticsearch.client.Request restRequest; - - try { - restRequest = createRestRequest(request, options); - } catch(Throwable thr) { - // Terminate early - future.completeExceptionally(thr); - return future; - } - - future.cancellable = restClient.performRequestAsync(restRequest, new ResponseListener() { - @Override - public void onSuccess(org.elasticsearch.client.Response response) { - future.complete(new RestResponse(response)); - } - - @Override - public void onFailure(Exception exception) { - future.completeExceptionally(exception); - } - }); - - return future; - } - - @Override - public void close() throws IOException { - this.restClient.close(); - } - - private org.elasticsearch.client.Request createRestRequest(TransportRequest request, RestClientOptions options) { - org.elasticsearch.client.Request clientReq = new org.elasticsearch.client.Request( - request.method, request.path - ); - - RequestOptions restOptions = options == null ? - transportOptions.restClientRequestOptions() : - RestClientOptions.of(options).restClientRequestOptions(); - - if (restOptions != null) { - clientReq.setOptions(restOptions); - } - - clientReq.addParameters(request.queryParams); - - Iterable body = request.body; - if (body != null) { - ContentType ct = ContentTypeCache.computeIfAbsent(request.contentType, ContentType::parse); - clientReq.setEntity(new MultiBufferEntity(body, ct)); - } - - // Request parameter intercepted by LLRC - clientReq.addParameter("ignore", "400,401,403,404,405"); - return clientReq; - } - - private static class RestResponse implements TransportResponse { - private final org.elasticsearch.client.Response restResponse; - - public RestResponse(org.elasticsearch.client.Response restResponse) { - this.restResponse = restResponse; - } - - @Override - public int statusCode() { - return restResponse.getStatusLine().getStatusCode(); - } - - @Override - public String getHeader(String name) { - return restResponse.getHeader(name); - } - - @Nullable - @Override - public BinaryData getBody() throws IOException { - HttpEntity entity = restResponse.getEntity(); - return entity == null ? null : new HttpEntityBinaryData(restResponse.getEntity()); - } - - @Override - public Throwable createException() throws IOException { - return new ResponseException(this.restResponse); - } - - @Override - public void close() throws IOException { - EntityUtils.consume(restResponse.getEntity()); - } - } - - private static class HttpEntityBinaryData implements BinaryData { - private final HttpEntity entity; - - public HttpEntityBinaryData(HttpEntity entity) { - this.entity = entity; - } - - @Override - public String contentType() { - Header h = entity.getContentType(); - return h == null ? "application/octet-stream" : h.getValue(); - } - - @Override - public void writeTo(OutputStream out) throws IOException { - entity.writeTo(out); - } - - @Override - public ByteBuffer asByteBuffer() throws IOException { - NoCopyByteArrayOutputStream out = new NoCopyByteArrayOutputStream(); - entity.writeTo(out); - return out.asByteBuffer(); - } - - @Override - public InputStream asInputStream() throws IOException { - return entity.getContent(); - } - - @Override - public boolean isRepeatable() { - return entity.isRepeatable(); - } - - @Override - public long size() { - long len = entity.getContentLength(); - return len < 0 ? -1 : entity.getContentLength(); - } - } } diff --git a/java-client/src/test/java/co/elastic/clients/elasticsearch/ElasticsearchTestServer.java b/java-client/src/test/java/co/elastic/clients/elasticsearch/ElasticsearchTestServer.java index 037fc028d..79331dab7 100644 --- a/java-client/src/test/java/co/elastic/clients/elasticsearch/ElasticsearchTestServer.java +++ b/java-client/src/test/java/co/elastic/clients/elasticsearch/ElasticsearchTestServer.java @@ -17,8 +17,10 @@ * under the License. */ -package co.elastic.clients.elasticsearch; +package co.elastic.clients.tests; +import co.elastic.clients.elasticsearch.ElasticsearchAsyncClient; +import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.ErrorResponse; import co.elastic.clients.json.JsonData; import co.elastic.clients.json.JsonpDeserializer; From ce82384dc859a2423ecc394b574bf39cb4462b67 Mon Sep 17 00:00:00 2001 From: Sylvain Wallez Date: Wed, 7 Jun 2023 19:27:42 +0200 Subject: [PATCH 3/5] More refactoring, add a simple Netty implementation --- example-transports/build.gradle.kts | 55 +++ .../transport/DefaultTransportOptions.java | 47 +++ .../transport/netty/DefaultOptions.java | 141 +++++++ .../netty/InputStreamBinaryData.java | 90 +++++ .../netty/NettyHttpClientExample.java | 159 ++++++++ .../transport/netty/NettyTransportClient.java | 362 ++++++++++++++++++ .../transport/TransportHttpClientTest.java | 139 +++++++ .../transport/netty/NettyClientTest.java | 30 ++ .../rest_client/RestTransportClientTest.java | 41 ++ .../transport/ElasticsearchTransportBase.java | 19 +- .../transport/http/TransportHttpClient.java | 74 +++- .../rest_client/RestClientHttpClient.java | 19 +- .../ElasticsearchTestServer.java | 4 +- 13 files changed, 1158 insertions(+), 22 deletions(-) create mode 100644 example-transports/build.gradle.kts create mode 100644 example-transports/src/main/java/co/elastic/clients/transport/DefaultTransportOptions.java create mode 100644 example-transports/src/main/java/co/elastic/clients/transport/netty/DefaultOptions.java create mode 100644 example-transports/src/main/java/co/elastic/clients/transport/netty/InputStreamBinaryData.java create mode 100644 example-transports/src/main/java/co/elastic/clients/transport/netty/NettyHttpClientExample.java create mode 100644 example-transports/src/main/java/co/elastic/clients/transport/netty/NettyTransportClient.java create mode 100644 example-transports/src/test/java/co/elastic/clients/transport/TransportHttpClientTest.java create mode 100644 example-transports/src/test/java/co/elastic/clients/transport/netty/NettyClientTest.java create mode 100644 example-transports/src/test/java/co/elastic/clients/transport/rest_client/RestTransportClientTest.java diff --git a/example-transports/build.gradle.kts b/example-transports/build.gradle.kts new file mode 100644 index 000000000..12849f582 --- /dev/null +++ b/example-transports/build.gradle.kts @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + java + `java-library` + `java-test-fixtures` +} + +tasks.withType { + useJUnitPlatform() +} + +java { + targetCompatibility = JavaVersion.VERSION_17 +} + + +dependencies { + val jacksonVersion = "2.13.3" + + api("io.netty", "netty-codec-http", "4.1.93.Final") + + implementation(project(":java-client")) + + // Apache 2.0 + // https://github.com/FasterXML/jackson + testImplementation("com.fasterxml.jackson.core", "jackson-core", jacksonVersion) + testImplementation("com.fasterxml.jackson.core", "jackson-databind", jacksonVersion) + + // EPL-2.0 + // https://junit.org/junit5/ + testImplementation("org.junit.jupiter:junit-jupiter-api:5.9.0") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.9.0") + +} +repositories { + mavenCentral() +} diff --git a/example-transports/src/main/java/co/elastic/clients/transport/DefaultTransportOptions.java b/example-transports/src/main/java/co/elastic/clients/transport/DefaultTransportOptions.java new file mode 100644 index 000000000..869836fd2 --- /dev/null +++ b/example-transports/src/main/java/co/elastic/clients/transport/DefaultTransportOptions.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public class DefaultTransportOptions implements TransportOptions { + @Override + public Collection> headers() { + return null; + } + + @Override + public Map queryParameters() { + return null; + } + + @Override + public Function, Boolean> onWarnings() { + return null; + } + + @Override + public Builder toBuilder() { + return null; + } +} diff --git a/example-transports/src/main/java/co/elastic/clients/transport/netty/DefaultOptions.java b/example-transports/src/main/java/co/elastic/clients/transport/netty/DefaultOptions.java new file mode 100644 index 000000000..ce5eed510 --- /dev/null +++ b/example-transports/src/main/java/co/elastic/clients/transport/netty/DefaultOptions.java @@ -0,0 +1,141 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport.netty; + +import co.elastic.clients.transport.TransportOptions; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public class DefaultOptions implements TransportOptions { + private final Map headers; + private final Map parameters; + private final Function, Boolean> onWarnings; + + public DefaultOptions() { + this(Collections.emptyMap(), Collections.emptyMap(), null); + } + + public DefaultOptions(TransportOptions options) { + this( + entriesToMap(options.headers()), + options.queryParameters(), + options.onWarnings() + ); + } + + public DefaultOptions( + Map headers, + Map parameters, + Function, Boolean> onWarnings + ) { + this.headers = headers.isEmpty() ? Collections.emptyMap() : Collections.unmodifiableMap(headers); + this.parameters = parameters.isEmpty() ? Collections.emptyMap() : Collections.unmodifiableMap(parameters); + this.onWarnings = onWarnings; + } + + @Override + public Collection> headers() { + return headers.entrySet(); + } + + @Override + public Map queryParameters() { + return parameters; + } + + @Override + public Function, Boolean> onWarnings() { + return onWarnings; + } + + @Override + public Builder toBuilder() { + return new Builder(this); + } + + private static Map entriesToMap(Collection> entries) { + if (entries.isEmpty()) { + return Collections.emptyMap(); + } else { + HashMap map = new HashMap<>(); + for (Map.Entry entry: entries) { + map.put(entry.getKey(), entry.getValue()); + } + return map; + } + } + + public static class Builder implements TransportOptions.Builder { + private Map headers; + private Map parameters; + private Function, Boolean> onWarnings; + + public Builder() { + } + + public Builder(DefaultOptions options) { + this.headers = copyOrNull(options.headers); + this.parameters = copyOrNull(options.parameters); + this.onWarnings = options.onWarnings; + } + + @Override + public TransportOptions.Builder addHeader(String name, String value) { + if (headers == null) { + headers = new HashMap<>(); + } + headers.put(name, value); + return this; + } + + @Override + public TransportOptions.Builder setParameter(String name, String value) { + if (parameters == null) { + parameters = new HashMap<>(); + } + parameters.put(name, value); + return this; + } + + @Override + public TransportOptions.Builder onWarnings(Function, Boolean> listener) { + this.onWarnings = listener; + return this; + } + + @Override + public TransportOptions build() { + return new DefaultOptions( + headers == null ? Collections.emptyMap() : headers, + parameters == null ? Collections.emptyMap() : parameters, + onWarnings + ); + } + + private Map copyOrNull(Map map) { + return map.isEmpty() ? null : new HashMap<>(map); + } + } +} diff --git a/example-transports/src/main/java/co/elastic/clients/transport/netty/InputStreamBinaryData.java b/example-transports/src/main/java/co/elastic/clients/transport/netty/InputStreamBinaryData.java new file mode 100644 index 000000000..47683bdb7 --- /dev/null +++ b/example-transports/src/main/java/co/elastic/clients/transport/netty/InputStreamBinaryData.java @@ -0,0 +1,90 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport.netty; + +import co.elastic.clients.util.BinaryData; +import co.elastic.clients.util.NoCopyByteArrayOutputStream; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +public class InputStreamBinaryData implements BinaryData { + + private final String contentType; + private final InputStream inputStream; + private boolean consumed = false; + + public InputStreamBinaryData(String contentType, InputStream inputStream) { + this.contentType = contentType; + this.inputStream = inputStream; + } + + @Override + public String contentType() { + return contentType; + } + + @Override + public void writeTo(OutputStream out) throws IOException { + consume(); + try { + byte[] buffer = new byte[8192]; + int len; + while ((len = inputStream.read(buffer)) > 0) { + out.write(buffer, 0, len); + } + } finally { + inputStream.close(); + } + } + + @Override + public ByteBuffer asByteBuffer() throws IOException { + consume(); + NoCopyByteArrayOutputStream baos = new NoCopyByteArrayOutputStream(); + writeTo(baos); + return baos.asByteBuffer(); + } + + @Override + public InputStream asInputStream() throws IOException { + consume(); + return inputStream; + } + + @Override + public boolean isRepeatable() { + return false; + } + + @Override + public long size() { + return -1; + } + + private void consume() throws IllegalStateException { + if (consumed) { + throw new IllegalStateException("Data has already been consumed"); + } + consumed = true; + } +} diff --git a/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyHttpClientExample.java b/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyHttpClientExample.java new file mode 100644 index 000000000..f7cbf3df9 --- /dev/null +++ b/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyHttpClientExample.java @@ -0,0 +1,159 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport.netty; + +import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpClientCodec; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpObject; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpUtil; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import io.netty.util.CharsetUtil; + +import java.net.URI; + +public class NettyHttpClientExample { + private static final String HOST = "example.com"; + private static final int PORT = 80; + + public static void main(String[] args) { + System.out.println("foo"); + //if (true) return; + try { + main0(args); + } catch (Throwable thr) { + thr.printStackTrace(); + } + System.out.println("done"); + } + + public static void main0(String[] args) throws Exception { + + NioEventLoopGroup workerGroup = new NioEventLoopGroup(); + + SslContext sslContext = SslContextBuilder.forClient() + .trustManager(InsecureTrustManagerFactory.INSTANCE) + .build(); + + try { + Bootstrap bootstrap = new Bootstrap(); + bootstrap.group(workerGroup) + .channel(NioSocketChannel.class) + .option(ChannelOption.SO_KEEPALIVE, true) + .handler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) throws Exception { + //ch.pipeline().addLast(sslContext.newHandler(ch.alloc())); + ch.pipeline().addLast(new HttpClientCodec()); + //ch.pipeline().addLast(new HttpObjectAggregator(8192)); + ch.pipeline().addLast(new SimpleChannelInboundHandler() { + + @Override + protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Exception { + if (msg instanceof HttpResponse) { + HttpResponse response = (HttpResponse) msg; + + System.err.println("STATUS: " + response.status()); + System.err.println("VERSION: " + response.protocolVersion()); + System.err.println(); + + if (!response.headers().isEmpty()) { + for (CharSequence name: response.headers().names()) { + for (CharSequence value: response.headers().getAll(name)) { + System.err.println("HEADER: " + name + " = " + value); + } + } + System.err.println(); + } + + if (HttpUtil.isTransferEncodingChunked(response)) { + System.err.println("CHUNKED CONTENT {"); + } else { + System.err.println("CONTENT {"); + } + } + if (msg instanceof HttpContent) { + HttpContent content = (HttpContent) msg; + + ByteBuf buf = content.content(); + System.err.println("Buffer: " + buf.readableBytes()); + //buf.retain(); + //buf.release(); + //System.err.print(buf.toString(CharsetUtil.UTF_8)); + //System.err.flush(); + + if (content instanceof LastHttpContent) { + System.err.println("} END OF CONTENT"); + ctx.close(); + } + } + + } + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + cause.printStackTrace(); + ctx.close(); + } + }); + } + }); + + Channel channelFuture = bootstrap.connect(HOST, PORT).sync().channel(); + + URI uri = new URI("http://" + HOST); + String msg = "Hello, Server!"; + FullHttpRequest request = new DefaultFullHttpRequest( + HttpVersion.HTTP_1_1, HttpMethod.GET, uri.toASCIIString(), + Unpooled.wrappedBuffer(msg.getBytes(CharsetUtil.UTF_8))); + + request.headers().set(HttpHeaders.Names.HOST, HOST); + request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.CLOSE); + //request.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP); + + System.err.println("after request 0"); + channelFuture.writeAndFlush(request); + System.err.println("after request 1"); + channelFuture.closeFuture().sync(); + System.err.println("after request 2"); + } finally { + workerGroup.shutdownGracefully().sync(); + System.err.println("after request 3"); + } + } +} diff --git a/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyTransportClient.java b/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyTransportClient.java new file mode 100644 index 000000000..a44f6f9d3 --- /dev/null +++ b/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyTransportClient.java @@ -0,0 +1,362 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport.netty; + +import co.elastic.clients.transport.TransportOptions; +import co.elastic.clients.transport.http.TransportHttpClient; +import co.elastic.clients.util.BinaryData; +import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufInputStream; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpClientCodec; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpObject; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.util.concurrent.Future; + +import javax.annotation.Nullable; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLException; +import java.io.IOException; +import java.net.URLEncoder; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +public class NettyTransportClient implements TransportHttpClient { + + private final NioEventLoopGroup workerGroup = new NioEventLoopGroup(); + private final SslContext sslContext; + + public NettyTransportClient() { + try { + sslContext = SslContextBuilder.forClient() + .sslContextProvider(SSLContext.getDefault().getProvider()) + .build(); + } catch (SSLException | NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + @Override + public DefaultOptions createOptions(@Nullable TransportOptions options) { + return options == null ? new DefaultOptions() : new DefaultOptions(options); + } + + @Override + public Response performRequest(String endpointId, Node node, Request request, TransportOptions options) throws IOException { + + try { + return performRequestAsync(endpointId, node, request, options).get(); + } catch (InterruptedException ie) { + throw new RuntimeException(ie); + } catch (ExecutionException ee) { + // Remove one nesting level + throw new RuntimeException(ee.getCause()); + } + } + + @Override + public CompletableFuture performRequestAsync(String endpointId, Node node, Request request, TransportOptions options) { + + CompletableFuture promise = new CompletableFuture<>() { + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + // TODO: cancel pending request + return super.cancel(mayInterruptIfRunning); + } + }; + + Bootstrap bootstrap = new Bootstrap(); + bootstrap.group(workerGroup) + .channel(NioSocketChannel.class) + .option(ChannelOption.SO_KEEPALIVE, true) + .handler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) throws Exception { + ChannelPipeline pipeline = ch.pipeline(); + if (node.uri().getScheme().equals("https")) { + pipeline.addLast(sslContext.newHandler(ch.alloc())); + } + pipeline.addLast(new HttpClientCodec()); + pipeline.addLast(new ChannelHandler(node, promise)); + } + }); + + + String path = request.path(); + String queryString = queryString(request, options); + if (queryString != null) { + path = path + "?" + queryString; + } + String uri = node.uriForPath(path).toString(); + + ByteBuf nettyBody; + Iterable body = request.body(); + if (body == null) { + nettyBody = Unpooled.buffer(0); + } else { + List bufs; + if (body instanceof List) { + bufs = (List)body; + } else { + bufs = new ArrayList<>(); + for (ByteBuffer buf: body) { + bufs.add(buf); + } + } + nettyBody = Unpooled.wrappedBuffer(bufs.toArray(new ByteBuffer[bufs.size()])); + } + + FullHttpRequest nettyRequest = new DefaultFullHttpRequest( + HttpVersion.HTTP_1_1, + HttpMethod.valueOf(request.method()), + uri, + nettyBody + ); + + // Netty doesn't set Content-Length automatically with FullRequest. + nettyRequest.headers().set(HttpHeaderNames.CONTENT_LENGTH, nettyBody.readableBytes()); + + if (request.contentType() != null) { + nettyRequest.headers().set(HttpHeaderNames.CONTENT_TYPE, request.contentType()); + } + + int port = node.uri().getPort(); + if (port == -1) { + port = node.uri().getScheme().equals("https") ? 443 : 80; + } + nettyRequest.headers().set(HttpHeaderNames.HOST, node.uri().getHost() + ":" + port); + + if (options != null) { + for (Map.Entry header: options.headers()) { + nettyRequest.headers().set(header.getKey(), header.getValue()); + } + } + + ChannelFuture future0 = bootstrap.connect(node.uri().getHost(), port); + future0.addListener((ChannelFutureListener) future1 -> { + if (checkSuccess(future1, promise)) { + ChannelFuture future2 = future1.channel().writeAndFlush(nettyRequest); + future2.addListener((ChannelFutureListener) future3 -> { + if (checkSuccess(future3, promise)) { + // Log request sent? + } + }); + } + }); + + future0.addListener(future4 -> { + if (future4.cause() != null) { + promise.completeExceptionally(future4.cause()); + } else if (future4.isCancelled()) { + promise.completeExceptionally(new RuntimeException("Request was cancelled")); + } + }); + + return promise; + } + + private String queryString(Request request, TransportOptions options) { + Map requestParams = request.queryParams(); + Map optionsParams = options == null ? Collections.emptyMap() : options.queryParameters(); + + Map allParams; + if (requestParams.isEmpty()) { + allParams = optionsParams; + } else if (optionsParams.isEmpty()) { + allParams = requestParams; + } else { + allParams = new HashMap<>(requestParams); + allParams.putAll(optionsParams); + } + + if (allParams.isEmpty()) { + return null; + } else { + return allParams + .entrySet() + .stream() + .map(e -> URLEncoder.encode(e.getKey(), StandardCharsets.UTF_8) + "=" + + URLEncoder.encode(e.getValue(), StandardCharsets.UTF_8)) + .collect(Collectors.joining("&")); + } + } + + private boolean checkSuccess(Future future, CompletableFuture promise) { + if (future.isSuccess()) { + return true; + } + + if (future.cause() != null) { + promise.completeExceptionally(future.cause()); + } else if (future.isCancelled()) { + promise.completeExceptionally(new RuntimeException("Request was cancelled")); + } else { + promise.completeExceptionally(new RuntimeException("Unexpected future state")); + } + return false; + } + + private static void dump(Future future, String name) { + System.err.println("Future " + name + " - " + future); + System.err.println(" Done : " + future.isDone()); + System.err.println(" Success : " + future.isSuccess()); + System.err.println(" Cancelled: " + future.isCancelled()); + if (future.cause() != null) { + System.err.println(" Cause : " + future.cause()); + } + System.err.flush(); + } + + private static class ChannelHandler extends SimpleChannelInboundHandler { + + private final CompletableFuture promise; + private final Node node; + private volatile HttpResponse response; + private volatile List body; + + public ChannelHandler(Node node, CompletableFuture promise) { + this.node = node; + this.promise = promise; + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Exception { + if (msg instanceof HttpResponse) { + this.response = (HttpResponse) msg; + + } else if(msg instanceof HttpContent) { + System.err.flush(); + HttpContent content = (HttpContent) msg; + ByteBuf buf = content.content(); + if (buf.readableBytes() > 0) { + buf.retain(); + if (this.body == null) { + this.body = new ArrayList<>(); + } + this.body.add(buf); + } + + if(msg instanceof LastHttpContent) { + promise.complete(new NettyResponse(node, response, body)); + ctx.close(); + } + } + } + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + promise.completeExceptionally(cause); + ctx.close(); + } + } + + @Override + public void close() throws IOException { + workerGroup.shutdownGracefully(); + } + + private static class NettyResponse implements TransportHttpClient.Response { + + private final Node node; + private final HttpResponse response; + @Nullable + private final List body; + + public NettyResponse(Node node, HttpResponse response, @Nullable List body) { + this.node = node; + this.response = response; + this.body = body; + } + + @Override + public Node node() { + return node; + } + + @Override + public int statusCode() { + return response.status().code(); + } + + @Override + public String header(String name) { + return response.headers().get(name); + } + + @Nullable + @Override + public BinaryData body() throws IOException { + if (body == null) { + return null; + } + + ByteBuf byteBuf = Unpooled.wrappedBuffer(body.size(), body.toArray(new ByteBuf[body.size()])); + return new InputStreamBinaryData( + response.headers().get(HttpHeaderNames.CONTENT_TYPE), + new ByteBufInputStream(byteBuf, true) + ); + } + + @Override + public Throwable createException() throws IOException { + // TODO + return null; + } + + @Override + public void close() throws IOException { + if (body != null) { + for (ByteBuf buf: body) { + buf.release(); + } + body.clear(); + } + } + } +} diff --git a/example-transports/src/test/java/co/elastic/clients/transport/TransportHttpClientTest.java b/example-transports/src/test/java/co/elastic/clients/transport/TransportHttpClientTest.java new file mode 100644 index 000000000..bc897f43b --- /dev/null +++ b/example-transports/src/test/java/co/elastic/clients/transport/TransportHttpClientTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport; + +import co.elastic.clients.transport.http.TransportHttpClient; +import co.elastic.clients.transport.netty.DefaultOptions; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public abstract class TransportHttpClientTest> extends Assertions { + + public static record EchoResponse(Map> headers, String body) { + } + + protected static HttpServer server; + protected final Client httpClient; + + @BeforeAll + public static void startEchoServer() throws Exception { + server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + + server.createContext("/root/echo", exchange -> { + + byte[] bytes = exchange.getRequestBody().readAllBytes(); + exchange.getRequestBody().close(); + + Headers requestHeaders = exchange.getRequestHeaders(); + + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, 0); + + var response = new EchoResponse( + requestHeaders, new String(bytes, StandardCharsets.UTF_8) + ); + + OutputStream out = exchange.getResponseBody(); + new ObjectMapper().writeValue(out, response); + out.close(); + }); + + server.start(); + } + + @AfterAll + public static void stopEchoServer() { + server.stop(0); + } + + public TransportHttpClientTest(Client httpClient) { + this.httpClient = httpClient; + } + + @Test + public void testClient() throws Exception { + + List requestBody = List.of( + ByteBuffer.wrap("Hello world\n".getBytes(StandardCharsets.UTF_8)), + ByteBuffer.wrap("Hello universe\n".getBytes(StandardCharsets.UTF_8)) + ); + + TransportHttpClient.Node node = new TransportHttpClient.Node( + "http://" + server.getAddress().getHostString() + ":" + server.getAddress().getPort() + "/" + ); + + TransportOptions options = new DefaultOptions.Builder().addHeader("X-Header", "some value").build(); + + TransportHttpClient.Response response = httpClient.performRequest( + "foo", + node, + new TransportHttpClient.Request("POST", "/root/echo", Map.of(), "text/plain", requestBody), + options + ); + + assertEquals("application/json", response.body().contentType()); + + EchoResponse echoResponse = new ObjectMapper().readValue(response.body().asInputStream(), EchoResponse.class); + + var echoHeaders = normalizeHeaders(echoResponse.headers()); + assertEquals("text/plain", echoHeaders.get("content-type")); + assertEquals("some value", echoHeaders.get("x-header")); + + dump(echoHeaders); + + assertEquals("Hello world\nHello universe\n", echoResponse.body); + } + + /** + * Set all header names to lowercase and only keep the 1st header value + */ + private static Map normalizeHeaders(Map> headers) { + var result = new HashMap(); + headers.forEach((k, v) -> { + if (v.size() != 1) { + fail("Header '" + k + "' should have a single value, but was: " + headers); + } + result.put(k.toLowerCase(), v.get(0)); + }); + return result; + } + + private static void dump(Map map) { + map.forEach((k, v) -> { + System.out.println(k + "=" + v); + }); + } +} diff --git a/example-transports/src/test/java/co/elastic/clients/transport/netty/NettyClientTest.java b/example-transports/src/test/java/co/elastic/clients/transport/netty/NettyClientTest.java new file mode 100644 index 000000000..72313296c --- /dev/null +++ b/example-transports/src/test/java/co/elastic/clients/transport/netty/NettyClientTest.java @@ -0,0 +1,30 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport.netty; + + +import co.elastic.clients.transport.TransportHttpClientTest; + +public class NettyClientTest extends TransportHttpClientTest { + + public NettyClientTest() { + super(new NettyTransportClient()); + } +} diff --git a/example-transports/src/test/java/co/elastic/clients/transport/rest_client/RestTransportClientTest.java b/example-transports/src/test/java/co/elastic/clients/transport/rest_client/RestTransportClientTest.java new file mode 100644 index 000000000..663829e75 --- /dev/null +++ b/example-transports/src/test/java/co/elastic/clients/transport/rest_client/RestTransportClientTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport.rest_client; + +import co.elastic.clients.transport.TransportHttpClientTest; +import co.elastic.clients.transport.http.TransportHttpClient; +import org.apache.http.HttpHost; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; + +public class RestTransportClientTest extends TransportHttpClientTest { + + public RestTransportClientTest() { + super(createClient()); + } + + private static RestClientHttpClient createClient() { + RestClient restClient = RestClient.builder( + new HttpHost(server.getAddress().getAddress(), server.getAddress().getPort(), "http") + ).build(); + + return new RestClientHttpClient(restClient); + } +} diff --git a/java-client/src/main/java/co/elastic/clients/transport/ElasticsearchTransportBase.java b/java-client/src/main/java/co/elastic/clients/transport/ElasticsearchTransportBase.java index 99e589039..48ceec44a 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/ElasticsearchTransportBase.java +++ b/java-client/src/main/java/co/elastic/clients/transport/ElasticsearchTransportBase.java @@ -104,7 +104,7 @@ public final ResponseT performRequest( ) throws IOException { TransportHttpClient.Request req = prepareTransportRequest(request, endpoint, options); Options opts = options == null ? transportOptions : httpClient.createOptions(options); - TransportHttpClient.Response resp = httpClient.performRequest(endpoint.id(), req, opts); + TransportHttpClient.Response resp = httpClient.performRequest(endpoint.id(), null, req, opts); return getApiResponse(resp, endpoint); } @@ -128,7 +128,7 @@ public final CompletableFuture performR boolean disableRequiredChecks = ApiTypeHelper.requiredPropertiesCheckDisabled(); CompletableFuture clientFuture = httpClient.performRequestAsync( - endpoint.id(), clientReq, httpClient.createOptions(options) + endpoint.id(), null, clientReq, httpClient.createOptions(options) ); // Cancelling the result will cancel the upstream future created by the http client, allowing to stop in-flight requests @@ -250,7 +250,7 @@ private ResponseT getApiResponse( ); } - BinaryData entity = clientResp.getBody(); + BinaryData entity = clientResp.body(); if (entity == null) { throw new TransportException( statusCode, @@ -264,8 +264,7 @@ private ResponseT getApiResponse( entity = new ByteArrayBinaryData(entity); } - try { - InputStream content = entity.asInputStream(); + try (InputStream content = entity.asInputStream()) { try (JsonParser parser = mapper.jsonProvider().createParser(content)) { ErrorT error = errorDeserializer.deserialize(parser, mapper); // TODO: have the endpoint provide the exception constructor @@ -285,7 +284,7 @@ private ResponseT getApiResponse( } } } else { - return decodeTransportResponse(statusCode, clientResp.getBody(), clientResp, endpoint); + return decodeTransportResponse(statusCode, clientResp.body(), clientResp, endpoint); } @@ -316,8 +315,10 @@ private ResponseT decodeTransportResponse( endpoint.id(), clientResp.createException() ); } - InputStream content = entity.asInputStream(); - try (JsonParser parser = mapper.jsonProvider().createParser(content)) { + try ( + InputStream content = entity.asInputStream(); + JsonParser parser = mapper.jsonProvider().createParser(content) + ) { response = responseParser.deserialize(parser, mapper); } catch (Exception e) { throw new TransportException( @@ -354,7 +355,7 @@ private ResponseT decodeTransportResponse( )); private void checkProductHeader(TransportHttpClient.Response clientResp, Endpoint endpoint) throws IOException { - String header = clientResp.getHeader("X-Elastic-Product"); + String header = clientResp.header("X-Elastic-Product"); if (header == null) { if (endpointsMissingProductHeader.contains(endpoint.id())) { return; diff --git a/java-client/src/main/java/co/elastic/clients/transport/http/TransportHttpClient.java b/java-client/src/main/java/co/elastic/clients/transport/http/TransportHttpClient.java index 8e6af31b2..1b11f86a2 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/http/TransportHttpClient.java +++ b/java-client/src/main/java/co/elastic/clients/transport/http/TransportHttpClient.java @@ -24,8 +24,11 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.net.URI; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; /** @@ -50,7 +53,7 @@ public interface TransportHttpClient { * * @return the request response */ - Response performRequest(String endpointId, Request request, Options options) throws IOException; + Response performRequest(String endpointId, @Nullable Node node, Request request, TransportOptions options) throws IOException; /** * Perform an asynchronous request. @@ -61,17 +64,74 @@ public interface TransportHttpClient { * * @return the request response */ - CompletableFuture performRequestAsync(String endpointId, Request request, Options options); + CompletableFuture performRequestAsync(String endpointId, @Nullable Node node, Request request, TransportOptions options); /** * Close this client, freeing associated resources. */ void close() throws IOException; + /** + * A node/host to send requests to. + */ + class Node { + private final URI uri; + private final Set roles; + private final Map attributes; + + /** + * Create a node with its URI, roles and attributes. + *

+ * If the URI doesn't end with a '{@code /}', then one is added. + */ + public Node(URI uri, Set roles, Map attributes) { + if (!uri.isAbsolute()) { + throw new IllegalArgumentException("Node URIs must be absolute: " + uri); + } + + if (!uri.getRawPath().endsWith("/")) { + uri = uri.resolve(uri.getRawPath() + "/"); + } + + this.uri = uri; + this.roles = roles; + this.attributes = attributes; + } + + public Node(URI uri) { + this(uri, Collections.emptySet(), Collections.emptyMap()); + } + + public Node(String uri) { + this(URI.create(uri), Collections.emptySet(), Collections.emptyMap()); + } + + public URI uri() { + return this.uri; + } + + public URI uriForPath(String path) { + // Make sure the path will be appended to the node's URI path if it exists, and will not replace it. + if (this.uri.getRawPath().length() > 1) { + if (path.charAt(0) == '/') { + path = path.substring(1); + } + } + + return uri.resolve(path); + } + + @Override + public String toString() { + return uri.toString(); + } + } + /** * An http request. */ class Request { + @Nullable private final String method; private final String path; private final Map queryParams; @@ -117,6 +177,12 @@ public Iterable body() { */ interface Response { + /** + * The host/node that was used to send the request. It may be different from the one that was provided with the request + * if the http client has an internal retry mechanism. + */ + Node node(); + /** * The response status code. */ @@ -125,13 +191,13 @@ interface Response { /** * Get a header value, or the first value if the header has multiple values. */ - String getHeader(String name); + String header(String name); /** * The response body, if any. */ @Nullable - BinaryData getBody() throws IOException; + BinaryData body() throws IOException; Throwable createException() throws IOException; diff --git a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientHttpClient.java b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientHttpClient.java index a887fcae2..be270a9c6 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientHttpClient.java +++ b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientHttpClient.java @@ -80,20 +80,22 @@ public RestClientOptions createOptions(@Nullable TransportOptions options) { } @Override - public Response performRequest(String endpointId, Request request, RestClientOptions options) throws IOException { - org.elasticsearch.client.Request restRequest = createRestRequest(request, options); + public Response performRequest(String endpointId, @Nullable Node node, Request request, TransportOptions options) throws IOException { + RestClientOptions rcOptions = RestClientOptions.of(options); + org.elasticsearch.client.Request restRequest = createRestRequest(request, rcOptions); org.elasticsearch.client.Response restResponse = restClient.performRequest(restRequest); return new RestResponse(restResponse); } @Override - public CompletableFuture performRequestAsync(String endpointId, Request request, RestClientOptions options) { + public CompletableFuture performRequestAsync(String endpointId, @Nullable Node node, Request request, TransportOptions options) { RequestFuture future = new RequestFuture<>(); org.elasticsearch.client.Request restRequest; try { - restRequest = createRestRequest(request, options); + RestClientOptions rcOptions = RestClientOptions.of(options); + restRequest = createRestRequest(request, rcOptions); } catch(Throwable thr) { // Terminate early future.completeExceptionally(thr); @@ -149,19 +151,24 @@ private static class RestResponse implements Response { this.restResponse = restResponse; } + @Override + public Node node() { + return new Node(restResponse.getHost().toURI()); + } + @Override public int statusCode() { return restResponse.getStatusLine().getStatusCode(); } @Override - public String getHeader(String name) { + public String header(String name) { return restResponse.getHeader(name); } @Nullable @Override - public BinaryData getBody() throws IOException { + public BinaryData body() throws IOException { HttpEntity entity = restResponse.getEntity(); return entity == null ? null : new HttpEntityBinaryData(restResponse.getEntity()); } diff --git a/java-client/src/test/java/co/elastic/clients/elasticsearch/ElasticsearchTestServer.java b/java-client/src/test/java/co/elastic/clients/elasticsearch/ElasticsearchTestServer.java index 79331dab7..037fc028d 100644 --- a/java-client/src/test/java/co/elastic/clients/elasticsearch/ElasticsearchTestServer.java +++ b/java-client/src/test/java/co/elastic/clients/elasticsearch/ElasticsearchTestServer.java @@ -17,10 +17,8 @@ * under the License. */ -package co.elastic.clients.tests; +package co.elastic.clients.elasticsearch; -import co.elastic.clients.elasticsearch.ElasticsearchAsyncClient; -import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.ErrorResponse; import co.elastic.clients.json.JsonData; import co.elastic.clients.json.JsonpDeserializer; From 7a42d5fa1fff79aa888e0b4b46e905c1eb42bfaa Mon Sep 17 00:00:00 2001 From: Sylvain Wallez Date: Fri, 9 Jun 2023 18:12:18 +0200 Subject: [PATCH 4/5] Finalize refactoring --- example-transports/README.md | 1 + .../transport/DefaultTransportOptions.java | 47 -- .../transport/netty/DefaultOptions.java | 141 ------ .../netty/NettyElasticsearchTransport.java | 70 +++ .../netty/NettyHttpClientExample.java | 159 ------- ...ent.java => NettyTransportHttpClient.java} | 92 ++-- .../transport/TransportHttpClientTest.java | 18 +- .../transport/netty/NettyClientTest.java | 6 +- .../rest_client/RestTransportClientTest.java | 2 - .../transport/DefaultTransportOptions.java | 209 +++++++++ .../transport/ElasticsearchTransportBase.java | 145 ++++-- .../clients/transport/TransportException.java | 30 +- .../clients/transport/TransportOptions.java | 6 + .../clients/transport/http/HeaderMap.java | 173 +++++++ .../transport/http/TransportHttpClient.java | 113 +++-- .../rest_client/HttpClientBinaryResponse.java | 64 --- .../rest_client/RestClientHttpClient.java | 75 ++- .../RestClientMonolithTransport.java | 442 ------------------ .../rest_client/RestClientOptions.java | 65 ++- .../rest_client/RestClientTransport.java | 4 +- .../LanguageRuntimeVersions.java | 4 +- .../clients/transport/TransportTest.java | 8 +- .../clients/transport/http/HeaderMapTest.java | 78 ++++ 23 files changed, 938 insertions(+), 1014 deletions(-) create mode 100644 example-transports/README.md delete mode 100644 example-transports/src/main/java/co/elastic/clients/transport/DefaultTransportOptions.java delete mode 100644 example-transports/src/main/java/co/elastic/clients/transport/netty/DefaultOptions.java create mode 100644 example-transports/src/main/java/co/elastic/clients/transport/netty/NettyElasticsearchTransport.java delete mode 100644 example-transports/src/main/java/co/elastic/clients/transport/netty/NettyHttpClientExample.java rename example-transports/src/main/java/co/elastic/clients/transport/netty/{NettyTransportClient.java => NettyTransportHttpClient.java} (81%) create mode 100644 java-client/src/main/java/co/elastic/clients/transport/DefaultTransportOptions.java create mode 100644 java-client/src/main/java/co/elastic/clients/transport/http/HeaderMap.java delete mode 100644 java-client/src/main/java/co/elastic/clients/transport/rest_client/HttpClientBinaryResponse.java delete mode 100644 java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientMonolithTransport.java rename java-client/src/main/java/co/elastic/clients/{transport/rest_client => util}/LanguageRuntimeVersions.java (98%) create mode 100644 java-client/src/test/java/co/elastic/clients/transport/http/HeaderMapTest.java diff --git a/example-transports/README.md b/example-transports/README.md new file mode 100644 index 000000000..a94b5dc3e --- /dev/null +++ b/example-transports/README.md @@ -0,0 +1 @@ +This directory contains experimental implementations of the `TransportHttpClient` interface. They are to be used as examples and inspiration and should not be considered production-ready. diff --git a/example-transports/src/main/java/co/elastic/clients/transport/DefaultTransportOptions.java b/example-transports/src/main/java/co/elastic/clients/transport/DefaultTransportOptions.java deleted file mode 100644 index 869836fd2..000000000 --- a/example-transports/src/main/java/co/elastic/clients/transport/DefaultTransportOptions.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package co.elastic.clients.transport; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -public class DefaultTransportOptions implements TransportOptions { - @Override - public Collection> headers() { - return null; - } - - @Override - public Map queryParameters() { - return null; - } - - @Override - public Function, Boolean> onWarnings() { - return null; - } - - @Override - public Builder toBuilder() { - return null; - } -} diff --git a/example-transports/src/main/java/co/elastic/clients/transport/netty/DefaultOptions.java b/example-transports/src/main/java/co/elastic/clients/transport/netty/DefaultOptions.java deleted file mode 100644 index ce5eed510..000000000 --- a/example-transports/src/main/java/co/elastic/clients/transport/netty/DefaultOptions.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package co.elastic.clients.transport.netty; - -import co.elastic.clients.transport.TransportOptions; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -public class DefaultOptions implements TransportOptions { - private final Map headers; - private final Map parameters; - private final Function, Boolean> onWarnings; - - public DefaultOptions() { - this(Collections.emptyMap(), Collections.emptyMap(), null); - } - - public DefaultOptions(TransportOptions options) { - this( - entriesToMap(options.headers()), - options.queryParameters(), - options.onWarnings() - ); - } - - public DefaultOptions( - Map headers, - Map parameters, - Function, Boolean> onWarnings - ) { - this.headers = headers.isEmpty() ? Collections.emptyMap() : Collections.unmodifiableMap(headers); - this.parameters = parameters.isEmpty() ? Collections.emptyMap() : Collections.unmodifiableMap(parameters); - this.onWarnings = onWarnings; - } - - @Override - public Collection> headers() { - return headers.entrySet(); - } - - @Override - public Map queryParameters() { - return parameters; - } - - @Override - public Function, Boolean> onWarnings() { - return onWarnings; - } - - @Override - public Builder toBuilder() { - return new Builder(this); - } - - private static Map entriesToMap(Collection> entries) { - if (entries.isEmpty()) { - return Collections.emptyMap(); - } else { - HashMap map = new HashMap<>(); - for (Map.Entry entry: entries) { - map.put(entry.getKey(), entry.getValue()); - } - return map; - } - } - - public static class Builder implements TransportOptions.Builder { - private Map headers; - private Map parameters; - private Function, Boolean> onWarnings; - - public Builder() { - } - - public Builder(DefaultOptions options) { - this.headers = copyOrNull(options.headers); - this.parameters = copyOrNull(options.parameters); - this.onWarnings = options.onWarnings; - } - - @Override - public TransportOptions.Builder addHeader(String name, String value) { - if (headers == null) { - headers = new HashMap<>(); - } - headers.put(name, value); - return this; - } - - @Override - public TransportOptions.Builder setParameter(String name, String value) { - if (parameters == null) { - parameters = new HashMap<>(); - } - parameters.put(name, value); - return this; - } - - @Override - public TransportOptions.Builder onWarnings(Function, Boolean> listener) { - this.onWarnings = listener; - return this; - } - - @Override - public TransportOptions build() { - return new DefaultOptions( - headers == null ? Collections.emptyMap() : headers, - parameters == null ? Collections.emptyMap() : parameters, - onWarnings - ); - } - - private Map copyOrNull(Map map) { - return map.isEmpty() ? null : new HashMap<>(map); - } - } -} diff --git a/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyElasticsearchTransport.java b/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyElasticsearchTransport.java new file mode 100644 index 000000000..3317eb3be --- /dev/null +++ b/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyElasticsearchTransport.java @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport.netty; + +import co.elastic.clients.json.JsonpMapper; +import co.elastic.clients.transport.ElasticsearchTransportBase; +import co.elastic.clients.transport.TransportOptions; +import co.elastic.clients.transport.http.TransportHttpClient; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.concurrent.CompletableFuture; + +public class NettyElasticsearchTransport extends ElasticsearchTransportBase { + + public NettyElasticsearchTransport(TransportHttpClient.Node node, TransportOptions options, JsonpMapper jsonpMapper) { + super(new SingleNodeHttpClient(new NettyTransportHttpClient(), node), options, jsonpMapper); + } + + public static class SingleNodeHttpClient implements TransportHttpClient { + private final TransportHttpClient client; + private final Node node; + + public SingleNodeHttpClient(TransportHttpClient client, Node node) { + this.client = client; + this.node = node; + } + + @Override + public TransportOptions createOptions(@Nullable TransportOptions options) { + return client.createOptions(options); + } + + @Override + public Response performRequest( + String endpointId, @Nullable Node ignoredNode, Request request, TransportOptions options + ) throws IOException { + return client.performRequest(endpointId, node, request, options); + } + + @Override + public CompletableFuture performRequestAsync( + String endpointId, @Nullable Node ignoredNode, Request request, TransportOptions options + ) { + return client.performRequestAsync(endpointId, node, request, options); + } + + @Override + public void close() throws IOException { + client.close(); + } + } +} diff --git a/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyHttpClientExample.java b/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyHttpClientExample.java deleted file mode 100644 index f7cbf3df9..000000000 --- a/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyHttpClientExample.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package co.elastic.clients.transport.netty; - -import io.netty.bootstrap.Bootstrap; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.channel.Channel; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioSocketChannel; -import io.netty.handler.codec.http.DefaultFullHttpRequest; -import io.netty.handler.codec.http.FullHttpRequest; -import io.netty.handler.codec.http.HttpClientCodec; -import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpObject; -import io.netty.handler.codec.http.HttpResponse; -import io.netty.handler.codec.http.HttpUtil; -import io.netty.handler.codec.http.HttpVersion; -import io.netty.handler.codec.http.LastHttpContent; -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslContextBuilder; -import io.netty.handler.ssl.util.InsecureTrustManagerFactory; -import io.netty.util.CharsetUtil; - -import java.net.URI; - -public class NettyHttpClientExample { - private static final String HOST = "example.com"; - private static final int PORT = 80; - - public static void main(String[] args) { - System.out.println("foo"); - //if (true) return; - try { - main0(args); - } catch (Throwable thr) { - thr.printStackTrace(); - } - System.out.println("done"); - } - - public static void main0(String[] args) throws Exception { - - NioEventLoopGroup workerGroup = new NioEventLoopGroup(); - - SslContext sslContext = SslContextBuilder.forClient() - .trustManager(InsecureTrustManagerFactory.INSTANCE) - .build(); - - try { - Bootstrap bootstrap = new Bootstrap(); - bootstrap.group(workerGroup) - .channel(NioSocketChannel.class) - .option(ChannelOption.SO_KEEPALIVE, true) - .handler(new ChannelInitializer() { - @Override - protected void initChannel(SocketChannel ch) throws Exception { - //ch.pipeline().addLast(sslContext.newHandler(ch.alloc())); - ch.pipeline().addLast(new HttpClientCodec()); - //ch.pipeline().addLast(new HttpObjectAggregator(8192)); - ch.pipeline().addLast(new SimpleChannelInboundHandler() { - - @Override - protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Exception { - if (msg instanceof HttpResponse) { - HttpResponse response = (HttpResponse) msg; - - System.err.println("STATUS: " + response.status()); - System.err.println("VERSION: " + response.protocolVersion()); - System.err.println(); - - if (!response.headers().isEmpty()) { - for (CharSequence name: response.headers().names()) { - for (CharSequence value: response.headers().getAll(name)) { - System.err.println("HEADER: " + name + " = " + value); - } - } - System.err.println(); - } - - if (HttpUtil.isTransferEncodingChunked(response)) { - System.err.println("CHUNKED CONTENT {"); - } else { - System.err.println("CONTENT {"); - } - } - if (msg instanceof HttpContent) { - HttpContent content = (HttpContent) msg; - - ByteBuf buf = content.content(); - System.err.println("Buffer: " + buf.readableBytes()); - //buf.retain(); - //buf.release(); - //System.err.print(buf.toString(CharsetUtil.UTF_8)); - //System.err.flush(); - - if (content instanceof LastHttpContent) { - System.err.println("} END OF CONTENT"); - ctx.close(); - } - } - - } - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - cause.printStackTrace(); - ctx.close(); - } - }); - } - }); - - Channel channelFuture = bootstrap.connect(HOST, PORT).sync().channel(); - - URI uri = new URI("http://" + HOST); - String msg = "Hello, Server!"; - FullHttpRequest request = new DefaultFullHttpRequest( - HttpVersion.HTTP_1_1, HttpMethod.GET, uri.toASCIIString(), - Unpooled.wrappedBuffer(msg.getBytes(CharsetUtil.UTF_8))); - - request.headers().set(HttpHeaders.Names.HOST, HOST); - request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.CLOSE); - //request.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP); - - System.err.println("after request 0"); - channelFuture.writeAndFlush(request); - System.err.println("after request 1"); - channelFuture.closeFuture().sync(); - System.err.println("after request 2"); - } finally { - workerGroup.shutdownGracefully().sync(); - System.err.println("after request 3"); - } - } -} diff --git a/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyTransportClient.java b/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyTransportHttpClient.java similarity index 81% rename from example-transports/src/main/java/co/elastic/clients/transport/netty/NettyTransportClient.java rename to example-transports/src/main/java/co/elastic/clients/transport/netty/NettyTransportHttpClient.java index a44f6f9d3..adad5d9be 100644 --- a/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyTransportClient.java +++ b/example-transports/src/main/java/co/elastic/clients/transport/netty/NettyTransportHttpClient.java @@ -19,6 +19,7 @@ package co.elastic.clients.transport.netty; +import co.elastic.clients.transport.DefaultTransportOptions; import co.elastic.clients.transport.TransportOptions; import co.elastic.clients.transport.http.TransportHttpClient; import co.elastic.clients.util.BinaryData; @@ -41,6 +42,7 @@ import io.netty.handler.codec.http.HttpClientCodec; import io.netty.handler.codec.http.HttpContent; import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpObject; import io.netty.handler.codec.http.HttpResponse; @@ -48,49 +50,51 @@ import io.netty.handler.codec.http.LastHttpContent; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; import io.netty.util.concurrent.Future; import javax.annotation.Nullable; -import javax.net.ssl.SSLContext; import javax.net.ssl.SSLException; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; -public class NettyTransportClient implements TransportHttpClient { +/** + * Prototype implementation of {@link TransportHttpClient} based on Netty. Not production-ready. + */ +public class NettyTransportHttpClient implements TransportHttpClient { private final NioEventLoopGroup workerGroup = new NioEventLoopGroup(); private final SslContext sslContext; - public NettyTransportClient() { + public NettyTransportHttpClient() { try { - sslContext = SslContextBuilder.forClient() - .sslContextProvider(SSLContext.getDefault().getProvider()) + // Trust any certificate. DO NOT USE IN PRODUCTION! + this.sslContext = SslContextBuilder + .forClient() + .trustManager(InsecureTrustManagerFactory.INSTANCE) .build(); - } catch (SSLException | NoSuchAlgorithmException e) { + } catch (SSLException e) { throw new RuntimeException(e); } } @Override - public DefaultOptions createOptions(@Nullable TransportOptions options) { - return options == null ? new DefaultOptions() : new DefaultOptions(options); + public DefaultTransportOptions createOptions(@Nullable TransportOptions options) { + return DefaultTransportOptions.of(options); } @Override public Response performRequest(String endpointId, Node node, Request request, TransportOptions options) throws IOException { - try { return performRequestAsync(endpointId, node, request, options).get(); } catch (InterruptedException ie) { @@ -104,7 +108,7 @@ public Response performRequest(String endpointId, Node node, Request request, Tr @Override public CompletableFuture performRequestAsync(String endpointId, Node node, Request request, TransportOptions options) { - CompletableFuture promise = new CompletableFuture<>() { + CompletableFuture promise = new CompletableFuture() { @Override public boolean cancel(boolean mayInterruptIfRunning) { // TODO: cancel pending request @@ -128,13 +132,22 @@ protected void initChannel(SocketChannel ch) throws Exception { } }); + String uri = request.path(); + + // If the node is not at the server root, prepend its path. + String nodePath = node.uri().getRawPath(); + if (nodePath.length() > 1) { + if (uri.charAt(0) == '/') { + uri = uri.substring(1); + } + uri = nodePath + uri; + } - String path = request.path(); + // Append query parameters String queryString = queryString(request, options); if (queryString != null) { - path = path + "?" + queryString; + uri = uri + "?" + queryString; } - String uri = node.uriForPath(path).toString(); ByteBuf nettyBody; Iterable body = request.body(); @@ -160,24 +173,19 @@ protected void initChannel(SocketChannel ch) throws Exception { nettyBody ); + HttpHeaders nettyHeaders = nettyRequest.headers(); // Netty doesn't set Content-Length automatically with FullRequest. - nettyRequest.headers().set(HttpHeaderNames.CONTENT_LENGTH, nettyBody.readableBytes()); - - if (request.contentType() != null) { - nettyRequest.headers().set(HttpHeaderNames.CONTENT_TYPE, request.contentType()); - } + nettyHeaders.set(HttpHeaderNames.CONTENT_LENGTH, nettyBody.readableBytes()); int port = node.uri().getPort(); if (port == -1) { port = node.uri().getScheme().equals("https") ? 443 : 80; } - nettyRequest.headers().set(HttpHeaderNames.HOST, node.uri().getHost() + ":" + port); - if (options != null) { - for (Map.Entry header: options.headers()) { - nettyRequest.headers().set(header.getKey(), header.getValue()); - } - } + nettyHeaders.set(HttpHeaderNames.HOST, node.uri().getHost() + ":" + port); + + request.headers().forEach(nettyHeaders::set); + options.headers().stream().forEach((kv) -> nettyHeaders.set(kv.getKey(), kv.getValue())); ChannelFuture future0 = bootstrap.connect(node.uri().getHost(), port); future0.addListener((ChannelFutureListener) future1 -> { @@ -222,8 +230,14 @@ private String queryString(Request request, TransportOptions options) { return allParams .entrySet() .stream() - .map(e -> URLEncoder.encode(e.getKey(), StandardCharsets.UTF_8) + "=" + - URLEncoder.encode(e.getValue(), StandardCharsets.UTF_8)) + .map(e -> { + try { + return URLEncoder.encode(e.getKey(), "UTF-8") + "=" + + URLEncoder.encode(e.getValue(), "UTF-8"); + } catch(UnsupportedEncodingException ex) { + throw new RuntimeException(ex); + } + }) .collect(Collectors.joining("&")); } } @@ -261,7 +275,7 @@ private static class ChannelHandler extends SimpleChannelInboundHandler body; - public ChannelHandler(Node node, CompletableFuture promise) { + ChannelHandler(Node node, CompletableFuture promise) { this.node = node; this.promise = promise; } @@ -308,7 +322,7 @@ private static class NettyResponse implements TransportHttpClient.Response { @Nullable private final List body; - public NettyResponse(Node node, HttpResponse response, @Nullable List body) { + NettyResponse(Node node, HttpResponse response, @Nullable List body) { this.node = node; this.response = response; this.body = body; @@ -329,6 +343,11 @@ public String header(String name) { return response.headers().get(name); } + @Override + public List headers(String name) { + return response.headers().getAll(name); // returns an empty list if no values + } + @Nullable @Override public BinaryData body() throws IOException { @@ -343,17 +362,20 @@ public BinaryData body() throws IOException { ); } + @Nullable @Override - public Throwable createException() throws IOException { - // TODO - return null; + public HttpResponse originalResponse() { + return this.response; } @Override public void close() throws IOException { if (body != null) { for (ByteBuf buf: body) { - buf.release(); + // May have been released already if body() was consumed + if (buf.refCnt() > 0) { + buf.release(); + } } body.clear(); } diff --git a/example-transports/src/test/java/co/elastic/clients/transport/TransportHttpClientTest.java b/example-transports/src/test/java/co/elastic/clients/transport/TransportHttpClientTest.java index bc897f43b..ba2a42956 100644 --- a/example-transports/src/test/java/co/elastic/clients/transport/TransportHttpClientTest.java +++ b/example-transports/src/test/java/co/elastic/clients/transport/TransportHttpClientTest.java @@ -19,8 +19,8 @@ package co.elastic.clients.transport; +import co.elastic.clients.transport.http.HeaderMap; import co.elastic.clients.transport.http.TransportHttpClient; -import co.elastic.clients.transport.netty.DefaultOptions; import com.fasterxml.jackson.databind.ObjectMapper; import com.sun.net.httpserver.Headers; import com.sun.net.httpserver.HttpServer; @@ -29,9 +29,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import java.io.ByteArrayOutputStream; import java.io.OutputStream; -import java.io.PrintStream; import java.net.InetAddress; import java.net.InetSocketAddress; import java.nio.ByteBuffer; @@ -40,7 +38,7 @@ import java.util.List; import java.util.Map; -public abstract class TransportHttpClientTest> extends Assertions { +public abstract class TransportHttpClientTest extends Assertions { public static record EchoResponse(Map> headers, String body) { } @@ -95,12 +93,17 @@ public void testClient() throws Exception { "http://" + server.getAddress().getHostString() + ":" + server.getAddress().getPort() + "/" ); - TransportOptions options = new DefaultOptions.Builder().addHeader("X-Header", "some value").build(); + TransportOptions options = new DefaultTransportOptions.Builder() + .addHeader("X-Options-Header", "options value") + .build(); + HeaderMap headers = new HeaderMap(); + headers.put("Content-Type", "text/plain"); + headers.put("X-Request-Header", "request value"); TransportHttpClient.Response response = httpClient.performRequest( "foo", node, - new TransportHttpClient.Request("POST", "/root/echo", Map.of(), "text/plain", requestBody), + new TransportHttpClient.Request("POST", "/root/echo", Map.of(), headers, requestBody), options ); @@ -110,7 +113,8 @@ public void testClient() throws Exception { var echoHeaders = normalizeHeaders(echoResponse.headers()); assertEquals("text/plain", echoHeaders.get("content-type")); - assertEquals("some value", echoHeaders.get("x-header")); + assertEquals("options value", echoHeaders.get("x-options-header")); + assertEquals("request value", echoHeaders.get("x-request-header")); dump(echoHeaders); diff --git a/example-transports/src/test/java/co/elastic/clients/transport/netty/NettyClientTest.java b/example-transports/src/test/java/co/elastic/clients/transport/netty/NettyClientTest.java index 72313296c..a41e007c2 100644 --- a/example-transports/src/test/java/co/elastic/clients/transport/netty/NettyClientTest.java +++ b/example-transports/src/test/java/co/elastic/clients/transport/netty/NettyClientTest.java @@ -22,9 +22,9 @@ import co.elastic.clients.transport.TransportHttpClientTest; -public class NettyClientTest extends TransportHttpClientTest { +public class NettyClientTest extends TransportHttpClientTest { - public NettyClientTest() { - super(new NettyTransportClient()); + public NettyClientTest() throws Exception { + super(new NettyTransportHttpClient()); } } diff --git a/example-transports/src/test/java/co/elastic/clients/transport/rest_client/RestTransportClientTest.java b/example-transports/src/test/java/co/elastic/clients/transport/rest_client/RestTransportClientTest.java index 663829e75..fe0866652 100644 --- a/example-transports/src/test/java/co/elastic/clients/transport/rest_client/RestTransportClientTest.java +++ b/example-transports/src/test/java/co/elastic/clients/transport/rest_client/RestTransportClientTest.java @@ -20,10 +20,8 @@ package co.elastic.clients.transport.rest_client; import co.elastic.clients.transport.TransportHttpClientTest; -import co.elastic.clients.transport.http.TransportHttpClient; import org.apache.http.HttpHost; import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; public class RestTransportClientTest extends TransportHttpClientTest { diff --git a/java-client/src/main/java/co/elastic/clients/transport/DefaultTransportOptions.java b/java-client/src/main/java/co/elastic/clients/transport/DefaultTransportOptions.java new file mode 100644 index 000000000..f199705c1 --- /dev/null +++ b/java-client/src/main/java/co/elastic/clients/transport/DefaultTransportOptions.java @@ -0,0 +1,209 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport; + +import co.elastic.clients.transport.http.HeaderMap; +import co.elastic.clients.util.ObjectBuilderBase; + +import javax.annotation.Nullable; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +/** + * Default implementation of {@link TransportOptions}. Extensions can use it as a base class to provide additional features. + */ +public class DefaultTransportOptions implements TransportOptions { + private final HeaderMap headers; + private final Map parameters; + private final Function, Boolean> onWarnings; + + public static final DefaultTransportOptions EMPTY = new DefaultTransportOptions(); + + public DefaultTransportOptions() { + this(new HeaderMap(), Collections.emptyMap(), null); + } + + public DefaultTransportOptions( + @Nullable HeaderMap headers, + @Nullable Map parameters, + @Nullable Function, Boolean> onWarnings + ) { + this.headers = headers == null ? HeaderMap.EMPTY : headers; + this.parameters = (parameters == null || parameters.isEmpty()) ? + Collections.emptyMap() : Collections.unmodifiableMap(parameters); + this.onWarnings = onWarnings; + } + + protected DefaultTransportOptions(AbstractBuilder builder) { + this(builder.headers, builder.parameters, builder.onWarnings); + } + + public static DefaultTransportOptions of(@Nullable TransportOptions options) { + if (options == null) { + return new DefaultTransportOptions(null, null, null); + } + if (options instanceof DefaultTransportOptions) { + return (DefaultTransportOptions) options; + } + return new DefaultTransportOptions( + new HeaderMap(entriesToMap(options.headers())), + options.queryParameters(), + options.onWarnings() + ); + } + + @Override + public Collection> headers() { + return Collections.unmodifiableSet(headers.entrySet()); + } + + @Override + public Map queryParameters() { + return parameters; + } + + @Override + public Function, Boolean> onWarnings() { + return onWarnings; + } + + @Override + public Builder toBuilder() { + return new Builder(this); + } + + private static Map entriesToMap(Collection> entries) { + if (entries.isEmpty()) { + return Collections.emptyMap(); + } else { + HashMap map = new HashMap<>(); + for (Map.Entry entry: entries) { + map.put(entry.getKey(), entry.getValue()); + } + return map; + } + } + + public abstract static class AbstractBuilder> + extends ObjectBuilderBase implements TransportOptions.Builder { + + private HeaderMap headers; + private Map parameters; + private Function, Boolean> onWarnings; + + public AbstractBuilder() { + } + + public AbstractBuilder(DefaultTransportOptions options) { + this.headers = new HeaderMap(options.headers); + this.parameters = copyOrNull(options.parameters); + this.onWarnings = options.onWarnings; + } + + protected abstract BuilderT self(); + + @Override + public BuilderT addHeader(String name, String value) { + if (name.equalsIgnoreCase(HeaderMap.CLIENT_META)) { + // Not overridable + return self(); + } + if (this.headers == null) { + this.headers = new HeaderMap(); + } + headers.add(name, value); + return self(); + } + + @Override + public BuilderT setHeader(String name, String value) { + if (name.equalsIgnoreCase(HeaderMap.CLIENT_META)) { + // Not overridable + return self(); + } + if (this.headers == null) { + this.headers = new HeaderMap(); + } + headers.put(name, value); + return self(); + } + + @Override + public BuilderT removeHeader(String name) { + if (this.headers != null) { + headers.remove(name); + } + return self(); + } + + @Override + public BuilderT setParameter(String name, String value) { + if (parameters == null) { + parameters = new HashMap<>(); + } + parameters.put(name, value); + return self(); + } + + @Override + public BuilderT removeParameter(String name) { + if (parameters != null) { + parameters.remove(name); + }; + return self(); + } + + @Override + public BuilderT onWarnings(Function, Boolean> listener) { + this.onWarnings = listener; + return self(); + } + + private Map copyOrNull(Map map) { + return map.isEmpty() ? null : new HashMap<>(map); + } + } + + public static class Builder extends AbstractBuilder { + + public Builder() { + super(); + } + + public Builder(DefaultTransportOptions options) { + super(options); + } + + @Override + protected Builder self() { + return this; + } + + @Override + public TransportOptions build() { + _checkSingleUse(); + return new DefaultTransportOptions(this); + } + } +} diff --git a/java-client/src/main/java/co/elastic/clients/transport/ElasticsearchTransportBase.java b/java-client/src/main/java/co/elastic/clients/transport/ElasticsearchTransportBase.java index 48ceec44a..2a27db247 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/ElasticsearchTransportBase.java +++ b/java-client/src/main/java/co/elastic/clients/transport/ElasticsearchTransportBase.java @@ -28,7 +28,9 @@ import co.elastic.clients.transport.endpoints.BinaryEndpoint; import co.elastic.clients.transport.endpoints.BooleanEndpoint; import co.elastic.clients.transport.endpoints.BooleanResponse; +import co.elastic.clients.transport.http.HeaderMap; import co.elastic.clients.transport.http.TransportHttpClient; +import co.elastic.clients.util.LanguageRuntimeVersions; import co.elastic.clients.util.ApiTypeHelper; import co.elastic.clients.util.BinaryData; import co.elastic.clients.util.ByteArrayBinaryData; @@ -50,27 +52,28 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; -public abstract class ElasticsearchTransportBase< - Options extends TransportOptions - > implements ElasticsearchTransport { +public abstract class ElasticsearchTransportBase implements ElasticsearchTransport { - public static final String JsonContentType; + private static final String USER_AGENT_VALUE = getUserAgent(); + private static final String CLIENT_META_VALUE = getClientMeta(); + public static final String JSON_CONTENT_TYPE; static { if (Version.VERSION == null) { - JsonContentType = ContentType.APPLICATION_JSON; + JSON_CONTENT_TYPE = ContentType.APPLICATION_JSON; } else { - JsonContentType = + JSON_CONTENT_TYPE = "application/vnd.elasticsearch+json; compatible-with=" + Version.VERSION.major(); } } - private final TransportHttpClient httpClient; + private final TransportHttpClient httpClient; @Override public void close() throws IOException { @@ -78,12 +81,12 @@ public void close() throws IOException { } private final JsonpMapper mapper; - protected final Options transportOptions; + protected final TransportOptions transportOptions; - public ElasticsearchTransportBase(JsonpMapper jsonpMapper, TransportHttpClient httpClient, Options options) { + public ElasticsearchTransportBase(TransportHttpClient httpClient, TransportOptions options, JsonpMapper jsonpMapper) { this.mapper = jsonpMapper; this.httpClient = httpClient; - this.transportOptions = options == null ? httpClient.createOptions(null) : options; + this.transportOptions = httpClient.createOptions(options); } @Override @@ -102,8 +105,8 @@ public final ResponseT performRequest( Endpoint endpoint, @Nullable TransportOptions options ) throws IOException { - TransportHttpClient.Request req = prepareTransportRequest(request, endpoint, options); - Options opts = options == null ? transportOptions : httpClient.createOptions(options); + TransportOptions opts = options == null ? transportOptions : options; + TransportHttpClient.Request req = prepareTransportRequest(request, endpoint); TransportHttpClient.Response resp = httpClient.performRequest(endpoint.id(), null, req, opts); return getApiResponse(resp, endpoint); } @@ -114,9 +117,10 @@ public final CompletableFuture performR Endpoint endpoint, @Nullable TransportOptions options ) { + TransportOptions opts = options == null ? transportOptions : options; TransportHttpClient.Request clientReq; try { - clientReq = prepareTransportRequest(request, endpoint, options); + clientReq = prepareTransportRequest(request, endpoint); } catch (Exception e) { // Terminate early CompletableFuture future = new CompletableFuture<>(); @@ -128,7 +132,7 @@ public final CompletableFuture performR boolean disableRequiredChecks = ApiTypeHelper.requiredPropertiesCheckDisabled(); CompletableFuture clientFuture = httpClient.performRequestAsync( - endpoint.id(), null, clientReq, httpClient.createOptions(options) + endpoint.id(), null, clientReq, opts ); // Cancelling the result will cancel the upstream future created by the http client, allowing to stop in-flight requests @@ -165,15 +169,14 @@ public boolean cancel(boolean mayInterruptIfRunning) { private TransportHttpClient.Request prepareTransportRequest( RequestT request, - Endpoint endpoint, - @Nullable TransportOptions options + Endpoint endpoint ) throws IOException { String method = endpoint.method(request); String path = endpoint.requestUrl(request); Map params = endpoint.queryParameters(request); List bodyBuffers = null; - String contentType = null; + HeaderMap headers = DefaultHeaders; Object body = endpoint.body(request); if (body != null) { @@ -181,7 +184,7 @@ private TransportHttpClient.Request prepareTranspo if (body instanceof NdJsonpSerializable) { bodyBuffers = new ArrayList<>(); collectNdJsonLines(bodyBuffers, (NdJsonpSerializable) request); - contentType = JsonContentType; + headers = JsonContentTypeHeaders; } else if (body instanceof BinaryData) { BinaryData data = (BinaryData)body; @@ -190,9 +193,10 @@ private TransportHttpClient.Request prepareTranspo String dataContentType = data.contentType(); if (ContentType.APPLICATION_JSON.equals(dataContentType)) { // Fast path - contentType = JsonContentType; + headers = JsonContentTypeHeaders; } else { - contentType = dataContentType; + headers = new HeaderMap(DefaultHeaders); + headers.put(HeaderMap.CONTENT_TYPE, dataContentType); } bodyBuffers = Collections.singletonList(data.asByteBuffer()); @@ -202,11 +206,19 @@ private TransportHttpClient.Request prepareTranspo mapper.serialize(body, generator); generator.close(); bodyBuffers = Collections.singletonList(baos.asByteBuffer()); - contentType = JsonContentType; + headers = JsonContentTypeHeaders; } } - return new TransportHttpClient.Request(method, path, params, contentType, bodyBuffers); + return new TransportHttpClient.Request(method, path, params, headers, bodyBuffers); + } + + private static final HeaderMap JsonContentTypeHeaders = new HeaderMap(); + private static final HeaderMap DefaultHeaders = new HeaderMap(); + static { + addStandardHeaders(DefaultHeaders); + addStandardHeaders(JsonContentTypeHeaders); + JsonContentTypeHeaders.put(HeaderMap.CONTENT_TYPE, JSON_CONTENT_TYPE); } private static final ByteBuffer NdJsonSeparator = ByteBuffer.wrap("\n".getBytes(StandardCharsets.UTF_8)); @@ -241,24 +253,27 @@ private ResponseT getApiResponse( } if (endpoint.isError(statusCode)) { + JsonpDeserializer errorDeserializer = endpoint.errorDeserializer(statusCode); if (errorDeserializer == null) { throw new TransportException( - statusCode, + clientResp, "Request failed with status code '" + statusCode + "'", - endpoint.id(), clientResp.createException() + endpoint.id() ); } BinaryData entity = clientResp.body(); if (entity == null) { throw new TransportException( - statusCode, + clientResp, "Expecting a response body, but none was sent", - endpoint.id(), clientResp.createException() + endpoint.id() ); } + checkJsonContentType(entity.contentType(), clientResp, endpoint); + // We may have to replay it. if (!entity.isRepeatable()) { entity = new ByteArrayBinaryData(entity); @@ -277,9 +292,11 @@ private ResponseT getApiResponse( return response; } catch(Exception respEx) { // No better luck: throw the original error decoding exception - throw new TransportException(statusCode, - "Failed to decode error response, check exception cause for additional details", endpoint.id(), - clientResp.createException() + throw new TransportException( + clientResp, + "Failed to decode error response, check exception cause for additional details", + endpoint.id(), + errorEx ); } } @@ -310,11 +327,12 @@ private ResponseT decodeTransportResponse( // Expecting a body if (entity == null) { throw new TransportException( - statusCode, + clientResp, "Expecting a response body, but none was sent", - endpoint.id(), clientResp.createException() + endpoint.id() ); } + checkJsonContentType(entity.contentType(), clientResp, endpoint); try ( InputStream content = entity.asInputStream(); JsonParser parser = mapper.jsonProvider().createParser(content) @@ -322,7 +340,7 @@ private ResponseT decodeTransportResponse( response = responseParser.deserialize(parser, mapper); } catch (Exception e) { throw new TransportException( - statusCode, + clientResp, "Failed to decode response", endpoint.id(), e @@ -345,7 +363,10 @@ private ResponseT decodeTransportResponse( return response; } else { - throw new TransportException(statusCode, "Unhandled endpoint type: '" + endpoint.getClass().getName() + "'", endpoint.id()); + throw new TransportException( + clientResp, + "Unhandled endpoint type: '" + endpoint.getClass().getName() + "'", endpoint.id() + ); } } @@ -361,21 +382,67 @@ private void checkProductHeader(TransportHttpClient.Response clientResp, Endpoin return; } throw new TransportException( - clientResp.statusCode(), + clientResp, "Missing [X-Elastic-Product] header. Please check that you are connecting to an Elasticsearch " + "instance, and that any networking filters are preserving that header.", - endpoint.id(), - clientResp.createException() + endpoint.id() ); } if (!"Elasticsearch".equals(header)) { throw new TransportException( - clientResp.statusCode(), + clientResp, "Invalid value '" + header + "' for 'X-Elastic-Product' header.", - endpoint.id(), - clientResp.createException() + endpoint.id() ); } } + + private void checkJsonContentType( + String contentType, TransportHttpClient.Response clientResp, Endpoint endpoint + ) throws IOException { + if (contentType == null) { + throw new TransportException(clientResp, "Response has no content-type", endpoint.id()); + } + + if (contentType.startsWith("application/json") || contentType.startsWith("application/vnd.elasticsearch+json")) { + return; + } + + throw new TransportException(clientResp, "Expecting JSON data but response content-type is: " + contentType, endpoint.id()); + } + + private static void addStandardHeaders(HeaderMap headers) { + headers.put(HeaderMap.USER_AGENT, USER_AGENT_VALUE); + headers.put(HeaderMap.CLIENT_META, CLIENT_META_VALUE); + headers.put(HeaderMap.ACCEPT, JSON_CONTENT_TYPE); + } + + private static String getUserAgent() { + return String.format( + Locale.ROOT, + "elastic-java/%s (Java/%s)", + Version.VERSION == null ? "Unknown" : Version.VERSION.toString(), + System.getProperty("java.version") + ); + } + + private static String getClientMeta() { + // Use a single 'p' suffix for all prerelease versions (snapshot, beta, etc). + String metaVersion = Version.VERSION == null ? "" : Version.VERSION.toString(); + int dashPos = metaVersion.indexOf('-'); + if (dashPos > 0) { + metaVersion = metaVersion.substring(0, dashPos) + "p"; + } + + // service, language, transport, followed by additional information + return "es=" + + metaVersion + + ",jv=" + + System.getProperty("java.specification.version") + + ",t=" + + metaVersion + + ",hl=2" + + LanguageRuntimeVersions.getRuntimeMetadata(); + } } diff --git a/java-client/src/main/java/co/elastic/clients/transport/TransportException.java b/java-client/src/main/java/co/elastic/clients/transport/TransportException.java index b1fbaa9e4..d7d65128a 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/TransportException.java +++ b/java-client/src/main/java/co/elastic/clients/transport/TransportException.java @@ -19,29 +19,42 @@ package co.elastic.clients.transport; +import co.elastic.clients.transport.http.TransportHttpClient; + import javax.annotation.Nullable; import java.io.IOException; public class TransportException extends IOException { - private final int statusCode; private final String endpointId; + private final TransportHttpClient.Response response; - public TransportException(int statusCode, String message, String endpointId) { - this(statusCode, message, endpointId, null); + public TransportException(TransportHttpClient.Response response, String message, String endpointId) { + this(response, message, endpointId, null); } - public TransportException(int statusCode, String message, String endpointId, Throwable cause) { - super("status: " + statusCode + ", " + (endpointId == null ? message : "[" + endpointId + "] " + message), cause); - this.statusCode = statusCode; + public TransportException(TransportHttpClient.Response response, String message, String endpointId, Throwable cause) { + super( + "node: " + response.node() + ", status: " + response.statusCode() + ", " + + (endpointId == null ? message : "[" + endpointId + "] " + message), + cause + ); + this.response = response; this.endpointId = endpointId; + + // Make sure the response is closed to free up resources. + try { + response.close(); + } catch (Exception e) { + this.addSuppressed(e); + } } /** * Status code returned by the http resquest */ public int statusCode() { - return statusCode; + return response.statusCode(); } /** @@ -52,4 +65,7 @@ public String endpointId() { return endpointId; } + public TransportHttpClient.Response response() { + return response; + } } diff --git a/java-client/src/main/java/co/elastic/clients/transport/TransportOptions.java b/java-client/src/main/java/co/elastic/clients/transport/TransportOptions.java index d9a6ddc88..d6c41f490 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/TransportOptions.java +++ b/java-client/src/main/java/co/elastic/clients/transport/TransportOptions.java @@ -50,8 +50,14 @@ interface Builder extends ObjectBuilder { Builder addHeader(String name, String value); + Builder setHeader(String name, String value); + + Builder removeHeader(String name); + Builder setParameter(String name, String value); + Builder removeParameter(String name); + Builder onWarnings(Function, Boolean> listener); } } diff --git a/java-client/src/main/java/co/elastic/clients/transport/http/HeaderMap.java b/java-client/src/main/java/co/elastic/clients/transport/http/HeaderMap.java new file mode 100644 index 000000000..45ae128b5 --- /dev/null +++ b/java-client/src/main/java/co/elastic/clients/transport/http/HeaderMap.java @@ -0,0 +1,173 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport.http; + +import javax.annotation.Nullable; +import java.util.AbstractMap; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +/** + * A (string, string) map with case-insensitive keys. + */ +public class HeaderMap extends AbstractMap { + + public static final String ACCEPT = "Accept"; + public static final String CONTENT_TYPE = "Content-Type"; + public static final String USER_AGENT = "User-Agent"; + public static final String CLIENT_META = "X-Elastic-Client-Meta"; + + @Nullable + protected Map map; + + public static HeaderMap EMPTY = new HeaderMap(null).locked(); + + public HeaderMap() { + this.map = null; + } + + /** + * Copy constructor + */ + public HeaderMap(@Nullable Map map) { + if (map == null || map.isEmpty()) { + this.map = null; + } else if (map instanceof HeaderMap) { + Map hmap = ((HeaderMap) map).map; + this.map = hmap == null ? null : new HashMap<>(hmap); + } else { + this.map = new HashMap<>(map); + } + } + + @Override + public int size() { + return map == null ? 0 : map.size(); + } + + @Override + public Set> entrySet() { + return map == null ? Collections.emptySet() : map.entrySet(); + } + + @Override + public String get(Object object) { + String key = (String)object; // throwing ClassCastException is allowed + if (map == null) { + return null; + } + for (Entry entry : map.entrySet()) { + if (entry.getKey().equalsIgnoreCase(key)) { + return entry.getValue(); + } + } + return null; + } + + @Override + public String put(String key, String value) { + String result; + if (map == null) { + map = new HashMap<>(); + result = null; + } else { + result = remove(key); + } + map.put(key, value); + return result; + } + + public String add(String key, String value) { + if (map == null) { + map = new HashMap<>(); + } else { + for (Entry entry : map.entrySet()) { + if (entry.getKey().equalsIgnoreCase(key)) { + String current = entry.getValue(); + entry.setValue(current + "; " + value); + return current; + } + } + } + return map.put(key, value); + } + + @Override + public String remove(Object object) { + String key = (String)object; // throwing ClassCastException is allowed + if (map == null) { + return null; + } else { + Iterator> entries = map.entrySet().iterator(); + while(entries.hasNext()) { + Map.Entry entry = entries.next(); + if (entry.getKey().equalsIgnoreCase(key)) { + entries.remove(); + return entry.getKey(); + } + } + } + return null; + } + + /** + * Return a locked copy of this header map that cannot be modified + */ + public HeaderMap locked() { + return new Locked(map); + } + + private static class Locked extends HeaderMap { + Locked(Map map) { + super(map); + } + + private String isLocked() { + throw new UnsupportedOperationException("HeaderMap is write locked"); + } + + @Override + public String put(String key, String value) { + return isLocked(); + } + + @Override + public String add(String key, String value) { + return isLocked(); + } + + @Override + public String remove(Object object) { + return isLocked(); + } + + @Override + public Set> entrySet() { + if (map == null) { + return Collections.emptySet(); + } else { + return Collections.unmodifiableSet(super.entrySet()); + } + } + } +} diff --git a/java-client/src/main/java/co/elastic/clients/transport/http/TransportHttpClient.java b/java-client/src/main/java/co/elastic/clients/transport/http/TransportHttpClient.java index 1b11f86a2..74ce8bd7a 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/http/TransportHttpClient.java +++ b/java-client/src/main/java/co/elastic/clients/transport/http/TransportHttpClient.java @@ -19,50 +19,69 @@ package co.elastic.clients.transport.http; +import co.elastic.clients.transport.DefaultTransportOptions; import co.elastic.clients.transport.TransportOptions; import co.elastic.clients.util.BinaryData; import javax.annotation.Nullable; +import java.io.Closeable; import java.io.IOException; import java.net.URI; import java.nio.ByteBuffer; import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.CompletableFuture; /** * Minimal http client interface needed to implement an Elasticsearch transport. - * - * @param the client's options type */ -public interface TransportHttpClient { +public interface TransportHttpClient { /** * Create a client-specific options value from an existing option object. If {@code null}, this must * create the default options to which additional options can be added. + *

+ * This method allows implementations to return subclasses with more features (that applications can use by downcasting the result). + * By default, it will use {@link DefaultTransportOptions}. */ - Options createOptions(@Nullable TransportOptions options); + default TransportOptions createOptions(@Nullable TransportOptions options) { + return options == null ? DefaultTransportOptions.EMPTY : options; + } /** * Perform a blocking request. * * @param endpointId the endpoint identifier. Can be used to have specific strategies depending on the endpoint. + * @param node the node to send the request to. If {@code null}, the implementation has to choose which node to send the request to, + * or throw an {@code IllegalArgumentException}. * @param request the request - * @param options additional options for the http client + * @param options additional options for the http client. Headers and request parameters set in the options have precedence over + * those defined by the request and should replace them in the final request sent. * - * @return the request response + * @return the response + * @throws IllegalArgumentException if {@code node} is {@code is null} and the implementation cannot decide of + * a node to use. */ Response performRequest(String endpointId, @Nullable Node node, Request request, TransportOptions options) throws IOException; /** * Perform an asynchronous request. + *

+ * Implementations should return a {@code CompletableFuture} whose cancellation also cancels any http request in flight and frees + * the associated resources. This allows applications to implement scenarios like timeouts or "first to respond" fan-out without + * leaking resources. * * @param endpointId the endpoint identifier. Can be used to have specific strategies depending on the endpoint. + * @param node the node to send the request to. If {@code null}, the implementation has to choose which node to send the request to, + * or throw an {@code IllegalArgumentException}. * @param request the request - * @param options additional options for the http client + * @param options additional options for the http client. Headers and request parameters set in the options have precedence over + * those defined by the request and should replace them in the final request sent. * - * @return the request response + * @return a future that will be completed with the response. */ CompletableFuture performRequestAsync(String endpointId, @Nullable Node node, Request request, TransportOptions options); @@ -83,6 +102,11 @@ class Node { * Create a node with its URI, roles and attributes. *

* If the URI doesn't end with a '{@code /}', then one is added. + * + * @param uri the node's URI + * @param roles the node's roles (such as "master", "ingest", etc). This can be used for routing decisions by multi-node + * implementations. + * @param attributes the node's attributes. This can be used for routing decisions by multi-node implementations. */ public Node(URI uri, Set roles, Map attributes) { if (!uri.isAbsolute()) { @@ -106,24 +130,35 @@ public Node(String uri) { this(URI.create(uri), Collections.emptySet(), Collections.emptyMap()); } + /** + * The URI of this node. This is an absolute URL with a path ending with a "/". + */ public URI uri() { return this.uri; } - public URI uriForPath(String path) { - // Make sure the path will be appended to the node's URI path if it exists, and will not replace it. - if (this.uri.getRawPath().length() > 1) { - if (path.charAt(0) == '/') { - path = path.substring(1); - } - } + @Override + public String toString() { + return uri.toString(); + } - return uri.resolve(path); + /** + * Two nodes are considered equal if their URIs are equal. Roles and attributes are ignored. + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Node)) return false; + Node node = (Node) o; + return Objects.equals(uri, node.uri); } + /** + * A node's hash code is that of its URI. Roles and attributes are ignored. + */ @Override - public String toString() { - return uri.toString(); + public int hashCode() { + return Objects.hash(uri); } } @@ -135,17 +170,21 @@ class Request { private final String method; private final String path; private final Map queryParams; - @Nullable - private final String contentType; + private final Map headers; @Nullable private final Iterable body; - public Request(String method, String path, Map queryParams, @Nullable String contentType, - @Nullable Iterable body) { + public Request( + String method, + String path, + Map queryParams, + Map headers, + @Nullable Iterable body + ) { this.method = method; this.path = path; this.queryParams = queryParams; - this.contentType = contentType; + this.headers = headers; this.body = body; } @@ -161,9 +200,8 @@ public Map queryParams() { return queryParams; } - @Nullable - public String contentType() { - return contentType; + public Map headers() { + return headers; } @Nullable @@ -175,11 +213,11 @@ public Iterable body() { /** * An http response. */ - interface Response { + interface Response extends Closeable { /** * The host/node that was used to send the request. It may be different from the one that was provided with the request - * if the http client has an internal retry mechanism. + * if the http client has a multi-node retry strategy. */ Node node(); @@ -190,20 +228,33 @@ interface Response { /** * Get a header value, or the first value if the header has multiple values. + *

+ * Note: header names are case-insensitive */ + @Nullable String header(String name); + /** + * Get all values for a given header name. + *

+ * Note: header names are case-insensitive + */ + List headers(String name); + /** * The response body, if any. */ @Nullable BinaryData body() throws IOException; - - Throwable createException() throws IOException; + /** + * The original response of the underlying http library, if available. + */ + @Nullable + Object originalResponse(); /** - * Close this response, freeing its associated resource, consuming the remaining body, if needed. + * Close this response, freeing its associated resources if needed, such as consuming the response body. */ void close() throws IOException; } diff --git a/java-client/src/main/java/co/elastic/clients/transport/rest_client/HttpClientBinaryResponse.java b/java-client/src/main/java/co/elastic/clients/transport/rest_client/HttpClientBinaryResponse.java deleted file mode 100644 index 27e4dad15..000000000 --- a/java-client/src/main/java/co/elastic/clients/transport/rest_client/HttpClientBinaryResponse.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package co.elastic.clients.transport.rest_client; - -import co.elastic.clients.transport.endpoints.BinaryResponse; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.util.EntityUtils; - -import java.io.IOException; -import java.io.InputStream; - -class HttpClientBinaryResponse implements BinaryResponse { - private final HttpEntity entity; - private boolean consumed = false; - - HttpClientBinaryResponse(HttpEntity entity) { - this.entity = entity; - } - - @Override - public String contentType() { - Header h = entity.getContentType(); - return h == null ? "application/octet-stream" : h.getValue(); - } - - @Override - public long contentLength() { - long len = entity.getContentLength(); - return len < 0 ? -1 : entity.getContentLength(); - } - - @Override - public InputStream content() throws IOException { - if (consumed) { - throw new IllegalStateException("Response content has already been consumed"); - } - consumed = true; - return entity.getContent(); - } - - @Override - public void close() throws IOException { - consumed = true; - EntityUtils.consume(entity); - } -} diff --git a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientHttpClient.java b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientHttpClient.java index be270a9c6..64b5aa08a 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientHttpClient.java +++ b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientHttpClient.java @@ -20,15 +20,16 @@ package co.elastic.clients.transport.rest_client; import co.elastic.clients.transport.TransportOptions; +import co.elastic.clients.transport.http.HeaderMap; import co.elastic.clients.transport.http.TransportHttpClient; import co.elastic.clients.util.BinaryData; import co.elastic.clients.util.NoCopyByteArrayOutputStream; import org.apache.http.Header; +import org.apache.http.HeaderElement; import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Cancellable; -import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.ResponseListener; import org.elasticsearch.client.RestClient; @@ -37,10 +38,14 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.AbstractList; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; -public class RestClientHttpClient implements TransportHttpClient { +public class RestClientHttpClient implements TransportHttpClient { private static final ConcurrentHashMap ContentTypeCache = new ConcurrentHashMap<>(); @@ -76,7 +81,7 @@ public RestClient restClient() { @Override public RestClientOptions createOptions(@Nullable TransportOptions options) { - return options == null ? RestClientOptions.initialOptions() : RestClientOptions.of(options); + return RestClientOptions.of(options); } @Override @@ -88,7 +93,9 @@ public Response performRequest(String endpointId, @Nullable Node node, Request r } @Override - public CompletableFuture performRequestAsync(String endpointId, @Nullable Node node, Request request, TransportOptions options) { + public CompletableFuture performRequestAsync( + String endpointId, @Nullable Node node, Request request, TransportOptions options + ) { RequestFuture future = new RequestFuture<>(); org.elasticsearch.client.Request restRequest; @@ -127,15 +134,41 @@ private org.elasticsearch.client.Request createRestRequest(Request request, Rest request.method(), request.path() ); + Iterable body = request.body(); + + Map requestHeaders = request.headers(); + if (!requestHeaders.isEmpty()) { + + int headerCount = requestHeaders.size(); + if ((body == null && headerCount != 3) || headerCount != 4) { + if (options == null) { + options = RestClientOptions.initialOptions(); + } + + RestClientOptions.Builder builder = options.toBuilder(); + for (Map.Entry header : requestHeaders.entrySet()) { + builder.setHeader(header.getKey(), header.getValue()); + } + // Original option headers have precedence + for (Map.Entry header : options.headers()) { + builder.setHeader(header.getKey(), header.getValue()); + } + options = builder.build(); + } + } + if (options != null) { clientReq.setOptions(options.restClientRequestOptions()); } clientReq.addParameters(request.queryParams()); - Iterable body = request.body(); if (body != null) { - ContentType ct = ContentTypeCache.computeIfAbsent(request.contentType(), ContentType::parse); + ContentType ct = null; + String ctStr; + if (( ctStr = requestHeaders.get(HeaderMap.CONTENT_TYPE)) != null) { + ct = ContentTypeCache.computeIfAbsent(ctStr, ContentType::parse); + } clientReq.setEntity(new MultiBufferEntity(body, ct)); } @@ -144,7 +177,7 @@ private org.elasticsearch.client.Request createRestRequest(Request request, Rest return clientReq; } - private static class RestResponse implements Response { + static class RestResponse implements Response { private final org.elasticsearch.client.Response restResponse; RestResponse(org.elasticsearch.client.Response restResponse) { @@ -166,6 +199,29 @@ public String header(String name) { return restResponse.getHeader(name); } + @Override + public List headers(String name) { + Header[] headers = restResponse.getHeaders(); + for (int i = 0; i < headers.length; i++) { + Header header = headers[i]; + if (header.getName().equalsIgnoreCase(name)) { + HeaderElement[] elements = header.getElements(); + return new AbstractList() { + @Override + public String get(int index) { + return elements[index].getValue(); + } + + @Override + public int size() { + return elements.length; + } + }; + } + } + return Collections.emptyList(); + } + @Nullable @Override public BinaryData body() throws IOException { @@ -173,9 +229,10 @@ public BinaryData body() throws IOException { return entity == null ? null : new HttpEntityBinaryData(restResponse.getEntity()); } + @Nullable @Override - public Throwable createException() throws IOException { - return new ResponseException(this.restResponse); + public org.elasticsearch.client.Response originalResponse() { + return this.restResponse; } @Override diff --git a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientMonolithTransport.java b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientMonolithTransport.java deleted file mode 100644 index 7877df636..000000000 --- a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientMonolithTransport.java +++ /dev/null @@ -1,442 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package co.elastic.clients.transport.rest_client; - -import co.elastic.clients.elasticsearch._types.ElasticsearchException; -import co.elastic.clients.elasticsearch._types.ErrorResponse; -import co.elastic.clients.json.JsonpDeserializer; -import co.elastic.clients.json.JsonpMapper; -import co.elastic.clients.json.NdJsonpSerializable; -import co.elastic.clients.transport.JsonEndpoint; -import co.elastic.clients.transport.TransportException; -import co.elastic.clients.transport.Version; -import co.elastic.clients.transport.endpoints.BinaryEndpoint; -import co.elastic.clients.transport.endpoints.BooleanEndpoint; -import co.elastic.clients.transport.endpoints.BooleanResponse; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.Endpoint; -import co.elastic.clients.transport.TransportOptions; -import co.elastic.clients.util.ApiTypeHelper; -import co.elastic.clients.util.BinaryData; -import co.elastic.clients.util.MissingRequiredPropertyException; -import jakarta.json.JsonException; -import jakarta.json.stream.JsonGenerator; -import jakarta.json.stream.JsonParser; -import org.apache.http.HttpEntity; -import org.apache.http.entity.BufferedHttpEntity; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.entity.ContentType; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; -import org.elasticsearch.client.Cancellable; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.ResponseListener; -import org.elasticsearch.client.RestClient; - -import javax.annotation.Nullable; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CompletableFuture; - -public class RestClientMonolithTransport implements ElasticsearchTransport { - - static final ContentType JsonContentType; - - static { - - if (Version.VERSION == null) { - JsonContentType = ContentType.APPLICATION_JSON; - } else { - JsonContentType = ContentType.create( - "application/vnd.elasticsearch+json", - new BasicNameValuePair("compatible-with", String.valueOf(Version.VERSION.major())) - ); - } - } - - /** - * The {@code Future} implementation returned by async requests. - * It wraps the RestClient's cancellable and propagates cancellation. - */ - private static class RequestFuture extends CompletableFuture { - private volatile Cancellable cancellable; - - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - boolean cancelled = super.cancel(mayInterruptIfRunning); - if (cancelled && cancellable != null) { - cancellable.cancel(); - } - return cancelled; - } - } - - private final RestClient restClient; - private final JsonpMapper mapper; - private final RestClientOptions transportOptions; - - public RestClientMonolithTransport(RestClient restClient, JsonpMapper mapper, @Nullable TransportOptions options) { - this.restClient = restClient; - this.mapper = mapper; - this.transportOptions = createOptions(options); - } - - protected RestClientOptions createOptions(@Nullable TransportOptions options) { - return options == null ? RestClientOptions.initialOptions() : RestClientOptions.of(options); - } - - public RestClientMonolithTransport(RestClient restClient, JsonpMapper mapper) { - this(restClient, mapper, null); - } - - /** - * Returns the underlying low level Rest Client used by this transport. - */ - public RestClient restClient() { - return this.restClient; - } - - /** - * Copies this {@link #RestClientMonolithTransport} with specific request options. - */ - public RestClientMonolithTransport withRequestOptions(@Nullable TransportOptions options) { - return new RestClientMonolithTransport(this.restClient, this.mapper, options); - } - - @Override - public JsonpMapper jsonpMapper() { - return mapper; - } - - @Override - public TransportOptions options() { - return transportOptions; - } - - @Override - public void close() throws IOException { - this.restClient.close(); - } - - public ResponseT performRequest( - RequestT request, - Endpoint endpoint, - @Nullable TransportOptions options - ) throws IOException { - - org.elasticsearch.client.Request clientReq = prepareLowLevelRequest(request, endpoint, options); - org.elasticsearch.client.Response clientResp = restClient.performRequest(clientReq); - return getHighLevelResponse(clientResp, endpoint); - } - - public CompletableFuture performRequestAsync( - RequestT request, - Endpoint endpoint, - @Nullable TransportOptions options - ) { - RequestFuture future = new RequestFuture<>(); - org.elasticsearch.client.Request clientReq; - try { - clientReq = prepareLowLevelRequest(request, endpoint, options); - } catch (Exception e) { - // Terminate early - future.completeExceptionally(e); - return future; - } - - // Propagate required property checks to the thread that will decode the response - boolean disableRequiredChecks = ApiTypeHelper.requiredPropertiesCheckDisabled(); - - future.cancellable = restClient.performRequestAsync(clientReq, new ResponseListener() { - @Override - public void onSuccess(Response clientResp) { - try (ApiTypeHelper.DisabledChecksHandle h = - ApiTypeHelper.DANGEROUS_disableRequiredPropertiesCheck(disableRequiredChecks)) { - - ResponseT response = getHighLevelResponse(clientResp, endpoint); - future.complete(response); - - } catch (Exception e) { - future.completeExceptionally(e); - } - } - - @Override - public void onFailure(Exception e) { - future.completeExceptionally(e); - } - }); - - return future; - } - - private org.elasticsearch.client.Request prepareLowLevelRequest( - RequestT request, - Endpoint endpoint, - @Nullable TransportOptions options - ) throws IOException { - String method = endpoint.method(request); - String path = endpoint.requestUrl(request); - Map params = endpoint.queryParameters(request); - - org.elasticsearch.client.Request clientReq = new org.elasticsearch.client.Request(method, path); - - RequestOptions restOptions = options == null ? - transportOptions.restClientRequestOptions() : - RestClientOptions.of(options).restClientRequestOptions(); - - if (restOptions != null) { - clientReq.setOptions(restOptions); - } - - clientReq.addParameters(params); - - Object body = endpoint.body(request); - if (body != null) { - // Request has a body - if (body instanceof NdJsonpSerializable) { - List lines = new ArrayList<>(); - collectNdJsonLines(lines, (NdJsonpSerializable) request); - clientReq.setEntity(new MultiBufferEntity(lines, JsonContentType)); - - } else if (body instanceof BinaryData) { - BinaryData data = (BinaryData)body; - - // ES expects the Accept and Content-Type headers to be consistent. - ContentType contentType; - String dataContentType = data.contentType(); - if (co.elastic.clients.util.ContentType.APPLICATION_JSON.equals(dataContentType)) { - // Fast path - contentType = JsonContentType; - } else { - contentType = ContentType.parse(dataContentType); - } - - clientReq.setEntity(new MultiBufferEntity( - Collections.singletonList(data.asByteBuffer()), - contentType - )); - - } else { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - JsonGenerator generator = mapper.jsonProvider().createGenerator(baos); - mapper.serialize(body, generator); - generator.close(); - clientReq.setEntity(new ByteArrayEntity(baos.toByteArray(), JsonContentType)); - } - } - - // Request parameter intercepted by LLRC - clientReq.addParameter("ignore", "400,401,403,404,405"); - return clientReq; - } - - private static final ByteBuffer NdJsonSeparator = ByteBuffer.wrap("\n".getBytes(StandardCharsets.UTF_8)); - - private void collectNdJsonLines(List lines, NdJsonpSerializable value) throws IOException { - Iterator values = value._serializables(); - while(values.hasNext()) { - Object item = values.next(); - if (item == null) { - // Skip - } else if (item instanceof NdJsonpSerializable && item != value) { // do not recurse on the item itself - collectNdJsonLines(lines, (NdJsonpSerializable)item); - } else { - // TODO: items that aren't already BinaryData could be serialized to ByteBuffers lazily - // to reduce the number of buffers to keep in memory - lines.add(BinaryData.of(item, this.mapper).asByteBuffer()); - lines.add(NdJsonSeparator); - } - } - } - -// /** -// * Write an nd-json value by serializing each of its items on a separate line, recursing if its items themselves implement -// * {@link NdJsonpSerializable} to flattening nested structures. -// */ -// private void writeNdJson(NdJsonpSerializable value, ByteArrayOutputStream baos) throws IOException { -// Iterator values = value._serializables(); -// while(values.hasNext()) { -// Object item = values.next(); -// if (item instanceof NdJsonpSerializable && item != value) { // do not recurse on the item itself -// writeNdJson((NdJsonpSerializable) item, baos); -// } else { -// JsonGenerator generator = mapper.jsonProvider().createGenerator(baos); -// mapper.serialize(item, generator); -// generator.close(); -// baos.write('\n'); -// } -// } -// } - - private ResponseT getHighLevelResponse( - org.elasticsearch.client.Response clientResp, - Endpoint endpoint - ) throws IOException { - - int statusCode = clientResp.getStatusLine().getStatusCode(); - try { - - if (statusCode == 200) { - checkProductHeader(clientResp, endpoint); - } - - if (endpoint.isError(statusCode)) { - JsonpDeserializer errorDeserializer = endpoint.errorDeserializer(statusCode); - if (errorDeserializer == null) { - throw new TransportException( - statusCode, - "Request failed with status code '" + statusCode + "'", - endpoint.id(), new ResponseException(clientResp) - ); - } - - HttpEntity entity = clientResp.getEntity(); - if (entity == null) { - throw new TransportException( - statusCode, - "Expecting a response body, but none was sent", - endpoint.id(), new ResponseException(clientResp) - ); - } - - // We may have to replay it. - entity = new BufferedHttpEntity(entity); - - try { - InputStream content = entity.getContent(); - try (JsonParser parser = mapper.jsonProvider().createParser(content)) { - ErrorT error = errorDeserializer.deserialize(parser, mapper); - // TODO: have the endpoint provide the exception constructor - throw new ElasticsearchException(endpoint.id(), (ErrorResponse) error); - } - } catch(JsonException | MissingRequiredPropertyException errorEx) { - // Could not decode exception, try the response type - try { - ResponseT response = decodeResponse(statusCode, entity, clientResp, endpoint); - return response; - } catch(Exception respEx) { - // No better luck: throw the original error decoding exception - throw new TransportException(statusCode, - "Failed to decode error response, check exception cause for additional details", endpoint.id(), - new ResponseException(clientResp) - ); - } - } - } else { - return decodeResponse(statusCode, clientResp.getEntity(), clientResp, endpoint); - } - } finally { - // Consume the entity unless this is a successful binary endpoint, where the user must consume the entity - if (!(endpoint instanceof BinaryEndpoint && !endpoint.isError(statusCode))) { - EntityUtils.consume(clientResp.getEntity()); - } - } - } - - private ResponseT decodeResponse( - int statusCode, @Nullable HttpEntity entity, Response clientResp, Endpoint endpoint - ) throws IOException { - - if (endpoint instanceof JsonEndpoint) { - @SuppressWarnings("unchecked") - JsonEndpoint jsonEndpoint = (JsonEndpoint) endpoint; - // Successful response - ResponseT response = null; - JsonpDeserializer responseParser = jsonEndpoint.responseDeserializer(); - if (responseParser != null) { - // Expecting a body - if (entity == null) { - throw new TransportException( - statusCode, - "Expecting a response body, but none was sent", - endpoint.id(), new ResponseException(clientResp) - ); - } - InputStream content = entity.getContent(); - try (JsonParser parser = mapper.jsonProvider().createParser(content)) { - response = responseParser.deserialize(parser, mapper); - } - } - return response; - - } else if(endpoint instanceof BooleanEndpoint) { - BooleanEndpoint bep = (BooleanEndpoint) endpoint; - - @SuppressWarnings("unchecked") - ResponseT response = (ResponseT) new BooleanResponse(bep.getResult(statusCode)); - return response; - - - } else if (endpoint instanceof BinaryEndpoint) { - BinaryEndpoint bep = (BinaryEndpoint) endpoint; - - @SuppressWarnings("unchecked") - ResponseT response = (ResponseT) new HttpClientBinaryResponse(entity); - return response; - - } else { - throw new TransportException(statusCode, "Unhandled endpoint type: '" + endpoint.getClass().getName() + "'", endpoint.id()); - } - } - - // Endpoints that (incorrectly) do not return the Elastic product header - private static final Set endpointsMissingProductHeader = new HashSet<>(Arrays.asList( - "es/snapshot.create" // #74 / elastic/elasticsearch#82358 - )); - - private void checkProductHeader(Response clientResp, Endpoint endpoint) throws IOException { - String header = clientResp.getHeader("X-Elastic-Product"); - if (header == null) { - if (endpointsMissingProductHeader.contains(endpoint.id())) { - return; - } - throw new TransportException( - clientResp.getStatusLine().getStatusCode(), - "Missing [X-Elastic-Product] header. Please check that you are connecting to an Elasticsearch " - + "instance, and that any networking filters are preserving that header.", - endpoint.id(), - new ResponseException(clientResp) - ); - } - - if (!"Elasticsearch".equals(header)) { - throw new TransportException( - clientResp.getStatusLine().getStatusCode(), - "Invalid value '" + header + "' for 'X-Elastic-Product' header.", - endpoint.id(), - new ResponseException(clientResp) - ); - } - } -} diff --git a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientOptions.java b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientOptions.java index 627d8fa11..455459c18 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientOptions.java +++ b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientOptions.java @@ -21,12 +21,15 @@ import co.elastic.clients.transport.TransportOptions; import co.elastic.clients.transport.Version; +import co.elastic.clients.transport.http.HeaderMap; +import co.elastic.clients.util.LanguageRuntimeVersions; import co.elastic.clients.util.VisibleForTesting; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.util.VersionInfo; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.WarningsHandler; +import javax.annotation.Nullable; import java.util.AbstractMap; import java.util.Collection; import java.util.List; @@ -39,25 +42,25 @@ public class RestClientOptions implements TransportOptions { private final RequestOptions options; - private static final String CLIENT_META_HEADER = "X-Elastic-Client-Meta"; - private static final String USER_AGENT_HEADER = "User-Agent"; - @VisibleForTesting static final String CLIENT_META_VALUE = getClientMeta(); @VisibleForTesting static final String USER_AGENT_VALUE = getUserAgent(); - static RestClientOptions of(TransportOptions options) { + static RestClientOptions of(@Nullable TransportOptions options) { + if (options == null) { + return initialOptions(); + } + if (options instanceof RestClientOptions) { return (RestClientOptions)options; - - } else { - final Builder builder = new Builder(RequestOptions.DEFAULT.toBuilder()); - options.headers().forEach(h -> builder.addHeader(h.getKey(), h.getValue())); - options.queryParameters().forEach(builder::setParameter); - builder.onWarnings(options.onWarnings()); - return builder.build(); } + + final Builder builder = new Builder(RequestOptions.DEFAULT.toBuilder()); + options.headers().forEach(h -> builder.addHeader(h.getKey(), h.getValue())); + options.queryParameters().forEach(builder::setParameter); + builder.onWarnings(options.onWarnings()); + return builder.build(); } public RestClientOptions(RequestOptions options) { @@ -118,24 +121,46 @@ public RequestOptions.Builder restClientRequestOptionsBuilder() { @Override public TransportOptions.Builder addHeader(String name, String value) { - if (name.equalsIgnoreCase(CLIENT_META_HEADER)) { + if (name.equalsIgnoreCase(HeaderMap.CLIENT_META)) { // Not overridable return this; } - if (name.equalsIgnoreCase(USER_AGENT_HEADER)) { + if (name.equalsIgnoreCase(HeaderMap.USER_AGENT)) { // We must remove our own user-agent from the options, or we'll end up with multiple values for the header - builder.removeHeader(USER_AGENT_HEADER); + builder.removeHeader(HeaderMap.USER_AGENT); } builder.addHeader(name, value); return this; } + @Override + public TransportOptions.Builder setHeader(String name, String value) { + if (name.equalsIgnoreCase(HeaderMap.CLIENT_META)) { + // Not overridable + return this; + } + builder.removeHeader(name).addHeader(name, value); + return this; + } + + @Override + public TransportOptions.Builder removeHeader(String name) { + builder.removeHeader(name); + return this; + } + @Override public TransportOptions.Builder setParameter(String name, String value) { + // Should be remove and add, but we can't remove. builder.addParameter(name, value); return this; } + @Override + public TransportOptions.Builder removeParameter(String name) { + throw new UnsupportedOperationException("This implementation does not support removing parameters"); + } + /** * Called if there are warnings to determine if those warnings should fail the request. */ @@ -167,13 +192,13 @@ static RestClientOptions initialOptions() { } private static RequestOptions.Builder addBuiltinHeaders(RequestOptions.Builder builder) { - builder.removeHeader(CLIENT_META_HEADER); - builder.addHeader(CLIENT_META_HEADER, CLIENT_META_VALUE); - if (builder.getHeaders().stream().noneMatch(h -> h.getName().equalsIgnoreCase(USER_AGENT_HEADER))) { - builder.addHeader(USER_AGENT_HEADER, USER_AGENT_VALUE); + builder.removeHeader(HeaderMap.CLIENT_META); + builder.addHeader(HeaderMap.CLIENT_META, CLIENT_META_VALUE); + if (builder.getHeaders().stream().noneMatch(h -> h.getName().equalsIgnoreCase(HeaderMap.USER_AGENT))) { + builder.addHeader(HeaderMap.USER_AGENT, USER_AGENT_VALUE); } - if (builder.getHeaders().stream().noneMatch(h -> h.getName().equalsIgnoreCase("Accept"))) { - builder.addHeader("Accept", RestClientTransport.JsonContentType); + if (builder.getHeaders().stream().noneMatch(h -> h.getName().equalsIgnoreCase(HeaderMap.ACCEPT))) { + builder.addHeader(HeaderMap.ACCEPT, RestClientTransport.JSON_CONTENT_TYPE); } return builder; diff --git a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientTransport.java b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientTransport.java index 0aa9021f2..563f59cd8 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientTransport.java +++ b/java-client/src/main/java/co/elastic/clients/transport/rest_client/RestClientTransport.java @@ -24,7 +24,7 @@ import org.elasticsearch.client.RestClient; -public class RestClientTransport extends ElasticsearchTransportBase { +public class RestClientTransport extends ElasticsearchTransportBase { private final RestClient restClient; @@ -33,7 +33,7 @@ public RestClientTransport(RestClient restClient, JsonpMapper jsonpMapper) { } public RestClientTransport(RestClient restClient, JsonpMapper jsonpMapper, RestClientOptions options) { - super(jsonpMapper, new RestClientHttpClient(restClient), options); + super(new RestClientHttpClient(restClient), options, jsonpMapper); this.restClient = restClient; } diff --git a/java-client/src/main/java/co/elastic/clients/transport/rest_client/LanguageRuntimeVersions.java b/java-client/src/main/java/co/elastic/clients/util/LanguageRuntimeVersions.java similarity index 98% rename from java-client/src/main/java/co/elastic/clients/transport/rest_client/LanguageRuntimeVersions.java rename to java-client/src/main/java/co/elastic/clients/util/LanguageRuntimeVersions.java index 5c9008e83..356ceebe0 100644 --- a/java-client/src/main/java/co/elastic/clients/transport/rest_client/LanguageRuntimeVersions.java +++ b/java-client/src/main/java/co/elastic/clients/util/LanguageRuntimeVersions.java @@ -17,14 +17,14 @@ * under the License. */ -package co.elastic.clients.transport.rest_client; +package co.elastic.clients.util; // Copied verbatim from https://github.com/elastic/jvm-languages-sniffer import java.lang.reflect.Field; import java.lang.reflect.Method; -class LanguageRuntimeVersions { +public class LanguageRuntimeVersions { /** * Returns runtime information by looking up classes identifying non-Java JVM diff --git a/java-client/src/test/java/co/elastic/clients/transport/TransportTest.java b/java-client/src/test/java/co/elastic/clients/transport/TransportTest.java index 4d3449984..b79b03893 100644 --- a/java-client/src/test/java/co/elastic/clients/transport/TransportTest.java +++ b/java-client/src/test/java/co/elastic/clients/transport/TransportTest.java @@ -24,7 +24,7 @@ import co.elastic.clients.transport.rest_client.RestClientTransport; import com.sun.net.httpserver.HttpServer; import org.apache.http.HttpHost; -import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -68,8 +68,8 @@ public void testXMLResponse() throws Exception { assertEquals(401, ex.statusCode()); assertEquals("es/cat.indices", ex.endpointId()); - // Cause is transport-dependent - ResponseException restException = (ResponseException) ex.getCause(); - assertEquals(401, restException.getResponse().getStatusLine().getStatusCode()); + // Original response is transport-dependent + Response restClientResponse = (Response)ex.response().originalResponse(); + assertEquals(401, restClientResponse.getStatusLine().getStatusCode()); } } diff --git a/java-client/src/test/java/co/elastic/clients/transport/http/HeaderMapTest.java b/java-client/src/test/java/co/elastic/clients/transport/http/HeaderMapTest.java new file mode 100644 index 000000000..83dee0f8e --- /dev/null +++ b/java-client/src/test/java/co/elastic/clients/transport/http/HeaderMapTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package co.elastic.clients.transport.http; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Iterator; +import java.util.Map; + +class HeaderMapTest extends Assertions { + + @Test + public void testCaseSensitivity() { + HeaderMap headers = new HeaderMap(); + + headers.put("Foo", "bar"); + assertEquals("bar", headers.get("Foo")); + assertEquals("bar", headers.get("foo")); + assertEquals("bar", headers.get("fOO")); + + headers.put("foo", "baz"); + assertEquals("baz", headers.get("Foo")); + assertEquals("baz", headers.get("foo")); + assertEquals("baz", headers.get("fOO")); + } + + @Test + public void testLock() { + HeaderMap headers = new HeaderMap(); + + headers.put("foo", "bar"); + + HeaderMap locked = headers.locked(); + assertEquals("bar", headers.get("Foo")); + + assertThrows(UnsupportedOperationException.class, () -> { + locked.put("foo", "baz"); + }); + + assertThrows(UnsupportedOperationException.class, () -> { + Iterator> iterator = locked.entrySet().iterator(); + assertEquals("bar", iterator.next().getValue()); + iterator.remove(); + }); + + headers.put("foo", "baz"); + assertEquals("baz", headers.get("Foo")); + assertEquals("bar", locked.get("Foo")); + } + + @Test + public void testAdd() { + HeaderMap headers = new HeaderMap(); + + headers.add("Foo", "bar"); + headers.add("foo", "baz"); + + assertEquals("bar; baz", headers.get("Foo")); + } +} From a5096cd7ceddb694494f235272194794945589bd Mon Sep 17 00:00:00 2001 From: Sylvain Wallez Date: Tue, 13 Jun 2023 17:20:03 +0200 Subject: [PATCH 5/5] Move checkstyle action to Java 17 --- .github/workflows/checkstyle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checkstyle.yml b/.github/workflows/checkstyle.yml index 89c3972f4..6bb8a1a6e 100644 --- a/.github/workflows/checkstyle.yml +++ b/.github/workflows/checkstyle.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java-version: [ 11 ] + java-version: [ 17 ] steps: - uses: actions/checkout@v2