Add documentation for Observability (#3896)

* Add documentation for Observability

* Adapt Observation code to the latest dependencies
* Add doc generation tasks for meters and spans
* Document new Observation API features
* Include generated meters and spans docs to a general `metrics.adoc` chapter

* * Adapt `ObservationPropagationChannelInterceptorTests` for the latest `SpansAssert` API

* * Adjust to the latest Micrometer SNAPSHOT
* Make Observation doc generation tasks only as local.
We don't need ambiguous changes to source code on CI

* * Automate metrics/spans docs generation as a part of `reference` build phase
* Replace 'org.springframework.integration' content in the generated files with a 'o.s.i'
to make it easier to read, especially in the tables
* Break `DefaultMessageReceiverObservationConvention <=> IntegrationObservation` classes tangle
using literal for `KeyValues` in the `DefaultMessageReceiverObservationConvention`
instead of nested enums from the `IntegrationObservation`
* Some other minor build script clean up

* Fix indent in `build.gradle` for `micrometerVersion` property code line

* Add new line after observation section in whats-new.adoc

* * Adapt to the latest Micrometer changes

* * Use Reactor `2022.0.0-SNAPSHOT` version

Author: artembilan

build.gradle

@@ -5,6 +5,9 @@ buildscript {
         mavenCentral()
         gradlePluginPortal()
         maven { url 'https://repo.spring.io/plugins-release-local' }
+        if (version.endsWith('SNAPSHOT')) {
+            maven { url 'https://repo.spring.io/snapshot' }
+        }
     }
     dependencies {
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion"
@@ -84,15 +87,15 @@ ext {
     lettuceVersion = '6.2.0.RELEASE'
     log4jVersion = '2.19.0'
     mailVersion = '2.0.1'
-	micrometerVersion = '1.10.0-M6'
-    micrometerTracingVersion = '1.0.0-M8'
+    micrometerVersion = '1.10.0-SNAPSHOT'
+    micrometerTracingVersion = '1.0.0-SNAPSHOT'
     mockitoVersion = '4.8.0'
     mongoDriverVersion = '4.7.1'
     mysqlVersion = '8.0.30'
     pahoMqttClientVersion = '1.2.5'
     postgresVersion = '42.5.0'
     r2dbch2Version = '1.0.0.RC1'
-    reactorVersion = '2022.0.0-M6'
+    reactorVersion = '2022.0.0-SNAPSHOT'
     resilience4jVersion = '1.7.1'
     romeToolsVersion = '1.18.0'
     rsocketVersion = '1.1.3'
@@ -534,7 +537,6 @@ project('spring-integration-core') {
             exclude group: 'io.opentelemetry'
             exclude group: 'com.wavefront'
             exclude group: 'io.micrometer', module: 'micrometer-tracing-bridge-otel'
-
         }
     }
 

gradle/docs.gradle

@@ -1,17 +1,21 @@
 ext {
 	backendVersion = '0.0.3'
+	micrometerDocsVersion='1.0.0-SNAPSHOT'
 }
 
 configurations {
 	asciidoctorExtensions
+	micrometerDocs
 }
 
 dependencies {
 	asciidoctorExtensions "io.spring.asciidoctor.backends:spring-asciidoctor-backends:$backendVersion"
+	micrometerDocs "io.micrometer:micrometer-docs-generator-spans:$micrometerDocsVersion"
+	micrometerDocs "io.micrometer:micrometer-docs-generator-metrics:$micrometerDocsVersion"
 }
 
 task checkAsciidocLinks {
-	inputs.dir("src/reference/asciidoc/")
+	inputs.dir('src/reference/asciidoc')
 	doLast {
 		def errors = new ArrayList<>();
 		errors.add('*** Anchor reference errors found:')
@@ -48,8 +52,42 @@ task checkAsciidocLinks {
 	}
 }
 
+def observationInputDir = file('spring-integration-core/src/main/java/org/springframework/integration/support/management/observation').absolutePath
+def generatedDocsDir = file("$buildDir/docs/generated").absolutePath
+
+task generateObservabilityMetricsDocs(type: JavaExec) {
+	inputs.dir(observationInputDir)
+	outputs.dir(generatedDocsDir)
+	classpath configurations.micrometerDocs
+	args observationInputDir, /.+/, generatedDocsDir
+	mainClass = 'io.micrometer.docs.metrics.DocsFromSources'
+}
+
+task generateObservabilitySpansDocs(type: JavaExec) {
+	inputs.dir(observationInputDir)
+	outputs.dir(generatedDocsDir)
+	classpath configurations.micrometerDocs
+	args observationInputDir, /.+/, generatedDocsDir
+	mainClass = 'io.micrometer.docs.spans.DocsFromSources'
+}
+
+task filterMetricsDocsContent(type: Copy) {
+	dependsOn generateObservabilitySpansDocs, generateObservabilityMetricsDocs
+	from generatedDocsDir
+	include '_*.adoc'
+	into generatedDocsDir
+	rename { filename -> filename.replace '_', '' }
+	filter { line -> line.replaceAll('org.springframework.integration', 'o.s.i') }
+}
+
+task prepareDocs(type: Copy) {
+	dependsOn checkAsciidocLinks, filterMetricsDocsContent
+	from 'src/reference/asciidoc'
+	into "$buildDir/docs"
+}
+
 asciidoctorPdf {
-	dependsOn checkAsciidocLinks
+	dependsOn prepareDocs
 
 	inProcess = JAVA_EXEC
 	forkOptions {
@@ -59,7 +97,7 @@ asciidoctorPdf {
 	baseDirFollowsSourceFile()
 
 	asciidoctorj {
-		sourceDir "src/reference/asciidoc/"
+		sourceDir "$buildDir/docs"
 		inputs.dir(sourceDir)
 		sources {
 			include 'index-single.adoc'
@@ -75,8 +113,28 @@ asciidoctorPdf {
 	}
 }
 
-asciidoctorj {
-	version = '2.5.2'
+asciidoctor {
+	dependsOn asciidoctorPdf
+
+	inProcess = JAVA_EXEC
+	forkOptions {
+		jvmArgs '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED', '--add-opens', 'java.base/java.io=ALL-UNNAMED'
+	}
+
+	baseDirFollowsSourceFile()
+
+	configurations 'asciidoctorExtensions'
+	sourceDir "$buildDir/docs"
+	inputs.dir(sourceDir)
+	outputOptions {
+		backends 'spring-html'
+	}
+	resources {
+		from(sourceDir) {
+			include 'images/*', 'css/**', 'js/**'
+		}
+	}
+
 	options doctype: 'book', eruby: 'erubis'
 	attributes 'docinfo': 'shared',
 			stylesdir: 'css/',
@@ -97,27 +155,6 @@ asciidoctorj {
 			'project-version': project.version
 }
 
-asciidoctor {
-	dependsOn asciidoctorPdf
-	inProcess = JAVA_EXEC
-	forkOptions {
-		jvmArgs '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED', '--add-opens', 'java.base/java.io=ALL-UNNAMED'
-	}
-	baseDirFollowsSourceFile()
-	configurations 'asciidoctorExtensions'
-	sourceDir "src/reference/asciidoc/"
-	inputs.dir(sourceDir)
-	outputOptions {
-		backends "spring-html"
-	}
-	resources {
-		from(sourceDir) {
-			include 'images/*', 'css/**', 'js/**'
-		}
-	}
-
-}
-
 task reference(dependsOn: asciidoctor) {
 	group = 'Documentation'
 	description = 'Generate the reference documentation'

spring-integration-core/src/main/java/org/springframework/integration/handler/AbstractMessageHandler.java

@@ -81,7 +81,7 @@ private void handleWithObservation(Message<?> message, ObservationRegistry obser
 		IntegrationObservation.HANDLER.observation(
 						this.observationConvention,
 						DefaultMessageReceiverObservationConvention.INSTANCE,
-						new MessageReceiverContext(message, getComponentName()),
+						() -> new MessageReceiverContext(message, getComponentName()),
 						observationRegistry)
 				.observe(() -> doHandleMessage(message));
 	}

spring-integration-core/src/main/java/org/springframework/integration/support/management/observation/DefaultMessageReceiverObservationConvention.java

@@ -36,9 +36,11 @@ public class DefaultMessageReceiverObservationConvention implements MessageRecei
 
 	@Override
 	public KeyValues getLowCardinalityKeyValues(MessageReceiverContext context) {
-		return KeyValues.of(
-				IntegrationObservation.HandlerTags.COMPONENT_NAME.withValue(context.getHandlerName()),
-				IntegrationObservation.HandlerTags.COMPONENT_TYPE.withValue("handler"));
+		return KeyValues
+				// See IntegrationObservation.HandlerTags.COMPONENT_NAME - to avoid class tangle
+				.of("spring.integration.name", context.getHandlerName())
+				// See IntegrationObservation.HandlerTags.COMPONENT_TYPE - to avoid class tangle
+				.and("spring.integration.type", "handler");
 	}
 
 }

spring-integration-core/src/main/java/org/springframework/integration/support/management/observation/IntegrationObservation.java

@@ -17,26 +17,21 @@
 package org.springframework.integration.support.management.observation;
 
 import io.micrometer.common.docs.KeyName;
-import io.micrometer.observation.docs.DocumentedObservation;
+import io.micrometer.observation.docs.ObservationDocumentation;
 
 /**
- * The {@link DocumentedObservation} implementation for Spring Integration infrastructure.
+ * The {@link ObservationDocumentation} implementation for Spring Integration infrastructure.
  *
  * @author Artem Bilan
  *
  * @since 6.0
  */
-public enum IntegrationObservation implements DocumentedObservation {
+public enum IntegrationObservation implements ObservationDocumentation {
 
 	/**
 	 * Observation for message handlers.
 	 */
 	HANDLER {
-		@Override
-		public String getName() {
-			return "spring.integration.handler";
-		}
-
 		@Override
 		public String getPrefix() {
 			return "spring.integration.";

spring-integration-core/src/main/java/org/springframework/integration/support/management/observation/MessageReceiverObservationConvention.java

@@ -29,8 +29,12 @@
  *
  * @since 6.0
  */
-public interface MessageReceiverObservationConvention
-		extends ObservationConvention<MessageReceiverContext> {
+public interface MessageReceiverObservationConvention extends ObservationConvention<MessageReceiverContext> {
+
+	@Override
+	default String getName() {
+		return "spring.integration.handler";
+	}
 
 	@Override
 	default boolean supportsContext(Observation.Context context) {

spring-integration-core/src/test/java/org/springframework/integration/channel/interceptor/ObservationPropagationChannelInterceptorTests.java

@@ -19,7 +19,6 @@
 import static org.assertj.core.api.Assertions.assertThat;
 
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executors;
@@ -67,7 +66,6 @@
 import io.micrometer.tracing.Span;
 import io.micrometer.tracing.TraceContext;
 import io.micrometer.tracing.Tracer;
-import io.micrometer.tracing.exporter.FinishedSpan;
 import io.micrometer.tracing.handler.DefaultTracingObservationHandler;
 import io.micrometer.tracing.handler.PropagatingReceiverTracingObservationHandler;
 import io.micrometer.tracing.handler.PropagatingSenderTracingObservationHandler;
@@ -223,7 +221,7 @@ void observationContextPropagatedOverExecutorChannel() {
 						.setHeader(MessageHeaders.REPLY_CHANNEL, replyChannel)
 						.build();
 
-		Observation.createNotStarted("sending", new MessageSenderContext(message), this.observationRegistry)
+		Observation.createNotStarted("sending", () -> new MessageSenderContext(message), this.observationRegistry)
 				.observe(() -> this.testTracingChannel.send(message));
 
 		Message<?> receive = replyChannel.receive();
@@ -240,7 +238,7 @@ void observationContextPropagatedOverExecutorChannel() {
 		TracerAssert.assertThat(this.simpleTracer)
 				.reportedSpans()
 				.hasSize(2)
-				.satisfies(simpleSpans -> assertSpans(simpleSpans)
+				.satisfies(simpleSpans -> SpansAssert.assertThat(simpleSpans)
 						.hasASpanWithName("sending")
 						.assertThatASpanWithNameEqualTo("testBridge receive")
 						.hasTag("foo", "some foo value")
@@ -259,11 +257,6 @@ void observationContextPropagatedOverExecutorChannel() {
 		assertThat(this.meterRegistry.get("spring.integration.handler").timer().count()).isEqualTo(1);
 	}
 
-	@SuppressWarnings("unchecked")
-	private static SpansAssert assertSpans(Collection<? extends FinishedSpan> actual) {
-		return SpansAssert.assertThat((Collection<FinishedSpan>) actual);
-	}
-
 	@Configuration
 	@EnableIntegration
 	public static class ContextConfiguration {

spring-integration-core/src/test/java/org/springframework/integration/support/management/observation/IntegrationObservabilityZipkinTests.java

@@ -73,13 +73,13 @@ public SampleTestRunnerConsumer yourCode() {
 				PollableChannel queueChannel = applicationContext.getBean("queueChannel", PollableChannel.class);
 				PollableChannel replyChannel = new QueueChannel();
 
-				MutableMessage<String> testMessage =
+				MutableMessage<String> message =
 						(MutableMessage<String>) MutableMessageBuilder.withPayload("test data")
 								.setHeader(MessageHeaders.REPLY_CHANNEL, replyChannel)
 								.build();
 
-				Observation.createNotStarted("Test send", new MessageSenderContext(testMessage), observationRegistry)
-						.observe(() -> queueChannel.send(testMessage));
+				Observation.createNotStarted("Test send", () -> new MessageSenderContext(message), observationRegistry)
+						.observe(() -> queueChannel.send(message));
 
 				Message<?> receive = replyChannel.receive(10_000);
 				assertThat(receive).isNotNull()

src/reference/asciidoc/metrics.adoc

@@ -145,6 +145,32 @@ registry.config().meterFilter(MeterFilter.deny(id ->
 ----
 ====
 
+[[micrometer-observation]]
+==== Micrometer Observation
+
+Starting with version 6.0, Spring Integration utilizes a Micrometer Observation abstraction which can handle metrics as well as https://micrometer.io/docs/tracing[tracing] via appropriate `ObservationHandler` configuration.
+
+The observation handling is enabled on the `IntegrationManagement` components whenever an `ObservationRegistry` bean is present in the application context.
+The meters are not gathered in this case independently, but delegated to an appropriate `ObservationHandler` configured on the provided `ObservationRegistry`.
+
+An observation production on the `IntegrationManagement` components can be customized via `ObservationConvention` configuration.
+For example an `AbstractMessageHandler` expects a `MessageReceiverObservationConvention` via its `setObservationConvention()` API.
+
+The following are supported metrics, spans and conventions for Observation API:
+
+include::./generated/metrics.adoc[leveloffset=+2]
+
+include::./generated/spans.adoc[leveloffset=+2]
+
+include::./generated/conventions.adoc[leveloffset=+2]
+
+==== Observation Propagation
+
+To supply a connected chain of spans in one trace, independently of the nature of the messaging flow, Spring Integration provides an `ObservationPropagationChannelInterceptor` implementation.
+This can be configured on `MessageChannnel` beans individually or as a `@GlobalChannelInterceptor` with respective `MessageChannnel` bean names pattern matching.
+The goal of this interceptor is to propagate an `Observation` from the producer thread to the consumer one independently of the `MessageChannnel` implementation and nature.
+A `DirectChannel`, though, is ignored since its consumer is executed directly on the producer thread.
+
 ==== Spring Integration JMX Support
 
 Also see <<./jmx.adoc#jmx,JMX Support>>.

src/reference/asciidoc/whats-new.adoc

@@ -56,6 +56,13 @@ See <<./amqp.adoc#rmq-streams,RabbitMQ Stream Queue Support>> for more informati
 The SFTP modules has been fully reworked from outdated JCraft JSch library to more robust and modern `org.apache.sshd:sshd-sftp` module of the Apache MINA project.
 
 See <<./sftp.adoc#sftp,SFTP Adapters>> for more information.
+
+[[x6.0-micrometer-observation]]
+==== Micrometer Observation
+
+Enabling observation for timers and tracing using Micrometer is now supported.
+See <<./metrics.adoc#micrometer-observation,Micrometer Observation>> for more information.
+
 [[x6.0-general]]
 === General Changes