Automatically handle TypeName for Elasticsearch version <7, 7 and 8+, when DetectElasticsearchVersion option is set to true

Author: Nenad Vicentic

src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs

@@ -143,6 +143,7 @@ public static LoggerConfiguration Elasticsearch(
         /// <param name="failureSink">Sink to use when Elasticsearch is unable to accept the events. This is optionally and depends on the EmitEventFailure setting.</param>   
         /// <param name="singleEventSizePostingLimit"><see cref="ElasticsearchSinkOptions.SingleEventSizePostingLimit"/>The maximum length of an event allowed to be posted to Elasticsearch.default null</param>
         /// <param name="templateCustomSettings">Add custom elasticsearch settings to the template</param>
+        /// <param name="detectElasticsearchVersion">Turns on detection of elasticsearch version via background HTTP call. This will also set `TypeName` automatically, according to the version of Elasticsearch.</param>
         /// <param name="batchAction">Configures the OpType being used when inserting document in batch. Must be set to create for data streams.</param>
         /// <returns>LoggerConfiguration object</returns>
         /// <exception cref="ArgumentNullException"><paramref name="nodeUris"/> is <see langword="null" />.</exception>
@@ -151,7 +152,7 @@ public static LoggerConfiguration Elasticsearch(
             string nodeUris,
             string indexFormat = null,
             string templateName = null,
-            string typeName = "logevent",
+            string typeName = null,
             int batchPostingLimit = 50,
             int period = 2,
             bool inlineFields = false,
@@ -182,7 +183,8 @@ public static LoggerConfiguration Elasticsearch(
             long? singleEventSizePostingLimit = null,
             int? bufferFileCountLimit = null,
             Dictionary<string,string> templateCustomSettings = null,
-            ElasticOpType batchAction = ElasticOpType.Index)
+            ElasticOpType batchAction = ElasticOpType.Index,
+            bool detectElasticsearchVersion = false)
         {
             if (string.IsNullOrEmpty(nodeUris))
                 throw new ArgumentNullException(nameof(nodeUris), "No Elasticsearch node(s) specified.");
@@ -204,8 +206,6 @@ public static LoggerConfiguration Elasticsearch(
                 options.TemplateName = templateName;
             }
 
-            options.TypeName = !string.IsNullOrWhiteSpace(typeName) ? typeName : null;
-
             options.BatchPostingLimit = batchPostingLimit;
             options.BatchAction = batchAction;
             options.SingleEventSizePostingLimit = singleEventSizePostingLimit;
@@ -272,6 +272,8 @@ public static LoggerConfiguration Elasticsearch(
 
             options.TemplateCustomSettings = templateCustomSettings;
 
+            options.DetectElasticsearchVersion = detectElasticsearchVersion;
+
             return Elasticsearch(loggerSinkConfiguration, options);
         }
     }

src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs

@@ -110,7 +110,7 @@ public class ElasticsearchSinkOptions
         public string DeadLetterIndexName { get; set; }
 
         ///<summary>
-        /// The default elasticsearch type name to use for the log events. Defaults to: "_doc".
+        /// The default elasticsearch type name to use for the log events. Defaults to: null.
         /// </summary>
         public string TypeName { get; set; }
 
@@ -288,7 +288,6 @@ public ElasticsearchSinkOptions()
         {
             this.IndexFormat = "logstash-{0:yyyy.MM.dd}";
             this.DeadLetterIndexName = "deadletter-{0:yyyy.MM.dd}";
-            this.TypeName = DefaultTypeName;
             this.Period = TimeSpan.FromSeconds(2);
             this.BatchPostingLimit = 50;
             this.SingleEventSizePostingLimit = null;
@@ -308,7 +307,7 @@ public ElasticsearchSinkOptions()
         /// The default Elasticsearch type name used for Elasticsearch versions prior to 7.
         /// <para>As of <c>Elasticsearch 7</c> and up <c>_type</c> has been removed.</para>
         /// </summary>
-        public static string DefaultTypeName { get; } = "_doc";
+        public static string DefaultTypeName { get; } = "logevent";
 
         /// <summary>
         /// Instructs the sink to auto detect the running Elasticsearch version.

src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs

@@ -241,10 +241,10 @@ private object GetTemplateData()
         public void DiscoverClusterVersion()
         {
             if (!_options.DetectElasticsearchVersion) return;
+            if (_options.TypeName != null) return; // Do not do anything if user explicitly configured `TypeName`
 
             try
             {
-
                 var response = _client.Cat.Nodes<StringResponse>(new CatNodesRequestParameters()
                 {
                     Headers = new[] { "v" }
@@ -254,8 +254,17 @@ public void DiscoverClusterVersion()
                 _discoveredVersion = response.Body.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)
                     .FirstOrDefault();
 
-                if (_discoveredVersion?.StartsWith("7.") ?? false)
-                    _options.TypeName = "_doc";
+                if (_discoveredVersion == null)
+                    return;
+
+                if (int.TryParse(_discoveredVersion.Substring(0, _discoveredVersion.IndexOf('.')), out int majorVersion))
+                {
+                    if (majorVersion < 7)
+                        _options.TypeName = ElasticsearchSinkOptions.DefaultTypeName; // "logevent"
+                    if (majorVersion == 7)
+                        _options.TypeName = "_doc";
+                    // Version 8 or higher leave `_options.TypeName` to be `null` (condition is already at the start of the method), so that `_type` is not sent to Elasticsearch at all.
+                }
             }
             catch (Exception ex)
             {

test/Serilog.Sinks.Elasticsearch.Tests/BulkActionTests.cs

@@ -30,7 +30,7 @@ public void DefaultBulkActionV7()
         public void BulkActionV7OverrideTypeName()
         {
             _options.IndexFormat = "logs";
-            _options.TypeName = "logevent"; // This is the default value when creating the sink via configuration
+            _options.TypeName = null; // This is the default value, starting v9.0.0
             _options.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7;
             _options.PipelineName = null;
             using (var sink = new ElasticsearchSink(_options))

test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs

@@ -75,7 +75,7 @@ protected IList<SerilogElasticsearchEvent> GetPostedLogEvents(int expectedCount)
                 }
                 action.IndexAction.Should().NotBeNull();
                 action.IndexAction.Index.Should().NotBeNullOrEmpty().And.StartWith("logstash-");
-                action.IndexAction.Type.Should().NotBeNullOrEmpty().And.Be("_doc");
+                action.IndexAction.Type.Should().BeNull();
 
                 SerilogElasticsearchEvent actionMetaData;
                 try