Enabled FileSink and RollingFileSink's output stream in another stream, such as a GZipStream

cocowalla · cocowalla · commit f7bfe4e672cd · 2019-02-01T14:57:12.000Z
diff --git a/src/Serilog.Sinks.File/FileLoggerConfigurationExtensions.cs b/src/Serilog.Sinks.File/FileLoggerConfigurationExtensions.cs
@@ -135,11 +135,12 @@ public static LoggerConfiguration File(
         /// <param name="shared">Allow the log file to be shared by multiple processes. The default is false.</param>
         /// <param name="flushToDiskInterval">If provided, a full disk flush will be performed periodically at the specified interval.</param>
         /// <param name="rollingInterval">The interval at which logging will roll over to a new file.</param>
-        /// <param name="rollOnFileSizeLimit">If <code>true</code>, a new file will be created when the file size limit is reached. Filenames 
+        /// <param name="rollOnFileSizeLimit">If <code>true</code>, a new file will be created when the file size limit is reached. Filenames
         /// will have a number appended in the format <code>_NNN</code>, with the first filename given no number.</param>
         /// <param name="retainedFileCountLimit">The maximum number of log files that will be retained,
         /// including the current log file. For unlimited retention, pass null. The default is 31.</param>
         /// <param name="encoding">Character encoding used to write the text file. The default is UTF-8 without BOM.</param>
+        /// <param name="wrapper">Optionally enables wrapping the output stream in another stream, such as a GZipStream.</param>
         /// <returns>Configuration object allowing method chaining.</returns>
         /// <remarks>The file will be written using the UTF-8 character set.</remarks>
         public static LoggerConfiguration File(
@@ -156,7 +157,8 @@ public static LoggerConfiguration File(
             RollingInterval rollingInterval = RollingInterval.Infinite,
             bool rollOnFileSizeLimit = false,
             int? retainedFileCountLimit = DefaultRetainedFileCountLimit,
-            Encoding encoding = null)
+            Encoding encoding = null,
+            StreamWrapper wrapper = null)
         {
             if (sinkConfiguration == null) throw new ArgumentNullException(nameof(sinkConfiguration));
             if (path == null) throw new ArgumentNullException(nameof(path));
@@ -165,7 +167,7 @@ public static LoggerConfiguration File(
             var formatter = new MessageTemplateTextFormatter(outputTemplate, formatProvider);
             return File(sinkConfiguration, formatter, path, restrictedToMinimumLevel, fileSizeLimitBytes,
                 levelSwitch, buffered, shared, flushToDiskInterval,
-                rollingInterval, rollOnFileSizeLimit, retainedFileCountLimit, encoding);
+                rollingInterval, rollOnFileSizeLimit, retainedFileCountLimit, encoding, wrapper);
         }
 
         /// <summary>
@@ -174,7 +176,7 @@ public static LoggerConfiguration File(
         /// <param name="sinkConfiguration">Logger sink configuration.</param>
         /// <param name="formatter">A formatter, such as <see cref="JsonFormatter"/>, to convert the log events into
         /// text for the file. If control of regular text formatting is required, use the other
-        /// overload of <see cref="File(LoggerSinkConfiguration, string, LogEventLevel, string, IFormatProvider, long?, LoggingLevelSwitch, bool, bool, TimeSpan?, RollingInterval, bool, int?, Encoding)"/>
+        /// overload of <see cref="File(LoggerSinkConfiguration, string, LogEventLevel, string, IFormatProvider, long?, LoggingLevelSwitch, bool, bool, TimeSpan?, RollingInterval, bool, int?, Encoding, StreamWrapper)"/>
         /// and specify the outputTemplate parameter instead.
         /// </param>
         /// <param name="path">Path to the file.</param>
@@ -190,11 +192,12 @@ public static LoggerConfiguration File(
         /// <param name="shared">Allow the log file to be shared by multiple processes. The default is false.</param>
         /// <param name="flushToDiskInterval">If provided, a full disk flush will be performed periodically at the specified interval.</param>
         /// <param name="rollingInterval">The interval at which logging will roll over to a new file.</param>
-        /// <param name="rollOnFileSizeLimit">If <code>true</code>, a new file will be created when the file size limit is reached. Filenames 
+        /// <param name="rollOnFileSizeLimit">If <code>true</code>, a new file will be created when the file size limit is reached. Filenames
         /// will have a number appended in the format <code>_NNN</code>, with the first filename given no number.</param>
         /// <param name="retainedFileCountLimit">The maximum number of log files that will be retained,
         /// including the current log file. For unlimited retention, pass null. The default is 31.</param>
         /// <param name="encoding">Character encoding used to write the text file. The default is UTF-8 without BOM.</param>
+        /// <param name="wrapper">Optionally enables wrapping the output stream in another stream, such as a GZipStream.</param>
         /// <returns>Configuration object allowing method chaining.</returns>
         /// <remarks>The file will be written using the UTF-8 character set.</remarks>
         public static LoggerConfiguration File(
@@ -210,10 +213,12 @@ public static LoggerConfiguration File(
             RollingInterval rollingInterval = RollingInterval.Infinite,
             bool rollOnFileSizeLimit = false,
             int? retainedFileCountLimit = DefaultRetainedFileCountLimit,
-            Encoding encoding = null)
+            Encoding encoding = null,
+            StreamWrapper wrapper = null)
         {
             return ConfigureFile(sinkConfiguration.Sink, formatter, path, restrictedToMinimumLevel, fileSizeLimitBytes, levelSwitch,
-                buffered, false, shared, flushToDiskInterval, encoding, rollingInterval, rollOnFileSizeLimit, retainedFileCountLimit);
+                buffered, false, shared, flushToDiskInterval, encoding, rollingInterval, rollOnFileSizeLimit,
+                retainedFileCountLimit, wrapper);
         }
 
         /// <summary>
@@ -270,7 +275,7 @@ public static LoggerConfiguration File(
             LoggingLevelSwitch levelSwitch = null)
         {
             return ConfigureFile(sinkConfiguration.Sink, formatter, path, restrictedToMinimumLevel, null, levelSwitch, false, true,
-                false, null, null, RollingInterval.Infinite, false, null);
+                false, null, null, RollingInterval.Infinite, false, null, null);
         }
 
         static LoggerConfiguration ConfigureFile(
@@ -287,7 +292,8 @@ static LoggerConfiguration ConfigureFile(
             Encoding encoding,
             RollingInterval rollingInterval,
             bool rollOnFileSizeLimit,
-            int? retainedFileCountLimit)
+            int? retainedFileCountLimit,
+            StreamWrapper wrapper)
         {
             if (addSink == null) throw new ArgumentNullException(nameof(addSink));
             if (formatter == null) throw new ArgumentNullException(nameof(formatter));
@@ -300,7 +306,7 @@ static LoggerConfiguration ConfigureFile(
 
             if (rollOnFileSizeLimit || rollingInterval != RollingInterval.Infinite)
             {
-                sink = new RollingFileSink(path, formatter, fileSizeLimitBytes, retainedFileCountLimit, encoding, buffered, shared, rollingInterval, rollOnFileSizeLimit);
+                sink = new RollingFileSink(path, formatter, fileSizeLimitBytes, retainedFileCountLimit, encoding, buffered, shared, rollingInterval, rollOnFileSizeLimit, wrapper);
             }
             else
             {
@@ -309,11 +315,16 @@ static LoggerConfiguration ConfigureFile(
 #pragma warning disable 618
                     if (shared)
                     {
+                        if (wrapper != null)
+                        {
+                            SelfLog.WriteLine("Unable to use output stream wrapper - these are not supported for shared log files");
+                        }
+
                         sink = new SharedFileSink(path, formatter, fileSizeLimitBytes);
                     }
                     else
                     {
-                        sink = new FileSink(path, formatter, fileSizeLimitBytes, buffered: buffered);
+                        sink = new FileSink(path, formatter, fileSizeLimitBytes, buffered: buffered, wrapper: wrapper);
                     }
 #pragma warning restore 618
                 }
diff --git a/src/Serilog.Sinks.File/Sinks/File/FileSink.cs b/src/Serilog.Sinks.File/Sinks/File/FileSink.cs
@@ -43,10 +43,12 @@ public sealed class FileSink : IFileSink, IDisposable
         /// <param name="encoding">Character encoding used to write the text file. The default is UTF-8 without BOM.</param>
         /// <param name="buffered">Indicates if flushing to the output file can be buffered or not. The default
         /// is false.</param>
+        /// <param name="wrapper">Optionally enables wrapping the output stream in another stream, such as a GZipStream.</param>
         /// <returns>Configuration object allowing method chaining.</returns>
         /// <remarks>The file will be written using the UTF-8 character set.</remarks>
         /// <exception cref="IOException"></exception>
-        public FileSink(string path, ITextFormatter textFormatter, long? fileSizeLimitBytes, Encoding encoding = null, bool buffered = false)
+        public FileSink(string path, ITextFormatter textFormatter, long? fileSizeLimitBytes, Encoding encoding = null, bool buffered = false,
+            StreamWrapper wrapper = null)
         {
             if (path == null) throw new ArgumentNullException(nameof(path));
             if (textFormatter == null) throw new ArgumentNullException(nameof(textFormatter));
@@ -68,6 +70,11 @@ public FileSink(string path, ITextFormatter textFormatter, long? fileSizeLimitBy
                 outputStream = _countingStreamWrapper = new WriteCountingStream(_underlyingStream);
             }
 
+            if (wrapper != null)
+            {
+                outputStream = wrapper.Wrap(outputStream);
+            }
+
             _output = new StreamWriter(outputStream, encoding ?? new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
         }
 
diff --git a/src/Serilog.Sinks.File/Sinks/File/RollingFileSink.cs b/src/Serilog.Sinks.File/Sinks/File/RollingFileSink.cs
@@ -35,6 +35,7 @@ sealed class RollingFileSink : ILogEventSink, IFlushableFileSink, IDisposable
         readonly bool _buffered;
         readonly bool _shared;
         readonly bool _rollOnFileSizeLimit;
+        readonly StreamWrapper _wrapper;
 
         readonly object _syncRoot = new object();
         bool _isDisposed;
@@ -50,7 +51,8 @@ public RollingFileSink(string path,
                               bool buffered,
                               bool shared,
                               RollingInterval rollingInterval,
-                              bool rollOnFileSizeLimit)
+                              bool rollOnFileSizeLimit,
+                              StreamWrapper wrapper = null)
         {
             if (path == null) throw new ArgumentNullException(nameof(path));
             if (fileSizeLimitBytes.HasValue && fileSizeLimitBytes < 0) throw new ArgumentException("Negative value provided; file size limit must be non-negative");
@@ -64,6 +66,7 @@ public RollingFileSink(string path,
             _buffered = buffered;
             _shared = shared;
             _rollOnFileSizeLimit = rollOnFileSizeLimit;
+            _wrapper = wrapper;
         }
 
         public void Emit(LogEvent logEvent)
@@ -144,7 +147,7 @@ void OpenFile(DateTime now, int? minSequence = null)
                 {
                     _currentFile = _shared ?
                         (IFileSink)new SharedFileSink(path, _textFormatter, _fileSizeLimitBytes, _encoding) :
-                        new FileSink(path, _textFormatter, _fileSizeLimitBytes, _encoding, _buffered);
+                        new FileSink(path, _textFormatter, _fileSizeLimitBytes, _encoding, _buffered, _wrapper);
                     _currentFileSequence = sequence;
                 }
                 catch (IOException ex)
diff --git a/src/Serilog.Sinks.File/StreamWrapper.cs b/src/Serilog.Sinks.File/StreamWrapper.cs
@@ -0,0 +1,17 @@
+using System.IO;
+
+namespace Serilog
+{
+    /// <summary>
+    /// Wraps the log file's output stream in another stream, such as a GZipStream
+    /// </summary>
+    public abstract class StreamWrapper
+    {
+        /// <summary>
+        /// Wraps <paramref name="sourceStream"/> in another stream, such as a GZipStream, then returns the wrapped stream
+        /// </summary>
+        /// <param name="sourceStream">The source log file stream</param>
+        /// <returns>The wrapped stream</returns>
+        public abstract Stream Wrap(Stream sourceStream);
+    }
+}
diff --git a/test/Serilog.Sinks.File.Tests/FileSinkTests.cs b/test/Serilog.Sinks.File.Tests/FileSinkTests.cs
@@ -1,9 +1,11 @@
-﻿using System.IO;
+﻿using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Text;
 using Xunit;
 using Serilog.Formatting.Json;
 using Serilog.Sinks.File.Tests.Support;
 using Serilog.Tests.Support;
-using System.Text;
 
 #pragma warning disable 618
 
@@ -141,6 +143,42 @@ public void WhenLimitIsNotSpecifiedAndEncodingHasNoPreambleDataIsCorrectlyAppend
             WriteTwoEventsAndCheckOutputFileLength(null, encoding);
         }
 
+        [Fact]
+        public void WhenStreamWrapperIsSpecifiedOutputStreamIsWrapped()
+        {
+            var gzipWrapper = new GZipStreamWrapper();
+
+            using (var tmp = TempFolder.ForCaller())
+            {
+                var nonexistent = tmp.AllocateFilename("txt");
+                var evt = Some.LogEvent("Hello, world!");
+
+                using (var sink = new FileSink(nonexistent, new JsonFormatter(), null, wrapper: gzipWrapper))
+                {
+                    sink.Emit(evt);
+                    sink.Emit(evt);
+                }
+
+                // Ensure the data was written through the wrapping GZipStream, by decompressing and comparing against
+                // what we wrote
+                var lines = new List<string>();
+                using (var textStream = new MemoryStream())
+                {
+                    using (var fs = System.IO.File.OpenRead(nonexistent))
+                    using (var decompressStream = new GZipStream(fs, CompressionMode.Decompress))
+                    {
+                        decompressStream.CopyTo(textStream);
+                    }
+
+                    textStream.Position = 0;
+                    lines = textStream.ReadAllLines();
+                }
+
+                Assert.Equal(2, lines.Count);
+                Assert.Contains("Hello, world!", lines[0]);
+            }
+        }
+
         static void WriteTwoEventsAndCheckOutputFileLength(long? maxBytes, Encoding encoding)
         {
             using (var tmp = TempFolder.ForCaller())
@@ -170,4 +208,3 @@ static void WriteTwoEventsAndCheckOutputFileLength(long? maxBytes, Encoding enco
         }
     }
 }
-
diff --git a/test/Serilog.Sinks.File.Tests/RollingFileSinkTests.cs b/test/Serilog.Sinks.File.Tests/RollingFileSinkTests.cs
@@ -1,6 +1,7 @@
 ﻿using System;
 using System.Collections.Generic;
 using System.IO;
+using System.IO.Compression;
 using System.Linq;
 using System.Reflection;
 using Xunit;
@@ -96,6 +97,64 @@ public void WhenSizeLimitIsBreachedNewFilesCreated()
             }
         }
 
+        [Fact]
+        public void WhenStreamWrapperSpecifiedIsUsedForRolledFiles()
+        {
+            var gzipWrapper = new GZipStreamWrapper();
+            var fileName = Some.String() + ".txt";
+
+            using (var temp = new TempFolder())
+            {
+                string[] files;
+                var logEvents = new[]
+                {
+                    Some.InformationEvent(),
+                    Some.InformationEvent(),
+                    Some.InformationEvent()
+                };
+
+                using (var log = new LoggerConfiguration()
+                    .WriteTo.File(Path.Combine(temp.Path, fileName), rollOnFileSizeLimit: true, fileSizeLimitBytes: 1, wrapper: gzipWrapper)
+                    .CreateLogger())
+                {
+
+                    foreach (var logEvent in logEvents)
+                    {
+                        log.Write(logEvent);
+                    }
+
+                    files = Directory.GetFiles(temp.Path)
+                        .OrderBy(p => p, StringComparer.OrdinalIgnoreCase)
+                        .ToArray();
+
+                    Assert.Equal(3, files.Length);
+                    Assert.True(files[0].EndsWith(fileName), files[0]);
+                    Assert.True(files[1].EndsWith("_001.txt"), files[1]);
+                    Assert.True(files[2].EndsWith("_002.txt"), files[2]);
+                }
+
+                // Ensure the data was written through the wrapping GZipStream, by decompressing and comparing against
+                // what we wrote
+                for (var i = 0; i < files.Length; i++)
+                {
+                    using (var textStream = new MemoryStream())
+                    {
+                        using (var fs = System.IO.File.OpenRead(files[i]))
+                        using (var decompressStream = new GZipStream(fs, CompressionMode.Decompress))
+                        {
+                            decompressStream.CopyTo(textStream);
+                        }
+
+                        textStream.Position = 0;
+                        var lines = textStream.ReadAllLines();
+
+                        Assert.Equal(1, lines.Count);
+                        Assert.True(lines[0].EndsWith(logEvents[i].MessageTemplate.Text));
+                    }
+                }
+            }
+        }
+
         [Fact]
         public void IfTheLogFolderDoesNotExistItWillBeCreated()
         {
diff --git a/test/Serilog.Sinks.File.Tests/Support/Extensions.cs b/test/Serilog.Sinks.File.Tests/Support/Extensions.cs
@@ -1,4 +1,6 @@
-﻿using Serilog.Events;
+﻿using System.Collections.Generic;
+using System.IO;
+using Serilog.Events;
 
 namespace Serilog.Sinks.File.Tests.Support
 {
@@ -8,5 +10,21 @@ public static object LiteralValue(this LogEventPropertyValue @this)
         {
             return ((ScalarValue)@this).Value;
         }
+
+        public static List<string> ReadAllLines(this Stream @this)
+        {
+            var lines = new List<string>();
+
+            using (var reader = new StreamReader(@this))
+            {
+                string line;
+                while ((line = reader.ReadLine()) != null)
+                {
+                    lines.Add(line);
+                }
+            }
+
+            return lines;
+        }
     }
 }
diff --git a/test/Serilog.Sinks.File.Tests/Support/GZipStreamWrapper.cs b/test/Serilog.Sinks.File.Tests/Support/GZipStreamWrapper.cs
@@ -0,0 +1,25 @@
+using System.IO;
+using System.IO.Compression;
+
+namespace Serilog.Sinks.File.Tests.Support
+{
+    /// <inheritdoc />
+    /// <summary>
+    /// Demonstrates the use of <seealso cref="T:Serilog.StreamWrapper" />, by compressing log output using GZip
+    /// </summary>
+    public class GZipStreamWrapper : StreamWrapper
+    {
+        readonly int _bufferSize;
+
+        public GZipStreamWrapper(int bufferSize = 1024 * 32)
+        {
+            _bufferSize = bufferSize;
+        }
+
+        public override Stream Wrap(Stream sourceStream)
+        {
+            var compressStream = new GZipStream(sourceStream, CompressionMode.Compress);
+            return new BufferedStream(compressStream, _bufferSize);
+        }
+    }
+}

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,6 @@`
`1`		`-using Serilog.Events;`
	`1`	`+using System.Collections.Generic;`
	`2`	`+using System.IO;`
	`3`	`+using Serilog.Events;`
`2`	`4`
`3`	`5`	`namespace Serilog.Sinks.File.Tests.Support`
`4`	`6`	`{`
`@@ -8,5 +10,21 @@ public static object LiteralValue(this LogEventPropertyValue @this)`
`8`	`10`	`{`
`9`	`11`	`return ((ScalarValue)@this).Value;`
`10`	`12`	`}`
	`13`	`+`
	`14`	`+ public static List<string> ReadAllLines(this Stream @this)`
	`15`	`+ {`
	`16`	`+ var lines = new List<string>();`
	`17`	`+`
	`18`	`+ using (var reader = new StreamReader(@this))`
	`19`	`+ {`
	`20`	`+ string line;`
	`21`	`+ while ((line = reader.ReadLine()) != null)`
	`22`	`+ {`
	`23`	`+ lines.Add(line);`
	`24`	`+ }`
	`25`	`+ }`
	`26`	`+`
	`27`	`+ return lines;`
	`28`	`+ }`
`11`	`29`	`}`
`12`	`30`	`}`