add support for durable logging

Support for durable logging is based on the implementation of the Seq sink.
ControlledSwithc, ExponentialBackoff and PortableTimes have been imported directly (except for the namespace change.
LogglySink uses the same strategy of Seq Sink to determine if the client is durable or not.
HttpLogShipper uses the LogglyClient from the loggly-csharp lib instead of the http client directly, and can use the bulk api correctly.
DurableSink uses rolling file sink (like Seq's durableSink) to store events, but serializes them as logglyEvents instead of serilog's logevent. This avoid problems reading the serialized events from the file.
Event conversion is in LogEventConverter class for reuse

.gitignore

@@ -6,6 +6,7 @@
 *.user
 *.userosscache
 *.sln.docstates
+.localHistory/*
 
 # User-specific files (MonoDevelop/Xamarin Studio)
 *.userprefs

sample/sampleDurableLogger/Program.cs

@@ -0,0 +1,98 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Loggly;
+using Loggly.Config;
+using Serilog;
+using Serilog.Core.Enrichers;
+using Serilog.Enrichers;
+using Serilog.Sinks.RollingFileAlternate;
+
+namespace sampleDurableLogger
+{
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            SetupLogglyConfiguration();
+            var logger = CreateLoggerFactory(@"c:\test\").CreateLogger();
+
+            logger.Information((Exception)null, "test message - app started");
+            logger.Warning((Exception)null, "test message with {@data}", new { p1 = "sample", p2=DateTime.Now });
+            logger.Warning((Exception)null, "test2 message with {@data}", new { p1 = "sample2", p2 = 10 });
+
+            Console.WriteLine("disconnect to test offline. Two messages will be sent. Press enter to send and wait a minute or so before reconnecting or use breakpoints to see that send fails.");
+            Console.ReadLine();
+
+            logger.Information((Exception)null, "second test message - app started");
+            logger.Warning((Exception)null, "second test message with {@data}", new { p1 = "sample2", p2 = DateTime.Now });
+
+
+            Console.WriteLine("Offline messages written. Once you have confirmed that messages have been written locally, reconnect to see messages go out. Press Enter  for more messages to be written.");
+            Console.ReadLine();
+
+            logger.Information((Exception)null, "third test message - app started");
+            logger.Warning((Exception)null, "third test message with {@data}", new { p1 = "sample3", p2 = DateTime.Now });
+
+            Console.WriteLine("back online messages written. Check loggly and files for data. wait a minute or so before reconnecting. Press Enter to temrinate");
+            Console.ReadLine();
+        }
+
+        public static LoggerConfiguration CreateLoggerFactory(string logFilePath)
+        {
+            return new LoggerConfiguration()
+                .MinimumLevel.Debug()
+                //Add enrichers
+                .Enrich.FromLogContext()
+                .Enrich.WithProcessId()
+                .Enrich.WithThreadId()
+                .Enrich.With(new EnvironmentUserNameEnricher())
+                .Enrich.With(new MachineNameEnricher())
+                .Enrich.With(new PropertyEnricher("Environment", GetLoggingEnvironment()))
+                //Add sinks
+                .WriteTo.Async(s => s.Loggly(
+                            bufferBaseFilename: logFilePath + "buffer")
+                        .MinimumLevel.Information()
+                )
+                .WriteTo.Async(s => s.RollingFileAlternate(
+                    logFilePath,
+                    outputTemplate:
+                        "[{ProcessId}] {Timestamp:yyyy-MM-dd HH:mm:ss.fff K} [{ThreadId}] [{Level}] [{SourceContext}] [{Category}] {Message}{NewLine}{Exception}",
+                    fileSizeLimitBytes: 10 * 1024 * 1024,
+                    retainedFileCountLimit: 100)
+                        .MinimumLevel.Debug()
+                );
+        }
+
+        private static string GetLoggingEnvironment()
+        {
+            return "development";
+        }
+
+        private static void SetupLogglyConfiguration()
+        {
+            ///CHANGE THESE TOO TO YOUR LOGGLY ACCOUNT: DO NOT COMMIT TO Source control!!!
+            const string appName = "AppNameHere";
+            const string customerToken = "yourkeyhere";
+
+            //Configure Loggly
+            var config = LogglyConfig.Instance;
+            config.CustomerToken = customerToken;
+            config.ApplicationName = appName;
+            config.Transport = new TransportConfiguration()
+            {
+                EndpointHostname = "logs-01.loggly.com",
+                EndpointPort = 443,
+                LogTransport = LogTransport.Https
+            };
+            config.ThrowExceptions = true;
+
+            //Define Tags sent to Loggly
+            config.TagConfig.Tags.AddRange(new ITag[]{
+                new ApplicationNameTag {Formatter = "application-{0}"},
+                new HostnameTag { Formatter = "host-{0}" }
+            });
+        }
+    }
+}

sample/sampleDurableLogger/Properties/AssemblyInfo.cs

@@ -0,0 +1,19 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("Microsoft")]
+[assembly: AssemblyProduct("sampleDurableLogger")]
+[assembly: AssemblyTrademark("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components.  If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("a47ed1ce-fe7c-444e-9391-10d6b60519c2")]

sample/sampleDurableLogger/project.json

@@ -0,0 +1,31 @@
+{
+  "version": "1.0.0-*",
+  "buildOptions": {
+    "emitEntryPoint": true
+  },
+
+  "dependencies": {
+    "Microsoft.NETCore.App": {
+      "type": "platform",
+      "version": "1.0.0"
+    },
+    "loggly-csharp": "4.6.0.5",
+    "Newtonsoft.Json": "9.0.1",
+    "Serilog": "2.3.0",
+    "Serilog.Sinks.PeriodicBatching": "2.0.0",
+    "Serilog.Sinks.File": "3.1.1",
+    "Serilog.Sinks.RollingFile": "3.2.0",
+    "Serilog.Sinks.RollingFileAlternate": "2.0.6",
+    "Serilog.Sinks.Async": "1.0.1",
+    "Serilog.Enrichers.Environment": "2.1.1",
+    "Serilog.Enrichers.Process": "2.0.0",
+    "Serilog.Enrichers.Thread": "3.0.0",
+    "Serilog.Sinks.Loggly": "3.0.3-*"
+  },
+
+  "frameworks": {
+    "netcoreapp1.0": {
+      "imports": "dnxcore50"
+    }
+  }
+}

sample/sampleDurableLogger/sampleDurableLogger.xproj

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+
+  <Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>a47ed1ce-fe7c-444e-9391-10d6b60519c2</ProjectGuid>
+    <RootNamespace>sampleDurableLogger</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">.\obj</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath>
+    <TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>

serilog-sinks-loggly.sln

@@ -20,6 +20,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{2B558B69-8
 EndProject
 Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Serilog.Sinks.Loggly.Tests", "test\Serilog.Sinks.Loggly.Tests\Serilog.Sinks.Loggly.Tests.xproj", "{120C431E-479C-48C7-9539-CFA32399769C}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "sample", "sample", "{FD377716-21BA-45D1-9580-02C2BECA5BAB}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "sampleDurableLogger", "sample\sampleDurableLogger\sampleDurableLogger.xproj", "{A47ED1CE-FE7C-444E-9391-10D6B60519C2}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -34,12 +38,17 @@ Global
 		{120C431E-479C-48C7-9539-CFA32399769C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{120C431E-479C-48C7-9539-CFA32399769C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{120C431E-479C-48C7-9539-CFA32399769C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A47ED1CE-FE7C-444E-9391-10D6B60519C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A47ED1CE-FE7C-444E-9391-10D6B60519C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A47ED1CE-FE7C-444E-9391-10D6B60519C2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A47ED1CE-FE7C-444E-9391-10D6B60519C2}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{94E6E098-11A0-43CF-B0CF-4BA270CE9DBD} = {037440DE-440B-4129-9F7A-09B42D00397E}
 		{120C431E-479C-48C7-9539-CFA32399769C} = {2B558B69-8F95-4F82-B223-EBF60F6F31EE}
+		{A47ED1CE-FE7C-444E-9391-10D6B60519C2} = {FD377716-21BA-45D1-9580-02C2BECA5BAB}
 	EndGlobalSection
 EndGlobal

src/Serilog.Sinks.Loggly/LoggerConfigurationLogglyExtensions.cs

@@ -13,7 +13,10 @@
 // limitations under the License.
 
 using System;
+using System.Net.Http;
+using Loggly.Config;
 using Serilog.Configuration;
+using Serilog.Core;
 using Serilog.Events;
 using Serilog.Sinks.Loggly;
 
@@ -32,22 +35,65 @@ public static class LoggerConfigurationLogglyExtensions
         /// <param name="formatProvider">Supplies culture-specific formatting information, or null.</param>
         /// <param name="batchPostingLimit">The maximum number of events to post in a single batch.</param>
         /// <param name="period">The time to wait between checking for event batches.</param>
+        /// <param name="bufferBaseFilename">Path for a set of files that will be used to buffer events until they
+        /// can be successfully transmitted across the network. Individual files will be created using the
+        /// pattern <paramref name="bufferBaseFilename"/>-{Date}.json.</param>
+        /// <param name="bufferFileSizeLimitBytes">The maximum size, in bytes, to which the buffer
+        /// log file for a specific date will be allowed to grow. By default no limit will be applied.</param>
+        /// <param name="eventBodyLimitBytes">The maximum size, in bytes, that the JSON representation of
+        /// an event may take before it is dropped rather than being sent to the Loggly server. Specify null for no limit.
+        /// The default is 1 MB.</param>
+        /// <param name="controlLevelSwitch">If provided, the switch will be updated based on the Seq server's level setting
+        /// for the corresponding API key. Passing the same key to MinimumLevel.ControlledBy() will make the whole pipeline
+        /// dynamically controlled. Do not specify <paramref name="restrictedToMinimumLevel"/> with this setting.</param>
+        /// <param name="retainedInvalidPayloadsLimitBytes">A soft limit for the number of bytes to use for storing failed requests.  
+        /// The limit is soft in that it can be exceeded by any single error payload, but in that case only that single error
+        /// payload will be retained.</param>
         /// <returns>Logger configuration, allowing configuration to continue.</returns>
         /// <exception cref="ArgumentNullException">A required parameter is null.</exception>
         public static LoggerConfiguration Loggly(
             this LoggerSinkConfiguration loggerConfiguration,
             LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum,
             int batchPostingLimit = LogglySink.DefaultBatchPostingLimit,
             TimeSpan? period = null,
-            IFormatProvider formatProvider = null)
+            IFormatProvider formatProvider = null,
+            string bufferBaseFilename = null,
+            long? bufferFileSizeLimitBytes = null,
+            long? eventBodyLimitBytes = 1024 * 1024,
+            LoggingLevelSwitch controlLevelSwitch = null,
+            long? retainedInvalidPayloadsLimitBytes = null)
         {
             if (loggerConfiguration == null) throw new ArgumentNullException("loggerConfiguration");
+            if (bufferFileSizeLimitBytes.HasValue && bufferFileSizeLimitBytes < 0)
+                throw new ArgumentOutOfRangeException(nameof(bufferFileSizeLimitBytes), "Negative value provided; file size limit must be non-negative.");
 
             var defaultedPeriod = period ?? LogglySink.DefaultPeriod;
 
-            return loggerConfiguration.Sink(
-                new LogglySink(formatProvider, batchPostingLimit, defaultedPeriod),
-                restrictedToMinimumLevel);
+            ILogEventSink sink;
+
+            if (bufferBaseFilename == null)
+            {
+                sink = new LogglySink(formatProvider, batchPostingLimit, defaultedPeriod);
+            }
+            else
+            {
+#if DURABLE
+                sink = new DurableLogglySink(
+                    bufferBaseFilename,
+                    batchPostingLimit,
+                    defaultedPeriod,
+                    bufferFileSizeLimitBytes,
+                    eventBodyLimitBytes,
+                    controlLevelSwitch,
+                    retainedInvalidPayloadsLimitBytes);
+#else
+                // We keep the API consistent for easier packaging and to support bait-and-switch.
+                throw new NotSupportedException("Durable log shipping is not supported on this platform.");
+#endif
+
+            }
+
+            return loggerConfiguration.Sink(sink, restrictedToMinimumLevel);
         }
 
     }

src/Serilog.Sinks.Loggly/Sinks/Loggly/ControlledSwitch.cs

@@ -0,0 +1,72 @@
+// Serilog.Sinks.Seq Copyright 2016 Serilog Contributors
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using Serilog.Core;
+using Serilog.Events;
+
+namespace Serilog.Sinks.Loggly
+{
+    /// <summary>
+    /// Instances of this type are single-threaded, generally only updated on a background
+    /// timer thread. An exception is <see cref="IsIncluded(LogEvent)"/>, which may be called
+    /// concurrently but performs no synchronization.
+    /// </summary>
+    class ControlledLevelSwitch
+    {
+        // If non-null, then background level checks will be performed; set either through the constructor
+        // or in response to a level specification from the server. Never set to null after being made non-null.
+        LoggingLevelSwitch _controlledSwitch;
+        LogEventLevel? _originalLevel;
+
+        public ControlledLevelSwitch(LoggingLevelSwitch controlledSwitch = null)
+        {
+            _controlledSwitch = controlledSwitch;
+        }
+
+        public bool IsActive => _controlledSwitch != null;
+
+        public bool IsIncluded(LogEvent evt)
+        {
+            // Concurrent, but not synchronized.
+            var controlledSwitch = _controlledSwitch;
+            return controlledSwitch == null ||
+                (int)controlledSwitch.MinimumLevel <= (int)evt.Level;
+        }
+
+        public void Update(LogEventLevel? minimumAcceptedLevel)
+        {
+            if (minimumAcceptedLevel == null)
+            {
+                if (_controlledSwitch != null && _originalLevel.HasValue)
+                    _controlledSwitch.MinimumLevel = _originalLevel.Value;
+
+                return;
+            }
+
+            if (_controlledSwitch == null)
+            {
+                // The server is controlling the logging level, but not the overall logger. Hence, if the server
+                // stops controlling the level, the switch should become transparent.
+                _originalLevel = LevelAlias.Minimum;
+                _controlledSwitch = new LoggingLevelSwitch(minimumAcceptedLevel.Value);
+                return;
+            }
+
+            if (!_originalLevel.HasValue)
+                _originalLevel = _controlledSwitch.MinimumLevel;
+
+            _controlledSwitch.MinimumLevel = minimumAcceptedLevel.Value;
+        }
+    }
+}