From f25f25047a3072192bf51cf157176f9c7d941a10 Mon Sep 17 00:00:00 2001 From: jericho Date: Mon, 27 May 2024 16:17:18 -0400 Subject: [PATCH 01/16] Upgrade the Logzio.DotNet.NLog reference This allows me to remove the workaround discussed here: https://github.com/logzio/logzio-dotnet/issues/72 --- .../StrongGrid.IntegrationTests.csproj | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Source/StrongGrid.IntegrationTests/StrongGrid.IntegrationTests.csproj b/Source/StrongGrid.IntegrationTests/StrongGrid.IntegrationTests.csproj index c784ce60..daeaec9d 100644 --- a/Source/StrongGrid.IntegrationTests/StrongGrid.IntegrationTests.csproj +++ b/Source/StrongGrid.IntegrationTests/StrongGrid.IntegrationTests.csproj @@ -12,16 +12,13 @@ - + - - - From f588654436724fccf13b2ef8362397f9ae2dde53 Mon Sep 17 00:00:00 2001 From: jericho Date: Mon, 27 May 2024 19:24:58 -0400 Subject: [PATCH 02/16] Fix the diagnostic log in the error handler Had to copy/paste 3 source files from the Microsoft.Extensions.Logging.Abstractions project because they are marked as internal Resolves #523 --- .../StrongGrid/Utilities/DiagnosticHandler.cs | 146 -------- Source/StrongGrid/Utilities/DiagnosticInfo.cs | 146 ++++++++ .../Utilities/LogValuesFormatter.cs | 291 ++++++++++++++++ Source/StrongGrid/Utilities/ThrowHelper.cs | 90 +++++ .../Utilities/ValueStringBuilder.cs | 319 ++++++++++++++++++ 5 files changed, 846 insertions(+), 146 deletions(-) create mode 100644 Source/StrongGrid/Utilities/DiagnosticInfo.cs create mode 100644 Source/StrongGrid/Utilities/LogValuesFormatter.cs create mode 100644 Source/StrongGrid/Utilities/ThrowHelper.cs create mode 100644 Source/StrongGrid/Utilities/ValueStringBuilder.cs diff --git a/Source/StrongGrid/Utilities/DiagnosticHandler.cs b/Source/StrongGrid/Utilities/DiagnosticHandler.cs index 69415a65..fac7ed31 100644 --- a/Source/StrongGrid/Utilities/DiagnosticHandler.cs +++ b/Source/StrongGrid/Utilities/DiagnosticHandler.cs @@ -4,12 +4,9 @@ using Pathoschild.Http.Client.Extensibility; using System; using System.Collections.Concurrent; -using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Net.Http; -using System.Text; -using System.Text.RegularExpressions; namespace StrongGrid.Utilities { @@ -19,149 +16,6 @@ namespace StrongGrid.Utilities /// internal class DiagnosticHandler : IHttpFilter { - internal class DiagnosticInfo - { - public WeakReference RequestReference { get; set; } - - public long RequestTimestamp { get; set; } - - public WeakReference ResponseReference { get; set; } - - public long ResponseTimestamp { get; set; } - - public DiagnosticInfo(WeakReference requestReference, long requestTimestamp, WeakReference responseReference, long responseTimestamp) - { - RequestReference = requestReference; - RequestTimestamp = requestTimestamp; - ResponseReference = responseReference; - ResponseTimestamp = responseTimestamp; - } - - public string GetLoggingTemplate() - { - RequestReference.TryGetTarget(out HttpRequestMessage request); - ResponseReference.TryGetTarget(out HttpResponseMessage response); - - var logTemplate = new StringBuilder(); - - if (request != null) - { - logTemplate.AppendLine("REQUEST SENT BY STRONGGRID: {Request_HttpMethod} {Request_Uri} HTTP/{Request_HttpVersion}"); - logTemplate.AppendLine("REQUEST HEADERS:"); - - var requestHeaders = response?.RequestMessage?.Headers ?? Enumerable.Empty>>(); - if (!requestHeaders.Any(kvp => string.Equals(kvp.Key, "Content-Length", StringComparison.OrdinalIgnoreCase))) - { - requestHeaders = requestHeaders.Append(new KeyValuePair>("Content-Length", new[] { "0" })); - } - - foreach (var header in requestHeaders.OrderBy(kvp => kvp.Key)) - { - logTemplate.AppendLine(" " + header.Key + ": {Request_Header_" + header.Key + "}"); - } - - logTemplate.AppendLine("REQUEST: {Request_Content}"); - logTemplate.AppendLine(); - } - - if (response != null) - { - logTemplate.AppendLine("RESPONSE FROM SENDGRID: HTTP/{Response_HttpVersion} {Response_StatusCode} {Response_ReasonPhrase}"); - logTemplate.AppendLine("RESPONSE HEADERS:"); - - var responseHeaders = response?.Headers ?? Enumerable.Empty>>(); - if (!responseHeaders.Any(kvp => string.Equals(kvp.Key, "Content-Length", StringComparison.OrdinalIgnoreCase))) - { - responseHeaders = responseHeaders.Append(new KeyValuePair>("Content-Length", new[] { "0" })); - } - - foreach (var header in responseHeaders.OrderBy(kvp => kvp.Key)) - { - logTemplate.AppendLine(" " + header.Key + ": {Response_Header_" + header.Key + "}"); - } - - logTemplate.AppendLine("RESPONSE: {Response_Content}"); - logTemplate.AppendLine(); - } - - logTemplate.AppendLine("DIAGNOSTIC: The request took {Diagnostic_Elapsed:N} milliseconds"); - - return logTemplate.ToString(); - } - - public object[] GetLoggingParameters() - { - RequestReference.TryGetTarget(out HttpRequestMessage request); - ResponseReference.TryGetTarget(out HttpResponseMessage response); - - // Get the content to the request/response and calculate how long it took to get the response - var elapsed = TimeSpan.FromTicks(ResponseTimestamp - RequestTimestamp); - var requestContent = request?.Content?.ReadAsStringAsync(null).GetAwaiter().GetResult(); - var responseContent = response?.Content?.ReadAsStringAsync(null).GetAwaiter().GetResult(); - - // Calculate the content size - var requestContentLength = requestContent?.Length ?? 0; - var responseContentLength = responseContent?.Length ?? 0; - - // Get the request headers (please note: intentionally getting headers from "response.RequestMessage" rather than "request") - var requestHeaders = response?.RequestMessage?.Headers ?? Enumerable.Empty>>(); - if (!requestHeaders.Any(kvp => string.Equals(kvp.Key, "Content-Length", StringComparison.OrdinalIgnoreCase))) - { - requestHeaders = requestHeaders.Append(new KeyValuePair>("Content-Length", new[] { requestContentLength.ToString() })); - } - - // Get the response headers - var responseHeaders = response?.Headers ?? Enumerable.Empty>>(); - if (!responseHeaders.Any(kvp => string.Equals(kvp.Key, "Content-Length", StringComparison.OrdinalIgnoreCase))) - { - responseHeaders = responseHeaders.Append(new KeyValuePair>("Content-Length", new[] { responseContentLength.ToString() })); - } - - // The order of these values must match the order in which they appear in the logging template - var logParams = new List(); - - if (request != null) - { - logParams.AddRange([request.Method.Method, request.RequestUri, request.Version]); - logParams.AddRange(requestHeaders - .OrderBy(kvp => kvp.Key) - .Select(kvp => kvp.Key.Equals("authorization", StringComparison.OrdinalIgnoreCase) ? "... omitted for security reasons ..." : string.Join(", ", kvp.Value)) - .ToArray()); - logParams.Add(requestContent?.TrimEnd('\r', '\n')); - } - - if (response != null) - { - logParams.AddRange([response.Version, (int)response.StatusCode, response.ReasonPhrase]); - logParams.AddRange(responseHeaders - .OrderBy(kvp => kvp.Key) - .Select(kvp => kvp.Key.Equals("authorization", StringComparison.OrdinalIgnoreCase) ? "... omitted for security reasons ..." : string.Join(", ", kvp.Value)) - .ToList()); - logParams.Add(responseContent?.TrimEnd('\r', '\n')); - } - - logParams.Add(elapsed.TotalMilliseconds); - - return logParams.ToArray(); - } - - public string GetFormattedLog() - { - var formattedLog = GetLoggingTemplate(); - var args = GetLoggingParameters(); - - var pattern = @"(.*?{)(\w+?.+?)(}.*)"; - for (var i = 0; i < args.Length; i++) - { - formattedLog = Regex.Replace(formattedLog, pattern, $"$1 {i} $3", RegexOptions.None); - } - - formattedLog = formattedLog.Replace("{ ", "{").Replace(" }", "}"); - - return string.Format(formattedLog, args); - } - } - #region FIELDS internal const string DIAGNOSTIC_ID_HEADER_NAME = "StrongGrid-Diagnostic-Id"; diff --git a/Source/StrongGrid/Utilities/DiagnosticInfo.cs b/Source/StrongGrid/Utilities/DiagnosticInfo.cs new file mode 100644 index 00000000..8b299eb4 --- /dev/null +++ b/Source/StrongGrid/Utilities/DiagnosticInfo.cs @@ -0,0 +1,146 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; + +namespace StrongGrid.Utilities +{ + internal class DiagnosticInfo + { + public WeakReference RequestReference { get; set; } + + public long RequestTimestamp { get; set; } + + public WeakReference ResponseReference { get; set; } + + public long ResponseTimestamp { get; set; } + + public DiagnosticInfo(WeakReference requestReference, long requestTimestamp, WeakReference responseReference, long responseTimestamp) + { + RequestReference = requestReference; + RequestTimestamp = requestTimestamp; + ResponseReference = responseReference; + ResponseTimestamp = responseTimestamp; + } + + public string GetLoggingTemplate() + { + RequestReference.TryGetTarget(out HttpRequestMessage request); + ResponseReference.TryGetTarget(out HttpResponseMessage response); + + var logTemplate = new StringBuilder(); + + if (request != null) + { + logTemplate.AppendLine("REQUEST SENT BY STRONGGRID: {Request_HttpMethod} {Request_Uri} HTTP/{Request_HttpVersion}"); + logTemplate.AppendLine("REQUEST HEADERS:"); + + var requestHeaders = response?.RequestMessage?.Headers ?? Enumerable.Empty>>(); + if (!requestHeaders.Any(kvp => string.Equals(kvp.Key, "Content-Length", StringComparison.OrdinalIgnoreCase))) + { + requestHeaders = requestHeaders.Append(new KeyValuePair>("Content-Length", new[] { "0" })); + } + + foreach (var header in requestHeaders.OrderBy(kvp => kvp.Key)) + { + logTemplate.AppendLine(" " + header.Key + ": {Request_Header_" + header.Key + "}"); + } + + logTemplate.AppendLine("REQUEST: {Request_Content}"); + logTemplate.AppendLine(); + } + + if (response != null) + { + logTemplate.AppendLine("RESPONSE FROM SENDGRID: HTTP/{Response_HttpVersion} {Response_StatusCode} {Response_ReasonPhrase}"); + logTemplate.AppendLine("RESPONSE HEADERS:"); + + var responseHeaders = response?.Headers ?? Enumerable.Empty>>(); + if (!responseHeaders.Any(kvp => string.Equals(kvp.Key, "Content-Length", StringComparison.OrdinalIgnoreCase))) + { + responseHeaders = responseHeaders.Append(new KeyValuePair>("Content-Length", new[] { "0" })); + } + + foreach (var header in responseHeaders.OrderBy(kvp => kvp.Key)) + { + logTemplate.AppendLine(" " + header.Key + ": {Response_Header_" + header.Key + "}"); + } + + logTemplate.AppendLine("RESPONSE: {Response_Content}"); + logTemplate.AppendLine(); + } + + logTemplate.AppendLine("DIAGNOSTIC: The request took {Diagnostic_Elapsed:N} milliseconds"); + + return logTemplate.ToString(); + } + + public object[] GetLoggingParameters() + { + RequestReference.TryGetTarget(out HttpRequestMessage request); + ResponseReference.TryGetTarget(out HttpResponseMessage response); + + // Get the content to the request/response and calculate how long it took to get the response + var elapsed = TimeSpan.FromTicks(ResponseTimestamp - RequestTimestamp); + var requestContent = request?.Content?.ReadAsStringAsync(null).GetAwaiter().GetResult(); + var responseContent = response?.Content?.ReadAsStringAsync(null).GetAwaiter().GetResult(); + + // Calculate the content size + var requestContentLength = requestContent?.Length ?? 0; + var responseContentLength = responseContent?.Length ?? 0; + + // Get the request headers (please note: intentionally getting headers from "response.RequestMessage" rather than "request") + var requestHeaders = response?.RequestMessage?.Headers ?? Enumerable.Empty>>(); + if (!requestHeaders.Any(kvp => string.Equals(kvp.Key, "Content-Length", StringComparison.OrdinalIgnoreCase))) + { + requestHeaders = requestHeaders.Append(new KeyValuePair>("Content-Length", new[] { requestContentLength.ToString() })); + } + + // Get the response headers + var responseHeaders = response?.Headers ?? Enumerable.Empty>>(); + if (!responseHeaders.Any(kvp => string.Equals(kvp.Key, "Content-Length", StringComparison.OrdinalIgnoreCase))) + { + responseHeaders = responseHeaders.Append(new KeyValuePair>("Content-Length", new[] { responseContentLength.ToString() })); + } + + // The order of these values must match the order in which they appear in the logging template + var logParams = new List(); + + if (request != null) + { + logParams.AddRange([request.Method.Method, request.RequestUri, request.Version]); + logParams.AddRange(requestHeaders + .OrderBy(kvp => kvp.Key) + .Select(kvp => kvp.Key.Equals("authorization", StringComparison.OrdinalIgnoreCase) ? "... omitted for security reasons ..." : string.Join(", ", kvp.Value)) + .ToArray()); + logParams.Add(requestContent?.TrimEnd('\r', '\n')); + } + + if (response != null) + { + logParams.AddRange([response.Version, (int)response.StatusCode, response.ReasonPhrase]); + logParams.AddRange(responseHeaders + .OrderBy(kvp => kvp.Key) + .Select(kvp => kvp.Key.Equals("authorization", StringComparison.OrdinalIgnoreCase) ? "... omitted for security reasons ..." : string.Join(", ", kvp.Value)) + .ToList()); + logParams.Add(responseContent?.TrimEnd('\r', '\n')); + } + + logParams.Add(elapsed.TotalMilliseconds); + + return logParams.ToArray(); + } + + public string GetFormattedLog() + { + var template = GetLoggingTemplate(); + var args = GetLoggingParameters(); + + var formater = new LogValuesFormatter(template); + var formattedLog = formater.Format(args); + + return formattedLog; + } + } +} diff --git a/Source/StrongGrid/Utilities/LogValuesFormatter.cs b/Source/StrongGrid/Utilities/LogValuesFormatter.cs new file mode 100644 index 00000000..0eb791dd --- /dev/null +++ b/Source/StrongGrid/Utilities/LogValuesFormatter.cs @@ -0,0 +1,291 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; + +namespace StrongGrid.Utilities +{ + /// + /// Formatter to convert the named format items like {NamedformatItem} to format. + /// + /// From the Microsoft.Extensions.Logging.Abstractions project. + internal sealed class LogValuesFormatter + { + private const string NullValue = "(null)"; + private readonly List _valueNames = new List(); +#if NET8_0_OR_GREATER + private readonly CompositeFormat _format; +#else + private readonly string _format; +#endif + + /* + * NOTE: If this assembly ever builds for netcoreapp, the below code should change to: + * - Be annotated as [SkipLocalsInit] to avoid zero'ing the stackalloc'd char span + * - Format _valueNames.Count directly into a span + */ + public LogValuesFormatter(string format) + { + ThrowHelper.ThrowIfNull(format); + + OriginalFormat = format; + + var vsb = new ValueStringBuilder(stackalloc char[256]); + int scanIndex = 0; + int endIndex = format.Length; + + while (scanIndex < endIndex) + { + int openBraceIndex = FindBraceIndex(format, '{', scanIndex, endIndex); + if (scanIndex == 0 && openBraceIndex == endIndex) + { + // No holes found. + _format = +#if NET8_0_OR_GREATER + CompositeFormat.Parse(format); +#else + format; +#endif + return; + } + + int closeBraceIndex = FindBraceIndex(format, '}', openBraceIndex, endIndex); + + if (closeBraceIndex == endIndex) + { + vsb.Append(format.AsSpan(scanIndex, endIndex - scanIndex)); + scanIndex = endIndex; + } + else + { + // Format item syntax : { index[,alignment][ :formatString] }. + int formatDelimiterIndex = format.AsSpan(openBraceIndex, closeBraceIndex - openBraceIndex).IndexOfAny(',', ':'); + formatDelimiterIndex = formatDelimiterIndex < 0 ? closeBraceIndex : formatDelimiterIndex + openBraceIndex; + + vsb.Append(format.AsSpan(scanIndex, openBraceIndex - scanIndex + 1)); + vsb.Append(_valueNames.Count.ToString()); + _valueNames.Add(format.Substring(openBraceIndex + 1, formatDelimiterIndex - openBraceIndex - 1)); + vsb.Append(format.AsSpan(formatDelimiterIndex, closeBraceIndex - formatDelimiterIndex + 1)); + + scanIndex = closeBraceIndex + 1; + } + } + + _format = +#if NET8_0_OR_GREATER + CompositeFormat.Parse(vsb.ToString()); +#else + vsb.ToString(); +#endif + } + + public string OriginalFormat { get; } + + public List ValueNames => _valueNames; + + public string Format(object[] values) + { + object[] formattedValues = values; + + if (values != null) + { + for (int i = 0; i < values.Length; i++) + { + object formattedValue = FormatArgument(values[i]); + + // If the formatted value is changed, we allocate and copy items to a new array to avoid mutating the array passed in to this method + if (!ReferenceEquals(formattedValue, values[i])) + { + formattedValues = new object[values.Length]; + Array.Copy(values, formattedValues, i); + formattedValues[i++] = formattedValue; + for (; i < values.Length; i++) + { + formattedValues[i] = FormatArgument(values[i]); + } + + break; + } + } + } + + return string.Format(CultureInfo.InvariantCulture, _format, formattedValues ?? Array.Empty()); + } + + public KeyValuePair GetValue(object[] values, int index) + { + if (index < 0 || index > _valueNames.Count) + { + throw new IndexOutOfRangeException(nameof(index)); + } + + if (_valueNames.Count > index) + { + return new KeyValuePair(_valueNames[index], values[index]); + } + + return new KeyValuePair("{OriginalFormat}", OriginalFormat); + } + + public IEnumerable> GetValues(object[] values) + { + var valueArray = new KeyValuePair[values.Length + 1]; + for (int index = 0; index != _valueNames.Count; ++index) + { + valueArray[index] = new KeyValuePair(_valueNames[index], values[index]); + } + + valueArray[valueArray.Length - 1] = new KeyValuePair("{OriginalFormat}", OriginalFormat); + return valueArray; + } + + // NOTE: This method mutates the items in the array if needed to avoid extra allocations, and should only be used when caller expects this to happen + internal string FormatWithOverwrite(object[] values) + { + if (values != null) + { + for (int i = 0; i < values.Length; i++) + { + values[i] = FormatArgument(values[i]); + } + } + + return string.Format(CultureInfo.InvariantCulture, _format, values ?? Array.Empty()); + } + + internal string Format() + { +#if NET8_0_OR_GREATER + return _format.Format; +#else + return _format; +#endif + } + +#if NET8_0_OR_GREATER + internal string Format(TArg0 arg0) + { + object arg0String = null; + return + !TryFormatArgumentIfNullOrEnumerable(arg0, ref arg0String) ? + string.Format(CultureInfo.InvariantCulture, _format, arg0) : + string.Format(CultureInfo.InvariantCulture, _format, arg0String); + } + + internal string Format(TArg0 arg0, TArg1 arg1) + { + object arg0String = null, arg1String = null; + return + !TryFormatArgumentIfNullOrEnumerable(arg0, ref arg0String) && + !TryFormatArgumentIfNullOrEnumerable(arg1, ref arg1String) ? + string.Format(CultureInfo.InvariantCulture, _format, arg0, arg1) : + string.Format(CultureInfo.InvariantCulture, _format, arg0String ?? arg0, arg1String ?? arg1); + } + + internal string Format(TArg0 arg0, TArg1 arg1, TArg2 arg2) + { + object arg0String = null, arg1String = null, arg2String = null; + return + !TryFormatArgumentIfNullOrEnumerable(arg0, ref arg0String) && + !TryFormatArgumentIfNullOrEnumerable(arg1, ref arg1String) && + !TryFormatArgumentIfNullOrEnumerable(arg2, ref arg2String) ? + string.Format(CultureInfo.InvariantCulture, _format, arg0, arg1, arg2) : + string.Format(CultureInfo.InvariantCulture, _format, arg0String ?? arg0, arg1String ?? arg1, arg2String ?? arg2); + } +#else + internal string Format(object arg0) => + string.Format(CultureInfo.InvariantCulture, _format, FormatArgument(arg0)); + + internal string Format(object arg0, object arg1) => + string.Format(CultureInfo.InvariantCulture, _format, FormatArgument(arg0), FormatArgument(arg1)); + + internal string Format(object arg0, object arg1, object arg2) => + string.Format(CultureInfo.InvariantCulture, _format, FormatArgument(arg0), FormatArgument(arg1), FormatArgument(arg2)); +#endif + + private static int FindBraceIndex(string format, char brace, int startIndex, int endIndex) + { + // Example: {{prefix{{{Argument}}}suffix}}. + int braceIndex = endIndex; + int scanIndex = startIndex; + int braceOccurrenceCount = 0; + + while (scanIndex < endIndex) + { + if (braceOccurrenceCount > 0 && format[scanIndex] != brace) + { + if (braceOccurrenceCount % 2 == 0) + { + // Even number of '{' or '}' found. Proceed search with next occurrence of '{' or '}'. + braceOccurrenceCount = 0; + braceIndex = endIndex; + } + else + { + // An unescaped '{' or '}' found. + break; + } + } + else if (format[scanIndex] == brace) + { + if (brace == '}') + { + if (braceOccurrenceCount == 0) + { + // For '}' pick the first occurrence. + braceIndex = scanIndex; + } + } + else + { + // For '{' pick the last occurrence. + braceIndex = scanIndex; + } + + braceOccurrenceCount++; + } + + scanIndex++; + } + + return braceIndex; + } + + private static object FormatArgument(object value) + { + object stringValue = null; + return TryFormatArgumentIfNullOrEnumerable(value, ref stringValue) ? stringValue : value!; + } + + private static bool TryFormatArgumentIfNullOrEnumerable(T value, ref object stringValue) + { + if (value == null) + { + stringValue = NullValue; + return true; + } + + // if the value implements IEnumerable but isn't itself a string, build a comma separated string. + if (value is not string && value is IEnumerable enumerable) + { + var vsb = new ValueStringBuilder(stackalloc char[256]); + bool first = true; + foreach (object e in enumerable) + { + if (!first) + { + vsb.Append(", "); + } + + vsb.Append(e != null ? e.ToString() : NullValue); + first = false; + } + + stringValue = vsb.ToString(); + return true; + } + + return false; + } + } +} diff --git a/Source/StrongGrid/Utilities/ThrowHelper.cs b/Source/StrongGrid/Utilities/ThrowHelper.cs new file mode 100644 index 00000000..f9ac97ac --- /dev/null +++ b/Source/StrongGrid/Utilities/ThrowHelper.cs @@ -0,0 +1,90 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +// This file is intended to be used by components that don't have access to ArgumentNullException.ThrowIfNull. +#pragma warning disable CS0436 // Type conflicts with imported type + +namespace StrongGrid.Utilities +{ + internal class ThrowHelper + { + /// Throws an if is null. + /// The reference type argument to validate as non-null. + /// The name of the parameter with which corresponds. + /// From the Microsoft.Extensions.Logging.Abstractions project. + public static void ThrowIfNull( +#if NET + [NotNull] +#endif + object argument, + [CallerArgumentExpression(nameof(argument))] string paramName = null) + { + if (argument is null) + { + Throw(paramName); + } + } + +#if NET + [DoesNotReturn] +#endif + public static void Throw(string paramName) => throw new ArgumentNullException(paramName); + + /// + /// Throws either an or an + /// if the specified string is or whitespace respectively. + /// + /// String to be checked for or whitespace. + /// The name of the parameter being checked. + /// The original value of . + [MethodImpl(MethodImplOptions.AggressiveInlining)] +#if NET + [return: NotNull] +#endif + public static string IfNullOrWhitespace( +#if NET + [NotNull] +#endif + string argument, + [CallerArgumentExpression(nameof(argument))] string paramName = "") + { +#if !NET + if (argument == null) + { + throw new ArgumentNullException(paramName); + } +#endif + + if (string.IsNullOrWhiteSpace(argument)) + { + if (argument == null) + { + throw new ArgumentNullException(paramName); + } + else + { + throw new ArgumentException(paramName, "Argument is whitespace"); + } + } + + return argument; + } + } +} + +#if !NET +namespace System.Runtime.CompilerServices +{ + [AttributeUsage(AttributeTargets.Parameter, AllowMultiple = false, Inherited = false)] + internal sealed class CallerArgumentExpressionAttribute : Attribute + { + public CallerArgumentExpressionAttribute(string parameterName) + { + ParameterName = parameterName; + } + + public string ParameterName { get; } + } +} +#endif diff --git a/Source/StrongGrid/Utilities/ValueStringBuilder.cs b/Source/StrongGrid/Utilities/ValueStringBuilder.cs new file mode 100644 index 00000000..901fb821 --- /dev/null +++ b/Source/StrongGrid/Utilities/ValueStringBuilder.cs @@ -0,0 +1,319 @@ +using System; +using System.Buffers; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace StrongGrid.Utilities +{ + /// From the Microsoft.Extensions.Logging.Abstractions project. + internal ref partial struct ValueStringBuilder + { + private char[] _arrayToReturnToPool; + private Span _chars; + private int _pos; + + public ValueStringBuilder(Span initialBuffer) + { + _arrayToReturnToPool = null; + _chars = initialBuffer; + _pos = 0; + } + + public ValueStringBuilder(int initialCapacity) + { + _arrayToReturnToPool = ArrayPool.Shared.Rent(initialCapacity); + _chars = _arrayToReturnToPool; + _pos = 0; + } + + public int Length + { + get => _pos; + set + { + Debug.Assert(value >= 0); + Debug.Assert(value <= _chars.Length); + _pos = value; + } + } + + public int Capacity => _chars.Length; + + public void EnsureCapacity(int capacity) + { + // This is not expected to be called this with negative capacity + Debug.Assert(capacity >= 0); + + // If the caller has a bug and calls this with negative capacity, make sure to call Grow to throw an exception. + if ((uint)capacity > (uint)_chars.Length) + Grow(capacity - _pos); + } + + /// + /// Get a pinnable reference to the builder. + /// Does not ensure there is a null char after + /// This overload is pattern matched in the C# 7.3+ compiler so you can omit + /// the explicit method call, and write eg "fixed (char* c = builder)". + /// + public ref char GetPinnableReference() + { + return ref MemoryMarshal.GetReference(_chars); + } + + /// + /// Get a pinnable reference to the builder. + /// + /// Ensures that the builder has a null char after . + public ref char GetPinnableReference(bool terminate) + { + if (terminate) + { + EnsureCapacity(Length + 1); + _chars[Length] = '\0'; + } + + return ref MemoryMarshal.GetReference(_chars); + } + + public ref char this[int index] + { + get + { + Debug.Assert(index < _pos); + return ref _chars[index]; + } + } + + public override string ToString() + { + string s = _chars.Slice(0, _pos).ToString(); + Dispose(); + return s; + } + + /// Gets the underlying storage of the builder. + public Span RawChars => _chars; + + /// + /// Returns a span around the contents of the builder. + /// + /// Ensures that the builder has a null char after . + public ReadOnlySpan AsSpan(bool terminate) + { + if (terminate) + { + EnsureCapacity(Length + 1); + _chars[Length] = '\0'; + } + + return _chars.Slice(0, _pos); + } + + public ReadOnlySpan AsSpan() => _chars.Slice(0, _pos); + + public ReadOnlySpan AsSpan(int start) => _chars.Slice(start, _pos - start); + + public ReadOnlySpan AsSpan(int start, int length) => _chars.Slice(start, length); + + public bool TryCopyTo(Span destination, out int charsWritten) + { + if (_chars.Slice(0, _pos).TryCopyTo(destination)) + { + charsWritten = _pos; + Dispose(); + return true; + } + else + { + charsWritten = 0; + Dispose(); + return false; + } + } + + public void Insert(int index, char value, int count) + { + if (_pos > _chars.Length - count) + { + Grow(count); + } + + int remaining = _pos - index; + _chars.Slice(index, remaining).CopyTo(_chars.Slice(index + count)); + _chars.Slice(index, count).Fill(value); + _pos += count; + } + + public void Insert(int index, string s) + { + if (s == null) + { + return; + } + + int count = s.Length; + + if (_pos > (_chars.Length - count)) + { + Grow(count); + } + + int remaining = _pos - index; + _chars.Slice(index, remaining).CopyTo(_chars.Slice(index + count)); + s +#if !NET + .AsSpan() +#endif + .CopyTo(_chars.Slice(index)); + _pos += count; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Append(char c) + { + int pos = _pos; + Span chars = _chars; + if ((uint)pos < (uint)chars.Length) + { + chars[pos] = c; + _pos = pos + 1; + } + else + { + GrowAndAppend(c); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Append(string s) + { + if (s == null) + { + return; + } + + int pos = _pos; + if (s.Length == 1 && (uint)pos < (uint)_chars.Length) // very common case, e.g. appending strings from NumberFormatInfo like separators, percent symbols, etc. + { + _chars[pos] = s[0]; + _pos = pos + 1; + } + else + { + AppendSlow(s); + } + } + + public void Append(char c, int count) + { + if (_pos > _chars.Length - count) + { + Grow(count); + } + + Span dst = _chars.Slice(_pos, count); + for (int i = 0; i < dst.Length; i++) + { + dst[i] = c; + } + + _pos += count; + } + + public void Append(scoped ReadOnlySpan value) + { + int pos = _pos; + if (pos > _chars.Length - value.Length) + { + Grow(value.Length); + } + + value.CopyTo(_chars.Slice(_pos)); + _pos += value.Length; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Span AppendSpan(int length) + { + int origPos = _pos; + if (origPos > _chars.Length - length) + { + Grow(length); + } + + _pos = origPos + length; + return _chars.Slice(origPos, length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Dispose() + { + char[] toReturn = _arrayToReturnToPool; + this = default; // for safety, to avoid using pooled array if this instance is erroneously appended to again + if (toReturn != null) + { + ArrayPool.Shared.Return(toReturn); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void GrowAndAppend(char c) + { + Grow(1); + Append(c); + } + + /// + /// Resize the internal buffer either by doubling current buffer size or + /// by adding to + /// whichever is greater. + /// + /// + /// Number of chars requested beyond current position. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private void Grow(int additionalCapacityBeyondPos) + { + Debug.Assert(additionalCapacityBeyondPos > 0); + Debug.Assert(_pos > _chars.Length - additionalCapacityBeyondPos, "Grow called incorrectly, no resize is needed."); + + const uint ArrayMaxLength = 0x7FFFFFC7; // same as Array.MaxLength + + // Increase to at least the required size (_pos + additionalCapacityBeyondPos), but try + // to double the size if possible, bounding the doubling to not go beyond the max array length. + int newCapacity = (int)Math.Max( + (uint)(_pos + additionalCapacityBeyondPos), + Math.Min((uint)_chars.Length * 2, ArrayMaxLength)); + + // Make sure to let Rent throw an exception if the caller has a bug and the desired capacity is negative. + // This could also go negative if the actual required length wraps around. + char[] poolArray = ArrayPool.Shared.Rent(newCapacity); + + _chars.Slice(0, _pos).CopyTo(poolArray); + + char[] toReturn = _arrayToReturnToPool; + _chars = _arrayToReturnToPool = poolArray; + if (toReturn != null) + { + ArrayPool.Shared.Return(toReturn); + } + } + + private void AppendSlow(string s) + { + int pos = _pos; + if (pos > _chars.Length - s.Length) + { + Grow(s.Length); + } + + s +#if !NET + .AsSpan() +#endif + .CopyTo(_chars.Slice(pos)); + _pos += s.Length; + } + } +} From a495270d5cbed32a9bc2cd6363dc6c191aaa6a67 Mon Sep 17 00:00:00 2001 From: jericho Date: Sun, 2 Jun 2024 13:55:33 -0400 Subject: [PATCH 03/16] Move files from the Microsoft.Extensions.Logging.Abstractions project to their own folder --- Source/StrongGrid/Utilities/DiagnosticInfo.cs | 1 + .../Utilities/{ => Log}/LogValuesFormatter.cs | 2 +- .../Utilities/{ => Log}/ThrowHelper.cs | 18 +----------------- .../Utilities/{ => Log}/ValueStringBuilder.cs | 2 +- 4 files changed, 4 insertions(+), 19 deletions(-) rename Source/StrongGrid/Utilities/{ => Log}/LogValuesFormatter.cs (99%) rename Source/StrongGrid/Utilities/{ => Log}/ThrowHelper.cs (84%) rename Source/StrongGrid/Utilities/{ => Log}/ValueStringBuilder.cs (99%) diff --git a/Source/StrongGrid/Utilities/DiagnosticInfo.cs b/Source/StrongGrid/Utilities/DiagnosticInfo.cs index 8b299eb4..6e6d4914 100644 --- a/Source/StrongGrid/Utilities/DiagnosticInfo.cs +++ b/Source/StrongGrid/Utilities/DiagnosticInfo.cs @@ -1,3 +1,4 @@ +using StrongGrid.Utilities.Log; using System; using System.Collections.Generic; using System.Linq; diff --git a/Source/StrongGrid/Utilities/LogValuesFormatter.cs b/Source/StrongGrid/Utilities/Log/LogValuesFormatter.cs similarity index 99% rename from Source/StrongGrid/Utilities/LogValuesFormatter.cs rename to Source/StrongGrid/Utilities/Log/LogValuesFormatter.cs index 0eb791dd..1c6df3c9 100644 --- a/Source/StrongGrid/Utilities/LogValuesFormatter.cs +++ b/Source/StrongGrid/Utilities/Log/LogValuesFormatter.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Globalization; -namespace StrongGrid.Utilities +namespace StrongGrid.Utilities.Log { /// /// Formatter to convert the named format items like {NamedformatItem} to format. diff --git a/Source/StrongGrid/Utilities/ThrowHelper.cs b/Source/StrongGrid/Utilities/Log/ThrowHelper.cs similarity index 84% rename from Source/StrongGrid/Utilities/ThrowHelper.cs rename to Source/StrongGrid/Utilities/Log/ThrowHelper.cs index f9ac97ac..3bfe2e67 100644 --- a/Source/StrongGrid/Utilities/ThrowHelper.cs +++ b/Source/StrongGrid/Utilities/Log/ThrowHelper.cs @@ -5,7 +5,7 @@ // This file is intended to be used by components that don't have access to ArgumentNullException.ThrowIfNull. #pragma warning disable CS0436 // Type conflicts with imported type -namespace StrongGrid.Utilities +namespace StrongGrid.Utilities.Log { internal class ThrowHelper { @@ -72,19 +72,3 @@ public static string IfNullOrWhitespace( } } } - -#if !NET -namespace System.Runtime.CompilerServices -{ - [AttributeUsage(AttributeTargets.Parameter, AllowMultiple = false, Inherited = false)] - internal sealed class CallerArgumentExpressionAttribute : Attribute - { - public CallerArgumentExpressionAttribute(string parameterName) - { - ParameterName = parameterName; - } - - public string ParameterName { get; } - } -} -#endif diff --git a/Source/StrongGrid/Utilities/ValueStringBuilder.cs b/Source/StrongGrid/Utilities/Log/ValueStringBuilder.cs similarity index 99% rename from Source/StrongGrid/Utilities/ValueStringBuilder.cs rename to Source/StrongGrid/Utilities/Log/ValueStringBuilder.cs index 901fb821..01acc893 100644 --- a/Source/StrongGrid/Utilities/ValueStringBuilder.cs +++ b/Source/StrongGrid/Utilities/Log/ValueStringBuilder.cs @@ -4,7 +4,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -namespace StrongGrid.Utilities +namespace StrongGrid.Utilities.Log { /// From the Microsoft.Extensions.Logging.Abstractions project. internal ref partial struct ValueStringBuilder From c416ee25880ad04fcddaa15c5daf3b9306449046 Mon Sep 17 00:00:00 2001 From: jericho Date: Sun, 2 Jun 2024 13:55:54 -0400 Subject: [PATCH 04/16] Refresh build script --- build.cake | 2 +- global.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.cake b/build.cake index 4bb43bff..241233c6 100644 --- a/build.cake +++ b/build.cake @@ -2,7 +2,7 @@ #tool dotnet:?package=GitVersion.Tool&version=5.12.0 #tool dotnet:?package=coveralls.net&version=4.0.1 #tool nuget:https://f.feedz.io/jericho/jericho/nuget/?package=GitReleaseManager&version=0.17.0-collaborators0004 -#tool nuget:?package=ReportGenerator&version=5.3.0 +#tool nuget:?package=ReportGenerator&version=5.3.5 #tool nuget:?package=xunit.runner.console&version=2.8.1 #tool nuget:?package=CodecovUploader&version=0.7.3 diff --git a/global.json b/global.json index 195d22f0..d3edfcb3 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.300", + "version": "8.0.301", "rollForward": "patch", "allowPrerelease": false } From 2b585fdb0e147a943c206b2a68a80d88069a5655 Mon Sep 17 00:00:00 2001 From: jericho Date: Mon, 3 Jun 2024 17:04:52 -0400 Subject: [PATCH 05/16] Fix Analyzer warnings which cause the build to fail when warnings are treated as errors --- Source/StrongGrid/Utilities/Log/ValueStringBuilder.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Source/StrongGrid/Utilities/Log/ValueStringBuilder.cs b/Source/StrongGrid/Utilities/Log/ValueStringBuilder.cs index 01acc893..1f7fde48 100644 --- a/Source/StrongGrid/Utilities/Log/ValueStringBuilder.cs +++ b/Source/StrongGrid/Utilities/Log/ValueStringBuilder.cs @@ -32,8 +32,8 @@ public int Length get => _pos; set { - Debug.Assert(value >= 0); - Debug.Assert(value <= _chars.Length); + Debug.Assert(value >= 0, "Value must be greater than zero"); + Debug.Assert(value <= _chars.Length, "Value cannot exceed the size of the buffer"); _pos = value; } } @@ -43,7 +43,7 @@ public int Length public void EnsureCapacity(int capacity) { // This is not expected to be called this with negative capacity - Debug.Assert(capacity >= 0); + Debug.Assert(capacity >= 0, "Capacity must be greater than zero"); // If the caller has a bug and calls this with negative capacity, make sure to call Grow to throw an exception. if ((uint)capacity > (uint)_chars.Length) @@ -80,7 +80,7 @@ public ref char this[int index] { get { - Debug.Assert(index < _pos); + Debug.Assert(index < _pos, "Index must less than the number of items currently in the buffer"); return ref _chars[index]; } } @@ -275,7 +275,7 @@ private void GrowAndAppend(char c) [MethodImpl(MethodImplOptions.NoInlining)] private void Grow(int additionalCapacityBeyondPos) { - Debug.Assert(additionalCapacityBeyondPos > 0); + Debug.Assert(additionalCapacityBeyondPos > 0, "Value must be greater than zero"); Debug.Assert(_pos > _chars.Length - additionalCapacityBeyondPos, "Grow called incorrectly, no resize is needed."); const uint ArrayMaxLength = 0x7FFFFFC7; // same as Array.MaxLength From 2adac41627024ad151162fe23ca2d498c231a4b2 Mon Sep 17 00:00:00 2001 From: jericho Date: Thu, 6 Jun 2024 09:14:38 -0400 Subject: [PATCH 06/16] Add file to source control which was inadvertently ignored --- .../Log/CallerArgumentExpressionAttribute.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 Source/StrongGrid/Utilities/Log/CallerArgumentExpressionAttribute.cs diff --git a/Source/StrongGrid/Utilities/Log/CallerArgumentExpressionAttribute.cs b/Source/StrongGrid/Utilities/Log/CallerArgumentExpressionAttribute.cs new file mode 100644 index 00000000..1fb93ee4 --- /dev/null +++ b/Source/StrongGrid/Utilities/Log/CallerArgumentExpressionAttribute.cs @@ -0,0 +1,16 @@ +#if !NET +namespace System.Runtime.CompilerServices +{ + /// From the Microsoft.Extensions.Logging.Abstractions project. + [AttributeUsage(AttributeTargets.Parameter, AllowMultiple = false, Inherited = false)] + internal sealed class CallerArgumentExpressionAttribute : Attribute + { + public CallerArgumentExpressionAttribute(string parameterName) + { + ParameterName = parameterName; + } + + public string ParameterName { get; } + } +} +#endif From a7d7eaec24410ae1663fcfe0c92c7ee6a6b8c532 Mon Sep 17 00:00:00 2001 From: jericho Date: Sun, 16 Jun 2024 10:11:02 -0400 Subject: [PATCH 07/16] Refresh build script --- build.cake | 2 +- global.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.cake b/build.cake index 241233c6..8a10e77d 100644 --- a/build.cake +++ b/build.cake @@ -2,7 +2,7 @@ #tool dotnet:?package=GitVersion.Tool&version=5.12.0 #tool dotnet:?package=coveralls.net&version=4.0.1 #tool nuget:https://f.feedz.io/jericho/jericho/nuget/?package=GitReleaseManager&version=0.17.0-collaborators0004 -#tool nuget:?package=ReportGenerator&version=5.3.5 +#tool nuget:?package=ReportGenerator&version=5.3.6 #tool nuget:?package=xunit.runner.console&version=2.8.1 #tool nuget:?package=CodecovUploader&version=0.7.3 diff --git a/global.json b/global.json index d3edfcb3..71b7811f 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.301", + "version": "8.0.302", "rollForward": "patch", "allowPrerelease": false } From 4f90078866798f7ca194cdb52a4b591c414564b0 Mon Sep 17 00:00:00 2001 From: jericho Date: Sun, 16 Jun 2024 19:15:09 -0400 Subject: [PATCH 08/16] Optimizations: Ensure we use only a single http client to download files --- Source/StrongGrid/Resources/Contacts.cs | 59 ++++++++++++------- .../StrongGrid/Resources/EmailActivities.cs | 31 ++++++++-- 2 files changed, 62 insertions(+), 28 deletions(-) diff --git a/Source/StrongGrid/Resources/Contacts.cs b/Source/StrongGrid/Resources/Contacts.cs index 7f0df6cf..fb24e6e8 100644 --- a/Source/StrongGrid/Resources/Contacts.cs +++ b/Source/StrongGrid/Resources/Contacts.cs @@ -24,8 +24,31 @@ namespace StrongGrid.Resources public class Contacts : IContacts { private const string _endpoint = "marketing/contacts"; + private static HttpClient _downloadFilesClient = null; private readonly Pathoschild.Http.Client.IClient _client; + private static HttpClient DownloadFilesClient + { + get + { + if (_downloadFilesClient == null) + { + var handler = new HttpClientHandler() + { +#if NET6_0_OR_GREATER + AutomaticDecompression = DecompressionMethods.All +#else + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate +#endif + }; + + _downloadFilesClient = new HttpClient(handler); + } + + return _downloadFilesClient; + } + } + /// /// Initializes a new instance of the class. /// @@ -464,33 +487,25 @@ public Task GetExportJobAsync(string jobId, CancellationToken cancell if (job == null) throw new ArgumentNullException(nameof(job)); if (job.Status != ExportJobStatus.Ready) throw new Exception("The job is not completed"); - var handler = new HttpClientHandler() - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate - }; - var result = new (string FileName, Stream Stream)[job.FileUrls.Length]; - using (var client = new HttpClient(handler)) + for (int i = 0; i < job.FileUrls.Length; i++) { - for (int i = 0; i < job.FileUrls.Length; i++) - { - var fileUri = new Uri(job.FileUrls[i]); - var fileName = Path.GetFileName(fileUri.AbsolutePath); - var stream = await client.GetStreamAsync(job.FileUrls[i]).ConfigureAwait(false); + var fileUri = new Uri(job.FileUrls[i]); + var fileName = Path.GetFileName(fileUri.AbsolutePath); + var stream = await DownloadFilesClient.GetStreamAsync(job.FileUrls[i]).ConfigureAwait(false); - const string gzipExtension = ".gzip"; - if (decompress && fileName.EndsWith(gzipExtension)) - { - result[i] = (fileName.Substring(0, fileName.Length - gzipExtension.Length), await stream.DecompressAsync().ConfigureAwait(false)); - } - else - { - result[i] = (fileName, stream); - } + const string gzipExtension = ".gzip"; + if (decompress && fileName.EndsWith(gzipExtension)) + { + result[i] = (fileName.Substring(0, fileName.Length - gzipExtension.Length), await stream.DecompressAsync().ConfigureAwait(false)); + } + else + { + result[i] = (fileName, stream); } - - return result; } + + return result; } private static StrongGridJsonObject ConvertToJson( diff --git a/Source/StrongGrid/Resources/EmailActivities.cs b/Source/StrongGrid/Resources/EmailActivities.cs index 45e41f6c..07b1fd2d 100644 --- a/Source/StrongGrid/Resources/EmailActivities.cs +++ b/Source/StrongGrid/Resources/EmailActivities.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -21,8 +22,31 @@ namespace StrongGrid.Resources public class EmailActivities : IEmailActivities { private const string _endpoint = "messages"; + private static HttpClient _downloadFilesClient = null; private readonly Pathoschild.Http.Client.IClient _client; + private static HttpClient DownloadFilesClient + { + get + { + if (_downloadFilesClient == null) + { + var handler = new HttpClientHandler() + { +#if NET6_0_OR_GREATER + AutomaticDecompression = DecompressionMethods.All +#else + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate +#endif + }; + + _downloadFilesClient = new HttpClient(handler); + } + + return _downloadFilesClient; + } + } + /// /// Initializes a new instance of the class. /// @@ -120,12 +144,7 @@ public Task GetCsvDownloadUrlAsync(string downloadUUID, CancellationToke public async Task DownloadCsvAsync(string downloadUUID, CancellationToken cancellationToken = default) { var url = await GetCsvDownloadUrlAsync(downloadUUID, cancellationToken); - - using (var client = new HttpClient()) - { - var responseStream = await client.GetStreamAsync(url).ConfigureAwait(false); - return responseStream; - } + return await DownloadFilesClient.GetStreamAsync(url).ConfigureAwait(false); } } } From 542810dd71d68a0a4459aa69f40b7604ddf29c3e Mon Sep 17 00:00:00 2001 From: jericho Date: Sun, 16 Jun 2024 19:15:49 -0400 Subject: [PATCH 09/16] Increase timeout to allow export job to complete during integration testing --- .../Tests/ContactsAndCustomFields.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/StrongGrid.IntegrationTests/Tests/ContactsAndCustomFields.cs b/Source/StrongGrid.IntegrationTests/Tests/ContactsAndCustomFields.cs index da100b5c..a690234f 100644 --- a/Source/StrongGrid.IntegrationTests/Tests/ContactsAndCustomFields.cs +++ b/Source/StrongGrid.IntegrationTests/Tests/ContactsAndCustomFields.cs @@ -176,7 +176,7 @@ public async Task RunAsync(IClient client, TextWriter log, CancellationToken can } // Make sure we don't loop indefinetly - if (elapsed.Elapsed >= TimeSpan.FromSeconds(5)) + if (elapsed.Elapsed >= TimeSpan.FromSeconds(10)) { elapsed.Stop(); await log.WriteLineAsync("\tThe job did not complete in a reasonable amount of time.").ConfigureAwait(false); From f4ffa5fb9e2cb8b922cc41ad9dcd904db3239de5 Mon Sep 17 00:00:00 2001 From: jericho Date: Tue, 18 Jun 2024 21:19:19 -0400 Subject: [PATCH 10/16] Fix typo --- .../email_with_attachments.txt | 0 .../raw_email.txt | 0 .../raw_email_with_attachments.txt | 0 Source/StrongGrid.UnitTests/StrongGrid.UnitTests.csproj | 8 ++++---- Source/StrongGrid.UnitTests/WebhookParserTests.cs | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) rename Source/StrongGrid.UnitTests/{InboudEmailTestData => InboundEmailTestData}/email_with_attachments.txt (100%) rename Source/StrongGrid.UnitTests/{InboudEmailTestData => InboundEmailTestData}/raw_email.txt (100%) rename Source/StrongGrid.UnitTests/{InboudEmailTestData => InboundEmailTestData}/raw_email_with_attachments.txt (100%) diff --git a/Source/StrongGrid.UnitTests/InboudEmailTestData/email_with_attachments.txt b/Source/StrongGrid.UnitTests/InboundEmailTestData/email_with_attachments.txt similarity index 100% rename from Source/StrongGrid.UnitTests/InboudEmailTestData/email_with_attachments.txt rename to Source/StrongGrid.UnitTests/InboundEmailTestData/email_with_attachments.txt diff --git a/Source/StrongGrid.UnitTests/InboudEmailTestData/raw_email.txt b/Source/StrongGrid.UnitTests/InboundEmailTestData/raw_email.txt similarity index 100% rename from Source/StrongGrid.UnitTests/InboudEmailTestData/raw_email.txt rename to Source/StrongGrid.UnitTests/InboundEmailTestData/raw_email.txt diff --git a/Source/StrongGrid.UnitTests/InboudEmailTestData/raw_email_with_attachments.txt b/Source/StrongGrid.UnitTests/InboundEmailTestData/raw_email_with_attachments.txt similarity index 100% rename from Source/StrongGrid.UnitTests/InboudEmailTestData/raw_email_with_attachments.txt rename to Source/StrongGrid.UnitTests/InboundEmailTestData/raw_email_with_attachments.txt diff --git a/Source/StrongGrid.UnitTests/StrongGrid.UnitTests.csproj b/Source/StrongGrid.UnitTests/StrongGrid.UnitTests.csproj index 09f4c2b0..5dd5da98 100644 --- a/Source/StrongGrid.UnitTests/StrongGrid.UnitTests.csproj +++ b/Source/StrongGrid.UnitTests/StrongGrid.UnitTests.csproj @@ -1,4 +1,4 @@ - + net48;net6.0;net7.0 @@ -42,13 +42,13 @@ - + Always - + Always - + Always diff --git a/Source/StrongGrid.UnitTests/WebhookParserTests.cs b/Source/StrongGrid.UnitTests/WebhookParserTests.cs index 57424ea8..658ac9a0 100644 --- a/Source/StrongGrid.UnitTests/WebhookParserTests.cs +++ b/Source/StrongGrid.UnitTests/WebhookParserTests.cs @@ -470,7 +470,7 @@ public async Task InboundEmailAsync() [Fact] public async Task InboundEmailWithAttachmentsAsync() { - using (var fileStream = File.OpenRead("InboudEmailTestData/email_with_attachments.txt")) + using (var fileStream = File.OpenRead("InboundEmailTestData/email_with_attachments.txt")) { var parser = new WebhookParser(); var inboundEmail = await parser.ParseInboundEmailWebhookAsync(fileStream); @@ -555,7 +555,7 @@ public async Task InboundEmailRawContentAsync() { // Arrange var parser = new WebhookParser(); - using (var fileStream = File.OpenRead("InboudEmailTestData/raw_email.txt")) + using (var fileStream = File.OpenRead("InboundEmailTestData/raw_email.txt")) { // Act var inboundEmail = await parser.ParseInboundEmailWebhookAsync(fileStream); @@ -580,7 +580,7 @@ public async Task InboundEmailRawContent_with_attachments_Async() { // Arrange var parser = new WebhookParser(); - using (var fileStream = File.OpenRead("InboudEmailTestData/raw_email_with_attachments.txt")) + using (var fileStream = File.OpenRead("InboundEmailTestData/raw_email_with_attachments.txt")) { // Act var inboundEmail = await parser.ParseInboundEmailWebhookAsync(fileStream); From a717f2879697d5273c11e72b1d23a78adad4c826 Mon Sep 17 00:00:00 2001 From: jericho Date: Tue, 18 Jun 2024 22:04:49 -0400 Subject: [PATCH 11/16] Analyzer recommendations --- .../Tests/ContactsAndCustomFields.cs | 6 +++--- Source/StrongGrid/Utilities/Log/ValueStringBuilder.cs | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Source/StrongGrid.IntegrationTests/Tests/ContactsAndCustomFields.cs b/Source/StrongGrid.IntegrationTests/Tests/ContactsAndCustomFields.cs index a690234f..383e47ce 100644 --- a/Source/StrongGrid.IntegrationTests/Tests/ContactsAndCustomFields.cs +++ b/Source/StrongGrid.IntegrationTests/Tests/ContactsAndCustomFields.cs @@ -82,7 +82,7 @@ public async Task RunAsync(IClient client, TextWriter log, CancellationToken can await log.WriteLineAsync($"\t{record.FirstName} {record.LastName}").ConfigureAwait(false); } - if (contacts.Any()) + if (contacts.Length > 0) { var batchById = await client.Contacts.GetMultipleAsync(contacts.Take(10).Select(c => c.Id), cancellationToken).ConfigureAwait(false); await log.WriteLineAsync($"Retrieved {batchById.Length} contacts by ID in a single API call.").ConfigureAwait(false); @@ -98,7 +98,7 @@ public async Task RunAsync(IClient client, TextWriter log, CancellationToken can await log.WriteLineAsync($"\t{record.FirstName} {record.LastName}").ConfigureAwait(false); } - var contact = await client.Contacts.GetAsync(contacts.First().Id).ConfigureAwait(false); + var contact = await client.Contacts.GetAsync(contacts.First().Id, cancellationToken).ConfigureAwait(false); await log.WriteLineAsync($"Retrieved contact {contact.Id}").ConfigureAwait(false); await log.WriteLineAsync($"\tEmail: {contact.Email}").ConfigureAwait(false); await log.WriteLineAsync($"\tFirst Name: {contact.FirstName}").ConfigureAwait(false); @@ -184,7 +184,7 @@ public async Task RunAsync(IClient client, TextWriter log, CancellationToken can } } - if (contacts.Any()) + if (contacts.Length > 0) { var contact = contacts.First(); await client.Contacts.DeleteAsync(contact.Id, cancellationToken).ConfigureAwait(false); diff --git a/Source/StrongGrid/Utilities/Log/ValueStringBuilder.cs b/Source/StrongGrid/Utilities/Log/ValueStringBuilder.cs index 1f7fde48..ca6661fa 100644 --- a/Source/StrongGrid/Utilities/Log/ValueStringBuilder.cs +++ b/Source/StrongGrid/Utilities/Log/ValueStringBuilder.cs @@ -194,8 +194,9 @@ public void Append(string s) } int pos = _pos; - if (s.Length == 1 && (uint)pos < (uint)_chars.Length) // very common case, e.g. appending strings from NumberFormatInfo like separators, percent symbols, etc. + if (s.Length == 1 && (uint)pos < (uint)_chars.Length) { + // very common case, e.g. appending strings from NumberFormatInfo like separators, percent symbols, etc. _chars[pos] = s[0]; _pos = pos + 1; } From 9f87a02a81dd9d62f2896cb2276fe0bf2166e678 Mon Sep 17 00:00:00 2001 From: jericho Date: Wed, 19 Jun 2024 11:19:38 -0400 Subject: [PATCH 12/16] Link segments to their parent list when creating a segment Resolves #525 --- Source/StrongGrid/Resources/Segments.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/StrongGrid/Resources/Segments.cs b/Source/StrongGrid/Resources/Segments.cs index 27f4aad6..a2fee5a7 100644 --- a/Source/StrongGrid/Resources/Segments.cs +++ b/Source/StrongGrid/Resources/Segments.cs @@ -41,7 +41,7 @@ public Task CreateAsync(string name, string query, string listId = null var data = new StrongGridJsonObject(); data.AddProperty("name", name); data.AddProperty("query_dsl", query); - data.AddProperty("parent_list_id", listId); + if (!string.IsNullOrEmpty(listId)) data.AddProperty("parent_list_ids", new[] { listId }); return _client .PostAsync($"{(queryLanguageVersion == QueryLanguageVersion.Version2 ? _endpoint_v2 : _endpoint)}") From ddef5bb6429a3ec597a65d6196f632bd4c53fa78 Mon Sep 17 00:00:00 2001 From: jericho Date: Wed, 19 Jun 2024 11:41:07 -0400 Subject: [PATCH 13/16] DateTime values are handled differently in query v1 versus v2 --- .../Tests/ContactsAndCustomFields.cs | 4 ++ .../Tests/ListsAndSegments.cs | 28 ++++++-- .../Utilities/QueryDslTests.cs | 66 +++++++++++++++---- .../Models/Search/ISearchCriteria.cs | 4 +- .../Models/Search/SearchCriteria.cs | 36 +++++++--- .../Models/Search/SearchCriteriaBetween.cs | 4 +- .../Models/Search/SearchCriteriaContains.cs | 4 +- .../Models/Search/SearchCriteriaIsNotNull.cs | 2 +- .../Models/Search/SearchCriteriaIsNull.cs | 2 +- .../Models/Search/SearchCriteriaNotBetween.cs | 4 +- .../Models/Search/SearchCriteriaUniqueArg.cs | 10 +-- .../Search/SearchCriteriaUniqueArgBetween.cs | 4 +- .../SearchCriteriaUniqueArgNotBetween.cs | 4 +- .../Search/SearchCriteriaUniqueArgNotNull.cs | 2 +- .../Search/SearchCriteriaUniqueArgNull.cs | 2 +- Source/StrongGrid/Utilities/Utils.cs | 8 ++- 16 files changed, 131 insertions(+), 53 deletions(-) diff --git a/Source/StrongGrid.IntegrationTests/Tests/ContactsAndCustomFields.cs b/Source/StrongGrid.IntegrationTests/Tests/ContactsAndCustomFields.cs index 383e47ce..39409025 100644 --- a/Source/StrongGrid.IntegrationTests/Tests/ContactsAndCustomFields.cs +++ b/Source/StrongGrid.IntegrationTests/Tests/ContactsAndCustomFields.cs @@ -124,6 +124,10 @@ public async Task RunAsync(IClient client, TextWriter log, CancellationToken can var searchResult = await client.Contacts.SearchAsync(new[] { firstNameCriteria, LastNameCriteria }, cancellationToken).ConfigureAwait(false); await log.WriteLineAsync($"Found {searchResult.Length} contacts named John Doe").ConfigureAwait(false); + var modifiedDuringPriorYearCriteria = new SearchCriteriaGreaterThan(ContactsFilterField.ModifiedOn, DateTime.UtcNow.AddYears(-1)); + searchResult = await client.Contacts.SearchAsync(modifiedDuringPriorYearCriteria, cancellationToken).ConfigureAwait(false); + await log.WriteLineAsync($"Found {searchResult.Length} contacts modified during the last year").ConfigureAwait(false); + var (totalCount, billableCount) = await client.Contacts.GetCountAsync(cancellationToken).ConfigureAwait(false); await log.WriteLineAsync("Record counts").ConfigureAwait(false); await log.WriteLineAsync($"\tTotal: {totalCount}").ConfigureAwait(false); diff --git a/Source/StrongGrid.IntegrationTests/Tests/ListsAndSegments.cs b/Source/StrongGrid.IntegrationTests/Tests/ListsAndSegments.cs index da89465d..7ae33b70 100644 --- a/Source/StrongGrid.IntegrationTests/Tests/ListsAndSegments.cs +++ b/Source/StrongGrid.IntegrationTests/Tests/ListsAndSegments.cs @@ -1,5 +1,6 @@ using StrongGrid.Models; using StrongGrid.Models.Search; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -72,17 +73,15 @@ public async Task RunAsync(IClient client, TextWriter log, CancellationToken can { new KeyValuePair>(SearchLogicalOperator.And, new[] { - new SearchCriteriaEqual(ContactsFilterField.FirstName, "Jane"), - new SearchCriteriaEqual(ContactsFilterField.LastName, "Doe") + new SearchCriteriaLike(ContactsFilterField.EmailAddress, "%hotmail.com") }) }; - var segment = await client.Segments.CreateAsync("StrongGrid Integration Testing: First Name is Jane and last name is Doe", filterConditions, list.Id, cancellationToken).ConfigureAwait(false); + var segment = await client.Segments.CreateAsync("StrongGrid Integration Testing: Recipients @ Hotmail", filterConditions, list.Id, cancellationToken).ConfigureAwait(false); await log.WriteLineAsync($"Segment '{segment.Name}' created. Id: {segment.Id}").ConfigureAwait(false); - // UPDATE THE SEGMENT (three contacts match the criteria) - var hotmailCriteria = new SearchCriteriaLike(ContactsFilterField.EmailAddress, "%hotmail.com"); - segment = await client.Segments.UpdateAsync(segment.Id, "StrongGrid Integration Testing: Recipients @ Hotmail", hotmailCriteria, cancellationToken).ConfigureAwait(false); - await log.WriteLineAsync($"Segment {segment.Id} updated. The new name is: '{segment.Name}'").ConfigureAwait(false); + // PLEASE NOTE: you must wait at least 5 minutes before updating a segment. + // If you attempt to update a segment too quickly, the SendGrid API will throw the following exception: + // "Update request came too soon, please wait 5 minutes before trying again" // GET THE SEGMENT segment = await client.Segments.GetAsync(segment.Id, cancellationToken).ConfigureAwait(false); @@ -91,6 +90,17 @@ public async Task RunAsync(IClient client, TextWriter log, CancellationToken can // GET THE CONTACTS contacts = await client.Contacts.GetMultipleByEmailAddressAsync(new[] { "dummy1@hotmail.com", "dummy2@hotmail.com", "dummy3@hotmail.com" }, cancellationToken).ConfigureAwait(false); + // CREATE ANOTHER SEGMENT + filterConditions = new[] + { + new KeyValuePair>(SearchLogicalOperator.And, new[] + { + new SearchCriteriaGreaterThan(ContactsFilterField.ModifiedOn, DateTime.UtcNow.AddYears(-1)), + }) + }; + var anotherSegment = await client.Segments.CreateAsync("StrongGrid Integration Testing: Modified in the prior year", filterConditions, list.Id, cancellationToken).ConfigureAwait(false); + await log.WriteLineAsync($"Segment '{anotherSegment.Name}' created. Id: {anotherSegment.Id}").ConfigureAwait(false); + // REMOVE THE CONTACTS FROM THE LIST (THEY WILL AUTOMATICALLY BE REMOVED FROM THE HOTMAIL SEGMENT) if (contacts.Any()) { @@ -109,6 +119,10 @@ public async Task RunAsync(IClient client, TextWriter log, CancellationToken can await client.Segments.DeleteAsync(segment.Id, false, cancellationToken).ConfigureAwait(false); await log.WriteLineAsync($"Segment {segment.Id} deleted").ConfigureAwait(false); + // DELETE THE OTHER SEGMENT + await client.Segments.DeleteAsync(anotherSegment.Id, false, cancellationToken).ConfigureAwait(false); + await log.WriteLineAsync($"Segment {anotherSegment.Id} deleted").ConfigureAwait(false); + // DELETE THE LIST await client.Lists.DeleteAsync(list.Id, false, cancellationToken).ConfigureAwait(false); await log.WriteLineAsync($"List {list.Id} deleted").ConfigureAwait(false); diff --git a/Source/StrongGrid.UnitTests/Utilities/QueryDslTests.cs b/Source/StrongGrid.UnitTests/Utilities/QueryDslTests.cs index 2412907e..2bcff05b 100644 --- a/Source/StrongGrid.UnitTests/Utilities/QueryDslTests.cs +++ b/Source/StrongGrid.UnitTests/Utilities/QueryDslTests.cs @@ -7,9 +7,9 @@ namespace StrongGrid.UnitTests.Utilities { - public class QueryDslTests + public class QueryDsl { - public class ToQueryDslVersion2 + public class Version2Tests { [Fact] public void One_condition_with_two_criteria() @@ -28,7 +28,7 @@ public void One_condition_with_two_criteria() var result = StrongGrid.Utilities.Utils.ToQueryDslVersion2(filter); // Assert - result.ShouldBe("SELECT contacts.contact_id, contacts.updated_at FROM contact_data AS contacts WHERE contacts.first_name='John' AND contacts.last_name='Doe'"); + result.ShouldBe("SELECT c.contact_id, c.updated_at FROM contact_data AS c WHERE c.first_name='John' AND c.last_name='Doe'"); } [Fact] @@ -41,7 +41,7 @@ public void All_contacts() var result = StrongGrid.Utilities.Utils.ToQueryDslVersion2(filter); // Assert - result.ShouldBe("SELECT contacts.contact_id, contacts.updated_at FROM contact_data AS contacts"); + result.ShouldBe("SELECT c.contact_id, c.updated_at FROM contact_data AS c"); } [Fact] @@ -60,7 +60,7 @@ public void All_contacts_with_firstname_Dave() var result = StrongGrid.Utilities.Utils.ToQueryDslVersion2(filter); // Assert - result.ShouldBe("SELECT contacts.contact_id, contacts.updated_at FROM contact_data AS contacts WHERE contacts.first_name='Dave'"); + result.ShouldBe("SELECT c.contact_id, c.updated_at FROM contact_data AS c WHERE c.first_name='Dave'"); } [Fact] @@ -79,7 +79,7 @@ public void All_contacts_in_Colorado() var result = StrongGrid.Utilities.Utils.ToQueryDslVersion2(filter); // Assert - result.ShouldBe("SELECT contacts.contact_id, contacts.updated_at FROM contact_data AS contacts WHERE contacts.state_province_region='CO'"); + result.ShouldBe("SELECT c.contact_id, c.updated_at FROM contact_data AS c WHERE c.state_province_region='CO'"); } [Fact] @@ -98,7 +98,7 @@ public void All_contacts_at_gmail() var result = StrongGrid.Utilities.Utils.ToQueryDslVersion2(filter); // Assert - result.ShouldBe("SELECT contacts.contact_id, contacts.updated_at FROM contact_data AS contacts WHERE contacts.email LIKE '%gmail.com%'"); + result.ShouldBe("SELECT c.contact_id, c.updated_at FROM contact_data AS c WHERE c.email LIKE '%gmail.com%'"); } [Fact] @@ -117,7 +117,7 @@ public void All_contacts_with_custom_text_field() var result = StrongGrid.Utilities.Utils.ToQueryDslVersion2(filter); // Assert - result.ShouldBe("SELECT contacts.contact_id, contacts.updated_at FROM contact_data AS contacts WHERE contacts.my_text_custom_field='abc'"); + result.ShouldBe("SELECT c.contact_id, c.updated_at FROM contact_data AS c WHERE c.my_text_custom_field='abc'"); } [Fact] @@ -136,7 +136,7 @@ public void All_contacts_with_custom_number_field() var result = StrongGrid.Utilities.Utils.ToQueryDslVersion2(filter); // Assert - result.ShouldBe("SELECT contacts.contact_id, contacts.updated_at FROM contact_data AS contacts WHERE contacts.my_number_custom_field=12"); + result.ShouldBe("SELECT c.contact_id, c.updated_at FROM contact_data AS c WHERE c.my_number_custom_field=12"); } [Fact] @@ -155,7 +155,7 @@ public void All_contacts_with_custom_date_field() var result = StrongGrid.Utilities.Utils.ToQueryDslVersion2(filter); // Assert - result.ShouldBe("SELECT contacts.contact_id, contacts.updated_at FROM contact_data AS contacts WHERE contacts.my_date_custom_field='2021-01-01T12:46:24Z'"); + result.ShouldBe("SELECT c.contact_id, c.updated_at FROM contact_data AS c WHERE c.my_date_custom_field='2021-01-01T12:46:24Z'"); } [Fact] @@ -174,7 +174,7 @@ public void All_contacts_where_alternate_email_contains() var result = StrongGrid.Utilities.Utils.ToQueryDslVersion2(filter); // Assert - result.ShouldBe("SELECT contacts.contact_id, contacts.updated_at FROM contact_data AS contacts WHERE array_contains(contacts.alternate_emails,'alternate@gmail.com')"); + result.ShouldBe("SELECT c.contact_id, c.updated_at FROM contact_data AS c WHERE array_contains(c.alternate_emails,'alternate@gmail.com')"); } [Fact] @@ -193,7 +193,7 @@ public void All_contacts_member_of_either_of_lists() var result = StrongGrid.Utilities.Utils.ToQueryDslVersion2(filter); // Assert - result.ShouldBe("SELECT contacts.contact_id, contacts.updated_at FROM contact_data AS contacts WHERE array_contains(contacts.list_ids,['aaa','bbb'])"); + result.ShouldBe("SELECT c.contact_id, c.updated_at FROM contact_data AS c WHERE array_contains(c.list_ids,['aaa','bbb'])"); } [Fact] @@ -213,11 +213,30 @@ public void All_contacts_not_Dave_or_first_name_is_null() var result = StrongGrid.Utilities.Utils.ToQueryDslVersion2(filter); // Assert - result.ShouldBe("SELECT contacts.contact_id, contacts.updated_at FROM contact_data AS contacts WHERE contacts.first_name!='Dave' OR contacts.first_name IS NULL"); + result.ShouldBe("SELECT c.contact_id, c.updated_at FROM contact_data AS c WHERE c.first_name!='Dave' OR c.first_name IS NULL"); + } + + [Fact] + public void All_contacts_modified_before_given_date() + { + // Arrange + var filter = new[] + { + new KeyValuePair>(SearchLogicalOperator.And, new[] + { + new SearchCriteriaLessThan(ContactsFilterField.ModifiedOn, new DateTime(2024, 6, 19, 0, 0, 0, DateTimeKind.Utc)) + }) + }; + + // Act + var result = StrongGrid.Utilities.Utils.ToQueryDslVersion2(filter); + + // Assert + result.ShouldBe("SELECT c.contact_id, c.updated_at FROM contact_data AS c WHERE c.updated_at<'2024-06-19T00:00:00Z'"); } } - public class ToQueryDslVersion1 + public class Version1Tests { [Fact] public void Filter_by_subject() @@ -275,6 +294,25 @@ public void Filter_by_bounced_email() // Assert result.ShouldBe("status=\"bounce\""); } + + [Fact] + public void Contacts_modified_prior_to_given_date() + { + // Arrange + var filter = new[] + { + new KeyValuePair>(SearchLogicalOperator.And, new[] + { + new SearchCriteriaLessThan(ContactsFilterField.ModifiedOn, new DateTime(2024, 6, 19, 0, 0, 0, DateTimeKind.Utc)), + }) + }; + + // Act + var result = StrongGrid.Utilities.Utils.ToQueryDslVersion1(filter); + + // Assert + result.ShouldBe("updated_at /// A representation of the search criteria. - /// This is used when generating a SGQL v1 query. For example, when searching for email activities. + /// This is used when generating a SGQL v1 query. For example, when searching for contacts or email activities. string ToString(); /// @@ -17,7 +17,7 @@ public interface ISearchCriteria /// /// The table alias. /// A representation of the search criteria. - /// This is used when generating a segmentation query v2. For example, when searching for contacts or when creating a new segment. + /// This is used when generating a segmentation query v2. For example, when creating a new segment. string ToString(string tableAlias); } } diff --git a/Source/StrongGrid/Models/Search/SearchCriteria.cs b/Source/StrongGrid/Models/Search/SearchCriteria.cs index 71d64ea9..7101273d 100644 --- a/Source/StrongGrid/Models/Search/SearchCriteria.cs +++ b/Source/StrongGrid/Models/Search/SearchCriteria.cs @@ -9,6 +9,9 @@ namespace StrongGrid.Models.Search /// public abstract class SearchCriteria : ISearchCriteria { + private const char QuoteV1 = '"'; + private const char QuoteV2 = '\''; + /// /// Gets or sets the name of the table. /// @@ -48,13 +51,28 @@ public SearchCriteria(FilterTable filterTable, string filterField, SearchCompari /// Returns the string representation of a given value as expected by the SendGrid segmenting API. /// /// The value. - /// The character used to quote string values. This character is a double-quote for v1 queries and a single-quote for v2 queries. + /// The desired query version. /// The representation of the value. - public static string ConvertToString(object value, char quote) + public static string ConvertToString(object value, QueryLanguageVersion queryLanguageVersion) { + var quote = queryLanguageVersion switch + { + QueryLanguageVersion.Unspecified => QuoteV1, + QueryLanguageVersion.Version1 => QuoteV1, + QueryLanguageVersion.Version2 => QuoteV2, + _ => throw new ArgumentOutOfRangeException(nameof(queryLanguageVersion), queryLanguageVersion, null) + }; + if (value is DateTime dateValue) { - return $"{quote}{dateValue.ToUniversalTime():yyyy-MM-ddTHH:mm:ssZ}{quote}"; + if (queryLanguageVersion == QueryLanguageVersion.Version1) + { + return $"TIMESTAMP {quote}{dateValue.ToUniversalTime():yyyy-MM-ddTHH:mm:ssZ}{quote}"; + } + else + { + return $"{quote}{dateValue.ToUniversalTime():yyyy-MM-ddTHH:mm:ssZ}{quote}"; + } } else if (value is string stringValue) { @@ -66,7 +84,7 @@ public static string ConvertToString(object value, char quote) } else if (value is IEnumerable values) { - return $"[{string.Join(",", values.Cast().Select(e => ConvertToString(e, quote)))}]"; + return $"[{string.Join(",", values.Cast().Select(e => ConvertToString(e, queryLanguageVersion)))}]"; } else if (value.IsNumber()) { @@ -98,11 +116,11 @@ public static string ConvertToString(object value, char quote) /// Converts the filter value into a string as expected by the SendGrid segmenting API. /// Can be overridden in subclasses if the value needs special formatting. /// - /// The character used to quote string values. This character is a double-quote for v1 queries and a single-quote for v2 queries. + /// The desired query version. /// The string representation of the value. - public virtual string ConvertValueToString(char quote) + public virtual string ConvertValueToString(QueryLanguageVersion queryLanguageVersion) { - return ConvertToString(FilterValue, quote); + return ConvertToString(FilterValue, queryLanguageVersion); } /// @@ -119,7 +137,7 @@ public virtual string ConvertOperatorToString() public override string ToString() { var filterOperator = ConvertOperatorToString(); - var filterValue = ConvertValueToString('"'); + var filterValue = ConvertValueToString(QueryLanguageVersion.Version1); return $"{FilterField}{filterOperator}{filterValue}"; } @@ -128,7 +146,7 @@ public override string ToString() public virtual string ToString(string tableAlias) { var filterOperator = ConvertOperatorToString(); - var filterValue = ConvertValueToString('\''); + var filterValue = ConvertValueToString(QueryLanguageVersion.Version2); var filterField = string.IsNullOrEmpty(tableAlias) ? FilterField : $"{tableAlias}.{FilterField}"; return $"{filterField}{filterOperator}{filterValue}"; diff --git a/Source/StrongGrid/Models/Search/SearchCriteriaBetween.cs b/Source/StrongGrid/Models/Search/SearchCriteriaBetween.cs index cfad5d56..c3d37767 100644 --- a/Source/StrongGrid/Models/Search/SearchCriteriaBetween.cs +++ b/Source/StrongGrid/Models/Search/SearchCriteriaBetween.cs @@ -46,9 +46,9 @@ public SearchCriteriaBetween(EmailActivitiesFilterField filterField, object lowe } /// - public override string ConvertValueToString(char quote) + public override string ConvertValueToString(QueryLanguageVersion queryLanguageVersion) { - return $"{ConvertToString(FilterValue, quote)} AND {ConvertToString(UpperValue, quote)}"; + return $"{ConvertToString(FilterValue, queryLanguageVersion)} AND {ConvertToString(UpperValue, queryLanguageVersion)}"; } /// diff --git a/Source/StrongGrid/Models/Search/SearchCriteriaContains.cs b/Source/StrongGrid/Models/Search/SearchCriteriaContains.cs index cf295d90..85c9e26f 100644 --- a/Source/StrongGrid/Models/Search/SearchCriteriaContains.cs +++ b/Source/StrongGrid/Models/Search/SearchCriteriaContains.cs @@ -40,7 +40,7 @@ public SearchCriteriaContains(EmailActivitiesFilterField filterField, object fil public override string ToString() { var filterOperator = "CONTAINS"; // This is the operator from SendGrid's query DSL version 1. "ARRAY_CONTAINS" is the operator from version 2. - var filterValue = ConvertValueToString('"'); + var filterValue = ConvertValueToString(QueryLanguageVersion.Version1); return $"{filterOperator}({FilterField},{filterValue})"; } @@ -49,7 +49,7 @@ public override string ToString() public override string ToString(string tableAlias) { var filterOperator = ConvertOperatorToString(); - var filterValue = ConvertValueToString('\''); + var filterValue = ConvertValueToString(QueryLanguageVersion.Version2); var filterField = string.IsNullOrEmpty(tableAlias) ? FilterField : $"{tableAlias}.{FilterField}"; return $"{filterOperator}({filterField},{filterValue})"; diff --git a/Source/StrongGrid/Models/Search/SearchCriteriaIsNotNull.cs b/Source/StrongGrid/Models/Search/SearchCriteriaIsNotNull.cs index 081633bc..af8ceec4 100644 --- a/Source/StrongGrid/Models/Search/SearchCriteriaIsNotNull.cs +++ b/Source/StrongGrid/Models/Search/SearchCriteriaIsNotNull.cs @@ -34,7 +34,7 @@ public SearchCriteriaIsNotNull(EmailActivitiesFilterField filterField) } /// - public override string ConvertValueToString(char quote) + public override string ConvertValueToString(QueryLanguageVersion queryLanguageVersion) { return string.Empty; } diff --git a/Source/StrongGrid/Models/Search/SearchCriteriaIsNull.cs b/Source/StrongGrid/Models/Search/SearchCriteriaIsNull.cs index 36257b3f..048c0cc0 100644 --- a/Source/StrongGrid/Models/Search/SearchCriteriaIsNull.cs +++ b/Source/StrongGrid/Models/Search/SearchCriteriaIsNull.cs @@ -34,7 +34,7 @@ public SearchCriteriaIsNull(EmailActivitiesFilterField filterField) } /// - public override string ConvertValueToString(char quote) + public override string ConvertValueToString(QueryLanguageVersion queryLanguageVersion) { return string.Empty; } diff --git a/Source/StrongGrid/Models/Search/SearchCriteriaNotBetween.cs b/Source/StrongGrid/Models/Search/SearchCriteriaNotBetween.cs index 592d6345..4cffc26e 100644 --- a/Source/StrongGrid/Models/Search/SearchCriteriaNotBetween.cs +++ b/Source/StrongGrid/Models/Search/SearchCriteriaNotBetween.cs @@ -46,9 +46,9 @@ public SearchCriteriaNotBetween(EmailActivitiesFilterField filterField, object l } /// - public override string ConvertValueToString(char quote) + public override string ConvertValueToString(QueryLanguageVersion queryLanguageVersion) { - return $"{ConvertToString(FilterValue, quote)} AND {ConvertToString(UpperValue, quote)}"; + return $"{ConvertToString(FilterValue, queryLanguageVersion)} AND {ConvertToString(UpperValue, queryLanguageVersion)}"; } /// diff --git a/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArg.cs b/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArg.cs index 14f55be0..be0749c5 100644 --- a/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArg.cs +++ b/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArg.cs @@ -37,11 +37,11 @@ public SearchCriteriaUniqueArg(string uniqueArgName, SearchComparisonOperator fi /// Converts the filter value into a string as expected by the SendGrid segmenting API. /// Can be overridden in subclasses if the value needs special formatting. /// - /// The character used to quote string values. This character is a double-quote for v1 queries and a single-quote for v2 queries. + /// The desired query version. /// The string representation of the value. - public virtual string ConvertValueToString(char quote) + public virtual string ConvertValueToString(QueryLanguageVersion queryLanguageVersion) { - return SearchCriteria.ConvertToString(FilterValue, quote); + return SearchCriteria.ConvertToString(FilterValue, queryLanguageVersion); } /// @@ -58,7 +58,7 @@ public virtual string ConvertOperatorToString() public override string ToString() { var filterOperator = ConvertOperatorToString(); - var filterValue = ConvertValueToString('"'); + var filterValue = ConvertValueToString(QueryLanguageVersion.Version1); return $"(unique_args['{UniqueArgName}']{filterOperator}{filterValue})"; } @@ -66,7 +66,7 @@ public override string ToString() public string ToString(string tableAlias) { var filterOperator = ConvertOperatorToString(); - var filterValue = ConvertValueToString('\''); + var filterValue = ConvertValueToString(QueryLanguageVersion.Version2); return $"{tableAlias}.DATA:payload.unique_args.{UniqueArgName}{filterOperator}{filterValue}"; } } diff --git a/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgBetween.cs b/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgBetween.cs index fc0a3644..98e42433 100644 --- a/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgBetween.cs +++ b/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgBetween.cs @@ -23,9 +23,9 @@ public SearchCriteriaUniqueArgBetween(string uniqueArgName, object lowerValue, o } /// - public override string ConvertValueToString(char quote) + public override string ConvertValueToString(QueryLanguageVersion queryLanguageVersion) { - return $"{SearchCriteria.ConvertToString(FilterValue, quote)} AND {SearchCriteria.ConvertToString(UpperValue, quote)}"; + return $"{SearchCriteria.ConvertToString(FilterValue, queryLanguageVersion)} AND {SearchCriteria.ConvertToString(UpperValue, queryLanguageVersion)}"; } /// diff --git a/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgNotBetween.cs b/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgNotBetween.cs index 9c473fa1..8a054dc3 100644 --- a/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgNotBetween.cs +++ b/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgNotBetween.cs @@ -23,9 +23,9 @@ public SearchCriteriaUniqueArgNotBetween(string uniqueArgName, object lowerValue } /// - public override string ConvertValueToString(char quote) + public override string ConvertValueToString(QueryLanguageVersion queryLanguageVersion) { - return $"{SearchCriteria.ConvertToString(FilterValue, quote)} AND {SearchCriteria.ConvertToString(UpperValue, quote)}"; + return $"{SearchCriteria.ConvertToString(FilterValue, queryLanguageVersion)} AND {SearchCriteria.ConvertToString(UpperValue, queryLanguageVersion)}"; } /// diff --git a/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgNotNull.cs b/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgNotNull.cs index 91bef97d..8101cf67 100644 --- a/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgNotNull.cs +++ b/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgNotNull.cs @@ -15,7 +15,7 @@ public SearchCriteriaUniqueArgNotNull(string uniqueArgName) } /// - public override string ConvertValueToString(char quote) + public override string ConvertValueToString(QueryLanguageVersion queryLanguageVersion) { return string.Empty; } diff --git a/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgNull.cs b/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgNull.cs index e2c63650..2eb7e720 100644 --- a/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgNull.cs +++ b/Source/StrongGrid/Models/Search/SearchCriteriaUniqueArgNull.cs @@ -15,7 +15,7 @@ public SearchCriteriaUniqueArgNull(string uniqueArgName) } /// - public override string ConvertValueToString(char quote) + public override string ConvertValueToString(QueryLanguageVersion queryLanguageVersion) { return string.Empty; } diff --git a/Source/StrongGrid/Utilities/Utils.cs b/Source/StrongGrid/Utilities/Utils.cs index 9a0e0638..e4e92b69 100644 --- a/Source/StrongGrid/Utilities/Utils.cs +++ b/Source/StrongGrid/Utilities/Utils.cs @@ -45,7 +45,11 @@ public static (byte[] X, byte[] Y) GetXYFromSecp256r1PublicKey(byte[] subjectPub } // As of August 2022, searching for contacts and searching for email activites still use the (old) version 1 query DSL. + // As of June 2024, this appears to still be the case. // You can also use query DSL v1 when segmenting contacts if you so desire, but by default StrongGrid uses v2. + // SendGrid's documentation states: "The Segmentation v1 API was deprecated on December 31, 2022. Following deprecation, + // all segments created in the Marketing Campaigns user interface began using the Segmentation v2 API.". + // My understanding of this statement is that it applies to segmentation of contacts, not to searching for contacts. public static string ToQueryDslVersion1(IEnumerable>> filterConditions) { if (filterConditions == null) return string.Empty; @@ -73,8 +77,8 @@ public static string ToQueryDslVersion2(IEnumerable(); foreach (var criteria in filterConditions) From 21be4e10e868629b79aa0d6c8e6b82bdf2bd7ca4 Mon Sep 17 00:00:00 2001 From: jericho Date: Wed, 19 Jun 2024 12:03:49 -0400 Subject: [PATCH 14/16] Add three new Contact filtering fields Resolves #526 --- .../Models/Search/ContactsFilterField.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Source/StrongGrid/Models/Search/ContactsFilterField.cs b/Source/StrongGrid/Models/Search/ContactsFilterField.cs index ee6bf9cf..3a8c19f0 100644 --- a/Source/StrongGrid/Models/Search/ContactsFilterField.cs +++ b/Source/StrongGrid/Models/Search/ContactsFilterField.cs @@ -55,6 +55,24 @@ public enum ContactsFilterField [EnumMember(Value = "email")] EmailAddress, + /// + /// The phone number. + /// + [EnumMember(Value = "phone_number_id")] + PhoneNumberId, + + /// + /// The external id. + /// + [EnumMember(Value = "external_id")] + ExternalId, + + /// + /// The anonymous id. + /// + [EnumMember(Value = "anonymous_id")] + AnonymousId, + /// /// The email domains. /// From e03b8cf70b2068c5b86b7be85eeddcf6ce7b28e5 Mon Sep 17 00:00:00 2001 From: jericho Date: Wed, 19 Jun 2024 12:08:33 -0400 Subject: [PATCH 15/16] Add missing EnumMember to EmailActivitiesFilterField.AsmGroupId --- Source/StrongGrid/Models/Search/EmailActivitiesFilterField.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Source/StrongGrid/Models/Search/EmailActivitiesFilterField.cs b/Source/StrongGrid/Models/Search/EmailActivitiesFilterField.cs index 8a24b950..d5317543 100644 --- a/Source/StrongGrid/Models/Search/EmailActivitiesFilterField.cs +++ b/Source/StrongGrid/Models/Search/EmailActivitiesFilterField.cs @@ -96,6 +96,7 @@ public enum EmailActivitiesFilterField /// /// The group id. /// + [EnumMember(Value = "asm_group_id")] AsmGroupId, /// From db3bdd8551f01a9737795e414c8268b04a4a7362 Mon Sep 17 00:00:00 2001 From: jericho Date: Wed, 19 Jun 2024 14:56:44 -0400 Subject: [PATCH 16/16] Allow developers to specify their own query when searching for EmailActivities Resolves #527 --- .../Resources/EmailActivitiesTests.cs | 2 +- Source/StrongGrid/Extensions/Public.cs | 20 +++++++++++++++++-- .../StrongGrid/Resources/EmailActivities.cs | 17 +++------------- .../StrongGrid/Resources/IEmailActivities.cs | 6 ++---- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Source/StrongGrid.UnitTests/Resources/EmailActivitiesTests.cs b/Source/StrongGrid.UnitTests/Resources/EmailActivitiesTests.cs index f2cc8cf7..d164ef5e 100644 --- a/Source/StrongGrid.UnitTests/Resources/EmailActivitiesTests.cs +++ b/Source/StrongGrid.UnitTests/Resources/EmailActivitiesTests.cs @@ -68,7 +68,7 @@ public async Task SearchMessages_without_criteria() var limit = 25; var mockHttp = new MockHttpMessageHandler(); - mockHttp.Expect(HttpMethod.Get, Utils.GetSendGridApiUri(ENDPOINT) + $"?limit={limit}&query=").Respond("application/json", NO_MESSAGES_FOUND); + mockHttp.Expect(HttpMethod.Get, Utils.GetSendGridApiUri(ENDPOINT) + $"?limit={limit}").Respond("application/json", NO_MESSAGES_FOUND); var client = Utils.GetFluentClient(mockHttp); var emailActivities = (IEmailActivities)new EmailActivities(client); diff --git a/Source/StrongGrid/Extensions/Public.cs b/Source/StrongGrid/Extensions/Public.cs index 1d8443af..aef4a254 100644 --- a/Source/StrongGrid/Extensions/Public.cs +++ b/Source/StrongGrid/Extensions/Public.cs @@ -1052,7 +1052,7 @@ public static Task SendToMultipleRecipientsAsync( /// /// The email activities resource. /// Filtering criteria. - /// Number of IP activity entries to return. + /// Maximum number of activity entries to return. /// Cancellation token. /// /// An array of . @@ -1068,7 +1068,7 @@ public static Task SearchAsync(this IEmailActivities ema /// /// The email activities resource. /// Filtering conditions. - /// Number of IP activity entries to return. + /// Maximum number of activity entries to return. /// Cancellation token. /// /// An array of . @@ -1080,6 +1080,22 @@ public static Task SearchAsync(this IEmailActivities ema return emailActivities.SearchAsync(filters, limit, cancellationToken); } + /// + /// Get all of the details about the messages matching the criteria. + /// + /// The email activities resource. + /// Filtering conditions. + /// Maximum number of activity entries to return. + /// Cancellation token. + /// + /// An array of . + /// + public static Task SearchAsync(this IEmailActivities emailActivities, IEnumerable>> filterConditions, int limit = 20, CancellationToken cancellationToken = default) + { + var query = Utils.ToQueryDslVersion1(filterConditions); + return emailActivities.SearchAsync(query, limit, cancellationToken); + } + /// /// Get all of the details about the contacts matching the criteria. /// diff --git a/Source/StrongGrid/Resources/EmailActivities.cs b/Source/StrongGrid/Resources/EmailActivities.cs index 07b1fd2d..872af68b 100644 --- a/Source/StrongGrid/Resources/EmailActivities.cs +++ b/Source/StrongGrid/Resources/EmailActivities.cs @@ -1,9 +1,6 @@ using Pathoschild.Http.Client; using StrongGrid.Models; -using StrongGrid.Models.Search; -using StrongGrid.Utilities; using System; -using System.Collections.Generic; using System.IO; using System.Net; using System.Net.Http; @@ -56,21 +53,13 @@ internal EmailActivities(Pathoschild.Http.Client.IClient client) _client = client; } - /// - /// Get all of the details about the messages matching the filtering conditions. - /// - /// Filtering conditions. - /// Number of IP activity entries to return. - /// Cancellation token. - /// - /// An array of . - /// - public Task SearchAsync(IEnumerable>> filterConditions, int limit = 20, CancellationToken cancellationToken = default) + /// + public Task SearchAsync(string query, int limit = 20, CancellationToken cancellationToken = default) { return _client .GetAsync(_endpoint) .WithArgument("limit", limit) - .WithArgument("query", Utils.ToQueryDslVersion1(filterConditions)) + .WithArgument("query", query) .WithCancellationToken(cancellationToken) .AsObject("messages"); } diff --git a/Source/StrongGrid/Resources/IEmailActivities.cs b/Source/StrongGrid/Resources/IEmailActivities.cs index 3b584622..a96c209e 100644 --- a/Source/StrongGrid/Resources/IEmailActivities.cs +++ b/Source/StrongGrid/Resources/IEmailActivities.cs @@ -1,6 +1,4 @@ using StrongGrid.Models; -using StrongGrid.Models.Search; -using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -19,13 +17,13 @@ public interface IEmailActivities /// /// Get all of the details about the messages matching the filtering conditions. /// - /// Filtering conditions. + /// The query. /// Maximum number of activity entries to return. /// Cancellation token. /// /// An array of . /// - Task SearchAsync(IEnumerable>> filterConditions, int limit = 20, CancellationToken cancellationToken = default); + Task SearchAsync(string query, int limit = 20, CancellationToken cancellationToken = default); /// /// Get all of the details about the specified message.