From fb63524fbb386268c4a9090fef634b30c81ec80b Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 10 Aug 2023 11:27:00 +0100 Subject: [PATCH 1/8] add message length check --- .../ToLoggingDatabaseDataLoadEventListener.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs b/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs index 937c463f72..e8f214df17 100644 --- a/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs +++ b/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs @@ -59,6 +59,10 @@ public virtual void StartLogging() _logManager.CreateDataLoadInfo(_loggingTask, _hostingApplication.ToString(), _runDescription, "", false); } + private int _maxMessageLength = 64000/ sizeof(Char); //loggingDB text column has max 64kb size, or 65535 chars + private bool IsMessageAValidLength(string message){ + return message.Length <= _maxMessageLength; + } public virtual void OnNotify(object sender, NotifyEventArgs e) { if (DataLoadInfo == null) @@ -70,20 +74,26 @@ public virtual void OnNotify(object sender, NotifyEventArgs e) case ProgressEventType.Debug: break; case ProgressEventType.Information: + if(IsMessageAValidLength(e.Message)){ DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation, sender.ToString(), e.Message); + } break; case ProgressEventType.Warning: var msg = e.Message + (e.Exception == null ? "" : Environment.NewLine + ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)); - DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, sender.ToString(), msg); + if(IsMessageAValidLength(msg)){ + DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, sender.ToString(), msg); + } break; case ProgressEventType.Error: var err = e.Message + (e.Exception == null ? "" : Environment.NewLine + ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)); - DataLoadInfo.LogFatalError(sender.ToString(), err); + if(IsMessageAValidLength(err)){ + DataLoadInfo.LogFatalError(sender.ToString(), err); + } break; default: throw new ArgumentOutOfRangeException(); From 854bf919f4567dd47fb43099e2fecccf4a2819c1 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 10 Aug 2023 11:29:37 +0100 Subject: [PATCH 2/8] fix tabbing --- .../Listeners/ToLoggingDatabaseDataLoadEventListener.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs b/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs index e8f214df17..71c2b864f3 100644 --- a/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs +++ b/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs @@ -74,10 +74,10 @@ public virtual void OnNotify(object sender, NotifyEventArgs e) case ProgressEventType.Debug: break; case ProgressEventType.Information: - if(IsMessageAValidLength(e.Message)){ - DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation, sender.ToString(), - e.Message); - } + if(IsMessageAValidLength(e.Message)){ + DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation, sender.ToString(), + e.Message); + } break; case ProgressEventType.Warning: var msg = e.Message + (e.Exception == null From c3da701afe8d1efe95e0af90a7c8d7cc80d9b450 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 10 Aug 2023 12:48:43 +0100 Subject: [PATCH 3/8] fix codescan alert --- .../Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs b/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs index 71c2b864f3..7158b5de6b 100644 --- a/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs +++ b/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs @@ -59,7 +59,7 @@ public virtual void StartLogging() _logManager.CreateDataLoadInfo(_loggingTask, _hostingApplication.ToString(), _runDescription, "", false); } - private int _maxMessageLength = 64000/ sizeof(Char); //loggingDB text column has max 64kb size, or 65535 chars + private readonly int _maxMessageLength = 64000/ sizeof(Char); //loggingDB text column has max 64kb size, or 65535 chars private bool IsMessageAValidLength(string message){ return message.Length <= _maxMessageLength; } From 1c62c702cef1ec63ab0e3d256faccb72d19fa998 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 16 Aug 2023 09:45:38 +0100 Subject: [PATCH 4/8] update max length and add env config --- .../ToLoggingDatabaseDataLoadEventListener.cs | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs b/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs index 7158b5de6b..9c938843a2 100644 --- a/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs +++ b/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs @@ -59,9 +59,24 @@ public virtual void StartLogging() _logManager.CreateDataLoadInfo(_loggingTask, _hostingApplication.ToString(), _runDescription, "", false); } - private readonly int _maxMessageLength = 64000/ sizeof(Char); //loggingDB text column has max 64kb size, or 65535 chars - private bool IsMessageAValidLength(string message){ - return message.Length <= _maxMessageLength; + private const string RDMP_LOGGING_STRING_LENGTH_LIMIT = "RDMP_LOGGING_STRING_LENGTH_LIMIT"; + private string strStringLengthLimit = Environment.GetEnvironmentVariable(RDMP_LOGGING_STRING_LENGTH_LIMIT); + + private uint GetMaxMessageLength() + { + uint maxMessageLength = 4294967295; //max length of a LONGTEXT column + if (strStringLengthLimit != null && strStringLengthLimit.Length > 0) + { + //don't bother if its not set or is an empty string + UInt32.TryParse(strStringLengthLimit, out maxMessageLength); + } + return maxMessageLength; + } + + // private uint _maxMessageLength = GetMaxMessageLength(); + private bool IsMessageAValidLength(string message) + { + return message.Length <= GetMaxMessageLength(); } public virtual void OnNotify(object sender, NotifyEventArgs e) { @@ -74,7 +89,8 @@ public virtual void OnNotify(object sender, NotifyEventArgs e) case ProgressEventType.Debug: break; case ProgressEventType.Information: - if(IsMessageAValidLength(e.Message)){ + if (IsMessageAValidLength(e.Message)) + { DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation, sender.ToString(), e.Message); } @@ -83,7 +99,8 @@ public virtual void OnNotify(object sender, NotifyEventArgs e) var msg = e.Message + (e.Exception == null ? "" : Environment.NewLine + ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)); - if(IsMessageAValidLength(msg)){ + if (IsMessageAValidLength(msg)) + { DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, sender.ToString(), msg); } break; @@ -91,7 +108,8 @@ public virtual void OnNotify(object sender, NotifyEventArgs e) var err = e.Message + (e.Exception == null ? "" : Environment.NewLine + ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)); - if(IsMessageAValidLength(err)){ + if (IsMessageAValidLength(err)) + { DataLoadInfo.LogFatalError(sender.ToString(), err); } break; From 9b5ba09dd1eb9fa9db2a320654c56f7bec2bba51 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 16 Aug 2023 10:07:09 +0100 Subject: [PATCH 5/8] attempt to allow RDMP_ --- .../ClassFileEvaluation/ExplicitDatabaseNameChecker.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/ExplicitDatabaseNameChecker.cs b/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/ExplicitDatabaseNameChecker.cs index f18da8ffd5..67e7560959 100644 --- a/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/ExplicitDatabaseNameChecker.cs +++ b/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/ExplicitDatabaseNameChecker.cs @@ -41,7 +41,8 @@ public static void FindProblems(List csFilesFound) "Settings.Designer.cs", "PlatformDatabaseCreationOptions.cs", "PackOptions.cs", - "PasswordEncryptionKeyLocation.cs" + "PasswordEncryptionKeyLocation.cs", + "ToLoggingDatabaseDataLoadEventListender.cs" }); //allowed because it's default arguments for CLI prohibitedStrings.Add("TEST_"); From 06305b99b2a46a82f26d11c659a6e61fd85411ff Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 16 Aug 2023 10:39:29 +0100 Subject: [PATCH 6/8] fix typo --- .../ClassFileEvaluation/ExplicitDatabaseNameChecker.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/ExplicitDatabaseNameChecker.cs b/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/ExplicitDatabaseNameChecker.cs index 67e7560959..cff0f78326 100644 --- a/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/ExplicitDatabaseNameChecker.cs +++ b/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/ExplicitDatabaseNameChecker.cs @@ -42,7 +42,7 @@ public static void FindProblems(List csFilesFound) "PlatformDatabaseCreationOptions.cs", "PackOptions.cs", "PasswordEncryptionKeyLocation.cs", - "ToLoggingDatabaseDataLoadEventListender.cs" + "ToLoggingDatabaseDataLoadEventListener.cs" }); //allowed because it's default arguments for CLI prohibitedStrings.Add("TEST_"); From cebd8a365f46ddda02242bf34d4efc2d95e43634 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 16 Aug 2023 13:29:46 +0100 Subject: [PATCH 7/8] add codeql update --- .../Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs b/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs index 9c938843a2..6326d985ce 100644 --- a/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs +++ b/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs @@ -60,7 +60,7 @@ public virtual void StartLogging() } private const string RDMP_LOGGING_STRING_LENGTH_LIMIT = "RDMP_LOGGING_STRING_LENGTH_LIMIT"; - private string strStringLengthLimit = Environment.GetEnvironmentVariable(RDMP_LOGGING_STRING_LENGTH_LIMIT); + private readonly string strStringLengthLimit = Environment.GetEnvironmentVariable(RDMP_LOGGING_STRING_LENGTH_LIMIT); private uint GetMaxMessageLength() { From 2576fc81cc6c77b9f6ec028450f0d0d9faad6e63 Mon Sep 17 00:00:00 2001 From: jas88 Date: Mon, 28 Aug 2023 13:01:04 -0500 Subject: [PATCH 8/8] Update ToLoggingDatabaseDataLoadEventListener.cs Adjust string trimming logic, handle ultra-short limits, only parse the environment variable once instead of per logging, avoid double-lookup in OnProgress handler --- .../ToLoggingDatabaseDataLoadEventListener.cs | 61 ++++++++----------- 1 file changed, 24 insertions(+), 37 deletions(-) diff --git a/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs b/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs index 6326d985ce..35984d12a6 100644 --- a/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs +++ b/Rdmp.Core/Logging/Listeners/ToLoggingDatabaseDataLoadEventListener.cs @@ -59,29 +59,25 @@ public virtual void StartLogging() _logManager.CreateDataLoadInfo(_loggingTask, _hostingApplication.ToString(), _runDescription, "", false); } - private const string RDMP_LOGGING_STRING_LENGTH_LIMIT = "RDMP_LOGGING_STRING_LENGTH_LIMIT"; - private readonly string strStringLengthLimit = Environment.GetEnvironmentVariable(RDMP_LOGGING_STRING_LENGTH_LIMIT); + private const string RDMPLoggingStringLengthLimit = "RDMP_LOGGING_STRING_LENGTH_LIMIT"; + private static readonly int StrStringLengthLimit; - private uint GetMaxMessageLength() + static ToLoggingDatabaseDataLoadEventListener() { - uint maxMessageLength = 4294967295; //max length of a LONGTEXT column - if (strStringLengthLimit != null && strStringLengthLimit.Length > 0) - { - //don't bother if its not set or is an empty string - UInt32.TryParse(strStringLengthLimit, out maxMessageLength); - } - return maxMessageLength; + StrStringLengthLimit = int.TryParse(Environment.GetEnvironmentVariable(RDMPLoggingStringLengthLimit),out var limit) ? limit : int.MaxValue; } - // private uint _maxMessageLength = GetMaxMessageLength(); - private bool IsMessageAValidLength(string message) + private static string EnsureMessageAValidLength(string message) { - return message.Length <= GetMaxMessageLength(); + return StrStringLengthLimit < 4 ? "" : + message.Length > StrStringLengthLimit ? message[..(StrStringLengthLimit - 3)] + "..." : message; } public virtual void OnNotify(object sender, NotifyEventArgs e) { if (DataLoadInfo == null) StartLogging(); + if (StrStringLengthLimit < 4) // Logging suppressed + return; switch (e.ProgressEventType) { @@ -89,32 +85,25 @@ public virtual void OnNotify(object sender, NotifyEventArgs e) case ProgressEventType.Debug: break; case ProgressEventType.Information: - if (IsMessageAValidLength(e.Message)) - { - DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation, sender.ToString(), - e.Message); - } + DataLoadInfo?.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnInformation, sender.ToString(), + EnsureMessageAValidLength(e.Message)); break; case ProgressEventType.Warning: var msg = e.Message + (e.Exception == null ? "" : Environment.NewLine + ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)); - if (IsMessageAValidLength(msg)) - { - DataLoadInfo.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, sender.ToString(), msg); - } + msg = EnsureMessageAValidLength(msg); + DataLoadInfo?.LogProgress(Logging.DataLoadInfo.ProgressEventType.OnWarning, sender.ToString(), msg); break; case ProgressEventType.Error: var err = e.Message + (e.Exception == null ? "" : Environment.NewLine + ExceptionHelper.ExceptionToListOfInnerMessages(e.Exception, true)); - if (IsMessageAValidLength(err)) - { - DataLoadInfo.LogFatalError(sender.ToString(), err); - } + err = EnsureMessageAValidLength(err); + DataLoadInfo?.LogFatalError(sender.ToString(), err); break; default: - throw new ArgumentOutOfRangeException(); + throw new ArgumentOutOfRangeException(nameof(e)); } } @@ -125,18 +114,16 @@ public virtual void OnProgress(object sender, ProgressEventArgs e) Debug.Assert(DataLoadInfo != null, "DataLoadInfo != null"); - if (e.Progress.UnitOfMeasurement == ProgressType.Records) + if (e.Progress.UnitOfMeasurement != ProgressType.Records) return; + + if (!TableLoads.TryGetValue(e.TaskDescription,out var t)) { - //if(!tableLoads.Any(tbl=>tbl.)) - if (!TableLoads.ContainsKey(e.TaskDescription)) - { - var t = DataLoadInfo.CreateTableLoadInfo("", e.TaskDescription, - new[] { new DataSource(sender.ToString()) }, e.Progress.KnownTargetValue); - TableLoads.Add(e.TaskDescription, t); - } - - TableLoads[e.TaskDescription].Inserts = e.Progress.Value; + t = DataLoadInfo.CreateTableLoadInfo("", e.TaskDescription, + new[] { new DataSource(sender.ToString()) }, e.Progress.KnownTargetValue); + TableLoads.Add(e.TaskDescription, t); } + + t.Inserts = e.Progress.Value; } public virtual void FinalizeTableLoadInfos()