Issue with appenders telling metrics to stop but they shouldn't be stopped. Caused queue to build up and use too much memory

Author: mwatson

Src/StackifyLib.log4net.v1_2_10/StackifyAppender.cs

@@ -46,7 +46,7 @@ protected override void OnClose()
             try
             {
                 _logClient.Close();
-                StackifyLib.Internal.Metrics.MetricClient.StopMetricsQueue();
+                StackifyLib.Internal.Metrics.MetricClient.StopMetricsQueue("log4net v1.2 OnClose");
 
             }
             catch

Src/StackifyLib.log4net/StackifyAppender.cs

@@ -38,9 +38,12 @@ protected override void OnClose()
         {
             try
             {
+                StackifyAPILogger.Log("log4net appender closing");
+
                 _logClient.Close();
-                StackifyLib.Internal.Metrics.MetricClient.StopMetricsQueue();
 
+                //This is to force the metrics queue to flush as well
+                StackifyLib.Internal.Metrics.MetricClient.StopMetricsQueue("log4net OnClose");
             }
             catch
             {

Src/StackifyLib.nLog/StackifyTarget.cs

@@ -43,7 +43,7 @@ protected override void CloseTarget()
             {
                 Utils.StackifyAPILogger.Log("NLog target closing");
                 _logClient.Close();
-                StackifyLib.Internal.Metrics.MetricClient.StopMetricsQueue();
+                StackifyLib.Internal.Metrics.MetricClient.StopMetricsQueue("NLog CloseTarget");
             }
             catch (Exception ex)
             {
@@ -53,6 +53,8 @@ protected override void CloseTarget()
 
         protected override void InitializeTarget()
         {
+            Utils.StackifyAPILogger.Log("NLog InitializeTarget");
+
             _logClient = new LogClient("StackifyLib.net-nlog", apiKey, uri);
             if (!String.IsNullOrEmpty(globalContextKeys))
             {

Src/StackifyLib/Internal/Metrics/MetricClient.cs

@@ -149,7 +149,11 @@ public static void QueueMetric(Metric metric)
         {
             try
             {
-                _MetricQueue.Enqueue(metric);
+                //set a sanity cap
+                if (_MetricQueue.Count < 100000)
+                {
+                    _MetricQueue.Enqueue(metric);
+                }
             }
             catch(Exception ex)
             {
@@ -205,10 +209,12 @@ private static void Aggregate(MetricAggregate aggregate)
         private static void ReadAllQueuedMetrics()
         {
             DateTime maxDate = DateTime.UtcNow; //read only up until now so it doesn't get stuck in an endless loop
-            //Loop through add sum up the totals of the counts and values by aggregate key then pass it all in at once to update the aggregate dictionary so it is done in one pass
+                                                //Loop through add sum up the totals of the counts and values by aggregate key then pass it all in at once to update the aggregate dictionary so it is done in one pass
 
             //key is the aggregate key which is the metric name, type and rounded minute of the occurrence
 
+            StackifyAPILogger.Log("ReadAllQueuedMetrics " + maxDate);
+
             var batches = new Dictionary<string, MetricAggregate>();
 
             long processed = 0;
@@ -293,6 +299,7 @@ private static void UploadMetricsCheck(object state)
 
                 DateTime currentMinute = DateTime.UtcNow.Floor(TimeSpan.FromMinutes(1));
 
+                StackifyAPILogger.Log("Calling UploadMetrics " + currentMinute);
                 allSuccess = UploadMetrics(currentMinute);
 
                 PurgeOldMetrics(purgeOlderThan);
@@ -302,24 +309,43 @@ private static void UploadMetricsCheck(object state)
                     seconds = .1;
                 }
             }
+            else
+            {
+                StackifyAPILogger.Log("Metrics processing canceled because stop was requested");
+            }
 
 
 
             _Timer.Change(TimeSpan.FromSeconds(seconds), TimeSpan.FromSeconds(seconds));
 
         }
 
-        public static void StopMetricsQueue()
+        
+        public static void StopMetricsQueue(string reason = "Unknown")
         {
+            StackifyAPILogger.Log("StopMetricsQueue called by " + reason, true);
+            
             //don't let t his method run more than once
             if (_StopRequested)
                 return;
 
             _StopRequested = true;
 
-            DateTime currentMinute = DateTime.UtcNow.AddMinutes(2).Floor(TimeSpan.FromMinutes(1));
+            try
+            {
+                DateTime currentMinute = DateTime.UtcNow.AddMinutes(2).Floor(TimeSpan.FromMinutes(1));
+
+                UploadMetrics(currentMinute);
+
+            }
+            catch (Exception ex)
+            {
+                StackifyAPILogger.Log("StopMetricsQueue error" + ex.ToString(), true);
+            }
+
 
-            UploadMetrics(currentMinute);
+            _StopRequested = false;
+            StackifyAPILogger.Log("StopMetricsQueue completed" + reason, true);
         }
 
         public static bool UploadMetrics(DateTime currentMinute)

Src/StackifyLib/Logger.cs

@@ -77,7 +77,7 @@ public static void Shutdown()
             _LogClient.Close();
 
             //flush any remaining metrics as well
-            MetricClient.StopMetricsQueue();
+            MetricClient.StopMetricsQueue("Logger Shutdown called");
         }
 
         /// <summary>