PendingTraceBuffer: Keep track of how often we write around the buffer

Author: deejgregor

dd-trace-core/src/main/java/datadog/trace/core/PendingTraceBuffer.java

@@ -64,6 +64,7 @@ private static class DelayingPendingTraceBuffer extends PendingTraceBuffer {
     private final MpscBlockingConsumerArrayQueue<Element> queue;
     private final Thread worker;
     private final TimeSource timeSource;
+    private final HealthMetrics healthMetrics;
 
     private volatile boolean closed = false;
     private final AtomicInteger flushCounter = new AtomicInteger(0);
@@ -102,6 +103,7 @@ public void enqueue(Element pendingTrace) {
           if (!pendingTrace.writeOnBufferFull()) {
             return;
           }
+          healthMetrics.onPendingWriteAround();
           pendingTrace.write();
         }
       }
@@ -324,6 +326,7 @@ public DelayingPendingTraceBuffer(
       this.queue = new MpscBlockingConsumerArrayQueue<>(bufferSize);
       this.worker = newAgentThread(TRACE_MONITOR, new Worker());
       this.timeSource = timeSource;
+      this.healthMetrics = healthMetrics;
       boolean runningSpansEnabled = config.isLongRunningTraceEnabled();
       this.runningTracesTracker =
           runningSpansEnabled

dd-trace-core/src/main/java/datadog/trace/core/monitor/HealthMetrics.java

@@ -71,6 +71,8 @@ public void onSend(
   public void onFailedSend(
       final int traceCount, final int sizeInBytes, final RemoteApi.Response response) {}
 
+  public void onPendingWriteAround() {}
+
   public void onLongRunningUpdate(
       final int dropped, final int write, final int expired, final int droppedSampling) {}
 

dd-trace-core/src/main/java/datadog/trace/core/monitor/TracerHealthMetrics.java

@@ -84,6 +84,8 @@ public class TracerHealthMetrics extends HealthMetrics implements AutoCloseable
   private final LongAdder scopeCloseErrors = new LongAdder();
   private final LongAdder userScopeCloseErrors = new LongAdder();
 
+  private final LongAdder pendingWriteAround = new LongAdder();
+
   private final LongAdder longRunningTracesWrite = new LongAdder();
   private final LongAdder longRunningTracesDropped = new LongAdder();
   private final LongAdder longRunningTracesExpired = new LongAdder();
@@ -295,6 +297,11 @@ public void onFailedSend(
     onSendAttempt(traceCount, sizeInBytes, response);
   }
 
+  @Override
+  public void onPendingWriteAround() {
+    pendingWriteAround.increment();
+  }
+
   @Override
   public void onLongRunningUpdate(
       final int dropped, final int write, final int expired, final int droppedSampling) {
@@ -473,6 +480,8 @@ public void run(TracerHealthMetrics target) {
         reportIfChanged(
             target.statsd, "scope.user.close.error", target.userScopeCloseErrors, NO_TAGS);
 
+        reportIfChanged(target.statsd, "pending.write_around", target.pendingWriteAround, NO_TAGS);
+
         reportIfChanged(
             target.statsd, "long-running.write", target.longRunningTracesWrite, NO_TAGS);
         reportIfChanged(
@@ -607,6 +616,9 @@ public String summary() {
         + "\nuserScopeCloseErrors="
         + userScopeCloseErrors.sum()
         + "\n"
+        + "\npendingWriteAround="
+        + pendingWriteAround.sum()
+        + "\n"
         + "\nlongRunningTracesWrite="
         + longRunningTracesWrite.sum()
         + "\nlongRunningTracesDropped="