OpenTelemetry monitoring (#103)

* Azure VM Updates

* chore: update submodules

* perf: add telemetry

* fix: hanging docker stop

* chore: update submodules

* perf: add thread monitoring

* chore(opentelemetry): update submodules

Author: andhreljaKern

app.py

@@ -1,3 +1,5 @@
+import os
+import logging
 from fastapi import FastAPI, responses, status
 
 
@@ -14,8 +16,25 @@
 from submodules.model.business_objects import general
 from submodules.model import enums
 from submodules.model import session
+from submodules.model import telemetry
 
-app = FastAPI()
+
+OTLP_GRPC_ENDPOINT = os.getenv("OTLP_GRPC_ENDPOINT", "tempo:4317")
+
+app_name = "refinery-tokenizer"
+app = FastAPI(title=app_name)
+
+if telemetry.ENABLE_TELEMETRY:
+    print("WARNING:  Running telemetry.", flush=True)
+    telemetry.setting_app_name(app_name)
+    telemetry.setting_otlp(app, app_name=app_name, endpoint=OTLP_GRPC_ENDPOINT)
+    app.add_middleware(telemetry.PrometheusMiddleware, app_name=app_name)
+    app.add_route("/metrics", telemetry.metrics)
+
+    # Filter out /metrics
+    logging.getLogger("uvicorn.access").addFilter(
+        lambda record: "GET /metrics" not in record.getMessage()
+    )
 
 
 @app.middleware("http")

controller/rats_manager.py

@@ -3,8 +3,7 @@
 from datetime import datetime
 from typing import Optional
 
-from misc import daemon
-from submodules.model import enums
+from submodules.model import enums, daemon
 from misc.notification import (
     send_notification_created,
 )
@@ -42,7 +41,7 @@ def trigger_rats_creation(
         attribute_name=tokenization_task.attribute_name,
         with_commit=True,
     )
-    daemon.run(
+    daemon.run_without_db_token(
         create_rats_entries,
         project_id,
         user_id,

controller/task_manager.py

@@ -4,11 +4,11 @@
     tokenize_calculated_attribute,
     tokenize_initial_project,
 )
-from submodules.model import enums
+from submodules.model import enums, daemon
 from submodules.model.business_objects import attribute, general, notification, record
 from submodules.model.business_objects import tokenization
 from submodules.model.business_objects.tokenization import create_tokenization_task
-from misc import daemon, notification as notification_util
+from misc import notification as notification_util
 from submodules.model.models import RecordTokenizationTask
 from fastapi import status
 
@@ -48,7 +48,7 @@ def start_tokenization_task(
             task = set_up_tokenization_task(
                 project_id, user_id, enums.RecordTokenizationScope.PROJECT.value
             )
-            daemon.run(
+            daemon.run_without_db_token(
                 tokenize_initial_project,
                 project_id,
                 user_id,
@@ -69,7 +69,7 @@ def start_tokenization_task(
             enums.RecordTokenizationScope.ATTRIBUTE.value,
             attribute_name,
         )
-        daemon.run(
+        daemon.run_without_db_token(
             tokenize_calculated_attribute,
             project_id,
             user_id,
@@ -115,7 +115,7 @@ def start_rats_task(
             attribute_name=attribute_name,
             with_commit=True,
         )
-        daemon.run(
+        daemon.run_without_db_token(
             create_rats_entries,
             project_id,
             user_id,

handler/config_handler.py

@@ -1,7 +1,7 @@
 from typing import Dict, Any, Optional, Union
 import requests
 import time
-from misc import daemon
+from submodules.model import daemon
 
 __config = None
 
@@ -25,7 +25,7 @@ def refresh_config():
         )
     global __config
     __config = response.json()
-    daemon.run(invalidate_after, 3600)  # one hour
+    daemon.run_without_db_token(invalidate_after, 3600)  # one hour
 
 
 def get_config_value(

misc/daemon.py

@@ -2,11 +2,13 @@
 
 DEBUG_MODE=false
 DEBUG_PORT=15671
+ENABLE_TELEMETRY=false
 
-while getopts d flag
+while getopts dg flag
 do
     case "${flag}" in
         d) DEBUG_MODE=true;;
+        g) ENABLE_TELEMETRY=true;;
     esac
 done
 
@@ -52,6 +54,7 @@ docker run -d --rm \
 -e S3_USE_SSL=0 \
 -e WS_NOTIFY_ENDPOINT=http://refinery-websocket:8080 \
 -e POSTGRES=postgresql://postgres:kern@graphql-postgres:5432 \
+-e ENABLE_TELEMETRY=$ENABLE_TELEMETRY \
 -v "$INFERENCE_DIR":/inference \
 --mount type=bind,source="$(pwd)"/,target=/app \
 -v /var/run/docker.sock:/var/run/docker.sock \