add plausibility check for trackers

Author: arch

funscript_editor/algorithms/funscriptgenerator.py

@@ -736,9 +736,9 @@ def init_trackers(self, ffmpeg_stream: FFmpegStream) -> tuple:
                     if StaticVideoTracker.is_bbox_in_tracking_area(bbox_woman, tracking_areas_woman[tracker_number]): break
                     self.logger.error("Invalid supervised tracking area selected")
                 preview_frame = self.draw_box(preview_frame, tracking_areas_woman[tracker_number], color=(0,255,0))
-                trackers_woman[tracker_number] = StaticVideoTracker(first_frame, bbox_woman, supervised_tracking_area = tracking_areas_woman[tracker_number])
+                trackers_woman[tracker_number] = StaticVideoTracker(first_frame, bbox_woman, self.video_info.fps, supervised_tracking_area = tracking_areas_woman[tracker_number])
             else:
-                trackers_woman[tracker_number] = StaticVideoTracker(first_frame, bbox_woman)
+                trackers_woman[tracker_number] = StaticVideoTracker(first_frame, bbox_woman, self.video_info.fps)
 
             if tracker_number == 0:
                 bboxes['Woman'][1] = { tracker_number: bbox_woman }
@@ -754,9 +754,9 @@ def init_trackers(self, ffmpeg_stream: FFmpegStream) -> tuple:
                         if StaticVideoTracker.is_bbox_in_tracking_area(bbox_men, tracking_areas_men[tracker_number]): break
                         self.logger.error("Invalid supervised tracking area selected")
                     preview_frame = self.draw_box(preview_frame, tracking_areas_men[tracker_number], color=(255,0,255))
-                    trackers_men[tracker_number] = StaticVideoTracker(first_frame, bbox_men, supervised_tracking_area = tracking_areas_men[tracker_number])
+                    trackers_men[tracker_number] = StaticVideoTracker(first_frame, bbox_men, self.video_info.fps, supervised_tracking_area = tracking_areas_men[tracker_number])
                 else:
-                    trackers_men[tracker_number] = StaticVideoTracker(first_frame, bbox_men)
+                    trackers_men[tracker_number] = StaticVideoTracker(first_frame, bbox_men, self.video_info.fps)
 
                 if tracker_number == 0:
                     bboxes['Men'][1] = { tracker_number: bbox_men }

funscript_editor/algorithms/videotracker.py

@@ -8,7 +8,7 @@
 from dataclasses import dataclass
 from threading import Thread
 from queue import Queue
-from funscript_editor.utils.config import SETTINGS
+from funscript_editor.utils.config import SETTINGS, HYPERPARAMETER
 
 import numpy as np
 
@@ -21,6 +21,7 @@ class StaticVideoTracker:
     Args:
         first_frame (np.ndarray): open cv image representing the start frame
         tracking_bbox (tuple): tuple with (x,y,w,h) of the init tracking box
+        fps (float): video fps
         limit_searchspace (dict) : only insert the specified region around the init box
         supervised_tracking_area (tuple, optional): tuple with (x,y,w,h) of the supervised tracking area
         queue_size (int): in (work) and out (result) queue size
@@ -29,20 +30,27 @@ class StaticVideoTracker:
     def __init__(self,
             first_frame: np.ndarray,
             tracking_bbox: tuple,
+            fps: float,
             limit_searchspace : dict = {'h': 0.45, 'w':0.4},
             supervised_tracking_area: tuple = None,
             queue_size : int = 2):
         self.first_frame = first_frame
         self.limit_searchspace = limit_searchspace
         self.first_tracking_bbox = tracking_bbox
         self.supervised_tracking_area = supervised_tracking_area
+        self.fps = max((1,fps))
         self.stopped = False
         self.sleep_time = 0.001
         self.queue_in = Queue(maxsize=queue_size)
         self.queue_out = Queue(maxsize=queue_size)
         self.thread = Thread(target=self.run, args=())
         self.thread.daemon = True
         self.thread.start()
+        self.tracking_points = []
+        self.tracking_counter = 0
+        self.cluster_center = [0, 0]
+        self.plausibility_thresholds = [0, 0]
+
 
 
     __logger = logging.getLogger(__name__)
@@ -53,6 +61,7 @@ class Status:
         OK :str = "OK"
         TRACKING_LOST :str = "Tracking Lost"
         FEATURE_OUTSIDE :str = "Feature outside the specified area"
+        IMPLAUSIBLE :str = "Tracking point is not plausible"
 
 
     @staticmethod
@@ -126,6 +135,47 @@ def __setup_tracker(self) -> None:
             self.tracker = cv2.TrackerCSRT_create()
 
 
+    def __is_plausible(self, box) -> bool:
+        tracking_init_phase_in_sec = HYPERPARAMETER['tracking_init_phase_in_sec']
+        tracking_plausibility_factor_x = HYPERPARAMETER['tracking_plausibility_factor_x']
+        tracking_plausibility_factor_y = HYPERPARAMETER['tracking_plausibility_factor_y']
+        if self.tracking_counter <= round(self.fps * tracking_init_phase_in_sec):
+            self.tracking_points.append([box[0] + box[2]/2, box[1] + box[3]/2])
+            if self.tracking_counter == round(self.fps * tracking_init_phase_in_sec):
+                self.__logger.info("Determine Plausibility Threshold for Tracker")
+                self.cluster_center = np.mean(np.array(self.tracking_points), axis = 0)
+                distances_x = [abs(self.cluster_center[0] - point[0]) for point in self.tracking_points]
+                distances_y = [abs(self.cluster_center[1] - point[1]) for point in self.tracking_points]
+                self.plausibility_thresholds = [ \
+                        max(distances_x) * tracking_plausibility_factor_x, \
+                        max(distances_y) * tracking_plausibility_factor_y \
+                ]
+        else:
+            point = [box[0] + box[2]/2, box[1] + box[3]/2]
+            if abs(point[0] - self.cluster_center[0]) > self.plausibility_thresholds[0]:
+                self.__logger.warning(
+                        "Tracking point x is not plausible (%d > %d +- %d)",
+                        round(point[0]),
+                        round(self.cluster_center[0]),
+                        round(self.plausibility_thresholds[0])
+                    )
+                return False
+
+            if abs(point[1] - self.cluster_center[1]) > self.plausibility_thresholds[1]:
+                self.__logger.warning(
+                        "Tracking point y is not plausible (%d > %d +- %d)",
+                        round(point[1]),
+                        round(self.cluster_center[1]),
+                        round(self.plausibility_thresholds[1])
+                    )
+                return False
+
+            self.cluster_center[0] = (self.cluster_center[0] * self.tracking_counter + point[0]) / (self.tracking_counter + 1)
+            self.cluster_center[1] = (self.cluster_center[1] * self.tracking_counter + point[1]) / (self.tracking_counter + 1)
+
+        return True
+
+
     def run(self) -> None:
         """ The Video Tracker Thread Function """
         self.__setup_tracker()
@@ -155,15 +205,18 @@ def run(self) -> None:
                 frame = self.queue_in.get()
                 frame_roi = frame[y0:y1, x0:x1]
                 success, bbox = self.tracker.update(frame_roi)
+                self.tracking_counter += 1
                 status = StaticVideoTracker.Status.TRACKING_LOST
-                if success:
+                if not success or bbox is None:
+                    bbox = None
+                else:
                     status = StaticVideoTracker.Status.OK
                     bbox = (int(bbox[0] + x0), int(bbox[1] + y0), int(bbox[2]), int(bbox[3]))
                     if not StaticVideoTracker.is_bbox_in_tracking_area(bbox, self.supervised_tracking_area):
                         status = StaticVideoTracker.Status.FEATURE_OUTSIDE
-                        bbox = None
-                else:
-                    bbox = None
+                    elif SETTINGS['tracking_plausibility_check']:
+                        if not self.__is_plausible(bbox):
+                            status = StaticVideoTracker.Status.IMPLAUSIBLE
 
                 self.queue_out.put((status, bbox))
 

funscript_editor/config/hyperparameter.yaml

@@ -37,7 +37,7 @@ user_reaction_time_in_milliseconds: 1000
 # Scene Detector #
 ##################
 
-# Specify the miniimum scene length in seconds
+# Specify the minimum scene length in seconds
 min_scene_len: 3
 
 # Threshold value for the content detector to detect an scene change
@@ -47,6 +47,20 @@ scene_content_detector_threshold: 30.0
 scene_threshold_detector_threshold: 12
 
 
+###########
+# Tracker #
+###########
+
+# Specify the init phase for the tracker (currently only used for the plausible check)
+tracking_init_phase_in_sec: 6
+
+# plausiblity factor to veriify a correct tracking position
+tracking_plausibility_factor_x: 4.0
+
+# plausiblity factor to veriify a correct tracking position
+tracking_plausibility_factor_y: 2.5
+
+
 ##############################
 # VR Movement in y Direction #
 ##############################

funscript_editor/config/settings.yaml

@@ -38,3 +38,6 @@ additional_changepoints: True
 
 # Force dark ui theme
 dark_theme: False
+
+# enable the tracking plausiblity check
+tracking_plausibility_check: True