add supervised ignore tracking

Author: arch

funscript_editor/algorithms/funscriptgenerator.py

@@ -34,6 +34,7 @@ class FunscriptGeneratorParameter:
     start_frame: int
     end_frame: int = -1 # default is video end (-1)
     number_of_trackers: int = 1
+    supervised_tracking_is_exit_condition: bool = True
 
     # Settings
     points: str = "local_min_max"
@@ -392,35 +393,91 @@ def init_trackers(self, ffmpeg_stream: FFmpegStream) -> tuple:
         first_frame = ffmpeg_stream.read()
         preview_frame = first_frame
         for tracker_number in range(self.params.number_of_trackers):
-            bbox_woman = self.ui.bbox_selector(preview_frame, "Select {} Feature #{}".format(self.get_target_name(0), tracker_number+1))
-            preview_frame = self.ui.draw_box_to_image(preview_frame, bbox_woman, color=(255,0,255))
+            bbox_woman = self.ui.bbox_selector(
+                    preview_frame,
+                    "Select {} Feature #{}".format(self.get_target_name(0), tracker_number+1)
+                )
+
+            preview_frame = self.ui.draw_box_to_image(
+                    preview_frame, 
+                    bbox_woman, 
+                    color=(255,0,255)
+                )
+
             if self.params.supervised_tracking:
                 while True:
-                    tracking_areas_woman[tracker_number] = self.ui.bbox_selector(preview_frame, "Select the Supervised Tracking Area for the {} Feature #{}".format(self.get_target_name(0), tracker_number+1))
-                    if StaticVideoTracker.is_bbox_in_tracking_area(bbox_woman, tracking_areas_woman[tracker_number]): break
+                    tracking_areas_woman[tracker_number] = self.ui.bbox_selector(
+                            preview_frame,
+                            "Select the Supervised Tracking Area for the {} Feature #{}".format(self.get_target_name(0), tracker_number+1)
+                        )
+
+                    if StaticVideoTracker.is_bbox_in_tracking_area(bbox_woman, tracking_areas_woman[tracker_number]):
+                        break
+
                     self.logger.error("Invalid supervised tracking area selected")
-                preview_frame = self.ui.draw_box_to_image(preview_frame, tracking_areas_woman[tracker_number], color=(0,255,0))
-                trackers_woman[tracker_number] = StaticVideoTracker(first_frame, bbox_woman, self.video_info.fps, supervised_tracking_area = tracking_areas_woman[tracker_number])
+
+                preview_frame = self.ui.draw_box_to_image(
+                        preview_frame,
+                        tracking_areas_woman[tracker_number],
+                        color=(0,255,0)
+                    )
+
+                trackers_woman[tracker_number] = StaticVideoTracker(
+                        first_frame,
+                        bbox_woman,
+                        self.video_info.fps,
+                        supervised_tracking_area = tracking_areas_woman[tracker_number],
+                        supervised_tracking_is_exit_condition=self.params.supervised_tracking_is_exit_condition
+                    )
             else:
-                trackers_woman[tracker_number] = StaticVideoTracker(first_frame, bbox_woman, self.video_info.fps)
+                trackers_woman[tracker_number] = StaticVideoTracker(
+                        first_frame,
+                        bbox_woman,
+                        self.video_info.fps
+                    )
 
             if tracker_number == 0:
                 bboxes['Woman'][1] = { tracker_number: bbox_woman }
             else:
                 bboxes['Woman'][1][tracker_number] = bbox_woman
 
             if self.params.track_men:
-                bbox_men = self.ui.bbox_selector(preview_frame, "Select {} Feature #{}".format(self.get_target_name(1), tracker_number+1))
+                bbox_men = self.ui.bbox_selector(
+                        preview_frame,
+                        "Select {} Feature #{}".format(self.get_target_name(1), tracker_number+1)
+                    )
                 preview_frame = self.ui.draw_box_to_image(preview_frame, bbox_men, color=(255,0,255))
                 if self.params.supervised_tracking:
                     while True:
-                        tracking_areas_men[tracker_number] = self.ui.bbox_selector(preview_frame, "Select the Supervised Tracking Area for the {} Feature #{}".format(self.get_target_name(1), tracker_number+1))
-                        if StaticVideoTracker.is_bbox_in_tracking_area(bbox_men, tracking_areas_men[tracker_number]): break
+                        tracking_areas_men[tracker_number] = self.ui.bbox_selector(
+                                preview_frame,
+                                "Select the Supervised Tracking Area for the {} Feature #{}".format(self.get_target_name(1), tracker_number+1)
+                            )
+
+                        if StaticVideoTracker.is_bbox_in_tracking_area(bbox_men, tracking_areas_men[tracker_number]):
+                            break
+
                         self.logger.error("Invalid supervised tracking area selected")
-                    preview_frame = self.ui.draw_box_to_image(preview_frame, tracking_areas_men[tracker_number], color=(255,0,255))
-                    trackers_men[tracker_number] = StaticVideoTracker(first_frame, bbox_men, self.video_info.fps, supervised_tracking_area = tracking_areas_men[tracker_number])
+
+                    preview_frame = self.ui.draw_box_to_image(
+                            preview_frame,
+                            tracking_areas_men[tracker_number],
+                            color=(255,0,255)
+                        )
+
+                    trackers_men[tracker_number] = StaticVideoTracker(
+                            first_frame,
+                            bbox_men,
+                            self.video_info.fps,
+                            supervised_tracking_area = tracking_areas_men[tracker_number],
+                            supervised_tracking_is_exit_condition = self.params.supervised_tracking_is_exit_condition
+                        )
                 else:
-                    trackers_men[tracker_number] = StaticVideoTracker(first_frame, bbox_men, self.video_info.fps)
+                    trackers_men[tracker_number] = StaticVideoTracker(
+                            first_frame,
+                            bbox_men,
+                            self.video_info.fps
+                        )
 
                 if tracker_number == 0:
                     bboxes['Men'][1] = { tracker_number: bbox_men }
@@ -434,7 +491,7 @@ def tracking(self) -> str:
         """ Tracking function to track the features in the video
 
         TODO:
-            Tracking lost for multiple tracker
+            bring back Tracking lost function
 
         Returns:
             str: a process status message e.g. 'end of video reached'
@@ -513,7 +570,6 @@ def tracking(self) -> str:
                                 boxes_to_draw.append(tracking_areas_men[tracker_number])
 
 
-
                     scene_change_quit_flag = False
                     if scene_detector.is_scene_change(frame_num-1 + self.params.start_frame):
                         self.logger.info("Scene change detected, Pause tracking")
@@ -615,16 +671,14 @@ def finished(self, status: str, success :bool) -> None:
         self.funscriptCompleted.emit(self.funscript, status, success)
 
 
-    def apply_shift(self, frame_number: int, metric: str, position: str) -> int:
-        """ Apply shift to predicted frame positions
+    def get_absolute_framenumber(self, frame_number: int) -> int:
+        """ Get the absoulte frame number
 
         Args:
             frame_number (int): relative frame number
-            metric (str): metric to apply the shift
-            position (str): keyword ['max', 'min', 'None']
 
         Returns:
-            int: real frame position
+            int: absolute frame position
         """
         return self.params.start_frame + frame_number
 
@@ -709,7 +763,7 @@ def create_funscript(self, idx_dict: dict) -> None:
             for idx in range(len(output_score)):
                 self.funscript.add_action(
                         output_score[idx],
-                        FFmpegStream.frame_to_millisec(self.apply_shift(idx, self.params.metric, 'None'), self.video_info.fps)
+                        FFmpegStream.frame_to_millisec(self.get_absolute_framenumber(idx), self.video_info.fps)
                     )
 
         else:
@@ -718,13 +772,13 @@ def create_funscript(self, idx_dict: dict) -> None:
             for idx in idx_dict['min']:
                 self.funscript.add_action(
                         round(output_score[idx]),
-                        FFmpegStream.frame_to_millisec(self.apply_shift(idx, self.params.metric, 'min'), self.video_info.fps)
+                        FFmpegStream.frame_to_millisec(self.get_absolute_framenumber(idx), self.video_info.fps)
                     )
 
             for idx in idx_dict['max']:
                 self.funscript.add_action(
                         round(output_score[idx]),
-                        FFmpegStream.frame_to_millisec(self.apply_shift(idx, self.params.metric, 'max'), self.video_info.fps)
+                        FFmpegStream.frame_to_millisec(self.get_absolute_framenumber(idx), self.video_info.fps)
                     )
 
 

funscript_editor/algorithms/videotracker.py

@@ -45,7 +45,9 @@ def __init__(self,
             fps: float,
             limit_searchspace : dict = {'h': 0.45, 'w':0.4},
             supervised_tracking_area: tuple = None,
+            supervised_tracking_is_exit_condition: bool = True,
             queue_size : int = 2):
+        self.logger = logging.getLogger(__name__)
         self.params = StaticVideoTrackerParameter()
         self.first_frame = first_frame
         self.limit_searchspace = limit_searchspace
@@ -63,10 +65,8 @@ def __init__(self,
         self.tracking_counter = 0
         self.cluster_center = [0, 0]
         self.plausibility_thresholds = [0, 0]
-
-
-
-    __logger = logging.getLogger(__name__)
+        self.last_valid_tracking_box = tracking_bbox
+        self.supervised_tracking_is_exit_condition = supervised_tracking_is_exit_condition
 
 
     @dataclass
@@ -137,14 +137,14 @@ def result(self) -> tuple:
     def __setup_tracker(self) -> None:
         """ Setup the tracker specified in the config """
         if self.params.tracking_algorithm.upper() == 'MIL':
-            self.__logger.info("Start MIL Tracker")
+            self.logger.info("Start MIL Tracker")
             self.tracker = cv2.TrackerMIL_create()
         elif self.params.tracking_algorithm.upper() == 'KCF':
-            self.__logger.info("Start KCF Tracker")
+            self.logger.info("Start KCF Tracker")
             self.tracker = cv2.TrackerKCF_create()
         else:
             # falback is CSRT tracker
-            self.__logger.info("Start CSRT Tracker")
+            self.logger.info("Start CSRT Tracker")
             self.tracker = cv2.TrackerCSRT_create()
 
 
@@ -154,7 +154,7 @@ def __is_plausible(self, box) -> bool:
             self.tracking_points.append([box[0] + box[2]/2, box[1] + box[3]/2])
 
             if self.tracking_counter == round(self.fps * self.params.tracking_init_phase_in_sec):
-                self.__logger.info("Determine Plausibility Threshold for Tracker")
+                self.logger.info("Determine Plausibility Threshold for Tracker")
                 self.cluster_center = np.mean(np.array(self.tracking_points), axis = 0)
 
                 distances_x = [abs(self.cluster_center[0] - point[0]) for point in self.tracking_points]
@@ -167,7 +167,7 @@ def __is_plausible(self, box) -> bool:
         else:
             point = [box[0] + box[2]/2, box[1] + box[3]/2]
             if abs(point[0] - self.cluster_center[0]) > self.plausibility_thresholds[0]:
-                self.__logger.warning(
+                self.logger.warning(
                         "Tracking point x is not plausible (%d > %d +- %d)",
                         round(point[0]),
                         round(self.cluster_center[0]),
@@ -176,7 +176,7 @@ def __is_plausible(self, box) -> bool:
                 return False
 
             if abs(point[1] - self.cluster_center[1]) > self.plausibility_thresholds[1]:
-                self.__logger.warning(
+                self.logger.warning(
                         "Tracking point y is not plausible (%d > %d +- %d)",
                         round(point[1]),
                         round(self.cluster_center[1]),
@@ -219,9 +219,9 @@ def run(self) -> None:
                 wait_counter += 1
                 if wait_counter == 2000:
                     if self.queue_in.qsize() == 0:
-                        self.__logger.error("Video Tracker still waiting for Input")
+                        self.logger.error("Video Tracker still waiting for Input")
                     else:
-                        self.__logger.error("Video Tracker output queue overrun!!!")
+                        self.logger.error("Video Tracker output queue overrun!!!")
             else:
                 frame = self.queue_in.get()
                 frame_roi = frame[y0:y1, x0:x1]
@@ -233,15 +233,22 @@ def run(self) -> None:
                 else:
                     status = StaticVideoTracker.Status.OK
                     bbox = (int(bbox[0] + x0), int(bbox[1] + y0), int(bbox[2]), int(bbox[3]))
-                    if not StaticVideoTracker.is_bbox_in_tracking_area(bbox, self.supervised_tracking_area):
-                        status = StaticVideoTracker.Status.FEATURE_OUTSIDE
-                    elif self.params.tracking_plausibility_check:
+                    if self.params.tracking_plausibility_check:
                         if not self.__is_plausible(bbox):
                             status = StaticVideoTracker.Status.IMPLAUSIBLE
+                    elif not StaticVideoTracker.is_bbox_in_tracking_area(bbox, self.supervised_tracking_area):
+                        if self.supervised_tracking_is_exit_condition:
+                            status = StaticVideoTracker.Status.FEATURE_OUTSIDE
+                        else:
+                            bbox = self.last_valid_tracking_box
+
 
                 self.queue_out.put((status, bbox))
 
+                if status == StaticVideoTracker.Status.OK:
+                    self.last_valid_tracking_box = bbox
+
         if platform.system() != 'Windows':
             # TODO logging here on windows cause open background process
-            self.__logger.info("Video Tracker Stoped")
+            self.logger.info("Video Tracker Stoped")
 

funscript_editor/ui/funscript_generator_window.py

@@ -126,6 +126,7 @@ def run(self) -> None:
                     video_path = self.video_file,
                     track_men = 'two' in self.settings['trackingMethod'],
                     supervised_tracking = 'Supervised' in self.settings['trackingMethod'],
+                    supervised_tracking_is_exit_condition = "stopping" in self.settings['trackingMethod'],
                     metric = self.settings['trackingMetric'].replace('inverted', '').strip(),
                     projection = self.settings['videoType'],
                     invert = "inverted" in self.settings['trackingMetric'],

funscript_editor/ui/settings_dialog.py

@@ -114,11 +114,43 @@ def __setup_combo_boxes(self):
         self.ui.videoTypeComboBox.addItems([PROJECTION[key]['name'] \
                 for key in PROJECTION.keys() \
                 if 'vr' not in key.lower() or self.include_vr])
-        self.ui.trackingMethodComboBox.addItems(['Unsupervised one moving person', 'Unsupervised two moving persons', 'Supervised one moving person', 'Supervised two moving persons']) # set before tracking metric
-        self.ui.trackingMetricComboBox.addItems(['y (up-down)', 'y inverted (down-up)', 'x (left-right)', 'x inverted (right-left)', 'distance (p1-p2)', 'distance inverted (p2-p1)', 'roll (rotation)', 'roll inverted (rotation)'])
-        self.ui.pointsComboBox.addItems(["Local Min Max", "Direction Changed"])
-        self.ui.additionalPointsComboBox.addItems(["None", "High Second Derivative", "Distance Minimization"])
-        self.ui.processingSpeedComboBox.addItems(["0 (accurate)", "1 (normal)", "2 (fast)"])
+
+        self.ui.trackingMethodComboBox.addItems([
+            'Unsupervised one moving person',
+            'Unsupervised two moving persons',
+            'Supervised stopping one moving person',
+            'Supervised stopping two moving persons',
+            'Supervised ignoring one moving person',
+            'Supervised ignoring two moving persons'
+            ]) # set before tracking metric!
+
+        self.ui.trackingMetricComboBox.addItems([
+            'y (up-down)',
+            'y inverted (down-up)',
+            'x (left-right)',
+            'x inverted (right-left)',
+            'distance (p1-p2)',
+            'distance inverted (p2-p1)',
+            'roll (rotation)',
+            'roll inverted (rotation)'
+        ])
+
+        self.ui.pointsComboBox.addItems([
+            "Local Min Max",
+            "Direction Changed"
+        ])
+
+        self.ui.additionalPointsComboBox.addItems([
+            "None",
+            "High Second Derivative",
+            "Distance Minimization"
+        ])
+
+        self.ui.processingSpeedComboBox.addItems([
+            "0 (accurate)",
+            "1 (normal)",
+            "2 (fast)"
+        ])
 
         self.ui.numberOfTrackerComboBox.addItems([str(i) for i in range(1, 6)])
 
@@ -129,11 +161,11 @@ def __set_tracking_metric(self, value):
 
         if value in ['x', 'y', 'x inverted', 'y inverted']:
             if 'Unsupervised one moving person' not in current_tracking_method_items:
-                self.ui.trackingMethodComboBox.addItems(['Unsupervised one moving person', 'Supervised one moving person'])
+                self.ui.trackingMethodComboBox.addItems(['Unsupervised one moving person', 'Supervised stopping one moving person'])
         else:
             if 'Unsupervised one moving person' in current_tracking_method_items:
                 self.ui.trackingMethodComboBox.clear()
-                self.ui.trackingMethodComboBox.addItems(['Unsupervised two moving persons', 'Supervised two moving persons'])
+                self.ui.trackingMethodComboBox.addItems(['Unsupervised two moving persons', 'Supervised stopping two moving persons'])
 
         self.__set_str_setting('trackingMetric', value)