improve change point detection (#10)

Author: arch

docs/app/docs/user-guide/build.md

@@ -1,4 +1,4 @@
-# Build from Source
+# Build Application from Source
 
 For Windows user i recommend to use the release version from [github release page](https://github.com/michael-mueller-git/Python-Funscript-Editor/releases)
 
@@ -55,3 +55,13 @@ python3 funscript-editor.py
 By default the latest code is used which may contain bugs. So you maybe want to switch to the latest release version with `` git checkout $(git describe --tags `git rev-list --tags --max-count=1`)``.
 
 You can always update the application to the latest version with `git checkout main && git pull` inside the repository directory.
+
+## Arch Linux and Arch-Based Distributions
+
+My development platform is Arch Linux. The code should run therefore without problems. When installing the dependencies, make sure you install the following packages from the repositories, as the wrapper will have problems if the versions differ from the pip packages and the local C, C++ libraries.
+
+```bash
+sudo pacman -Sy python-opencv python-pyqt5
+```
+
+All other required python packages can be installed from pip.

docs/app/docs/user-guide/config.md

@@ -27,6 +27,8 @@ Config Files:
 - `notification_sound` (str) Specify the wav file to play when tracking finished (write 'off' to disable the sound notification).
 - `tracking_lost_time` (int) Time in milliseconds at which the tracking is stopped if the selected feature is not found.
 - `scene_detector` (str): Specify the scene detector algorithm. Available options: `'CSV'`, `'CONTENT'`, `'THRESHOLD'`
+- `raw_output` (bool): Output an position for each frame. This option disable post processing!
+- `additional_changepoints` (bool): Insert change points at high second derivative
 
 #### `hyperparameter.yaml`
 
@@ -35,12 +37,14 @@ Config Files:
 - `local_max_delta_in_percent` (float): Specify the maximum deviation for a max point in percent (recommend range: `0.0 - 10.0`)
 - `local_min_delta_in_percent` (float): Specify the maximum deviation for a min point in percent (recommend range: `0.0 - 10.0`)
 - `min_frames` (int): Specify the minimum required frames for the tracking. Wee need this parameter to ensure there is at leas two strokes in the tracking result.
+- `changepoint_detection_threshold` (float): threshold value tor get additional change points by comparing second derivative with the rolling standard deviation and given threshold
+- `additional_changepoints_merge_threshold_in_ms` (int): threshold value in milliseconds to merge additional change points
 - `shift_top_points` (int): Shift predicted top points by given frame number. Positive values delay the position and negative values result in an earlier position.
 - `shift_bottom_points` (int): Shift predicted bottom points by given frame number. Positive values delay the position and negative values result in an earlier position.
 - `top_points_offset` (float): An fix offset to the top points (positive values move the point up and negative values move the point down). The offset respect the user defined upper and lower limit.
 - `bottom_points_offset` (float): An fix offset to the bottom points (positive values move the point up and negative values move the point down). The offset respect the user defined upper and lower limit.
 - `top_threshold` (float): Define the top threshold. All top points greater than `(max - threshold)` will be set to the specified max value. Set 0.0 to disable this function.
 - `bottom_threshold` (float): Define the bottom threshold. All bottom points lower than `(min + threshold)` will be set to the specified min value. Set 0.0 to disable this function.
-- `min_scene_len` (int): Specify the miniimum scene length in seconds for the scene detector
-- `scene_content_detector_threshold` (float): Thresold value for the content detector to detect an scene change
-- `scene_threshold_detector_threshold` (int): Thresold value for the threshold detector to detect an scene change
+- `min_scene_len` (int): Specify the minimum scene length in seconds for the scene detector
+- `scene_content_detector_threshold` (float): Threshold value for the content detector to detect an scene change
+- `scene_threshold_detector_threshold` (int): Threshold value for the threshold detector to detect an scene change

funscript-editor.py

@@ -6,6 +6,8 @@
 if __name__ == '__main__':
     try:
         main()
+    except SystemExit as e:
+        pass
     except:
         traceback.print_exc()
         input()

funscript_editor/algorithms/signalprocessing.py

@@ -1,7 +1,8 @@
 """ Signal Processing Algorithms """
 
 import numpy as np
-from funscript_editor.utils.config import HYPERPARAMETER
+import logging
+from funscript_editor.utils.config import HYPERPARAMETER, SETTINGS
 
 def scale_signal(signal :list, lower: float = 0, upper: float = 99) -> list:
     """ Scale an signal (list of float or int) between given lower and upper value
@@ -66,6 +67,69 @@ def moving_average(x :list, w: int) -> list:
             +[avg[-1] for _ in range(len(avg)+w,len(x))]
 
 
+def moving_standard_deviation(x: list, w: int) -> list:
+    """ Get 2 seconds moving standard deviation of given signal
+
+    Args:
+        x (list): input signal
+        w (int): window size
+
+    Returns:
+        list: moving standard deviation
+    """
+    w = round(w)
+    std = [np.std(x[ii-w:ii+w+1]) for ii in range(w,len(x)-w)]
+    # TODO use extrapolation function
+    return [std[0] for _ in range(w)]\
+            +list(std)\
+            +[std[-1] for _ in range(w)]
+
+
+def second_derivative(x: list, w: int = 1) -> list:
+    """ Get second derivative of given signal
+
+    Args:
+        x (list): input signal
+        w (int): window size for signal smothing
+
+    Returns:
+        list: second derivative
+    """
+    if w < 0: w = 1
+    if w % 2 == 0: w += 1
+    return moving_average(np.diff(moving_average(np.diff(x, 1).tolist(), w), 1).tolist(), w)
+
+
+def get_changepoints(score: list, fps: int, alpha: float) -> list:
+    """ Get change points by comparing second derivative with the rolling standard deviation
+
+    Args:
+        score (list): list with float or int signal values
+        fps (int): rounded fps of the video
+        alpha (float): threshold value for standard deviation comparing
+
+    Returns:
+        list: idx list with changepoints
+    """
+    dx2 = second_derivative(score)
+    std = moving_standard_deviation(dx2, round(fps * HYPERPARAMETER['avg_sec_for_local_min_max_extraction']))
+    dx2_abs = abs(np.array(dx2))
+    tmp_max_idx = -1
+    changepoints = []
+    for pos in range(len(dx2_abs)):
+        if abs(dx2_abs[pos]) > alpha*std[pos]:
+            if tmp_max_idx < 0: tmp_max_idx = pos
+            elif dx2_abs[tmp_max_idx] <= dx2_abs[pos]: tmp_max_idx = pos
+        elif tmp_max_idx >= 0:
+            changepoints.append(tmp_max_idx)
+            tmp_max_idx = -1
+
+    #if tmp_max_idx > 0:
+    #    changepoints.append((tmp_max_idx))
+
+    return changepoints
+
+
 def get_local_max_and_min_idx(score :list, fps: int, shift_min :int = 0, shift_max :int = 0) -> dict:
     """ Get the local max and min positions in given signal
 
@@ -78,6 +142,7 @@ def get_local_max_and_min_idx(score :list, fps: int, shift_min :int = 0, shift_m
     Returns:
         dict: dict with 2 lists with all local max and min indexes ({'min':[], 'max':[]})
     """
+    logger = logging.getLogger("changepoints")
     avg = moving_average(score, w=round(fps * HYPERPARAMETER['avg_sec_for_local_min_max_extraction']))
     changepoints = {'min': [], 'max': []}
     tmp_min_idx, tmp_max_idx = -1, -1
@@ -96,6 +161,12 @@ def get_local_max_and_min_idx(score :list, fps: int, shift_min :int = 0, shift_m
             changepoints['max'].append(tmp_max_idx)
             tmp_max_idx = -1
 
+    #if tmp_min_idx > 0:
+    #    changepoints['min'].append((tmp_min_idx))
+
+    #if tmp_max_idx > 0:
+    #    changepoints['max'].append((tmp_max_idx))
+
     delta_max = (max(score) - min(score)) * 0.01 * min((10.0, float(HYPERPARAMETER['local_max_delta_in_percent'])))
     delta_min = (max(score) - min(score)) * 0.01 * min((10.0, float(HYPERPARAMETER['local_min_delta_in_percent'])))
 
@@ -112,6 +183,26 @@ def get_local_max_and_min_idx(score :list, fps: int, shift_min :int = 0, shift_m
             new_pos += 1
         changepoints['max'][k] = new_pos
 
+
+    if SETTINGS['additional_changepoints']:
+        logger.info("add additional change points")
+        merge_threshold = max(1, round(fps * float(HYPERPARAMETER['additional_changepoints_merge_threshold_in_ms']) / 1000.0))
+        additional_changepoints = get_changepoints(score, fps, float(HYPERPARAMETER['changepoint_detection_threshold']))
+        for cp_idx in additional_changepoints:
+            if len(list(filter(lambda x: abs(cp_idx - x) <= merge_threshold, changepoints['min']))) > 0:
+                continue
+
+            if len(list(filter(lambda x: abs(cp_idx - x) <= merge_threshold, changepoints['max']))) > 0:
+                continue
+
+            if score[cp_idx] < avg[cp_idx]:
+                logger.debug("add additional min changepoint at idx %d", cp_idx)
+                changepoints['min'].append(cp_idx)
+            else:
+                logger.debug("add additional max changepoint at idx %d", cp_idx)
+                changepoints['max'].append(cp_idx)
+
+
     # apply manual shift
     if shift_min != 0:
         for k, idx in enumerate(changepoints['min']):

funscript_editor/config/hyperparameter.yaml

@@ -23,6 +23,12 @@ local_min_delta_in_percent: 0.1
 # ensure there is at leas two strokes in the tracking result.
 min_frames: 90
 
+# threshold value tor get additional change points by comparing second derivative with the rolling standard deviation and given threshold
+changepoint_detection_threshold: 1.2
+
+# threshold value in milliseconds to merge additional change points
+additional_changepoints_merge_threshold_in_ms: 100
+
 
 ##################
 # Scene Detector #
@@ -31,10 +37,10 @@ min_frames: 90
 # Specify the miniimum scene length in seconds
 min_scene_len: 3
 
-# Thresold value for the content detector to detect an scene change
+# Threshold value for the content detector to detect an scene change
 scene_content_detector_threshold: 30.0
 
-# Thresold value for the threshold detector to detect an scene change
+# Threshold value for the threshold detector to detect an scene change
 scene_threshold_detector_threshold: 12
 
 

funscript_editor/config/settings.yaml

@@ -31,3 +31,6 @@ scene_detector: 'CSV'
 
 # Output an position for each frame. This option disable post processing!
 raw_output: False
+
+# Insert change points at high second derivative
+additional_changepoints: True