From 9ad1f0547d1c940428e86da76031699e390edc68 Mon Sep 17 00:00:00 2001 From: Breakthrough Date: Sun, 21 Apr 2024 10:16:04 -0400 Subject: [PATCH] Revert "[scene_detector] Make SceneDetector a proper interface" Need to create separate branch for breaking v0.7 API changes. This reverts commit 755c94152fce9254d198992ba06feac6ebd65a79. --- scenedetect/detectors/adaptive_detector.py | 12 ++- scenedetect/detectors/content_detector.py | 10 +-- scenedetect/detectors/hash_detector.py | 12 +-- scenedetect/detectors/histogram_detector.py | 10 ++- scenedetect/detectors/threshold_detector.py | 7 +- scenedetect/scene_detector.py | 97 ++++++++++++--------- scenedetect/scene_manager.py | 40 +++++++-- website/pages/changelog.md | 10 +-- 8 files changed, 115 insertions(+), 83 deletions(-) diff --git a/scenedetect/detectors/adaptive_detector.py b/scenedetect/detectors/adaptive_detector.py index c26f592c..85778158 100644 --- a/scenedetect/detectors/adaptive_detector.py +++ b/scenedetect/detectors/adaptive_detector.py @@ -106,10 +106,13 @@ def event_buffer_length(self) -> int: """Number of frames any detected cuts will be behind the current frame due to buffering.""" return self.window_width - @property - def metric_keys(self) -> List[str]: + def get_metrics(self) -> List[str]: """Combines base ContentDetector metric keys with the AdaptiveDetector one.""" - return super().metric_keys + [self._adaptive_ratio_key] + return super().get_metrics() + [self._adaptive_ratio_key] + + def stats_manager_required(self) -> bool: + """Not required for AdaptiveDetector.""" + return False def process_frame(self, frame_num: int, frame_img: Optional[np.ndarray]) -> List[int]: """Process the next frame. `frame_num` is assumed to be sequential. @@ -123,6 +126,9 @@ def process_frame(self, frame_num: int, frame_img: Optional[np.ndarray]) -> List List[int]: List of frames where scene cuts have been detected. There may be 0 or more frames in the list, and not necessarily the same as frame_num. """ + + # TODO(#283): Merge this with ContentDetector and turn it on by default. + super().process_frame(frame_num=frame_num, frame_img=frame_img) # Initialize last scene cut point at the beginning of the frames of interest. diff --git a/scenedetect/detectors/content_detector.py b/scenedetect/detectors/content_detector.py index fc5bdd27..954a91d7 100644 --- a/scenedetect/detectors/content_detector.py +++ b/scenedetect/detectors/content_detector.py @@ -138,15 +138,11 @@ def __init__( self._frame_score: Optional[float] = None self._flash_filter = FlashFilter(mode=filter_mode, length=min_scene_len) - @property - def metric_keys(self) -> List[str]: + def get_metrics(self): return ContentDetector.METRIC_KEYS - @property - def event_buffer_length(self) -> int: - """Number of frames any detected cuts will be behind the current frame due to buffering.""" - # TODO(v0.7): Fixup private variables with properties. - return self._min_scene_len if self._flash_filter._mode == FlashFilter.Mode.MERGE else 0 + def is_processing_required(self, frame_num): + return True def _calculate_frame_score(self, frame_num: int, frame_img: numpy.ndarray) -> float: """Calculate score representing relative amount of motion in `frame_img` compared to diff --git a/scenedetect/detectors/hash_detector.py b/scenedetect/detectors/hash_detector.py index c9b991aa..7b94bed0 100644 --- a/scenedetect/detectors/hash_detector.py +++ b/scenedetect/detectors/hash_detector.py @@ -34,8 +34,6 @@ `detect-hash` command. """ -import typing as ty - # Third-Party Library Imports import numpy import cv2 @@ -115,10 +113,12 @@ def __init__( self._last_hash = numpy.array([]) self._metric_keys = ['hash_dist'] - @property - def metric_keys(self) -> ty.List[str]: + def get_metrics(self): return self._metric_keys + def is_processing_required(self, frame_num): + return True + def process_frame(self, frame_num, frame_img): """ Similar to ContentDetector, but using a perceptual hashing algorithm to calculate a hash for each frame and then calculate a hash difference @@ -127,7 +127,9 @@ def process_frame(self, frame_num, frame_img): Arguments: frame_num (int): Frame number of frame that is being passed. - frame_img (numpy.ndarray): Decoded frame image (BGR) to perform scene detection on. + frame_img (Optional[int]): Decoded frame image (numpy.ndarray) to perform scene + detection on. Can be None *only* if the self.is_processing_required() method + (inhereted from the base SceneDetector class) returns True. Returns: List[int]: List of frames where scene cuts have been detected. There may be 0 diff --git a/scenedetect/detectors/histogram_detector.py b/scenedetect/detectors/histogram_detector.py index 141f4ad0..937b7e13 100644 --- a/scenedetect/detectors/histogram_detector.py +++ b/scenedetect/detectors/histogram_detector.py @@ -16,7 +16,7 @@ This detector is available from the command-line as the `detect-hist` command. """ -import typing as ty +from typing import List import numpy @@ -49,7 +49,7 @@ def __init__(self, threshold: float = 20000.0, bits: int = 4, min_scene_len: int self._last_hist = None self._last_scene_cut = None - def process_frame(self, frame_num: int, frame_img: numpy.ndarray) -> ty.List[int]: + def process_frame(self, frame_num: int, frame_img: numpy.ndarray) -> List[int]: """First, compress the image according to the self.bits value, then build a histogram for the input frame. Afterward, compare against the previously analyzed frame and check if the difference is large enough to trigger a cut. @@ -185,6 +185,8 @@ def _shift_images(self, img, img_shift): return shifted_img - @property - def metric_keys(self) -> ty.List[str]: + def is_processing_required(self, frame_num: int) -> bool: + return True + + def get_metrics(self) -> List[str]: return HistogramDetector.METRIC_KEYS diff --git a/scenedetect/detectors/threshold_detector.py b/scenedetect/detectors/threshold_detector.py index ce00ec02..784bd1f9 100644 --- a/scenedetect/detectors/threshold_detector.py +++ b/scenedetect/detectors/threshold_detector.py @@ -18,7 +18,7 @@ from enum import Enum from logging import getLogger -import typing as ty +from typing import List, Optional import numpy @@ -114,11 +114,10 @@ def __init__( } self._metric_keys = [ThresholdDetector.THRESHOLD_VALUE_KEY] - @property - def metric_keys(self) -> ty.List[str]: + def get_metrics(self) -> List[str]: return self._metric_keys - def process_frame(self, frame_num: int, frame_img: numpy.ndarray) -> ty.List[int]: + def process_frame(self, frame_num: int, frame_img: numpy.ndarray) -> List[int]: """Process the next frame. `frame_num` is assumed to be sequential. Args: diff --git a/scenedetect/scene_detector.py b/scenedetect/scene_detector.py index 5ba1c2e9..ded5d35d 100644 --- a/scenedetect/scene_detector.py +++ b/scenedetect/scene_detector.py @@ -25,7 +25,6 @@ event (in, out, cut, etc...). """ -from abc import ABC, abstractmethod from enum import Enum import typing as ty @@ -34,37 +33,60 @@ from scenedetect.stats_manager import StatsManager -class SceneDetector(ABC): - """Base class to inherit from when implementing a scene detection algorithm. +# pylint: disable=unused-argument, no-self-use +class SceneDetector: + """ Base class to inherit from when implementing a scene detection algorithm. - This API is not yet stable and subject to change. Currently has a very simple interface, where - on each frame, a detector emits a list of points where scene cuts are detected. + This API is not yet stable and subject to change. - Also see the implemented scene detectors in the scenedetect.detectors module to get an idea of - how a particular detector can be created. In the future, this will be changed to support - different types of detections (e.g. fades versus cuts) and confidence scores of each event. + This represents a "dense" scene detector, which returns a list of frames where + the next scene/shot begins in a video. + + Also see the implemented scene detectors in the scenedetect.detectors module + to get an idea of how a particular detector can be created. """ + # TODO(v0.7): Make this a proper abstract base class. - def __init__(self): - self._stats_manager = None + stats_manager: ty.Optional[StatsManager] = None + """Optional :class:`StatsManager ` to + use for caching frame metrics to and from.""" - @property - def stats_manager(self) -> ty.Optional[StatsManager]: - """Optional :class:`StatsManager ` to - use for caching frame metrics to and from.""" - return self._stats_manager + # TODO(v1.0): Remove - this is a rarely used case for what is now a neglegible performance gain. + def is_processing_required(self, frame_num: int) -> bool: + """[DEPRECATED] DO NOT USE - @stats_manager.setter - def stats_manager(self, new_manager): - self._stats_manager = new_manager + Test if all calculations for a given frame are already done. - @property - @abstractmethod - def metric_keys(self) -> ty.List[str]: - """List of all metric names/keys used by the detector.""" - raise NotImplementedError + Returns: + False if the SceneDetector has assigned _metric_keys, and the + stats_manager property is set to a valid StatsManager object containing + the required frame metrics/calculations for the given frame - thus, not + needing the frame to perform scene detection. + + True otherwise (i.e. the frame_img passed to process_frame is required + to be passed to process_frame for the given frame_num). + """ + metric_keys = self.get_metrics() + return not metric_keys or not (self.stats_manager is not None + and self.stats_manager.metrics_exist(frame_num, metric_keys)) + + def stats_manager_required(self) -> bool: + """Stats Manager Required: Prototype indicating if detector requires stats. + + Returns: + True if a StatsManager is required for the detector, False otherwise. + """ + return False + + def get_metrics(self) -> ty.List[str]: + """Get Metrics: Get a list of all metric names/keys used by the detector. + + Returns: + List of strings of frame metric key names that will be used by + the detector when a StatsManager is passed to process_frame. + """ + return [] - @abstractmethod def process_frame(self, frame_num: int, frame_img: numpy.ndarray) -> ty.List[int]: """Process the next frame. `frame_num` is assumed to be sequential. @@ -80,12 +102,12 @@ def process_frame(self, frame_num: int, frame_img: numpy.ndarray) -> ty.List[int Returns: List of frame numbers of cuts to be added to the cutting list. """ - raise NotImplementedError + return [] def post_process(self, frame_num: int) -> ty.List[int]: """Post Process: Performs any processing after the last frame has been read. - Default implementation is a no-op. + Prototype method, no actual detection. Returns: List of frame numbers of cuts to be added to the cutting list. @@ -99,26 +121,16 @@ def event_buffer_length(self) -> int: """ return 0 - # DEPRECATED METHODS TO BE REMOVED IN v1.0 - - def is_processing_required(self, frame_num: int) -> bool: - """[DEPRECATED] DO NOT USE""" - return True - - def stats_manager_required(self) -> bool: - """[DEPRECATED] DO NOT USE""" - return False - def get_metrics(self) -> ty.List[str]: - """[DEPRECATED] USE `metric_keys` PROPERTY INSTEAD""" - return self.metric_keys +class SparseSceneDetector(SceneDetector): + """Base class to inherit from when implementing a sparse scene detection algorithm. + This class will be removed in v1.0 and should not be used. -class SparseSceneDetector(SceneDetector): - """[DEPRECATED - DO NOT USE] + Unlike dense detectors, sparse detectors detect "events" and return a *pair* of frames, + as opposed to just a single cut. - This class will be removed in v1.0, with the goal being the SceneDetector interface will emit - event types and confidence scores rather than having different interfaces. + An example of a SparseSceneDetector is the MotionDetector. """ def process_frame(self, frame_num: int, @@ -145,7 +157,6 @@ def post_process(self, frame_num: int) -> ty.List[ty.Tuple[int, int]]: return [] -# TODO(v0.7): Add documentation. class FlashFilter: class Mode(Enum): diff --git a/scenedetect/scene_manager.py b/scenedetect/scene_manager.py index a138ec3b..3d3bd435 100644 --- a/scenedetect/scene_manager.py +++ b/scenedetect/scene_manager.py @@ -639,9 +639,15 @@ def add_detector(self, detector: SceneDetector) -> None: Arguments: detector (SceneDetector): Scene detector to add to the SceneManager. """ + if self._stats_manager is None and detector.stats_manager_required(): + # Make sure the lists are empty so that the detectors don't get + # out of sync (require an explicit statsmanager instead) + assert not self._detector_list and not self._sparse_detector_list + self._stats_manager = StatsManager() + detector.stats_manager = self._stats_manager if self._stats_manager is not None: - self._stats_manager.register_metrics(detector.metric_keys) + self._stats_manager.register_metrics(detector.get_metrics()) if not issubclass(type(detector), SparseSceneDetector): self._detector_list.append(detector) @@ -902,14 +908,24 @@ def _decode_thread( ): try: while not self._stop.is_set(): - frame_im = video.read() - if frame_im is False: - break - if downscale_factor > 1: - frame_im = cv2.resize( - frame_im, (round(frame_im.shape[1] / downscale_factor), - round(frame_im.shape[0] / downscale_factor)), - interpolation=self._interpolation.value) + frame_im = None + # We don't do any kind of locking here since the worst-case of this being wrong + # is that we do some extra work, and this function should never mutate any data + # (all of which should be modified under the GIL). + # TODO(v1.0): This optimization should be removed as it is an uncommon use case and + # greatly increases the complexity of detection algorithms using it. + if self._is_processing_required(video.position.frame_num): + frame_im = video.read() + if frame_im is False: + break + if downscale_factor > 1: + frame_im = cv2.resize( + frame_im, (round(frame_im.shape[1] / downscale_factor), + round(frame_im.shape[0] / downscale_factor)), + interpolation=self._interpolation.value) + else: + if video.read(decode=False) is False: + break # Set the start position now that we decoded at least the first frame. if self._start_pos is None: @@ -1002,3 +1018,9 @@ def get_event_list( return self._get_event_list() # pylint: enable=unused-argument + + def _is_processing_required(self, frame_num: int) -> bool: + """True if frame metrics not in StatsManager, False otherwise.""" + if self.stats_manager is None: + return True + return all([detector.is_processing_required(frame_num) for detector in self._detector_list]) diff --git a/website/pages/changelog.md b/website/pages/changelog.md index 1fe3545a..f304cf2f 100644 --- a/website/pages/changelog.md +++ b/website/pages/changelog.md @@ -2,21 +2,15 @@ Releases ========================================================== -## PySceneDetect 0.7 +## PySceneDetect 0.6 -### 0.7 (In Development) +### 0.6.4 (In Development) - [feature] New detector: `detect-hist` / `HistogramDetector`, [thanks @wjs018](https://github.com/Breakthrough/PySceneDetect/pull/295) [#53](https://github.com/Breakthrough/PySceneDetect/issues/53) - [feature] Add new flash suppression filter with `filter-mode` config option, reduces number of cuts generated during strobing/flashing effects [#35](https://github.com/Breakthrough/PySceneDetect/pull/295) [#53](https://github.com/Breakthrough/PySceneDetect/issues/35) - `filter-mode = merge`, the new default mode, merges consecutive scenes shorter than `min-scene-len` - `filter-mode = suppress`, the previous behavior, disables generating new scenes until `min-scene-len` has passed - [bugfix] Remove extraneous console output when using `--drop-short-scenes` - - [api] Major changes to `SceneDetector` interface: - - Replace `get_metrics()` function with abstract property `metric_keys` to avoid confusion with `StatsManager.get_metrics()` function - - Deprecate `is_processing_required()` and `stats_manager_required()` functions - - Replace public `stats_manager` class variable with property including setter/getter - -## PySceneDetect 0.6 ### 0.6.3 (March 9, 2024)