From d3dd51dee4d54004d00ed98d820fda900ef156e6 Mon Sep 17 00:00:00 2001 From: Daphne Demekas Date: Mon, 12 Feb 2024 09:12:32 -0700 Subject: [PATCH 1/8] remove two labeling methods and add in maximum probabilities before viterbi --- .../predictors/supervised_fpe/labelers.py | 372 +++++++++--------- .../supervised_frame_pass_engine.py | 4 +- .../supervised_segmented_frame_pass_engine.py | 14 +- 3 files changed, 199 insertions(+), 191 deletions(-) diff --git a/diplomat/predictors/supervised_fpe/labelers.py b/diplomat/predictors/supervised_fpe/labelers.py index 8d0b9c6..0f17f6b 100644 --- a/diplomat/predictors/supervised_fpe/labelers.py +++ b/diplomat/predictors/supervised_fpe/labelers.py @@ -335,189 +335,189 @@ def supports_multi_label(cls) -> bool: return True -class ApproximateSourceOnly(Approximate): - @staticmethod - def _absorb_frame_data(p1, c1, off1, p2, c2, off2): - comb_c = np.concatenate([c1.T, c2.T]) - comb_p = np.concatenate([p1, p2]) - comb_off = np.concatenate([off1.T, off2.T]) - from_dlc = np.repeat([True, False], [len(p1), len(p2)]) - - sort_idx = np.lexsort([comb_c[:, 1], comb_c[:, 0]]) - comb_c = comb_c[sort_idx] - comb_p = comb_p[sort_idx] - comb_off = comb_off[sort_idx] - from_dlc = from_dlc[sort_idx] - - match_idx, = np.nonzero(np.all(comb_c[1:] == comb_c[:-1], axis=1)) - match_idx_after = match_idx + 1 - - comb_p[match_idx] = comb_p[match_idx] + comb_p[match_idx_after] - - return ( - np.delete(comb_p, ~from_dlc, axis=0), - np.delete(comb_c, ~from_dlc, axis=0).T, - np.delete(comb_off, ~from_dlc, axis=0).T - ) - - @staticmethod - def _filter_cell_count( - x: np.ndarray, - y: np.ndarray, - probs: np.ndarray, - x_off: np.ndarray, - y_off: np.ndarray, - max_cell_count: int - ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: - return (x, y, probs, x_off, y_off) - - -class NearestPeakInSource(labeler_lib.PoseLabeler): - """ - Nearest labeling mode, similar to approximate but always selects the nearest peak location in the source probability - map, assigning it a fixed value, and all the other locations a lower value... - """ - - def __init__(self, frame_engine: EditableFramePassEngine): - super().__init__() - self._frame_engine = frame_engine - self._settings = labeler_lib.SettingCollection( - minimum_peak_value=labeler_lib.FloatSpin(0, 1, 0.05, 0.001, 4), - selected_peak_value=labeler_lib.FloatSpin(0.5, 1, 0.95, 0.001, 4), - unselected_peak_value=labeler_lib.FloatSpin(0, 0.5, 0.05, 0.001, 4) - ) - - def predict_location( - self, - frame_idx: int, - bp_idx: int, - x: float, - y: float, - probability: float - ) -> Tuple[Any, Tuple[float, float, float]]: - meta = self._frame_engine.frame_data.metadata - config = self._settings.get_values() - - modified_frames = self._frame_engine.changed_frames - if((frame_idx, bp_idx) in modified_frames): - frame = modified_frames[(frame_idx, bp_idx)] - else: - frame = self._frame_engine.frame_data.frames[frame_idx][bp_idx] - - if(x is None): - x, y, prob = self._frame_engine.scmap_to_video_coord( - *self._frame_engine.get_maximum_with_defaults(frame), - meta.down_scaling - ) - return ((frame_idx, bp_idx, None, (x, y)), (x, y, 0)) - - ys, xs, probs, off_xs, off_ys = frame.orig_data.unpack() - - peak_locs = find_peaks(xs, ys, probs, meta.width) - peak_locs = peak_locs[probs[peak_locs] >= config.minimum_peak_value] - print(peak_locs) - if(len(peak_locs) <= 1): - # No peaks, or only one peak, perform basically a no-op, return prior frame state... - x, y, prob = self._frame_engine.scmap_to_video_coord( - *self._frame_engine.get_maximum_with_defaults(frame), - meta.down_scaling - ) - return ((frame_idx, bp_idx, frame, (x, y)), (x, y, prob)) - - def to_exact(_x, _y, _x_off, _y_off): - return _x + 0.5 + (_x_off / meta.down_scaling), _y + 0.5 + (_y_off / meta.down_scaling) - - # Compute nearest location... - xp, yp, pp, xp_off, yp_off = self._frame_engine.video_to_scmap_coord((x, y, probability)) - - xp_ex, yp_ex = to_exact(xp, yp, xp_off, yp_off) - x_ex, y_ex = to_exact(xs[peak_locs], ys[peak_locs], off_xs[peak_locs], off_ys[peak_locs]) - - dists = (xp_ex - x_ex) ** 2 + (yp_ex - y_ex) ** 2 - nearest_idx = np.argmin(dists) - - # Compute belonging of every cell... - owner_peak = np.argmin( - ((np.expand_dims(xs, 1) - np.expand_dims(xs[peak_locs], 0)) ** 2) - + ((np.expand_dims(ys, 1) - np.expand_dims(ys[peak_locs], 0)) ** 2), - axis=-1 - ) - - # Compute how much we need to scale each peak and it's neighbors by to get the configured weighting... - multipliers = config.unselected_peak_value / probs[peak_locs] - multipliers[nearest_idx] = config.selected_peak_value / probs[peak_locs[nearest_idx]] - - # Apply scaling to all peaks... - probs = probs * multipliers[owner_peak] - - temp_f = ForwardBackwardFrame( - src_data=SparseTrackingData().pack(ys, xs, probs, off_xs, off_ys), frame_probs=probs - ) - - x, y, prob = self._frame_engine.scmap_to_video_coord( - *self._frame_engine.get_maximum_with_defaults(temp_f), - meta.down_scaling - ) - - return ((frame_idx, bp_idx, temp_f, (x, y)), (x, y, prob)) - - def pose_change(self, new_state: Any) -> Any: - frm, bp, suggested_frame, coord = new_state - changed_frames = self._frame_engine.changed_frames - frames = self._frame_engine.frame_data.frames - - old_frame_data = frames[frm][bp] - is_orig = False - - idx = (frm, bp) - if(idx not in changed_frames): - changed_frames[idx] = old_frame_data - is_orig = True - - if(suggested_frame is None): - new_data = SparseTrackingData() - x, y, off_x, off_y, prob = self._frame_engine.video_to_scmap_coord( - coord + (0,) - ) - new_data.pack(*[np.array([item]) for item in [y, x, prob, off_x, off_y]]) - else: - new_data = suggested_frame.src_data - - new_frame = ForwardBackwardFrame() - new_frame.orig_data = new_data - new_frame.src_data = new_data - new_frame.disable_occluded = True - new_frame.ignore_clustering = True - - frames[frm][bp] = new_frame - - return (frm, bp, is_orig, old_frame_data) - - def undo(self, data: Any) -> Any: - frames = self._frame_engine.frame_data.frames - changed_frames = self._frame_engine.changed_frames - frm, bp, is_orig, frame_data = data - - idx = (frm, bp) - new_is_orig = False - new_old_frame_data = frames[frm][bp] - - if (idx not in changed_frames): - changed_frames[idx] = new_old_frame_data - new_is_orig = True - elif (is_orig): - del changed_frames[idx] - - frames[frm][bp] = frame_data - - return (frm, bp, new_is_orig, new_old_frame_data) - - def redo(self, data: Any) -> Any: - return self.undo(data) - - def get_settings(self) -> Optional[labeler_lib.SettingCollection]: - return self._settings - - @classmethod - def supports_multi_label(cls) -> bool: - return True +# class ApproximateSourceOnly(Approximate): +# @staticmethod +# def _absorb_frame_data(p1, c1, off1, p2, c2, off2): +# comb_c = np.concatenate([c1.T, c2.T]) +# comb_p = np.concatenate([p1, p2]) +# comb_off = np.concatenate([off1.T, off2.T]) +# from_dlc = np.repeat([True, False], [len(p1), len(p2)]) + +# sort_idx = np.lexsort([comb_c[:, 1], comb_c[:, 0]]) +# comb_c = comb_c[sort_idx] +# comb_p = comb_p[sort_idx] +# comb_off = comb_off[sort_idx] +# from_dlc = from_dlc[sort_idx] + +# match_idx, = np.nonzero(np.all(comb_c[1:] == comb_c[:-1], axis=1)) +# match_idx_after = match_idx + 1 + +# comb_p[match_idx] = comb_p[match_idx] + comb_p[match_idx_after] + +# return ( +# np.delete(comb_p, ~from_dlc, axis=0), +# np.delete(comb_c, ~from_dlc, axis=0).T, +# np.delete(comb_off, ~from_dlc, axis=0).T +# ) + +# @staticmethod +# def _filter_cell_count( +# x: np.ndarray, +# y: np.ndarray, +# probs: np.ndarray, +# x_off: np.ndarray, +# y_off: np.ndarray, +# max_cell_count: int +# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: +# return (x, y, probs, x_off, y_off) + + +# class NearestPeakInSource(labeler_lib.PoseLabeler): +# """ +# Nearest labeling mode, similar to approximate but always selects the nearest peak location in the source probability +# map, assigning it a fixed value, and all the other locations a lower value... +# """ + +# def __init__(self, frame_engine: EditableFramePassEngine): +# super().__init__() +# self._frame_engine = frame_engine +# self._settings = labeler_lib.SettingCollection( +# minimum_peak_value=labeler_lib.FloatSpin(0, 1, 0.05, 0.001, 4), +# selected_peak_value=labeler_lib.FloatSpin(0.5, 1, 0.95, 0.001, 4), +# unselected_peak_value=labeler_lib.FloatSpin(0, 0.5, 0.05, 0.001, 4) +# ) + +# def predict_location( +# self, +# frame_idx: int, +# bp_idx: int, +# x: float, +# y: float, +# probability: float +# ) -> Tuple[Any, Tuple[float, float, float]]: +# meta = self._frame_engine.frame_data.metadata +# config = self._settings.get_values() + +# modified_frames = self._frame_engine.changed_frames +# if((frame_idx, bp_idx) in modified_frames): +# frame = modified_frames[(frame_idx, bp_idx)] +# else: +# frame = self._frame_engine.frame_data.frames[frame_idx][bp_idx] + +# if(x is None): +# x, y, prob = self._frame_engine.scmap_to_video_coord( +# *self._frame_engine.get_maximum_with_defaults(frame), +# meta.down_scaling +# ) +# return ((frame_idx, bp_idx, None, (x, y)), (x, y, 0)) + +# ys, xs, probs, off_xs, off_ys = frame.orig_data.unpack() + +# peak_locs = find_peaks(xs, ys, probs, meta.width) +# peak_locs = peak_locs[probs[peak_locs] >= config.minimum_peak_value] +# print(peak_locs) +# if(len(peak_locs) <= 1): +# # No peaks, or only one peak, perform basically a no-op, return prior frame state... +# x, y, prob = self._frame_engine.scmap_to_video_coord( +# *self._frame_engine.get_maximum_with_defaults(frame), +# meta.down_scaling +# ) +# return ((frame_idx, bp_idx, frame, (x, y)), (x, y, prob)) + +# def to_exact(_x, _y, _x_off, _y_off): +# return _x + 0.5 + (_x_off / meta.down_scaling), _y + 0.5 + (_y_off / meta.down_scaling) + +# # Compute nearest location... +# xp, yp, pp, xp_off, yp_off = self._frame_engine.video_to_scmap_coord((x, y, probability)) + +# xp_ex, yp_ex = to_exact(xp, yp, xp_off, yp_off) +# x_ex, y_ex = to_exact(xs[peak_locs], ys[peak_locs], off_xs[peak_locs], off_ys[peak_locs]) + +# dists = (xp_ex - x_ex) ** 2 + (yp_ex - y_ex) ** 2 +# nearest_idx = np.argmin(dists) + +# # Compute belonging of every cell... +# owner_peak = np.argmin( +# ((np.expand_dims(xs, 1) - np.expand_dims(xs[peak_locs], 0)) ** 2) +# + ((np.expand_dims(ys, 1) - np.expand_dims(ys[peak_locs], 0)) ** 2), +# axis=-1 +# ) + +# # Compute how much we need to scale each peak and it's neighbors by to get the configured weighting... +# multipliers = config.unselected_peak_value / probs[peak_locs] +# multipliers[nearest_idx] = config.selected_peak_value / probs[peak_locs[nearest_idx]] + +# # Apply scaling to all peaks... +# probs = probs * multipliers[owner_peak] + +# temp_f = ForwardBackwardFrame( +# src_data=SparseTrackingData().pack(ys, xs, probs, off_xs, off_ys), frame_probs=probs +# ) + +# x, y, prob = self._frame_engine.scmap_to_video_coord( +# *self._frame_engine.get_maximum_with_defaults(temp_f), +# meta.down_scaling +# ) + +# return ((frame_idx, bp_idx, temp_f, (x, y)), (x, y, prob)) + +# def pose_change(self, new_state: Any) -> Any: +# frm, bp, suggested_frame, coord = new_state +# changed_frames = self._frame_engine.changed_frames +# frames = self._frame_engine.frame_data.frames + +# old_frame_data = frames[frm][bp] +# is_orig = False + +# idx = (frm, bp) +# if(idx not in changed_frames): +# changed_frames[idx] = old_frame_data +# is_orig = True + +# if(suggested_frame is None): +# new_data = SparseTrackingData() +# x, y, off_x, off_y, prob = self._frame_engine.video_to_scmap_coord( +# coord + (0,) +# ) +# new_data.pack(*[np.array([item]) for item in [y, x, prob, off_x, off_y]]) +# else: +# new_data = suggested_frame.src_data + +# new_frame = ForwardBackwardFrame() +# new_frame.orig_data = new_data +# new_frame.src_data = new_data +# new_frame.disable_occluded = True +# new_frame.ignore_clustering = True + +# frames[frm][bp] = new_frame + +# return (frm, bp, is_orig, old_frame_data) + +# def undo(self, data: Any) -> Any: +# frames = self._frame_engine.frame_data.frames +# changed_frames = self._frame_engine.changed_frames +# frm, bp, is_orig, frame_data = data + +# idx = (frm, bp) +# new_is_orig = False +# new_old_frame_data = frames[frm][bp] + +# if (idx not in changed_frames): +# changed_frames[idx] = new_old_frame_data +# new_is_orig = True +# elif (is_orig): +# del changed_frames[idx] + +# frames[frm][bp] = frame_data + +# return (frm, bp, new_is_orig, new_old_frame_data) + +# def redo(self, data: Any) -> Any: +# return self.undo(data) + +# def get_settings(self) -> Optional[labeler_lib.SettingCollection]: +# return self._settings + +# @classmethod +# def supports_multi_label(cls) -> bool: +# return True diff --git a/diplomat/predictors/supervised_fpe/supervised_frame_pass_engine.py b/diplomat/predictors/supervised_fpe/supervised_frame_pass_engine.py index 7484dc6..e9a4afa 100644 --- a/diplomat/predictors/supervised_fpe/supervised_frame_pass_engine.py +++ b/diplomat/predictors/supervised_fpe/supervised_frame_pass_engine.py @@ -5,7 +5,7 @@ from diplomat.wx_gui.progress_dialog import FBProgressDialog from ..fpe.frame_pass_engine import FramePassEngine, SparseTrackingData from ..fpe.sparse_storage import ForwardBackwardFrame, ForwardBackwardData -from .labelers import Approximate, Point, NearestPeakInSource, ApproximateSourceOnly +from .labelers import Approximate, Point#, NearestPeakInSource, ApproximateSourceOnly from .scorers import EntropyOfTransitions, MaximumJumpInStandardDeviations import wx @@ -67,7 +67,7 @@ def on_end(self, progress_bar: ProgressBar) -> Union[None, Pose]: self._get_names(), self.video_metadata, self._get_crop_box(), - [Approximate(self), ApproximateSourceOnly(self), Point(self), NearestPeakInSource(self)], + [Approximate(self), Point(self)], [EntropyOfTransitions(self), MaximumJumpInStandardDeviations(self)], None, list(range(1, self.num_outputs + 1)) * (self._num_total_bp // self.num_outputs) diff --git a/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py b/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py index a2afe09..0ea096a 100644 --- a/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py +++ b/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py @@ -6,7 +6,7 @@ from diplomat.predictors.sfpe.disk_sparse_storage import DiskBackedForwardBackwardData from diplomat.wx_gui.progress_dialog import FBProgressDialog -from diplomat.predictors.supervised_fpe.labelers import Approximate, Point, NearestPeakInSource, ApproximateSourceOnly +from diplomat.predictors.supervised_fpe.labelers import Approximate, Point#, NearestPeakInSource, ApproximateSourceOnly from diplomat.predictors.supervised_fpe.scorers import EntropyOfTransitions, MaximumJumpInStandardDeviations from typing import Optional, Dict, Tuple, List, MutableMapping, Iterator, Iterable from diplomat.predictors.sfpe.segmented_frame_pass_engine import SegmentedFramePassEngine, AntiCloseObject @@ -531,7 +531,15 @@ def _partial_rerun( for (s_i, e_i, f_i), seg_ord in zip(self._segments, self._segment_bp_order): poses[s_i:e_i, :] = poses[s_i:e_i, seg_ord] old_poses.get_all()[:] = poses.reshape(old_poses.get_frame_count(), old_poses.get_bodypart_count() * 3) - + + # For each changed frame and each body part, take the maximum probability coordinates and set them to one + for (frame_idx, bp_idx), frame in changed_frames.items(): + max_prob_coord = np.unravel_index(frame.frame_probs.argmax(), frame.frame_probs.shape) + new_frame_probs = np.zeros_like(frame.frame_probs) #copy because this is read only + new_frame_probs[max_prob_coord] = 1 + frame.frame_probs = new_frame_probs + + return ( self.get_maximums( self._frame_holder, @@ -672,7 +680,7 @@ def _on_end(self, progress_bar: ProgressBar) -> Optional[Pose]: self._get_names(), self.video_metadata, self._get_crop_box(), - [Approximate(self), ApproximateSourceOnly(self), Point(self), NearestPeakInSource(self)], + [Approximate(self), Point(self)], [EntropyOfTransitions(self), MaximumJumpInStandardDeviations(self)], None, list(range(1, self.num_outputs + 1)) * (self._num_total_bp // self.num_outputs), From 828e19d51d112fdfbf0724c62a7adc7c26903afb Mon Sep 17 00:00:00 2001 From: Daphne Demekas Date: Mon, 12 Feb 2024 17:52:47 -0700 Subject: [PATCH 2/8] windows error resolving --- diplomat/predictors/supervised_fpe/labelers.py | 2 +- diplomat/wx_gui/fpe_editor.py | 2 ++ diplomat/wx_gui/point_edit.py | 7 +++++-- diplomat/wx_gui/settings_dialog.py | 6 +++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/diplomat/predictors/supervised_fpe/labelers.py b/diplomat/predictors/supervised_fpe/labelers.py index 0f17f6b..e18522b 100644 --- a/diplomat/predictors/supervised_fpe/labelers.py +++ b/diplomat/predictors/supervised_fpe/labelers.py @@ -206,7 +206,7 @@ def predict_location( bp_idx: int, x: float, y: float, - probability: float + probability: float, ) -> Tuple[Any, Tuple[float, float, float]]: info = self._settings.get_values() user_amp = info.user_input_strength / 1000 diff --git a/diplomat/wx_gui/fpe_editor.py b/diplomat/wx_gui/fpe_editor.py index 1920824..842da49 100644 --- a/diplomat/wx_gui/fpe_editor.py +++ b/diplomat/wx_gui/fpe_editor.py @@ -288,6 +288,7 @@ def __init__( bp_names=names, labeling_modes=labeling_modes, group_list=part_groups, + # skeleton_info = self.skeleton_info **ps ) self.video_controls = VideoController(self._sub_panel, video_player=self.video_player.video_viewer) @@ -336,6 +337,7 @@ def __init__( self.video_controls.Bind(PointViewNEdit.EVT_FRAME_CHANGE, self._on_frame_chg) + def _on_close_caller(self, event: wx.CloseEvent): self._on_close(event, self._was_save_button_flag) self._was_save_button_flag = False diff --git a/diplomat/wx_gui/point_edit.py b/diplomat/wx_gui/point_edit.py index 0204661..74655ab 100644 --- a/diplomat/wx_gui/point_edit.py +++ b/diplomat/wx_gui/point_edit.py @@ -2,7 +2,7 @@ Provides a point editing widget. This is a video player with a body part and labeler selections available on the side. """ from typing import Tuple, List, Optional, Union, Any, Iterable - +from diplomat.predictors.fpe.skeleton_structures import StorageGraph import numpy as np import wx from diplomat.processing import * @@ -139,6 +139,7 @@ def __init__( video_hdl: cv2.VideoCapture, crop_box: Box, poses: Pose, + skeleton_info: StorageGraph, colormap: Union[str, list, Colormap] = DEF_MAP, shape_list: Iterable[str] = None, plot_threshold: float = 0.1, @@ -190,6 +191,7 @@ def __init__( self._fast_m_speed = ctrl_speed_divider self._pose_label_modes = {} self._current_pose_labeling_mode = "" + self.skeleton_info = skeleton_info BasicDataFields.__init__(self, colormap, plot_threshold, point_radius, point_alpha, line_thickness, shape_list) @@ -1087,6 +1089,7 @@ def __init__( bp_names: List[str], labeling_modes: List[PoseLabeler], group_list: Optional[List[int]] = None, + skeleton_info: StorageGraph = None, colormap: str = PointViewNEdit.DEF_MAP, shape_list: str = None, plot_threshold: float = 0.1, @@ -1133,7 +1136,7 @@ def __init__( self._main_sizer = wx.BoxSizer(wx.HORIZONTAL) self._side_bar_sizer = wx.BoxSizer(wx.VERTICAL) - self.video_viewer = PointViewNEdit(self, video_hdl, crop_box, poses, colormap, shape_list, plot_threshold, point_radius, + self.video_viewer = PointViewNEdit(self, video_hdl, crop_box, poses, skeleton_info, colormap, shape_list, plot_threshold, point_radius, point_alpha, line_thickness) for p in labeling_modes: diff --git a/diplomat/wx_gui/settings_dialog.py b/diplomat/wx_gui/settings_dialog.py index 506f323..2418ff7 100644 --- a/diplomat/wx_gui/settings_dialog.py +++ b/diplomat/wx_gui/settings_dialog.py @@ -43,7 +43,11 @@ def set_hook(self, hook: Callable[[str], None]): self._hook = hook def get_new_widget(self, parent=None) -> wx.Control: - text_list = wx.Choice(parent, choices=self._option_names, style=wx.LB_SINGLE, **self._kwargs) + # Check if the platform is Windows + if platform.system() != 'Windows': + # If not Windows, add the style flag to kwargs + self.kwargs['style'] = wx.LB_SINGLE + text_list = wx.Choice(parent, choices=self._option_names, **self._kwargs) text_list.SetSelection(self._default) def val_change(evt): From 44da609fc43abaa407b7e54f731194b1fe817d1d Mon Sep 17 00:00:00 2001 From: Daphne Demekas Date: Tue, 13 Feb 2024 11:31:38 -0700 Subject: [PATCH 3/8] put pose labelers back in --- .../predictors/supervised_fpe/labelers.py | 372 +++++++++--------- .../supervised_frame_pass_engine.py | 4 +- .../supervised_segmented_frame_pass_engine.py | 5 +- 3 files changed, 190 insertions(+), 191 deletions(-) diff --git a/diplomat/predictors/supervised_fpe/labelers.py b/diplomat/predictors/supervised_fpe/labelers.py index e18522b..7a2a81a 100644 --- a/diplomat/predictors/supervised_fpe/labelers.py +++ b/diplomat/predictors/supervised_fpe/labelers.py @@ -335,189 +335,189 @@ def supports_multi_label(cls) -> bool: return True -# class ApproximateSourceOnly(Approximate): -# @staticmethod -# def _absorb_frame_data(p1, c1, off1, p2, c2, off2): -# comb_c = np.concatenate([c1.T, c2.T]) -# comb_p = np.concatenate([p1, p2]) -# comb_off = np.concatenate([off1.T, off2.T]) -# from_dlc = np.repeat([True, False], [len(p1), len(p2)]) - -# sort_idx = np.lexsort([comb_c[:, 1], comb_c[:, 0]]) -# comb_c = comb_c[sort_idx] -# comb_p = comb_p[sort_idx] -# comb_off = comb_off[sort_idx] -# from_dlc = from_dlc[sort_idx] - -# match_idx, = np.nonzero(np.all(comb_c[1:] == comb_c[:-1], axis=1)) -# match_idx_after = match_idx + 1 - -# comb_p[match_idx] = comb_p[match_idx] + comb_p[match_idx_after] - -# return ( -# np.delete(comb_p, ~from_dlc, axis=0), -# np.delete(comb_c, ~from_dlc, axis=0).T, -# np.delete(comb_off, ~from_dlc, axis=0).T -# ) - -# @staticmethod -# def _filter_cell_count( -# x: np.ndarray, -# y: np.ndarray, -# probs: np.ndarray, -# x_off: np.ndarray, -# y_off: np.ndarray, -# max_cell_count: int -# ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: -# return (x, y, probs, x_off, y_off) - - -# class NearestPeakInSource(labeler_lib.PoseLabeler): -# """ -# Nearest labeling mode, similar to approximate but always selects the nearest peak location in the source probability -# map, assigning it a fixed value, and all the other locations a lower value... -# """ - -# def __init__(self, frame_engine: EditableFramePassEngine): -# super().__init__() -# self._frame_engine = frame_engine -# self._settings = labeler_lib.SettingCollection( -# minimum_peak_value=labeler_lib.FloatSpin(0, 1, 0.05, 0.001, 4), -# selected_peak_value=labeler_lib.FloatSpin(0.5, 1, 0.95, 0.001, 4), -# unselected_peak_value=labeler_lib.FloatSpin(0, 0.5, 0.05, 0.001, 4) -# ) - -# def predict_location( -# self, -# frame_idx: int, -# bp_idx: int, -# x: float, -# y: float, -# probability: float -# ) -> Tuple[Any, Tuple[float, float, float]]: -# meta = self._frame_engine.frame_data.metadata -# config = self._settings.get_values() - -# modified_frames = self._frame_engine.changed_frames -# if((frame_idx, bp_idx) in modified_frames): -# frame = modified_frames[(frame_idx, bp_idx)] -# else: -# frame = self._frame_engine.frame_data.frames[frame_idx][bp_idx] - -# if(x is None): -# x, y, prob = self._frame_engine.scmap_to_video_coord( -# *self._frame_engine.get_maximum_with_defaults(frame), -# meta.down_scaling -# ) -# return ((frame_idx, bp_idx, None, (x, y)), (x, y, 0)) - -# ys, xs, probs, off_xs, off_ys = frame.orig_data.unpack() - -# peak_locs = find_peaks(xs, ys, probs, meta.width) -# peak_locs = peak_locs[probs[peak_locs] >= config.minimum_peak_value] -# print(peak_locs) -# if(len(peak_locs) <= 1): -# # No peaks, or only one peak, perform basically a no-op, return prior frame state... -# x, y, prob = self._frame_engine.scmap_to_video_coord( -# *self._frame_engine.get_maximum_with_defaults(frame), -# meta.down_scaling -# ) -# return ((frame_idx, bp_idx, frame, (x, y)), (x, y, prob)) - -# def to_exact(_x, _y, _x_off, _y_off): -# return _x + 0.5 + (_x_off / meta.down_scaling), _y + 0.5 + (_y_off / meta.down_scaling) - -# # Compute nearest location... -# xp, yp, pp, xp_off, yp_off = self._frame_engine.video_to_scmap_coord((x, y, probability)) - -# xp_ex, yp_ex = to_exact(xp, yp, xp_off, yp_off) -# x_ex, y_ex = to_exact(xs[peak_locs], ys[peak_locs], off_xs[peak_locs], off_ys[peak_locs]) - -# dists = (xp_ex - x_ex) ** 2 + (yp_ex - y_ex) ** 2 -# nearest_idx = np.argmin(dists) - -# # Compute belonging of every cell... -# owner_peak = np.argmin( -# ((np.expand_dims(xs, 1) - np.expand_dims(xs[peak_locs], 0)) ** 2) -# + ((np.expand_dims(ys, 1) - np.expand_dims(ys[peak_locs], 0)) ** 2), -# axis=-1 -# ) - -# # Compute how much we need to scale each peak and it's neighbors by to get the configured weighting... -# multipliers = config.unselected_peak_value / probs[peak_locs] -# multipliers[nearest_idx] = config.selected_peak_value / probs[peak_locs[nearest_idx]] - -# # Apply scaling to all peaks... -# probs = probs * multipliers[owner_peak] - -# temp_f = ForwardBackwardFrame( -# src_data=SparseTrackingData().pack(ys, xs, probs, off_xs, off_ys), frame_probs=probs -# ) - -# x, y, prob = self._frame_engine.scmap_to_video_coord( -# *self._frame_engine.get_maximum_with_defaults(temp_f), -# meta.down_scaling -# ) - -# return ((frame_idx, bp_idx, temp_f, (x, y)), (x, y, prob)) - -# def pose_change(self, new_state: Any) -> Any: -# frm, bp, suggested_frame, coord = new_state -# changed_frames = self._frame_engine.changed_frames -# frames = self._frame_engine.frame_data.frames - -# old_frame_data = frames[frm][bp] -# is_orig = False - -# idx = (frm, bp) -# if(idx not in changed_frames): -# changed_frames[idx] = old_frame_data -# is_orig = True - -# if(suggested_frame is None): -# new_data = SparseTrackingData() -# x, y, off_x, off_y, prob = self._frame_engine.video_to_scmap_coord( -# coord + (0,) -# ) -# new_data.pack(*[np.array([item]) for item in [y, x, prob, off_x, off_y]]) -# else: -# new_data = suggested_frame.src_data - -# new_frame = ForwardBackwardFrame() -# new_frame.orig_data = new_data -# new_frame.src_data = new_data -# new_frame.disable_occluded = True -# new_frame.ignore_clustering = True - -# frames[frm][bp] = new_frame - -# return (frm, bp, is_orig, old_frame_data) - -# def undo(self, data: Any) -> Any: -# frames = self._frame_engine.frame_data.frames -# changed_frames = self._frame_engine.changed_frames -# frm, bp, is_orig, frame_data = data - -# idx = (frm, bp) -# new_is_orig = False -# new_old_frame_data = frames[frm][bp] - -# if (idx not in changed_frames): -# changed_frames[idx] = new_old_frame_data -# new_is_orig = True -# elif (is_orig): -# del changed_frames[idx] - -# frames[frm][bp] = frame_data - -# return (frm, bp, new_is_orig, new_old_frame_data) - -# def redo(self, data: Any) -> Any: -# return self.undo(data) - -# def get_settings(self) -> Optional[labeler_lib.SettingCollection]: -# return self._settings - -# @classmethod -# def supports_multi_label(cls) -> bool: -# return True +class ApproximateSourceOnly(Approximate): + @staticmethod + def _absorb_frame_data(p1, c1, off1, p2, c2, off2): + comb_c = np.concatenate([c1.T, c2.T]) + comb_p = np.concatenate([p1, p2]) + comb_off = np.concatenate([off1.T, off2.T]) + from_dlc = np.repeat([True, False], [len(p1), len(p2)]) + + sort_idx = np.lexsort([comb_c[:, 1], comb_c[:, 0]]) + comb_c = comb_c[sort_idx] + comb_p = comb_p[sort_idx] + comb_off = comb_off[sort_idx] + from_dlc = from_dlc[sort_idx] + + match_idx, = np.nonzero(np.all(comb_c[1:] == comb_c[:-1], axis=1)) + match_idx_after = match_idx + 1 + + comb_p[match_idx] = comb_p[match_idx] + comb_p[match_idx_after] + + return ( + np.delete(comb_p, ~from_dlc, axis=0), + np.delete(comb_c, ~from_dlc, axis=0).T, + np.delete(comb_off, ~from_dlc, axis=0).T + ) + + @staticmethod + def _filter_cell_count( + x: np.ndarray, + y: np.ndarray, + probs: np.ndarray, + x_off: np.ndarray, + y_off: np.ndarray, + max_cell_count: int + ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + return (x, y, probs, x_off, y_off) + + +class NearestPeakInSource(labeler_lib.PoseLabeler): + """ + Nearest labeling mode, similar to approximate but always selects the nearest peak location in the source probability + map, assigning it a fixed value, and all the other locations a lower value... + """ + + def __init__(self, frame_engine: EditableFramePassEngine): + super().__init__() + self._frame_engine = frame_engine + self._settings = labeler_lib.SettingCollection( + minimum_peak_value=labeler_lib.FloatSpin(0, 1, 0.05, 0.001, 4), + selected_peak_value=labeler_lib.FloatSpin(0.5, 1, 0.95, 0.001, 4), + unselected_peak_value=labeler_lib.FloatSpin(0, 0.5, 0.05, 0.001, 4) + ) + + def predict_location( + self, + frame_idx: int, + bp_idx: int, + x: float, + y: float, + probability: float + ) -> Tuple[Any, Tuple[float, float, float]]: + meta = self._frame_engine.frame_data.metadata + config = self._settings.get_values() + + modified_frames = self._frame_engine.changed_frames + if((frame_idx, bp_idx) in modified_frames): + frame = modified_frames[(frame_idx, bp_idx)] + else: + frame = self._frame_engine.frame_data.frames[frame_idx][bp_idx] + + if(x is None): + x, y, prob = self._frame_engine.scmap_to_video_coord( + *self._frame_engine.get_maximum_with_defaults(frame), + meta.down_scaling + ) + return ((frame_idx, bp_idx, None, (x, y)), (x, y, 0)) + + ys, xs, probs, off_xs, off_ys = frame.orig_data.unpack() + + peak_locs = find_peaks(xs, ys, probs, meta.width) + peak_locs = peak_locs[probs[peak_locs] >= config.minimum_peak_value] + print(peak_locs) + if(len(peak_locs) <= 1): + # No peaks, or only one peak, perform basically a no-op, return prior frame state... + x, y, prob = self._frame_engine.scmap_to_video_coord( + *self._frame_engine.get_maximum_with_defaults(frame), + meta.down_scaling + ) + return ((frame_idx, bp_idx, frame, (x, y)), (x, y, prob)) + + def to_exact(_x, _y, _x_off, _y_off): + return _x + 0.5 + (_x_off / meta.down_scaling), _y + 0.5 + (_y_off / meta.down_scaling) + + # Compute nearest location... + xp, yp, pp, xp_off, yp_off = self._frame_engine.video_to_scmap_coord((x, y, probability)) + + xp_ex, yp_ex = to_exact(xp, yp, xp_off, yp_off) + x_ex, y_ex = to_exact(xs[peak_locs], ys[peak_locs], off_xs[peak_locs], off_ys[peak_locs]) + + dists = (xp_ex - x_ex) ** 2 + (yp_ex - y_ex) ** 2 + nearest_idx = np.argmin(dists) + + # Compute belonging of every cell... + owner_peak = np.argmin( + ((np.expand_dims(xs, 1) - np.expand_dims(xs[peak_locs], 0)) ** 2) + + ((np.expand_dims(ys, 1) - np.expand_dims(ys[peak_locs], 0)) ** 2), + axis=-1 + ) + + # Compute how much we need to scale each peak and it's neighbors by to get the configured weighting... + multipliers = config.unselected_peak_value / probs[peak_locs] + multipliers[nearest_idx] = config.selected_peak_value / probs[peak_locs[nearest_idx]] + + # Apply scaling to all peaks... + probs = probs * multipliers[owner_peak] + + temp_f = ForwardBackwardFrame( + src_data=SparseTrackingData().pack(ys, xs, probs, off_xs, off_ys), frame_probs=probs + ) + + x, y, prob = self._frame_engine.scmap_to_video_coord( + *self._frame_engine.get_maximum_with_defaults(temp_f), + meta.down_scaling + ) + + return ((frame_idx, bp_idx, temp_f, (x, y)), (x, y, prob)) + + def pose_change(self, new_state: Any) -> Any: + frm, bp, suggested_frame, coord = new_state + changed_frames = self._frame_engine.changed_frames + frames = self._frame_engine.frame_data.frames + + old_frame_data = frames[frm][bp] + is_orig = False + + idx = (frm, bp) + if(idx not in changed_frames): + changed_frames[idx] = old_frame_data + is_orig = True + + if(suggested_frame is None): + new_data = SparseTrackingData() + x, y, off_x, off_y, prob = self._frame_engine.video_to_scmap_coord( + coord + (0,) + ) + new_data.pack(*[np.array([item]) for item in [y, x, prob, off_x, off_y]]) + else: + new_data = suggested_frame.src_data + + new_frame = ForwardBackwardFrame() + new_frame.orig_data = new_data + new_frame.src_data = new_data + new_frame.disable_occluded = True + new_frame.ignore_clustering = True + + frames[frm][bp] = new_frame + + return (frm, bp, is_orig, old_frame_data) + + def undo(self, data: Any) -> Any: + frames = self._frame_engine.frame_data.frames + changed_frames = self._frame_engine.changed_frames + frm, bp, is_orig, frame_data = data + + idx = (frm, bp) + new_is_orig = False + new_old_frame_data = frames[frm][bp] + + if (idx not in changed_frames): + changed_frames[idx] = new_old_frame_data + new_is_orig = True + elif (is_orig): + del changed_frames[idx] + + frames[frm][bp] = frame_data + + return (frm, bp, new_is_orig, new_old_frame_data) + + def redo(self, data: Any) -> Any: + return self.undo(data) + + def get_settings(self) -> Optional[labeler_lib.SettingCollection]: + return self._settings + + @classmethod + def supports_multi_label(cls) -> bool: + return True diff --git a/diplomat/predictors/supervised_fpe/supervised_frame_pass_engine.py b/diplomat/predictors/supervised_fpe/supervised_frame_pass_engine.py index e9a4afa..60588fa 100644 --- a/diplomat/predictors/supervised_fpe/supervised_frame_pass_engine.py +++ b/diplomat/predictors/supervised_fpe/supervised_frame_pass_engine.py @@ -5,7 +5,7 @@ from diplomat.wx_gui.progress_dialog import FBProgressDialog from ..fpe.frame_pass_engine import FramePassEngine, SparseTrackingData from ..fpe.sparse_storage import ForwardBackwardFrame, ForwardBackwardData -from .labelers import Approximate, Point#, NearestPeakInSource, ApproximateSourceOnly +from .labelers import Approximate, Point, NearestPeakInSource, ApproximateSourceOnly from .scorers import EntropyOfTransitions, MaximumJumpInStandardDeviations import wx @@ -67,7 +67,7 @@ def on_end(self, progress_bar: ProgressBar) -> Union[None, Pose]: self._get_names(), self.video_metadata, self._get_crop_box(), - [Approximate(self), Point(self)], + [Approximate(self), Point(self), NearestPeakInSource(self), ApproximateSourceOnly(self)], [EntropyOfTransitions(self), MaximumJumpInStandardDeviations(self)], None, list(range(1, self.num_outputs + 1)) * (self._num_total_bp // self.num_outputs) diff --git a/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py b/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py index 0ea096a..0d950ba 100644 --- a/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py +++ b/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py @@ -6,7 +6,7 @@ from diplomat.predictors.sfpe.disk_sparse_storage import DiskBackedForwardBackwardData from diplomat.wx_gui.progress_dialog import FBProgressDialog -from diplomat.predictors.supervised_fpe.labelers import Approximate, Point#, NearestPeakInSource, ApproximateSourceOnly +from diplomat.predictors.supervised_fpe.labelers import Approximate, Point, NearestPeakInSource, ApproximateSourceOnly from diplomat.predictors.supervised_fpe.scorers import EntropyOfTransitions, MaximumJumpInStandardDeviations from typing import Optional, Dict, Tuple, List, MutableMapping, Iterator, Iterable from diplomat.predictors.sfpe.segmented_frame_pass_engine import SegmentedFramePassEngine, AntiCloseObject @@ -538,7 +538,6 @@ def _partial_rerun( new_frame_probs = np.zeros_like(frame.frame_probs) #copy because this is read only new_frame_probs[max_prob_coord] = 1 frame.frame_probs = new_frame_probs - return ( self.get_maximums( @@ -680,7 +679,7 @@ def _on_end(self, progress_bar: ProgressBar) -> Optional[Pose]: self._get_names(), self.video_metadata, self._get_crop_box(), - [Approximate(self), Point(self)], + [Approximate(self), Point(self), NearestPeakInSource(self), ApproximateSourceOnly(self)], [EntropyOfTransitions(self), MaximumJumpInStandardDeviations(self)], None, list(range(1, self.num_outputs + 1)) * (self._num_total_bp // self.num_outputs), From e75a119c90db43823952412777258ce52050960d Mon Sep 17 00:00:00 2001 From: Daphne Demekas Date: Wed, 14 Feb 2024 11:53:50 -0700 Subject: [PATCH 4/8] move update lines a bit further above so that the overriding occurs before run segemnted pasess --- .../supervised_segmented_frame_pass_engine.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py b/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py index 0d950ba..94926fd 100644 --- a/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py +++ b/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py @@ -516,6 +516,13 @@ def _partial_rerun( old_poses: Pose, progress_bar: ProgressBar ) -> Tuple[Pose, Iterable[int]]: + + # For each changed frame and each body part, take the maximum probability coordinates and set them to one + for (frame_idx, bp_idx), frame in changed_frames.items(): + max_prob_coord = np.unravel_index(frame.frame_probs.argmax(), frame.frame_probs.shape) + new_frame_probs = np.zeros_like(frame.frame_probs) #copy because this is read only + new_frame_probs[max_prob_coord] = 1 + frame.frame_probs = new_frame_probs # Determine what segments have been manipulated... segment_indexes = sorted({np.searchsorted(self._segments[:, 1], f_i, "right") for f_i, b_i in changed_frames}) @@ -532,12 +539,6 @@ def _partial_rerun( poses[s_i:e_i, :] = poses[s_i:e_i, seg_ord] old_poses.get_all()[:] = poses.reshape(old_poses.get_frame_count(), old_poses.get_bodypart_count() * 3) - # For each changed frame and each body part, take the maximum probability coordinates and set them to one - for (frame_idx, bp_idx), frame in changed_frames.items(): - max_prob_coord = np.unravel_index(frame.frame_probs.argmax(), frame.frame_probs.shape) - new_frame_probs = np.zeros_like(frame.frame_probs) #copy because this is read only - new_frame_probs[max_prob_coord] = 1 - frame.frame_probs = new_frame_probs return ( self.get_maximums( From c21abdad6ac6a33ba5f2c279e72a2537bcde1a4b Mon Sep 17 00:00:00 2001 From: Daphne Demekas Date: Mon, 19 Feb 2024 12:13:02 -0700 Subject: [PATCH 5/8] changes in pose change code --- diplomat/predictors/supervised_fpe/labelers.py | 11 +++++++++-- .../supervised_segmented_frame_pass_engine.py | 12 ++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/diplomat/predictors/supervised_fpe/labelers.py b/diplomat/predictors/supervised_fpe/labelers.py index 7a2a81a..f11fbc5 100644 --- a/diplomat/predictors/supervised_fpe/labelers.py +++ b/diplomat/predictors/supervised_fpe/labelers.py @@ -57,6 +57,7 @@ def predict_location( frame = self._frame_engine.frame_data.frames[frame_idx][bp_idx] if(x is None): + #should we be returning this prob value or the probability value? x, y, prob = self._frame_engine.scmap_to_video_coord( *self._frame_engine.get_maximum_with_defaults(frame), meta.down_scaling @@ -293,7 +294,10 @@ def pose_change(self, new_state: Any) -> Any: ) new_data.pack(*[np.array([item]) for item in [y, x, prob, off_x, off_y]]) else: - new_data = suggested_frame.src_data + y, x, prob, x_offset, y_offset = suggested_frame.src_data.unpack() + max_prob_idx = np.argmax(prob) + new_data = SparseTrackingData() + new_data.pack(*[np.array([item]) for item in [y[max_prob_idx], x[max_prob_idx], 1, x_offset[max_prob_idx], y_offset[max_prob_idx]]]) new_frame = ForwardBackwardFrame() new_frame.orig_data = new_data @@ -481,7 +485,10 @@ def pose_change(self, new_state: Any) -> Any: ) new_data.pack(*[np.array([item]) for item in [y, x, prob, off_x, off_y]]) else: - new_data = suggested_frame.src_data + y, x, prob, x_offset, y_offset = suggested_frame.src_data.unpack() + max_prob_idx = np.argmax(prob) + new_data = SparseTrackingData() + new_data.pack(*[np.array([item]) for item in [y[max_prob_idx], x[max_prob_idx], 1, x_offset[max_prob_idx], y_offset[max_prob_idx]]]) new_frame = ForwardBackwardFrame() new_frame.orig_data = new_data diff --git a/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py b/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py index 94926fd..6d38e0a 100644 --- a/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py +++ b/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py @@ -517,12 +517,12 @@ def _partial_rerun( progress_bar: ProgressBar ) -> Tuple[Pose, Iterable[int]]: - # For each changed frame and each body part, take the maximum probability coordinates and set them to one - for (frame_idx, bp_idx), frame in changed_frames.items(): - max_prob_coord = np.unravel_index(frame.frame_probs.argmax(), frame.frame_probs.shape) - new_frame_probs = np.zeros_like(frame.frame_probs) #copy because this is read only - new_frame_probs[max_prob_coord] = 1 - frame.frame_probs = new_frame_probs + # # For each changed frame and each body part, take the maximum probability coordinates and set them to one + # for (frame_idx, bp_idx), frame in changed_frames.items(): + # max_prob_coord = np.unravel_index(frame.frame_probs.argmax(), frame.frame_probs.shape) + # new_frame_probs = np.zeros_like(frame.frame_probs) #copy because this is read only + # new_frame_probs[max_prob_coord] = 1 + # frame.frame_probs = new_frame_probs # Determine what segments have been manipulated... segment_indexes = sorted({np.searchsorted(self._segments[:, 1], f_i, "right") for f_i, b_i in changed_frames}) From ffddaf677894b6b6833569bd8ce457f7e4473957 Mon Sep 17 00:00:00 2001 From: Daphne Demekas Date: Tue, 20 Feb 2024 10:43:12 -0700 Subject: [PATCH 6/8] new version number --- diplomat/__init__.py | 2 +- .../supervised_sfpe/supervised_segmented_frame_pass_engine.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/diplomat/__init__.py b/diplomat/__init__.py index db43578..2e166b8 100644 --- a/diplomat/__init__.py +++ b/diplomat/__init__.py @@ -2,7 +2,7 @@ A tool providing multi-animal tracking capabilities on top of other Deep learning based tracking software. """ -__version__ = "0.1.0" +__version__ = "0.1.1" # Can be used by functions to determine if diplomat was invoked through it's CLI interface. CLI_RUN = False diff --git a/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py b/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py index 6d38e0a..2db0d92 100644 --- a/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py +++ b/diplomat/predictors/supervised_sfpe/supervised_segmented_frame_pass_engine.py @@ -517,6 +517,8 @@ def _partial_rerun( progress_bar: ProgressBar ) -> Tuple[Pose, Iterable[int]]: + #TODO : delete below lines, not doing as expected + # # For each changed frame and each body part, take the maximum probability coordinates and set them to one # for (frame_idx, bp_idx), frame in changed_frames.items(): # max_prob_coord = np.unravel_index(frame.frame_probs.argmax(), frame.frame_probs.shape) From 3044b8b4be3fce8869c7607f9a642537b2d30aeb Mon Sep 17 00:00:00 2001 From: Daphne Demekas Date: Wed, 21 Feb 2024 10:03:33 -0700 Subject: [PATCH 7/8] change issue with gear button --- diplomat/wx_gui/settings_dialog.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/diplomat/wx_gui/settings_dialog.py b/diplomat/wx_gui/settings_dialog.py index 2418ff7..bb0d413 100644 --- a/diplomat/wx_gui/settings_dialog.py +++ b/diplomat/wx_gui/settings_dialog.py @@ -6,6 +6,7 @@ import wx from diplomat.processing import Config from diplomat.wx_gui.labeler_lib import SettingWidget, SettingCollection, SettingCollectionWidget +import platform class DropDown(SettingWidget): @@ -46,7 +47,7 @@ def get_new_widget(self, parent=None) -> wx.Control: # Check if the platform is Windows if platform.system() != 'Windows': # If not Windows, add the style flag to kwargs - self.kwargs['style'] = wx.LB_SINGLE + self._kwargs['style'] = wx.LB_SINGLE text_list = wx.Choice(parent, choices=self._option_names, **self._kwargs) text_list.SetSelection(self._default) From 20e32c0f451a828c292b32c96677858af0d4c555 Mon Sep 17 00:00:00 2001 From: Daphne Demekas Date: Tue, 5 Mar 2024 10:42:21 -0700 Subject: [PATCH 8/8] add comments --- .../fpe/frame_passes/mit_viterbi.py | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/diplomat/predictors/fpe/frame_passes/mit_viterbi.py b/diplomat/predictors/fpe/frame_passes/mit_viterbi.py index e82fbf4..b14bfaa 100644 --- a/diplomat/predictors/fpe/frame_passes/mit_viterbi.py +++ b/diplomat/predictors/fpe/frame_passes/mit_viterbi.py @@ -72,6 +72,9 @@ def __init__( self._enter_exit_prob = to_log_space(enter_exit_prob) self._enter_stay_prob = to_log_space(enter_stay_prob) + """The ViterbiTransitionTable class is used to manage transition probabilities, + including those modified by the dominance relationship and the "flat-topped" Gaussian distribution.""" + @staticmethod def _is_enter_state(coords: Coords) -> bool: return len(coords[0]) == 1 and np.isneginf(coords[0][0]) @@ -132,6 +135,7 @@ def _init_gaussian_table(self, metadata: AttributeDict): else: self._scaled_std = (std if (std != "auto") else 1) / metadata.down_scaling + #flat topped gaussian self._flatten_std = None if (conf.gaussian_plateau is None) else self._scaled_std * conf.gaussian_plateau self._gaussian_table = norm(fpe_math.gaussian_table( self.height, self.width, self._scaled_std, conf.amplitude, @@ -153,6 +157,27 @@ def _init_gaussian_table(self, metadata: AttributeDict): metadata.include_soft_domination = self.config.include_soft_domination def _init_skeleton(self, data: ForwardBackwardData): + """If skeleton data is available, this function initializes the skeleton tables, + which are used to enhance tracking by considering the structural + relationships between different body parts. + + The _skeleton_tables is a StorageGraph object that stores the relationship between different body parts + as defined in the skeleton data from the metadata. Each entry in this table represents a connection + between two body parts (nodes) and contains the statistical data (bin_val, freq, avg) related to that connection. + This data is used to enhance tracking accuracy by considering the structural relationships between body parts. + + Specifically, it stores: + # - The names of the nodes (body parts) involved in the skeleton structure. + # - A matrix for each pair of connected nodes, which is computed based on the skeleton formula. This matrix + # represents the likelihood of transitioning from one body part to another, taking into account the average + # distance and frequency of such transitions as observed in the training data. + # - The configuration parameters used for calculating these matrices, which include adjustments for log space + # calculations and other statistical considerations. + # This structure is crucial for the Viterbi algorithm to accurately model the movement and relationships + # between different parts of the body during tracking. + + """ + if("skeleton" in data.metadata): meta = data.metadata self._skeleton_tables = StorageGraph(meta.skeleton.node_names()) @@ -188,6 +213,10 @@ def run_pass( in_place: bool = True, reset_bar: bool = True ) -> ForwardBackwardData: + """ + This is the main function that orchestrates the forward and backward passes of the Viterbi algorithm. + It initializes the necessary tables and states, then runs the forward pass to calculate probabilities, + followed by a backtrace to determine the most probable paths.""" with warnings.catch_warnings(): warnings.filterwarnings("ignore") if("fixed_frame_index" not in fb_data.metadata): @@ -343,6 +372,9 @@ def _run_backtrace( @staticmethod def _get_pool(): # Check globals for a pool... + """This function sets up a multiprocessing pool for parallel processing, + improving the efficiency of the algorithm by allowing it to process + multiple parts of the frame or multiple frames simultaneously.""" if(FramePass.GLOBAL_POOL is not None): return FramePass.GLOBAL_POOL @@ -431,6 +463,42 @@ def _compute_backtrace_step( soft_dom_weight: float = 0, skeleton_weight: float = 0 ) -> List[np.ndarray]: + """This method is responsible for computing the transition probabilities from the prior maximum locations + (highest probability states) of all body parts in the prior frame to the current frame's states. + It's where the algorithm determines the most probable path that leads to each pixel + in the current frame based on the accumulated probabilities from previous frames. + + Parameters + prior: A list of lists containing tuples. + Each tuple represents the probability and coordinates (x, y) of the prior maximum locations + for all body parts in the prior frame. + This data structure allows the method to consider multiple potential origins for each body part's current position. + + current: A list of tuples containing the probability and coordinates (x, y) of the current frame's states + This represents the possible current positions and their associated probabilities. + + bp_idx: The index of the body part being processed. This is used to identify which part of the data corresponds to the current body part in multi-body part tracking scenarios. + + metadata: The metadata from the ForwardBackwardData object. + An AttributeDict containing metadata that might be necessary for the computation, such as configuration parameters or additional data needed for probability calculations. + + transition_function: A function or callable object that calculates the transition probabilities between states. This is crucial for determining how likely it is to move from one state to another. + + resist_transition_function: A function or callable object that calculates the resistance to transitioning between states. + Similar to transition_function, but used for calculating resistive transitions, which might be part of handling interactions between different tracked objects or body parts. + + skeleton_table: A StorageGraph object that stores the relationship between different body parts as defined in the skeleton data from the metadata. + An optional parameter that, if provided, contains skeleton information that can be used to enhance the tracking by considering the structural relationships between different body parts. + + soft_dom_weight: A float representing the weight of the soft domination factor. + + skeleton_weight: A float representing the weight of the skeleton factor. + + """ + + # If skeleton information is available, the method first computes the influence of skeletal connections + # on the transition probabilities. + # This involves considering the structural relationships between body parts and adjusting probabilities accordingly. skel_res = cls._compute_from_skeleton( prior, current, @@ -439,6 +507,11 @@ def _compute_backtrace_step( skeleton_table ) + #The method then calculates the effect of soft domination, + # which is a technique used to handle the dominance relationship between different paths. + # This step adjusts the probabilities to favor more likely paths and suppress less likely ones, + # based on the configured soft domination weight. + from_soft_dom = cls._compute_soft_domination( prior, current, @@ -447,12 +520,22 @@ def _compute_backtrace_step( resist_transition_function, ) + #The core of the method involves calculating the transition probabilities from the prior states to the current states. + # This is done using the transition_function, which takes into account the distances between states and other factors + # to determine how likely it is to transition from one state to another. trans_res = cls.log_viterbi_between( current, prior[bp_idx], transition_function ) + #The calculated probabilities from the skeleton influence, soft domination, + # and direct transitions are then combined to determine the overall probability of transitioning + # to each current state from the prior states. + # This involves weighting each component according to the configured weights and summing them up to get the final probabilities. + + #Normalization: Finally, the probabilities are normalized to ensure they are within a valid range + # and to facilitate comparison between different paths. return norm_together([ t + s * skeleton_weight + d * soft_dom_weight for t, s, d in zip(trans_res, skel_res, from_soft_dom) ]) @@ -527,6 +610,8 @@ def _compute_from_skeleton( merge_internal: Callable[[np.ndarray, int], np.ndarray] = np.max, merge_results: bool = True ) -> Union[List[Tuple[int, List[NumericArray]]], List[NumericArray]]: + + #TODO: Add docstring and notes in coda if(skeleton_table is None): return [0] * len(current_data) if(merge_results) else [] @@ -597,6 +682,45 @@ def _compute_soft_domination( merge_internal: Callable[[np.ndarray, int], np.ndarray] = np.max, merge_results: bool = True ) -> Union[List[Tuple[int, List[NumericArray]]], List[NumericArray]]: + """ + Computes the soft domination for a given body part across frames, considering prior and current data. + + This method calculates the soft domination values by comparing the probabilities of a body part being in + a certain state in the current frame against its probabilities in the prior frames. It uses a transition + function to determine the likelihood of transitioning from each state in the prior frames to each state in + the current frame. The results are merged using specified merging functions to find the most probable state + transitions. This method can optionally merge the results across all body parts to find the overall most + probable states. + + Parameters: + - prior: Union[List[ForwardBackwardFrame], List[List[Tuple[np.ndarray, Tuple[np.ndarray, np.ndarray]]]]] + The prior frame data or computed probabilities and coordinates for each body part. + - current_data: List[Tuple[np.ndarray, Tuple[np.ndarray, np.ndarray]]] + The current frame data including probabilities and coordinates for each body part. + - bp_idx: int + The index of the body part being processed. + - metadata: AttributeDict + Metadata containing configuration and state information for the current processing. + - transition_func: Optional[TransitionFunction] + The function used to compute the transition probabilities between states. + - merge_arrays: Callable[[Iterable[np.ndarray]], np.ndarray] + A function to merge arrays of probabilities from different transitions. + - merge_internal: Callable[[np.ndarray, int], np.ndarray] + A function to merge probabilities within a single transition. + - merge_results: bool + A flag indicating whether to merge the results across all body parts. + + Returns: + - Union[List[Tuple[int, List[NumericArray]]], List[NumericArray]]: + The computed soft domination values for the specified body part, either as a list of numeric arrays + (if merge_results is False) or as a list of tuples containing the body part index and the list of + numeric arrays (if merge_results is True). + + This method is crucial for optimizing the Viterbi path selection by considering not only the most probable + paths but also how these paths compare when considering potential transitions from prior states. It helps + in refining the selection of paths that are not only probable in isolation but also in the context of the + sequence of frames being analyzed. + """ if(transition_func is None or metadata.num_outputs <= 1): return [0] * len(current_data) if(merge_results) else [] @@ -669,6 +793,12 @@ def _compute_normal_frame( soft_dom_weight: float = 0, skeleton_weight: float = 0 ) -> List[ForwardBackwardFrame]: + + """processes a single frame in the context of tracking multiple body parts or individuals, + calculating the probabilities of each body part being in each position based on prior information, + current observations, and various transition models. + It integrates several key concepts, including handling occlusions, leveraging skeleton information, + and applying soft domination to refine the tracking process.""" group_range = range( bp_group * metadata.num_outputs, (bp_group + 1) * metadata.num_outputs @@ -814,6 +944,21 @@ def log_viterbi_between( merge_arrays: Callable[[Iterable[np.ndarray]], np.ndarray] = np.maximum.reduce, merge_internal: Callable[[np.ndarray, int], np.ndarray] = np.nanmax ) -> List[np.ndarray]: + """ + This method calculates the transition probabilities between the prior and current data points for each body part. + It utilizes a transition function to compute the probabilities of moving from each prior state to each current state. + The method then merges these probabilities across all body parts to determine the most likely transitions. + + Parameters: + - current_data: A sequence of tuples containing the current probabilities and coordinates for each body part. + - prior_data: A sequence of tuples containing the prior probabilities and coordinates for each body part. + - transition_function: A callable that computes the transition probabilities between prior and current states. + - merge_arrays: A callable that merges arrays of probabilities across all body parts. + - merge_internal: A callable that merges probabilities within each body part. + + Returns: + A list of numpy arrays representing the merged transition probabilities for each body part. + """ return [ merge_arrays([ merge_internal( @@ -851,6 +996,11 @@ def generate_occluded( @classmethod def get_config_options(cls) -> ConfigSpec: # Class to enforce that probabilities are between 0 and 1.... + """This function returns a dictionary of configuration options that can be adjusted to + customize the behavior of the algorithm. + These options include parameters for the Gaussian distribution, + probabilities for obscured and edge states, + and weights for the dominance relationship and skeleton data.""" return { "standard_deviation": ( "auto", tc.Union(float, tc.Literal("auto")),