diff --git a/docs/api/lightning_pose.utils.fiftyone.FiftyOneFactory.rst b/docs/api/lightning_pose.utils.fiftyone.FiftyOneFactory.rst deleted file mode 100644 index 960baba7..00000000 --- a/docs/api/lightning_pose.utils.fiftyone.FiftyOneFactory.rst +++ /dev/null @@ -1,17 +0,0 @@ -FiftyOneFactory -=============== - -.. currentmodule:: lightning_pose.utils.fiftyone - -.. autoclass:: FiftyOneFactory - :show-inheritance: - - .. rubric:: Methods Summary - - .. autosummary:: - - ~FiftyOneFactory.__call__ - - .. rubric:: Methods Documentation - - .. automethod:: __call__ diff --git a/docs/api/lightning_pose.utils.fiftyone.FiftyOneImagePlotter.rst b/docs/api/lightning_pose.utils.fiftyone.FiftyOneImagePlotter.rst index f77f678f..d5dabf36 100644 --- a/docs/api/lightning_pose.utils.fiftyone.FiftyOneImagePlotter.rst +++ b/docs/api/lightning_pose.utils.fiftyone.FiftyOneImagePlotter.rst @@ -11,19 +11,39 @@ FiftyOneImagePlotter .. autosummary:: ~FiftyOneImagePlotter.image_paths + ~FiftyOneImagePlotter.img_height + ~FiftyOneImagePlotter.img_width + ~FiftyOneImagePlotter.model_names + ~FiftyOneImagePlotter.num_keypoints .. rubric:: Methods Summary .. autosummary:: + ~FiftyOneImagePlotter.build_single_frame_keypoints ~FiftyOneImagePlotter.create_dataset + ~FiftyOneImagePlotter.dataset_info_print ~FiftyOneImagePlotter.get_gt_keypoints_list + ~FiftyOneImagePlotter.get_keypoints_per_image + ~FiftyOneImagePlotter.get_model_abs_paths + ~FiftyOneImagePlotter.get_pred_keypoints_dict + ~FiftyOneImagePlotter.load_model_predictions .. rubric:: Attributes Documentation .. autoattribute:: image_paths + .. autoattribute:: img_height + .. autoattribute:: img_width + .. autoattribute:: model_names + .. autoattribute:: num_keypoints .. rubric:: Methods Documentation + .. automethod:: build_single_frame_keypoints .. automethod:: create_dataset + .. automethod:: dataset_info_print .. automethod:: get_gt_keypoints_list + .. automethod:: get_keypoints_per_image + .. automethod:: get_model_abs_paths + .. automethod:: get_pred_keypoints_dict + .. automethod:: load_model_predictions diff --git a/docs/api/lightning_pose.utils.fiftyone.FiftyOneKeypointBase.rst b/docs/api/lightning_pose.utils.fiftyone.FiftyOneKeypointBase.rst deleted file mode 100644 index 17f52e09..00000000 --- a/docs/api/lightning_pose.utils.fiftyone.FiftyOneKeypointBase.rst +++ /dev/null @@ -1,47 +0,0 @@ -FiftyOneKeypointBase -==================== - -.. currentmodule:: lightning_pose.utils.fiftyone - -.. autoclass:: FiftyOneKeypointBase - :show-inheritance: - - .. rubric:: Attributes Summary - - .. autosummary:: - - ~FiftyOneKeypointBase.build_speed - ~FiftyOneKeypointBase.img_height - ~FiftyOneKeypointBase.img_width - ~FiftyOneKeypointBase.model_names - ~FiftyOneKeypointBase.num_keypoints - - .. rubric:: Methods Summary - - .. autosummary:: - - ~FiftyOneKeypointBase.build_single_frame_keypoints - ~FiftyOneKeypointBase.create_dataset - ~FiftyOneKeypointBase.dataset_info_print - ~FiftyOneKeypointBase.get_keypoints_per_image - ~FiftyOneKeypointBase.get_model_abs_paths - ~FiftyOneKeypointBase.get_pred_keypoints_dict - ~FiftyOneKeypointBase.load_model_predictions - - .. rubric:: Attributes Documentation - - .. autoattribute:: build_speed - .. autoattribute:: img_height - .. autoattribute:: img_width - .. autoattribute:: model_names - .. autoattribute:: num_keypoints - - .. rubric:: Methods Documentation - - .. automethod:: build_single_frame_keypoints - .. automethod:: create_dataset - .. automethod:: dataset_info_print - .. automethod:: get_keypoints_per_image - .. automethod:: get_model_abs_paths - .. automethod:: get_pred_keypoints_dict - .. automethod:: load_model_predictions diff --git a/docs/api/lightning_pose.utils.fiftyone.FiftyOneKeypointVideoPlotter.rst b/docs/api/lightning_pose.utils.fiftyone.FiftyOneKeypointVideoPlotter.rst deleted file mode 100644 index b9c81c84..00000000 --- a/docs/api/lightning_pose.utils.fiftyone.FiftyOneKeypointVideoPlotter.rst +++ /dev/null @@ -1,19 +0,0 @@ -FiftyOneKeypointVideoPlotter -============================ - -.. currentmodule:: lightning_pose.utils.fiftyone - -.. autoclass:: FiftyOneKeypointVideoPlotter - :show-inheritance: - - .. rubric:: Methods Summary - - .. autosummary:: - - ~FiftyOneKeypointVideoPlotter.check_inputs - ~FiftyOneKeypointVideoPlotter.create_dataset - - .. rubric:: Methods Documentation - - .. automethod:: check_inputs - .. automethod:: create_dataset diff --git a/docs/api/lightning_pose.utils.fiftyone.check_unique_tags.rst b/docs/api/lightning_pose.utils.fiftyone.check_unique_tags.rst deleted file mode 100644 index 8601918f..00000000 --- a/docs/api/lightning_pose.utils.fiftyone.check_unique_tags.rst +++ /dev/null @@ -1,6 +0,0 @@ -check_unique_tags -================= - -.. currentmodule:: lightning_pose.utils.fiftyone - -.. autofunction:: check_unique_tags diff --git a/docs/source/faqs.rst b/docs/source/faqs.rst index e341d157..50665d9c 100644 --- a/docs/source/faqs.rst +++ b/docs/source/faqs.rst @@ -24,3 +24,27 @@ Note that both semi-supervised and context models will increase memory usage If you encounter this error, reduce batch sizes during training or inference. You can find the relevant parameters to adjust in :ref:`The configuration file ` section. + +.. _faq_nan_heatmaps: + +**Q: Why does the network produce high confidence values for keypoints even when they are occluded?** + +Generally, when a keypoint is briefly occluded and its location can be resolved by the network, we are fine with +high confidence values (this will happen, for example, when using temporal context frames). +However, there may be scenarios where the goal is to explicitly track whether a keypoint is visible or hidden using +confidence values (e.g., quantifying whether a tongue is in or out of the mouth). +In this case, if the confidence values are too high during occlusions, try the suggestions below. + +First, note that including a keypoint in the unsupervised losses - especially the PCA losses - +will generally increase confidence values even during occlusions (by design). +If a low confidence value is desired during occlusions, ensure the keypoint in question is not +included in those losses. + +If this does not fix the issue, another option is to set the following field in the config file: +``training.uniform_heatmaps_for_nan_keypoints: true``. +[This field is not visible in the default config but can be added.] +This option will force the model to output a uniform heatmap for any keypoint that does not have +a ground truth label in the training data. +The model will therefore not try to guess where the occluded keypoint is located. +This approach requires a set of training frames that include both visible and occluded examples +of the keypoint in question. diff --git a/docs/source/user_guide/config_file.rst b/docs/source/user_guide/config_file.rst index d2ddab56..0e206700 100644 --- a/docs/source/user_guide/config_file.rst +++ b/docs/source/user_guide/config_file.rst @@ -21,6 +21,25 @@ The config file contains several sections: * ``losses``: hyperparameters for unsupervised losses * ``eval``: paths for video inference and fiftyone app +Data parameters +=============== + +* ``data.imaged_orig_dims.height/width``: the current version of Lightning Pose requires all training images to be the same size. We are working on an updated version without this requirement. However, if you plan to use the PCA losses (Pose PCA or multiview PCA) then all training images **must** be the same size, otherwise the PCA subspace will erroneously contain variance related to image size. + +* ``data.image_resize_dims.height/width``: images (and videos) will be resized to the specified height and width before being processed by the network. Supported values are {64, 128, 256, 384, 512}. The height and width need not be identical. Some points to keep in mind when selecting +these values: if the resized images are too small, you will lose resolution/details; if they are too large, the model takes longer to train and might not train as well. + +* ``data.data_dir/video_dir``: update these to reflect your local paths + +* ``data.num_keypoints``: the number of body parts. If using a mirrored setup, this should be the number of body parts summed across all views. If using a multiview setup, this number should indicate the number of keyponts per view (must be the same across all views). + +* ``data.keypoint_names``: keypoint names should reflect the actual names/order in the csv file. This field is necessary if, for example, you are running inference on a machine that does not have the training data saved on it. + +* ``data.columns_for_singleview_pca``: see the :ref:`Pose PCA documentation ` + +* ``data.mirrored_column_matches``: see the :ref:`Multiview PCA documentation ` + + Model/training parameters ========================= diff --git a/docs/source/user_guide_advanced/unsupervised_losses.rst b/docs/source/user_guide_advanced/unsupervised_losses.rst index 25856456..96f160fd 100644 --- a/docs/source/user_guide_advanced/unsupervised_losses.rst +++ b/docs/source/user_guide_advanced/unsupervised_losses.rst @@ -12,9 +12,9 @@ and brief descriptions of some of the available losses. #. :ref:`Data requirements ` #. :ref:`The configuration file ` #. :ref:`Loss options ` - * :ref:`Temporal continuity ` - * :ref:`Pose plausibility ` - * :ref:`Multiview consistency ` + * :ref:`Temporal difference ` + * :ref:`Pose PCA ` + * :ref:`Multiview PCA ` .. _unsup_data: @@ -122,9 +122,18 @@ losses across multiple datasets, but we encourage users to test out several valu data for best effect. The inverse of this weight is actually used for the final weight, so smaller values indicate stronger penalties. +We are particularly interested in preventing, and having the network learn from, severe violations +of the different losses. +Therefore, we enforce our losses only when they exceed a tolerance threshold :math:`\epsilon`, +rendering them :math:`\epsilon`-insensitive: + +.. math:: + + \mathscr{L}(\epsilon) = \textrm{max}(0, \mathscr{L} - \epsilon). + .. _unsup_loss_temporal: -Temporal continuity +Temporal difference ------------------- This loss penalizes the difference in predictions between successive timepoints for each keypoint independently. @@ -133,16 +142,17 @@ independently. temporal: log_weight: 5.0 - epsilon: 20.0 prob_threshold: 0.05 + epsilon: 20.0 + * ``log_weight``: weight of the loss in the final cost function -* ``epsilon``: in pixels; temporal differences below this threshold are not penalized, which keeps natural movements from being penalized. The value of epsilon will depend on the size of the video frames, framerate (how much does the animal move from one frame to the next), the size of the animal in the frame, etc. * ``prob_threshold``: predictions with a probability below this threshold are not included in the loss. This is desirable if, for example, a keypoint is occluded and the prediction has low probability. +* ``epsilon``: in pixels; temporal differences below this threshold are not penalized, which keeps natural movements from being penalized. The value of epsilon will depend on the size of the video frames, framerate (how much does the animal move from one frame to the next), the size of the animal in the frame, etc. .. _unsup_loss_pcasv: -Pose plausibility +Pose PCA ----------------- This loss penalizes deviations away from a low-dimensional subspace of plausible poses computed on labeled data. @@ -186,7 +196,7 @@ If instead you want to include the ears and tailbase: columns_for_singleview_pca: [1, 2, 4] See -`these config files `_ +`these config files `_ for more examples. Below are the various hyperparameters and their descriptions. @@ -197,19 +207,15 @@ Besides the ``log_weight`` none of the provided values need to be tested for new pca_singleview: log_weight: 5.0 components_to_keep: 0.99 - empirical_epsilon_percentile: 1.00 - empirical_epsilon_multiplier: 1.0 epsilon: null * ``log_weight``: weight of the loss in the final cost function * ``components_to_keep``: predictions should lie within the low-d subspace spanned by components that describe this fraction of variance -* ``empirical_epsilon_percentile``: the reprojecton error on labeled training data is computed to arrive at a noise ceiling; reprojection errors from the video data are not penalized if they fall below this percentile of labeled data error (replaces ``epsilon``) -* ``empirical_epsilon_multiplier``: this allows the user to increase the epsilon relative the the empirical epsilon error; with the multiplier the effective epsilon is `eff_epsilon = percentile(error, empirical_epsilon_percentile) * empirical_epsilon_multiplier` -* ``epsilon``: absolute error (in pixels) below which pca loss is zeroed out; if not null, this parameter takes precedence over ``empirical_epsilon_percentile`` +* ``epsilon``: if not null, this parameter is automatically computed from the labeled data .. _unsup_loss_pcamv: -Multiview consistency +Multiview PCA --------------------- This loss penalizes deviations of predictions across all available views away from a 3-dimensional subspace computed on labeled data. @@ -273,12 +279,8 @@ Besides the ``log_weight`` none of the provided values need to be tested for new pca_multiview: log_weight: 5.0 components_to_keep: 3 - empirical_epsilon_percentile: 1.00 - empirical_epsilon_multiplier: 1.0 epsilon: null * ``log_weight``: weight of the loss in the final cost function -* ``components_to_keep``: predictions should lie within the 3D subspace -* ``empirical_epsilon_percentile``: the reprojecton error on labeled training data is computed to arrive at a noise ceiling; reprojection errors from the video data are not penalized if they fall below this percentile of labeled data error (replaces ``epsilon``) -* ``empirical_epsilon_multiplier``: this allows the user to increase the epsilon relative the the empirical epsilon error; with the multiplier the effective epsilon is `eff_epsilon = percentile(error, empirical_epsilon_percentile) * empirical_epsilon_multiplier` -* ``epsilon``: absolute error (in pixels) below which pca loss is zeroed out; if not null, this parameter takes precedence over ``empirical_epsilon_percentile`` +* ``components_to_keep``: should be set to 3 so that predictions lie within a 3D subspace +* ``epsilon``: if not null, this parameter is automatically computed from the labeled data diff --git a/lightning_pose/data/dali.py b/lightning_pose/data/dali.py index 76efa51d..8fe8ba59 100644 --- a/lightning_pose/data/dali.py +++ b/lightning_pose/data/dali.py @@ -112,7 +112,7 @@ def video_pipe( else: # choose arbitrary scalar (rather than a matrix) so that downstream operations know there # is no geometric transforms to undo - matrix = -1 + matrix = np.array([-1]) # video pixel range is [0, 255]; transform it to [0, 1]. # happens naturally in the torchvision transform to tensor. video = video / 255.0 diff --git a/lightning_pose/utils/fiftyone.py b/lightning_pose/utils/fiftyone.py index 0816bbaa..e5b00b9c 100644 --- a/lightning_pose/utils/fiftyone.py +++ b/lightning_pose/utils/fiftyone.py @@ -1,5 +1,5 @@ import os -from typing import Dict, List, Literal, Optional, Union +from typing import Dict, List, Optional import fiftyone as fo import numpy as np @@ -15,14 +15,10 @@ __all__ = [ "check_lists_equal", "remove_string_w_substring_from_list", - "check_unique_tags", "check_dataset", "get_image_tags", - "FiftyOneKeypointBase", "FiftyOneImagePlotter", - "FiftyOneKeypointVideoPlotter", "dfConverter", - "FiftyOneFactory", ] @@ -39,17 +35,6 @@ def remove_string_w_substring_from_list(strings: List[str], substring: str) -> L return strings -@typechecked -def check_unique_tags(data_pt_tags: List[str]) -> bool: - uniques = list(np.unique(data_pt_tags)) - cond_list = ["test", "train", "validation"] - cond_list_with_unused_images = ["test", "train", "validation", "unused"] - flag = check_lists_equal(uniques, cond_list) or check_lists_equal( - uniques, cond_list_with_unused_images - ) - return flag - - @typechecked def check_dataset(dataset: fo.Dataset) -> None: pretty_print_str("Checking FiftyOne.Dataset by computing metadata... ") @@ -58,9 +43,7 @@ def check_dataset(dataset: fo.Dataset) -> None: except ValueError: print("Encountered error in metadata computation. See print:") print(dataset.exists("metadata", False)) - print( - "The above print should indicate bad image samples, e.g., with bad paths." - ) + print("The above print should indicate bad image samples, e.g., with bad paths.") @typechecked @@ -76,12 +59,13 @@ def get_image_tags(pred_df: pd.DataFrame) -> pd.Series: # #@typechecked # force typechecking over the entire class. right now fails due to some # list/listconfig issue -class FiftyOneKeypointBase: +class FiftyOneImagePlotter: def __init__( self, cfg: DictConfig, keypoints_to_plot: Optional[List[str]] = None, + csv_filename: str = "predictions.csv", ) -> None: self.cfg = cfg @@ -98,9 +82,7 @@ def __init__( ) if self.keypoints_to_plot is None: # plot all keypoints that appear in the ground-truth dataframe - self.keypoints_to_plot: List[str] = list( - self.ground_truth_df.columns.levels[0] - ) + self.keypoints_to_plot: List[str] = list(self.ground_truth_df.columns.levels[0]) # remove "bodyparts" if "bodyparts" in self.keypoints_to_plot: self.keypoints_to_plot.remove("bodyparts") @@ -115,13 +97,29 @@ def __init__( self.gt_data_dict: Dict[str, Dict[str, np.array]] = dfConverter( df=self.ground_truth_df, keypoint_names=self.keypoints_to_plot )() - # self.model_abs_paths = self.get_model_abs_paths() - # - self.pred_csv_files = [] # override in subclasses + + model_abs_paths = self.get_model_abs_paths() + self.pred_csv_files = [ + os.path.join(model_dir, csv_filename) for model_dir in model_abs_paths + ] @property - def build_speed(self) -> str: - return self.cfg.eval.fiftyone.build_speed + def image_paths(self) -> List[str]: + """extract absolute paths for all the images in the ground truth csv file + + Returns: + List[str]: absolute paths per image, checked before returning. + """ + relative_list = list(self.ground_truth_df.iloc[:, 0]) + absolute_list = [ + os.path.join(self.data_dir, im_path) for im_path in relative_list + ] + # assert that the images are indeed files + for im in absolute_list: + if not os.path.isfile(im): + raise FileNotFoundError(im) + + return absolute_list @property def img_width(self) -> int: @@ -161,6 +159,11 @@ def get_model_abs_paths(self) -> List[str]: assert os.path.isdir(mod_path) return model_abs_paths + def get_gt_keypoints_list(self) -> List[fo.Keypoints]: + # for each frame, extract ground-truth keypoint information + print("Collecting ground-truth keypoints...") + return self.get_keypoints_per_image(self.gt_data_dict) + def load_model_predictions(self) -> None: # take the abs paths, and load the models into a dictionary self.model_preds_dict = {} @@ -169,13 +172,10 @@ def load_model_predictions(self) -> None: # assuming that each path of saved logs has a predictions.csv file in it # always assume [1, 2] since our code generated the predictions temp_df = pd.read_csv(pred_csv_file, header=[1, 2]) - self.model_preds_dict[model_name] = dfConverter( - temp_df, self.keypoints_to_plot - )() + self.model_preds_dict[model_name] = dfConverter(temp_df, self.keypoints_to_plot)() self.preds_pandas_df_dict[model_name] = temp_df - # @typechecked - def _slow_single_frame_build( + def build_single_frame_keypoints( self, data_dict: Dict[str, Dict[str, np.array]], frame_idx: int, @@ -198,44 +198,6 @@ def _slow_single_frame_build( ) return keypoints_list - # @typechecked - def _fast_single_frame_build( - self, - data_dict: Dict[str, Dict[str, np.array]], - frame_idx: int, - ) -> List[fo.Keypoint]: - # output: the positions of all keypoints in a single frame for a single model - keypoint = [ - fo.Keypoint( - points=[ - ( - data_dict[kp_name]["coords"][frame_idx, 0] / self.img_width, - data_dict[kp_name]["coords"][frame_idx, 1] / self.img_height, - ) - for kp_name in self.keypoints_to_plot - ], - confidence=[ - data_dict[kp_name]["likelihood"][frame_idx] - for kp_name in self.keypoints_to_plot] - ) - ] - return keypoint - - # have two options here, "fast" and "slow" - # @typechecked - def build_single_frame_keypoints( - self, data_dict: Dict[str, Dict[str, np.array]], frame_idx: int - ) -> List[fo.Keypoint]: - if self.build_speed == "fast": - return self._fast_single_frame_build( - data_dict=data_dict, frame_idx=frame_idx - ) - else: # slow - return self._slow_single_frame_build( - data_dict=data_dict, frame_idx=frame_idx - ) - - # @typechecked def get_keypoints_per_image( self, data_dict: Dict[str, Dict[str, np.array]] ) -> List[fo.Keypoints]: @@ -249,7 +211,6 @@ def get_keypoints_per_image( keypoints_list.append(fo.Keypoints(keypoints=single_frame_keypoints_list)) return keypoints_list - # @typechecked def get_pred_keypoints_dict(self) -> Dict[str, List[fo.Keypoints]]: pred_keypoints_dict = {} # loop over the dictionary with predictions per model @@ -259,50 +220,6 @@ def get_pred_keypoints_dict(self) -> Dict[str, List[fo.Keypoints]]: return pred_keypoints_dict - def create_dataset(self): - # subclasses build their own - raise NotImplementedError - - -# @typechecked -class FiftyOneImagePlotter(FiftyOneKeypointBase): - - def __init__( - self, - cfg: DictConfig, - keypoints_to_plot: Optional[List[str]] = None, - csv_filename: str = "predictions.csv", - ) -> None: - super().__init__(cfg=cfg, keypoints_to_plot=keypoints_to_plot) - - model_abs_paths = self.get_model_abs_paths() - self.pred_csv_files = [ - os.path.join(model_dir, csv_filename) for model_dir in model_abs_paths - ] - - @property - def image_paths(self) -> List[str]: - """extract absolute paths for all the images in the ground truth csv file - - Returns: - List[str]: absolute paths per image, checked before returning. - """ - relative_list = list(self.ground_truth_df.iloc[:, 0]) - absolute_list = [ - os.path.join(self.data_dir, im_path) for im_path in relative_list - ] - # assert that the images are indeed files - for im in absolute_list: - if not os.path.isfile(im): - raise FileNotFoundError(im) - - return absolute_list - - def get_gt_keypoints_list(self) -> List[fo.Keypoints]: - # for each frame, extract ground-truth keypoint information - print("Collecting ground-truth keypoints...") - return self.get_keypoints_per_image(self.gt_data_dict) - def create_dataset(self) -> fo.Dataset: samples = [] # read each model's csv into a pandas dataframe @@ -314,9 +231,7 @@ def create_dataset(self) -> fo.Dataset: gt_keypoints_list = self.get_gt_keypoints_list() # do the same for each model's predictions (lists are stored in a dict) pred_keypoints_dict = self.get_pred_keypoints_dict() - pretty_print_str( - "Appending fiftyone.Keypoints to fiftyone.Sample objects, for each image..." - ) + pretty_print_str("Appending fo.Keypoints to fo.Sample objects for each image...") for img_idx, img_path in enumerate(tqdm(self.image_paths)): # create a "sample" with an image and a tag (should be appended to self.samples) sample = fo.Sample(filepath=img_path, tags=[self.data_tags[img_idx]]) @@ -335,57 +250,6 @@ def create_dataset(self) -> fo.Dataset: return fiftyone_dataset -# @typechecked -class FiftyOneKeypointVideoPlotter(FiftyOneKeypointBase): - - def __init__( - self, - cfg: DictConfig, - keypoints_to_plot: Optional[List[str]] = None, - **kwargs # make robust to other inputs - ) -> None: - # initialize FiftyOneKeypointBase - super().__init__(cfg=cfg, keypoints_to_plot=keypoints_to_plot) - - self.video: str = cfg.eval.video_file_to_plot - self.pred_csv_files: List[str] = self.cfg.eval.pred_csv_files_to_plot - # self.pred_csv_files overrides the attribute in FiftyOneKeypointBase - self.check_inputs() - self.dataset_name = self.dataset_name + "_video" - - def check_inputs(self) -> None: - for f in self.pred_csv_files: - if not os.path.isfile(f): - raise FileNotFoundError(f) - if not os.path.isfile(self.video): - raise FileNotFoundError(self.video) - - def create_dataset(self) -> fo.Dataset: - # read each model's csv into a pandas dataframe, save in self.model_preds_dict - self.load_model_predictions() - # modify the predictions into fiftyone format - pred_keypoints_dict = self.get_pred_keypoints_dict() - # inherited from FiftyOneKeypointBase - dataset = fo.Dataset(self.dataset_name, persistent=True) - # adding _videos so as to not overwrite existing datasets with images. - # NOTE: for now, one sample only in the dataset (one video) - video_sample = fo.Sample(filepath=self.video) - first_model_name = list(pred_keypoints_dict.keys())[0] - pretty_print_str( - "Appending fiftyone.Keypoints to a fiftyone.Sample object for video, for each frame..." - ) - for frame_idx in tqdm(range(len(pred_keypoints_dict[first_model_name]))): - for model_field_name, model_preds in pred_keypoints_dict.items(): - video_sample.frames[frame_idx + 1][ - model_field_name + "_preds" - ] = model_preds[frame_idx] - # fo.Frame(keypoints=model_preds[frame_idx]) raised some issues - pretty_print_str("Adding a fiftyone.Sample to fiftyone.Dataset...") - dataset.add_sample(video_sample) - pretty_print_str("Done!") - return dataset - - # @typechecked class dfConverter: @@ -409,16 +273,3 @@ def __call__(self) -> Dict[str, Dict[str, np.array]]: full_dict[kp_name] = self.dict_per_bp(kp_name) return full_dict - - -# @typechecked -class FiftyOneFactory: - - def __init__(self, dataset_to_create: Literal["images", "videos"]) -> None: - self.dataset_to_create = dataset_to_create - - def __call__(self) -> Union[FiftyOneImagePlotter, FiftyOneKeypointVideoPlotter]: - if self.dataset_to_create == "images": - return FiftyOneImagePlotter - else: - return FiftyOneKeypointVideoPlotter diff --git a/lightning_pose/utils/scripts.py b/lightning_pose/utils/scripts.py index 1b7270ac..05e90be6 100644 --- a/lightning_pose/utils/scripts.py +++ b/lightning_pose/utils/scripts.py @@ -517,7 +517,8 @@ def compute_metrics( loss_type="pca_singleview", data_module=data_module, components_to_keep=cfg.losses.pca_singleview.components_to_keep, - empirical_epsilon_percentile=cfg.losses.pca_singleview.empirical_epsilon_percentile, + empirical_epsilon_percentile=cfg.losses.pca_singleview.get( + "empirical_epsilon_percentile", 1.0), columns_for_singleview_pca=cfg.data.columns_for_singleview_pca, ) # re-fit pca on the labeled data to get params @@ -537,7 +538,8 @@ def compute_metrics( loss_type="pca_multiview", data_module=data_module, components_to_keep=cfg.losses.pca_singleview.components_to_keep, - empirical_epsilon_percentile=cfg.losses.pca_singleview.empirical_epsilon_percentile, + empirical_epsilon_percentile=cfg.losses.pca_singleview.get( + "empirical_epsilon_percentile", 1.0), mirrored_column_matches=cfg.data.mirrored_column_matches, ) # re-fit pca on the labeled data to get params diff --git a/scripts/configs/config_crim13.yaml b/scripts/configs/config_crim13.yaml index 2e7f88be..3ad9466b 100644 --- a/scripts/configs/config_crim13.yaml +++ b/scripts/configs/config_crim13.yaml @@ -126,12 +126,8 @@ losses: log_weight: 5.0 # predictions should lie within the low-d subspace spanned by components that describe this fraction of variance components_to_keep: 0.99 - # percentile of reprojection errors on train data below which pca loss is zeroed out - empirical_epsilon_percentile: 1.00 - # doing eff_epsilon = percentile(error, empirical_epsilon_percentile) * empirical_epsilon_multiplier - empirical_epsilon_multiplier: 1.0 - # absolute error (in pixels) below which pca loss is zeroed out; if not null, this - # parameter takes precedence over `empirical_epsilon_percentile` + # absolute error (in pixels) below which pca loss is zeroed out; if null, an empirical + # epsilon is computed using the labeled data epsilon: null # loss = norm of distance between successive timepoints temporal: @@ -154,7 +150,6 @@ eval: fiftyone: # will be the name of the dataset (Mongo DB) created by FiftyOne. for video dataset, we will append dataset_name + "_video" dataset_name: crim13 - build_speed: slow # "slow"/"fast". "fast" drops keypoint name and confidence information for faster processing. # if you want to manually provide a different model name to be displayed in FiftyOne model_display_names: ["test_model"] # whether to launch the app from the script (True), or from ipython (and have finer control over the outputs) @@ -164,8 +159,6 @@ eval: address: 127.0.0.1 # ip to launch the app on. port: 5151 # port to launch the app on. - # whether to create a "videos" or "images" dataset, since the processes are the same - dataset_to_create: "images" # str with an absolute path to a directory containing videos for prediction. # (it's not absolute just for the toy example) test_videos_directory: /home/zeus/content/data/crim13/videos diff --git a/scripts/configs/config_default.yaml b/scripts/configs/config_default.yaml index 51861e09..dd62a131 100644 --- a/scripts/configs/config_default.yaml +++ b/scripts/configs/config_default.yaml @@ -119,12 +119,8 @@ losses: log_weight: 5.0 # predictions should lie within the low-d subspace spanned by these components components_to_keep: 3 - # percentile of reprojection errors on train data below which pca loss is zeroed out - empirical_epsilon_percentile: 1.00 - # doing eff_epsilon = percentile(error, empirical_epsilon_percentile) * empirical_epsilon_multiplier - empirical_epsilon_multiplier: 1.0 - # absolute error (in pixels) below which pca loss is zeroed out; if not null, this - # parameter takes precedence over `empirical_epsilon_percentile` + # absolute error (in pixels) below which pca loss is zeroed out; if null, an empirical + # epsilon is computed using the labeled data epsilon: null # loss = projection onto the discarded eigenvectors pca_singleview: @@ -132,12 +128,8 @@ losses: log_weight: 5.0 # predictions should lie within the low-d subspace spanned by components that describe this fraction of variance components_to_keep: 0.99 - # percentile of reprojection errors on train data below which pca loss is zeroed out - empirical_epsilon_percentile: 1.00 - # doing eff_epsilon = percentile(error, empirical_epsilon_percentile) * empirical_epsilon_multiplier - empirical_epsilon_multiplier: 1.0 - # absolute error (in pixels) below which pca loss is zeroed out; if not null, this - # parameter takes precedence over `empirical_epsilon_percentile` + # absolute error (in pixels) below which pca loss is zeroed out; if null, an empirical + # epsilon is computed using the labeled data epsilon: null # loss = norm of distance between successive timepoints temporal: @@ -160,7 +152,6 @@ eval: fiftyone: # will be the name of the dataset (Mongo DB) created by FiftyOne. for video dataset, we will append dataset_name + "_video" dataset_name: test - build_speed: slow # "slow"/"fast". "fast" drops keypoint name and confidence information for faster processing. # if you want to manually provide a different model name to be displayed in FiftyOne model_display_names: ["test_model"] # whether to launch the app from the script (True), or from ipython (and have finer control over the outputs) @@ -170,8 +161,6 @@ eval: address: 127.0.0.1 # ip to launch the app on. port: 5151 # port to launch the app on. - # whether to create a "videos" or "images" dataset, since the processes are the same - dataset_to_create: images # str with an absolute path to a directory containing videos for prediction. # set to null to skip automatic video prediction from train_hydra.py script test_videos_directory: null diff --git a/scripts/configs/config_ibl-paw.yaml b/scripts/configs/config_ibl-paw.yaml index 9eeeacda..95c2c1b9 100644 --- a/scripts/configs/config_ibl-paw.yaml +++ b/scripts/configs/config_ibl-paw.yaml @@ -112,12 +112,8 @@ losses: log_weight: 5.0 # predictions should lie within the low-d subspace spanned by components that describe this fraction of variance components_to_keep: 0.99 - # percentile of reprojection errors on train data below which pca loss is zeroed out - empirical_epsilon_percentile: 1.00 - # doing eff_epsilon = percentile(error, empirical_epsilon_percentile) * empirical_epsilon_multiplier - empirical_epsilon_multiplier: 1.0 - # absolute error (in pixels) below which pca loss is zeroed out; if not null, this - # parameter takes precedence over `empirical_epsilon_percentile` + # absolute error (in pixels) below which pca loss is zeroed out; if null, an empirical + # epsilon is computed using the labeled data epsilon: null # loss = norm of distance between successive timepoints temporal: @@ -140,7 +136,6 @@ eval: fiftyone: # will be the name of the dataset (Mongo DB) created by FiftyOne. for video dataset, we will append dataset_name + "_video" dataset_name: ibl-paw - build_speed: slow # "slow"/"fast". "fast" drops keypoint name and confidence information for faster processing. # if you want to manually provide a different model name to be displayed in FiftyOne model_display_names: ["test_model"] # whether to launch the app from the script (True), or from ipython (and have finer control over the outputs) @@ -150,8 +145,6 @@ eval: address: 127.0.0.1 # ip to launch the app on. port: 5151 # port to launch the app on. - # whether to create a "videos" or "images" dataset, since the processes are the same - dataset_to_create: images # str with an absolute path to a directory containing videos for prediction. test_videos_directory: /home/zeus/content/data/ibl-paw/videos_new # str with an absolute path to directory in which you want to save .csv with predictions diff --git a/scripts/configs/config_ibl-pupil.yaml b/scripts/configs/config_ibl-pupil.yaml index d8ebda94..0174befb 100644 --- a/scripts/configs/config_ibl-pupil.yaml +++ b/scripts/configs/config_ibl-pupil.yaml @@ -113,12 +113,8 @@ losses: log_weight: 5.0 # predictions should lie within the low-d subspace spanned by components that describe this fraction of variance components_to_keep: 0.99 - # percentile of reprojection errors on train data below which pca loss is zeroed out - empirical_epsilon_percentile: 1.00 - # doing eff_epsilon = percentile(error, empirical_epsilon_percentile) * empirical_epsilon_multiplier - empirical_epsilon_multiplier: 1.0 - # absolute error (in pixels) below which pca loss is zeroed out; if not null, this - # parameter takes precedence over `empirical_epsilon_percentile` + # absolute error (in pixels) below which pca loss is zeroed out; if null, an empirical + # epsilon is computed using the labeled data epsilon: null # loss = norm of distance between successive timepoints temporal: @@ -142,7 +138,6 @@ eval: fiftyone: # will be the name of the dataset (Mongo DB) created by FiftyOne. for video dataset, we will append dataset_name + "_video" dataset_name: ibl-pupil - build_speed: slow # "slow"/"fast". "fast" drops keypoint name and confidence information for faster processing. # if you want to manually provide a different model name to be displayed in FiftyOne model_display_names: ["test_model"] # whether to launch the app from the script (True), or from ipython (and have finer control over the outputs) @@ -152,8 +147,6 @@ eval: address: 127.0.0.1 # ip to launch the app on. port: 5151 # port to launch the app on. - # whether to create a "videos" or "images" dataset, since the processes are the same - dataset_to_create: images # str with an absolute path to a directory containing videos for prediction. test_videos_directory: /home/zeus/content/data/ibl-pupil/videos_new # str with an absolute path to directory in which you want to save .csv with predictions diff --git a/scripts/configs/config_mirror-fish.yaml b/scripts/configs/config_mirror-fish.yaml index 4705dd81..2c5307e8 100644 --- a/scripts/configs/config_mirror-fish.yaml +++ b/scripts/configs/config_mirror-fish.yaml @@ -172,12 +172,8 @@ losses: log_weight: 5.0 # predictions should lie within the low-d subspace spanned by these components components_to_keep: 3 - # percentile of reprojection errors on train data below which pca loss is zeroed out - empirical_epsilon_percentile: 1.00 - # doing eff_epsilon = percentile(error, empirical_epsilon_percentile) * empirical_epsilon_multiplier - empirical_epsilon_multiplier: 1.0 - # absolute error (in pixels) below which pca loss is zeroed out; if not null, this - # parameter takes precedence over `empirical_epsilon_percentile` + # absolute error (in pixels) below which pca loss is zeroed out; if null, an empirical + # epsilon is computed using the labeled data epsilon: null # loss = projection onto the discarded eigenvectors pca_singleview: @@ -185,12 +181,8 @@ losses: log_weight: 5.0 # predictions should lie within the low-d subspace spanned by components that describe this fraction of variance components_to_keep: 0.99 - # percentile of reprojection errors on train data below which pca loss is zeroed out - empirical_epsilon_percentile: 1.00 - # doing eff_epsilon = percentile(error, empirical_epsilon_percentile) * empirical_epsilon_multiplier - empirical_epsilon_multiplier: 1.0 - # absolute error (in pixels) below which pca loss is zeroed out; if not null, this - # parameter takes precedence over `empirical_epsilon_percentile` + # absolute error (in pixels) below which pca loss is zeroed out; if null, an empirical + # epsilon is computed using the labeled data epsilon: null # loss = norm of distance between successive timepoints temporal: @@ -213,7 +205,6 @@ eval: fiftyone: # will be the name of the dataset (Mongo DB) created by FiftyOne. for video dataset, we will append dataset_name + "_video" dataset_name: mirror-fish - build_speed: slow # "slow"/"fast". "fast" drops keypoint name and confidence information for faster processing. # if you want to manually provide a different model name to be displayed in FiftyOne model_display_names: ["test_model"] # whether to launch the app from the script (True), or from ipython (and have finer control over the outputs) @@ -223,8 +214,6 @@ eval: address: 127.0.0.1 # ip to launch the app on. port: 5151 # port to launch the app on. - # whether to create a "videos" or "images" dataset, since the processes are the same - dataset_to_create: images # str with an absolute path to a directory containing videos for prediction. test_videos_directory: /home/zeus/content/data/mirror-fish/videos_new # str with an absolute path to directory in which you want to save .csv with predictions diff --git a/scripts/configs/config_mirror-mouse-example.yaml b/scripts/configs/config_mirror-mouse-example.yaml index 5501ed20..a62316ed 100644 --- a/scripts/configs/config_mirror-mouse-example.yaml +++ b/scripts/configs/config_mirror-mouse-example.yaml @@ -141,12 +141,8 @@ losses: log_weight: 5.0 # predictions should lie within the low-d subspace spanned by these components components_to_keep: 3 - # percentile of reprojection errors on train data below which pca loss is zeroed out - empirical_epsilon_percentile: 1.00 - # doing eff_epsilon = percentile(error, empirical_epsilon_percentile) * empirical_epsilon_multiplier - empirical_epsilon_multiplier: 1.0 - # absolute error (in pixels) below which pca loss is zeroed out; if not null, this - # parameter takes precedence over `empirical_epsilon_percentile` + # absolute error (in pixels) below which pca loss is zeroed out; if null, an empirical + # epsilon is computed using the labeled data epsilon: null # loss = projection onto the discarded eigenvectors pca_singleview: @@ -154,12 +150,8 @@ losses: log_weight: 5.0 # predictions should lie within the low-d subspace spanned by components that describe this fraction of variance components_to_keep: 0.99 - # percentile of reprojection errors on train data below which pca loss is zeroed out - empirical_epsilon_percentile: 1.00 - # doing eff_epsilon = percentile(error, empirical_epsilon_percentile) * empirical_epsilon_multiplier - empirical_epsilon_multiplier: 1.0 - # absolute error (in pixels) below which pca loss is zeroed out; if not null, this - # parameter takes precedence over `empirical_epsilon_percentile` + # absolute error (in pixels) below which pca loss is zeroed out; if null, an empirical + # epsilon is computed using the labeled data epsilon: null # loss = norm of distance between successive timepoints temporal: @@ -201,7 +193,6 @@ eval: fiftyone: # will be the name of the dataset (Mongo DB) created by FiftyOne. for video dataset, we will append dataset_name + "_video" dataset_name: rick_data_test - build_speed: slow # slow/fast. "fast" drops keypoint name and confidence information for faster processing. # if you want to manually provide a different model name to be displayed in FiftyOne model_display_names: ["test_model"] # whether to launch the app from the script (True), or from ipython (and have finer control over the outputs) @@ -209,8 +200,6 @@ eval: remote: true # for LAI, must be False address: 127.0.0.1 # ip to launch the app on. port: 5151 # port to launch the app on. - # whether to create a "videos" or "images" dataset, since the processes are the same - dataset_to_create: images # str with an absolute path to a directory containing videos for prediction. # (it's not absolute just for the toy example) test_videos_directory: data/mirror-mouse-example/videos diff --git a/scripts/configs/config_mirror-mouse.yaml b/scripts/configs/config_mirror-mouse.yaml index 975632bd..690af338 100644 --- a/scripts/configs/config_mirror-mouse.yaml +++ b/scripts/configs/config_mirror-mouse.yaml @@ -137,12 +137,8 @@ losses: log_weight: 5.0 # predictions should lie within the low-d subspace spanned by these components components_to_keep: 3 - # percentile of reprojection errors on train data below which pca loss is zeroed out - empirical_epsilon_percentile: 1.00 - # doing eff_epsilon = percentile(error, empirical_epsilon_percentile) * empirical_epsilon_multiplier - empirical_epsilon_multiplier: 1.0 - # absolute error (in pixels) below which pca loss is zeroed out; if not null, this - # parameter takes precedence over `empirical_epsilon_percentile` + # absolute error (in pixels) below which pca loss is zeroed out; if null, an empirical + # epsilon is computed using the labeled data epsilon: null # loss = projection onto the discarded eigenvectors pca_singleview: @@ -150,12 +146,8 @@ losses: log_weight: 5.0 # predictions should lie within the low-d subspace spanned by components that describe this fraction of variance components_to_keep: 0.99 - # percentile of reprojection errors on train data below which pca loss is zeroed out - empirical_epsilon_percentile: 1.00 - # doing eff_epsilon = percentile(error, empirical_epsilon_percentile) * empirical_epsilon_multiplier - empirical_epsilon_multiplier: 1.0 - # absolute error (in pixels) below which pca loss is zeroed out; if not null, this - # parameter takes precedence over `empirical_epsilon_percentile` + # absolute error (in pixels) below which pca loss is zeroed out; if null, an empirical + # epsilon is computed using the labeled data epsilon: null # loss = norm of distance between successive timepoints temporal: @@ -178,7 +170,6 @@ eval: fiftyone: # will be the name of the dataset (Mongo DB) created by FiftyOne. for video dataset, we will append dataset_name + "_video" dataset_name: mirror-mouse - build_speed: slow # "slow"/"fast". "fast" drops keypoint name and confidence information for faster processing. # if you want to manually provide a different model name to be displayed in FiftyOne model_display_names: ["test_model"] # whether to launch the app from the script (True), or from ipython (and have finer control over the outputs) @@ -188,8 +179,6 @@ eval: address: 127.0.0.1 # ip to launch the app on. port: 5151 # port to launch the app on. - # whether to create a "videos" or "images" dataset, since the processes are the same - dataset_to_create: images # str with an absolute path to a directory containing videos for prediction. test_videos_directory: /home/zeus/content/data/mirror-mouse/videos # str with an absolute path to directory in which you want to save .csv with predictions diff --git a/scripts/create_fiftyone_dataset.py b/scripts/create_fiftyone_dataset.py index acfb20ef..283d56d1 100755 --- a/scripts/create_fiftyone_dataset.py +++ b/scripts/create_fiftyone_dataset.py @@ -5,24 +5,15 @@ from omegaconf import DictConfig from lightning_pose.utils import pretty_print_str -from lightning_pose.utils.fiftyone import ( - FiftyOneFactory, - FiftyOneImagePlotter, - FiftyOneKeypointVideoPlotter, - check_dataset, -) +from lightning_pose.utils.fiftyone import FiftyOneImagePlotter, check_dataset @hydra.main(config_path="configs", config_name="config_mirror-mouse-example") def build_fo_dataset(cfg: DictConfig) -> None: - pretty_print_str( - "Launching a job that creates %s FiftyOne.Dataset" - % cfg.eval.fiftyone.dataset_to_create - ) - FiftyOneClass = FiftyOneFactory( - dataset_to_create=cfg.eval.fiftyone.dataset_to_create - )() - fo_plotting_instance = FiftyOneClass(cfg=cfg) # initializes everything + + pretty_print_str("Launching a job that creates FiftyOne.Dataset") + + fo_plotting_instance = FiftyOneImagePlotter(cfg=cfg) # initializes everything dataset = fo_plotting_instance.create_dataset() # internally loops over models check_dataset(dataset) # create metadata and print if there are problems fo_plotting_instance.dataset_info_print() # print the name of the dataset diff --git a/scripts/litpose_training_demo.ipynb b/scripts/litpose_training_demo.ipynb index b217940d..b787ffe9 100644 --- a/scripts/litpose_training_demo.ipynb +++ b/scripts/litpose_training_demo.ipynb @@ -903,9 +903,7 @@ "source": [ "# Override the default configs here:\n", "cfg.eval.hydra_paths=[output_directory] # you can add multiple output_directory2, output_directory3 to compare \n", - "cfg.eval.fiftyone.dataset_to_create=\"images\"\n", "cfg.eval.fiftyone.dataset_name=\"lightning-demo-colab\"\n", - "cfg.eval.fiftyone.build_speed=\"fast\"\n", "cfg.eval.fiftyone.model_display_names=[\"semi\"]" ] }, @@ -922,18 +920,10 @@ "outputs": [], "source": [ "import fiftyone as fo\n", - "from lightning_pose.utils.fiftyone import (\n", - " FiftyOneImagePlotter,\n", - " FiftyOneKeypointVideoPlotter,\n", - " check_dataset,\n", - " FiftyOneFactory,\n", - ")\n", + "from lightning_pose.utils.fiftyone import check_dataset, FiftyOneImagePlotter\n", "\n", "# initializes everything\n", - "FiftyOneClass = FiftyOneFactory(\n", - " dataset_to_create=cfg.eval.fiftyone.dataset_to_create\n", - " )()\n", - "fo_plotting_instance = FiftyOneClass(cfg=cfg)\n", + "fo_plotting_instance = FiftyOneImagePlotter(cfg=cfg)\n", "\n", "# internally loops over models\n", "dataset = fo_plotting_instance.create_dataset()\n", diff --git a/tests/utils/test_fiftyone.py b/tests/utils/test_fiftyone.py new file mode 100644 index 00000000..b51609a9 --- /dev/null +++ b/tests/utils/test_fiftyone.py @@ -0,0 +1,38 @@ +"""Test the FiftyOne module.""" + +import os +import shutil + +import pandas as pd + + +def test_fiftyone_image_plotter(cfg, tmpdir): + + from lightning_pose.utils.fiftyone import FiftyOneImagePlotter + + # copy ground truth labels to a "predictions" file + data_dir_abs = os.path.abspath(cfg.data.data_dir) + gt_file = os.path.join(data_dir_abs, cfg.data.csv_file) + model_dir = os.path.join(tmpdir, "date", "time") + os.makedirs(model_dir, exist_ok=True) + pred_file = os.path.join(model_dir, "predictions.csv") + shutil.copyfile(gt_file, pred_file) + + # add final column to "predictions" + df = pd.read_csv(pred_file, header=[0, 1, 2], index_col=0) + df.loc[:, ("set", "", "")] = "train" + df.to_csv(pred_file) + n_preds = df.shape[0] + + # update config + cfg_new = cfg.copy() + cfg_new.data.data_dir = data_dir_abs + cfg_new.eval.model_display_names = ["test_model"] + cfg_new.eval.hydra_paths = [model_dir] + cfg_new.eval.fiftyone.dataset_name = str(tmpdir) # get unique dataset name + + # make fiftyone dataset and check + plotter = FiftyOneImagePlotter(cfg=cfg_new) + dataset = plotter.create_dataset() + + assert len(dataset) == n_preds diff --git a/tests/utils/test_pca.py b/tests/utils/test_pca.py index 05ffb4b7..9e0bcd63 100644 --- a/tests/utils/test_pca.py +++ b/tests/utils/test_pca.py @@ -5,13 +5,10 @@ import torch from lightning.pytorch.utilities import CombinedLoader +from lightning_pose.utils.fiftyone import check_lists_equal from lightning_pose.utils.pca import KeypointPCA -def check_lists_equal(list_1, list_2): - return len(list_1) == len(list_2) and sorted(list_1) == sorted(list_2) - - def test_train_loader_iter(base_data_module_combined): # TODO: this is just messing around with dataloaders