From a165cb859cdff11ba951085208a66e8d15240b48 Mon Sep 17 00:00:00 2001 From: themattinthehatt Date: Mon, 17 Jun 2024 17:32:33 -0400 Subject: [PATCH] add pretrained backbone on human hand dataset --- docs/source/user_guide/config_file.rst | 2 +- lightning_pose/models/backbones/torchvision.py | 2 ++ lightning_pose/models/base.py | 1 + scripts/configs/config_crim13.yaml | 9 +-------- scripts/configs/config_default.yaml | 4 ++-- scripts/configs/config_ibl-paw.yaml | 8 +------- scripts/configs/config_ibl-pupil.yaml | 8 +------- scripts/configs/config_mirror-fish.yaml | 8 +------- scripts/configs/config_mirror-mouse-example.yaml | 10 +++------- scripts/configs/config_mirror-mouse.yaml | 8 +------- 10 files changed, 14 insertions(+), 46 deletions(-) diff --git a/docs/source/user_guide/config_file.rst b/docs/source/user_guide/config_file.rst index ad7d6cce..13bb7bfd 100644 --- a/docs/source/user_guide/config_file.rst +++ b/docs/source/user_guide/config_file.rst @@ -103,11 +103,11 @@ Below is a list of some commonly modified arguments related to model architectur * resnet50_animal_apose: ResNet-50 pretrained on an animal pose dataset (Cao et al 2019, Cross-Domain Adaptation for Animal Pose Estimation) * resnet50_human_jhmdb: ResNet-50 pretrained on JHMDB dataset (Jhuang et al 2013, Towards Understanding Action Recognition) * resnet50_human_res_rle: ResNet-50 pretrained on MPii dataset (Andriluka et al 2014, 2D Human Pose Estimation: New Benchmark and State of the Art Analysis) + * resnet50_human_hand: ResNet-50 pretrained on OneHand10k dataset (Wang et al 2018, Mask-pose Cascaded CNN for 2d Hand Pose Estimation from Single Color Image) * efficientnet_b0: EfficientNet-B0 pretrained on ImageNet * efficientnet_b1: EfficientNet-B1 pretrained on ImageNet * efficientnet_b2: EfficientNet-B2 pretrained on ImageNet * vit_b_sam: Segment Anything Model (Vision Transformer Base) - * vit_h_sam: Segment Anything Model (Vision Transformer Huge) See the :ref:`Unsupervised losses ` section for more details on the various losses and their associated hyperparameters. diff --git a/lightning_pose/models/backbones/torchvision.py b/lightning_pose/models/backbones/torchvision.py index 97e1384a..14a65728 100644 --- a/lightning_pose/models/backbones/torchvision.py +++ b/lightning_pose/models/backbones/torchvision.py @@ -74,6 +74,8 @@ def build_backbone( ckpt_url = "https://download.openmmlab.com/mmpose/top_down/deeppose/deeppose_res50_mpii_256x256_rle-5f92a619_20220504.pth" # noqa: E501 elif backbone_type == "top_res": ckpt_url = "https://download.openmmlab.com/mmpose/top_down/resnet/res50_mpii_256x256-418ffc88_20200812.pth" # noqa: E501 + elif backbone_type == "hand": + ckpt_url = "https://download.openmmlab.com/mmpose/hand/resnet/res50_onehand10k_256x256-739c8639_20210330.pth" # noqa: E501 state_dict = torch.hub.load_state_dict_from_url(ckpt_url)["state_dict"] new_state_dict = OrderedDict() diff --git a/lightning_pose/models/base.py b/lightning_pose/models/base.py index 47523270..1743f2ec 100644 --- a/lightning_pose/models/base.py +++ b/lightning_pose/models/base.py @@ -42,6 +42,7 @@ "resnet50_human_jhmdb", "resnet50_human_res_rle", "resnet50_human_top_res", + "resnet50_human_hand", "efficientnet_b0", "efficientnet_b1", "efficientnet_b2", diff --git a/scripts/configs/config_crim13.yaml b/scripts/configs/config_crim13.yaml index 1e79ba2d..ebd63a95 100644 --- a/scripts/configs/config_crim13.yaml +++ b/scripts/configs/config_crim13.yaml @@ -87,19 +87,12 @@ training: model: # list of unsupervised losses - # "pca_singleview" | "pca_multiview" | "temporal" | "unimodal_mse" | "unimodal_kl" losses_to_use: [] - # backbone network: - # resnet18 | resnet34 | resnet50 | resnet101 | resnet152 | resnet50_contrastive - # resnet50_animalpose_apose | resnet50_animal_ap10k - # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res - # efficientnet_b0 | efficientnet_b1 | efficientnet_b2 - # vit_b_sam | vit_h_sam + # backbone network backbone: resnet50_animal_ap10k # prediction mode: regression | heatmap | heatmap_mhcrnn (context) model_type: heatmap # which heatmap loss to use - # "mse" | "kl" | "js" heatmap_loss_type: mse # directory name for model saving model_name: rebuttal23a diff --git a/scripts/configs/config_default.yaml b/scripts/configs/config_default.yaml index 321231b0..54ad1327 100644 --- a/scripts/configs/config_default.yaml +++ b/scripts/configs/config_default.yaml @@ -86,8 +86,8 @@ model: losses_to_use: [] # backbone network: # resnet18 | resnet34 | resnet50 | resnet101 | resnet152 | resnet50_contrastive - # resnet50_animalpose_apose | resnet50_animal_ap10k - # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res + # resnet50_animal_apose | resnet50_animal_ap10k + # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res | resnet50_human_hand # efficientnet_b0 | efficientnet_b1 | efficientnet_b2 # vit_b_sam | vit_h_sam backbone: resnet50_animal_ap10k diff --git a/scripts/configs/config_ibl-paw.yaml b/scripts/configs/config_ibl-paw.yaml index 35eb5213..69612940 100644 --- a/scripts/configs/config_ibl-paw.yaml +++ b/scripts/configs/config_ibl-paw.yaml @@ -73,18 +73,12 @@ training: model: # list of unsupervised losses - # "pca_singleview" | "pca_multiview" | "temporal" | "unimodal_mse" losses_to_use: [] - # backbone network: - # resnet18 | resnet34 | resnet50 | resnet101 | resnet152 | resnet50_contrastive - # resnet50_animalpose_apose | resnet50_animal_ap10k - # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res - # efficientnet_b0 | efficientnet_b1 | efficientnet_b2 + # backbone network backbone: resnet50_animal_ap10k # prediction mode: regression | heatmap | heatmap_mhcrnn (context) model_type: heatmap # which heatmap loss to use - # "mse" | "kl" | "js" heatmap_loss_type: mse # tt expt name model_name: rebuttal23a diff --git a/scripts/configs/config_ibl-pupil.yaml b/scripts/configs/config_ibl-pupil.yaml index 02caa5b7..4fe99376 100644 --- a/scripts/configs/config_ibl-pupil.yaml +++ b/scripts/configs/config_ibl-pupil.yaml @@ -75,18 +75,12 @@ training: model: # list of unsupervised losses - # "pca_singleview" | "pca_multiview" | "temporal" | "unimodal_mse" | "" losses_to_use: [] - # backbone network: - # resnet18 | resnet34 | resnet50 | resnet101 | resnet152 | resnet50_contrastive - # resnet50_animalpose_apose | resnet50_animal_ap10k - # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res - # efficientnet_b0 | efficientnet_b1 | efficientnet_b2 + # backbone network backbone: resnet50_animal_ap10k # prediction mode: regression | heatmap | heatmap_mhcrnn (context) model_type: heatmap # which heatmap loss to use - # "mse" | "kl" | "js" heatmap_loss_type: mse # tt expt name model_name: rebuttal23a diff --git a/scripts/configs/config_mirror-fish.yaml b/scripts/configs/config_mirror-fish.yaml index 8906fcde..8ede0012 100644 --- a/scripts/configs/config_mirror-fish.yaml +++ b/scripts/configs/config_mirror-fish.yaml @@ -134,18 +134,12 @@ training: model: # list of unsupervised losses - # "pca_singleview" | "pca_multiview" | "temporal" | "unimodal" | "" losses_to_use: [] - # backbone network: - # resnet18 | resnet34 | resnet50 | resnet101 | resnet152 | resnet50_contrastive - # resnet50_animalpose_apose | resnet50_animal_ap10k - # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res - # efficientnet_b0 | efficientnet_b1 | efficientnet_b2 + # backbone network backbone: resnet50 # DO NOT USE resnet50_animal_ap10k # prediction mode: regression | heatmap | heatmap_mhcrnn (context) model_type: heatmap # which heatmap loss to use - # "mse" | "wasserstein" heatmap_loss_type: mse # tt expt name model_name: rebuttal23a diff --git a/scripts/configs/config_mirror-mouse-example.yaml b/scripts/configs/config_mirror-mouse-example.yaml index 0e89bc4b..ac5f3b3b 100644 --- a/scripts/configs/config_mirror-mouse-example.yaml +++ b/scripts/configs/config_mirror-mouse-example.yaml @@ -107,15 +107,11 @@ model: # - "unimodal_kl" # - null: if null, assume a model is supervised. losses_to_use: ["pca_singleview"] - # backbone network: - # resnet18 | resnet34 | resnet50 | resnet101 | resnet152 | resnet50_contrastive - # resnet50_animalpose_apose | resnet50_animal_ap10k - # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res - # efficientnet_b0 | efficientnet_b1 | efficientnet_b2 - backbone: resnet50 + # backbone network + backbone: resnet50_animal_ap10k # prediction mode: regression | heatmap | heatmap_mhcrnn (context) model_type: heatmap - # which heatmap loss to use: mse | kl | js + # which heatmap loss to use heatmap_loss_type: mse # directory name for model saving model_name: my_base_toy_model diff --git a/scripts/configs/config_mirror-mouse.yaml b/scripts/configs/config_mirror-mouse.yaml index 6299dfd7..21df939a 100644 --- a/scripts/configs/config_mirror-mouse.yaml +++ b/scripts/configs/config_mirror-mouse.yaml @@ -98,18 +98,12 @@ training: model: # list of unsupervised losses - # "pca_singleview" | "pca_multiview" | "temporal" | "unimodal_mse" | "unimodal_kl" losses_to_use: [] - # backbone network: - # resnet18 | resnet34 | resnet50 | resnet101 | resnet152 | resnet50_contrastive - # resnet50_animalpose_apose | resnet50_animal_ap10k - # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res - # efficientnet_b0 | efficientnet_b1 | efficientnet_b2 + # backbone network backbone: resnet50_animal_ap10k # prediction mode: regression | heatmap | heatmap_mhcrnn (context) model_type: heatmap # which heatmap loss to use - # "mse" | "kl" | "js" heatmap_loss_type: mse # tt expt name model_name: rebuttal23a