add pretrained backbone on human hand dataset

docs/source/user_guide/config_file.rst

@@ -103,11 +103,11 @@ Below is a list of some commonly modified arguments related to model architectur
     * resnet50_animal_apose: ResNet-50 pretrained on an animal pose dataset (Cao et al 2019, Cross-Domain Adaptation for Animal Pose Estimation)
     * resnet50_human_jhmdb: ResNet-50 pretrained on JHMDB dataset (Jhuang et al 2013, Towards Understanding Action Recognition)
     * resnet50_human_res_rle: ResNet-50 pretrained on MPii dataset (Andriluka et al 2014, 2D Human Pose Estimation: New Benchmark and State of the Art Analysis)
+    * resnet50_human_hand: ResNet-50 pretrained on OneHand10k dataset (Wang et al 2018, Mask-pose Cascaded CNN for 2d Hand Pose Estimation from Single Color Image)
     * efficientnet_b0: EfficientNet-B0 pretrained on ImageNet
     * efficientnet_b1: EfficientNet-B1 pretrained on ImageNet
     * efficientnet_b2: EfficientNet-B2 pretrained on ImageNet
     * vit_b_sam: Segment Anything Model (Vision Transformer Base)
-    * vit_h_sam: Segment Anything Model (Vision Transformer Huge)
 
 See the :ref:`Unsupervised losses <unsupervised_losses>` section for more details on the various
 losses and their associated hyperparameters.

lightning_pose/models/backbones/torchvision.py

@@ -74,6 +74,8 @@ def build_backbone(
             ckpt_url = "https://download.openmmlab.com/mmpose/top_down/deeppose/deeppose_res50_mpii_256x256_rle-5f92a619_20220504.pth"  # noqa: E501
         elif backbone_type == "top_res":
             ckpt_url = "https://download.openmmlab.com/mmpose/top_down/resnet/res50_mpii_256x256-418ffc88_20200812.pth"  # noqa: E501
+        elif backbone_type == "hand":
+            ckpt_url = "https://download.openmmlab.com/mmpose/hand/resnet/res50_onehand10k_256x256-739c8639_20210330.pth"  # noqa: E501
 
         state_dict = torch.hub.load_state_dict_from_url(ckpt_url)["state_dict"]
         new_state_dict = OrderedDict()

lightning_pose/models/base.py

@@ -42,6 +42,7 @@
     "resnet50_human_jhmdb",
     "resnet50_human_res_rle",
     "resnet50_human_top_res",
+    "resnet50_human_hand",
     "efficientnet_b0",
     "efficientnet_b1",
     "efficientnet_b2",

scripts/configs/config_crim13.yaml

@@ -87,19 +87,12 @@ training:
 
 model:
   # list of unsupervised losses
-  # "pca_singleview" | "pca_multiview" | "temporal" | "unimodal_mse" | "unimodal_kl"
   losses_to_use: []
-  # backbone network:
-  # resnet18 | resnet34 | resnet50 | resnet101 | resnet152 | resnet50_contrastive
-  # resnet50_animalpose_apose | resnet50_animal_ap10k
-  # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res
-  # efficientnet_b0 | efficientnet_b1 | efficientnet_b2
-  # vit_b_sam | vit_h_sam
+  # backbone network
   backbone: resnet50_animal_ap10k
   # prediction mode: regression | heatmap | heatmap_mhcrnn (context)
   model_type: heatmap
   # which heatmap loss to use
-  # "mse" | "kl" | "js"
   heatmap_loss_type: mse
   # directory name for model saving
   model_name: rebuttal23a

scripts/configs/config_default.yaml

@@ -86,8 +86,8 @@ model:
   losses_to_use: []
   # backbone network:
   # resnet18 | resnet34 | resnet50 | resnet101 | resnet152 | resnet50_contrastive
-  # resnet50_animalpose_apose | resnet50_animal_ap10k
-  # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res
+  # resnet50_animal_apose | resnet50_animal_ap10k
+  # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res | resnet50_human_hand
   # efficientnet_b0 | efficientnet_b1 | efficientnet_b2
   # vit_b_sam | vit_h_sam
   backbone: resnet50_animal_ap10k

scripts/configs/config_ibl-paw.yaml

@@ -73,18 +73,12 @@ training:
 
 model:
   # list of unsupervised losses
-  # "pca_singleview" | "pca_multiview" | "temporal" | "unimodal_mse"
   losses_to_use: []
-  # backbone network:
-  # resnet18 | resnet34 | resnet50 | resnet101 | resnet152 | resnet50_contrastive
-  # resnet50_animalpose_apose | resnet50_animal_ap10k
-  # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res
-  # efficientnet_b0 | efficientnet_b1 | efficientnet_b2
+  # backbone network
   backbone: resnet50_animal_ap10k
   # prediction mode: regression | heatmap | heatmap_mhcrnn (context)
   model_type: heatmap
   # which heatmap loss to use
-  # "mse" | "kl" | "js"
   heatmap_loss_type: mse
   # tt expt name
   model_name: rebuttal23a

scripts/configs/config_ibl-pupil.yaml

@@ -75,18 +75,12 @@ training:
 
 model:
   # list of unsupervised losses
-  # "pca_singleview" | "pca_multiview" | "temporal" | "unimodal_mse" | ""
   losses_to_use: []
-  # backbone network:
-  # resnet18 | resnet34 | resnet50 | resnet101 | resnet152 | resnet50_contrastive
-  # resnet50_animalpose_apose | resnet50_animal_ap10k
-  # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res
-  # efficientnet_b0 | efficientnet_b1 | efficientnet_b2
+  # backbone network
   backbone: resnet50_animal_ap10k
   # prediction mode: regression | heatmap | heatmap_mhcrnn (context)
   model_type: heatmap
   # which heatmap loss to use
-  # "mse" | "kl" | "js"
   heatmap_loss_type: mse
   # tt expt name
   model_name: rebuttal23a

scripts/configs/config_mirror-fish.yaml

@@ -134,18 +134,12 @@ training:
 
 model:
   # list of unsupervised losses
-  # "pca_singleview" | "pca_multiview" | "temporal" | "unimodal" | ""
   losses_to_use: []
-  # backbone network:
-  # resnet18 | resnet34 | resnet50 | resnet101 | resnet152 | resnet50_contrastive
-  # resnet50_animalpose_apose | resnet50_animal_ap10k
-  # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res
-  # efficientnet_b0 | efficientnet_b1 | efficientnet_b2
+  # backbone network
   backbone: resnet50  # DO NOT USE resnet50_animal_ap10k
   # prediction mode: regression | heatmap | heatmap_mhcrnn (context)
   model_type: heatmap
   # which heatmap loss to use
-  # "mse" | "wasserstein"
   heatmap_loss_type: mse
   # tt expt name
   model_name: rebuttal23a

scripts/configs/config_mirror-mouse-example.yaml

@@ -107,15 +107,11 @@ model:
   # - "unimodal_kl"
   # - null: if null, assume a model is supervised.
   losses_to_use: ["pca_singleview"]
-  # backbone network:
-  # resnet18 | resnet34 | resnet50 | resnet101 | resnet152 | resnet50_contrastive
-  # resnet50_animalpose_apose | resnet50_animal_ap10k
-  # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res
-  # efficientnet_b0 | efficientnet_b1 | efficientnet_b2
-  backbone: resnet50
+  # backbone network
+  backbone: resnet50_animal_ap10k
   # prediction mode: regression | heatmap | heatmap_mhcrnn (context)
   model_type: heatmap
-  # which heatmap loss to use: mse | kl | js
+  # which heatmap loss to use
   heatmap_loss_type: mse
   # directory name for model saving
   model_name: my_base_toy_model

scripts/configs/config_mirror-mouse.yaml

@@ -98,18 +98,12 @@ training:
 
 model:
   # list of unsupervised losses
-  # "pca_singleview" | "pca_multiview" | "temporal" | "unimodal_mse" | "unimodal_kl"
   losses_to_use: []
-  # backbone network:
-  # resnet18 | resnet34 | resnet50 | resnet101 | resnet152 | resnet50_contrastive
-  # resnet50_animalpose_apose | resnet50_animal_ap10k
-  # resnet50_human_jhmdb | resnet50_human_res_rle | resnet50_human_top_res
-  # efficientnet_b0 | efficientnet_b1 | efficientnet_b2
+  # backbone network
   backbone: resnet50_animal_ap10k
   # prediction mode: regression | heatmap | heatmap_mhcrnn (context)
   model_type: heatmap
   # which heatmap loss to use
-  # "mse" | "kl" | "js"
   heatmap_loss_type: mse
   # tt expt name
   model_name: rebuttal23a