add a validation set, seems to work ok (#53)

r-pad · May 16, 2024 · a9a7a83 · a9a7a83
1 parent 97cdd2b
commit a9a7a83
Show file tree

Hide file tree

Showing 32 changed files with 142 additions and 207 deletions.
diff --git a/configs/commands/rlbench/_eval_metrics.yaml b/configs/commands/rlbench/_eval_metrics.yaml
@@ -7,5 +7,5 @@ defaults:
 
 dm:
   train_dset:
-    num_points: 256
+    num_points: 512
     synthetic_occlusion: False
diff --git a/configs/commands/rlbench/_train.yaml b/configs/commands/rlbench/_train.yaml
@@ -19,5 +19,5 @@ training:
 
 dm:
   train_dset:
-    num_points: 256
+    num_points: 512
     synthetic_occlusion: False
diff --git a/configs/commands/rlbench/pick_and_lift/taxpose_tc/precision_eval/precision_eval.sh b/configs/commands/rlbench/pick_and_lift/taxpose_tc/precision_eval/precision_eval.sh
@@ -10,21 +10,18 @@ echo "--------------------------------------------------------------------------
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/pick_and_lift/taxpose_tc/precision_eval/pregrasp data_root=/data wandb.group=rlbench_pick_and_lift $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/pick_and_lift/taxpose_tc/precision_eval/pregrasp data_root=/data wandb.group=rlbench_pick_and_lift $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating grasp'
 echo "--------------------------------------------------------------------------------"
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/pick_and_lift/taxpose_tc/precision_eval/grasp data_root=/data wandb.group=rlbench_pick_and_lift $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/pick_and_lift/taxpose_tc/precision_eval/grasp data_root=/data wandb.group=rlbench_pick_and_lift $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating lift'
 echo "--------------------------------------------------------------------------------"
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/pick_and_lift/taxpose_tc/precision_eval/lift data_root=/data wandb.group=rlbench_pick_and_lift $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/pick_and_lift/taxpose_tc/precision_eval/lift data_root=/data wandb.group=rlbench_pick_and_lift $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating final'
 echo "--------------------------------------------------------------------------------"

diff --git a/configs/commands/rlbench/pick_up_cup/taxpose_tc/precision_eval/precision_eval.sh b/configs/commands/rlbench/pick_up_cup/taxpose_tc/precision_eval/precision_eval.sh
@@ -10,14 +10,12 @@ echo "--------------------------------------------------------------------------
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/pick_up_cup/taxpose_tc/precision_eval/pregrasp data_root=/data wandb.group=rlbench_pick_up_cup $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/pick_up_cup/taxpose_tc/precision_eval/pregrasp data_root=/data wandb.group=rlbench_pick_up_cup $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating grasp'
 echo "--------------------------------------------------------------------------------"
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/pick_up_cup/taxpose_tc/precision_eval/grasp data_root=/data wandb.group=rlbench_pick_up_cup $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/pick_up_cup/taxpose_tc/precision_eval/grasp data_root=/data wandb.group=rlbench_pick_up_cup $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating lift'
 echo "--------------------------------------------------------------------------------"

diff --git a/configs/commands/rlbench/push_button/taxpose_tc/precision_eval/precision_eval.sh b/configs/commands/rlbench/push_button/taxpose_tc/precision_eval/precision_eval.sh
@@ -10,7 +10,6 @@ echo "--------------------------------------------------------------------------
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/push_button/taxpose_tc/precision_eval/prepush data_root=/data wandb.group=rlbench_push_button $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/push_button/taxpose_tc/precision_eval/prepush data_root=/data wandb.group=rlbench_push_button $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating postpush'
 echo "--------------------------------------------------------------------------------"

diff --git a/.../commands/rlbench/put_knife_on_chopping_board/taxpose_tc/precision_eval/precision_eval.sh b/.../commands/rlbench/put_knife_on_chopping_board/taxpose_tc/precision_eval/precision_eval.sh
@@ -10,21 +10,18 @@ echo "--------------------------------------------------------------------------
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/put_knife_on_chopping_board/taxpose_tc/precision_eval/pregrasp data_root=/data wandb.group=rlbench_put_knife_on_chopping_board $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/put_knife_on_chopping_board/taxpose_tc/precision_eval/pregrasp data_root=/data wandb.group=rlbench_put_knife_on_chopping_board $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating grasp'
 echo "--------------------------------------------------------------------------------"
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/put_knife_on_chopping_board/taxpose_tc/precision_eval/grasp data_root=/data wandb.group=rlbench_put_knife_on_chopping_board $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/put_knife_on_chopping_board/taxpose_tc/precision_eval/grasp data_root=/data wandb.group=rlbench_put_knife_on_chopping_board $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating lift'
 echo "--------------------------------------------------------------------------------"
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/put_knife_on_chopping_board/taxpose_tc/precision_eval/lift data_root=/data wandb.group=rlbench_put_knife_on_chopping_board $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/put_knife_on_chopping_board/taxpose_tc/precision_eval/lift data_root=/data wandb.group=rlbench_put_knife_on_chopping_board $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating place'
 echo "--------------------------------------------------------------------------------"

diff --git a/configs/commands/rlbench/put_money_in_safe/taxpose_tc/precision_eval/precision_eval.sh b/configs/commands/rlbench/put_money_in_safe/taxpose_tc/precision_eval/precision_eval.sh
@@ -10,28 +10,24 @@ echo "--------------------------------------------------------------------------
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/put_money_in_safe/taxpose_tc/precision_eval/pregrasp data_root=/data wandb.group=rlbench_put_money_in_safe $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/put_money_in_safe/taxpose_tc/precision_eval/pregrasp data_root=/data wandb.group=rlbench_put_money_in_safe $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating grasp'
 echo "--------------------------------------------------------------------------------"
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/put_money_in_safe/taxpose_tc/precision_eval/grasp data_root=/data wandb.group=rlbench_put_money_in_safe $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/put_money_in_safe/taxpose_tc/precision_eval/grasp data_root=/data wandb.group=rlbench_put_money_in_safe $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating lift'
 echo "--------------------------------------------------------------------------------"
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/put_money_in_safe/taxpose_tc/precision_eval/lift data_root=/data wandb.group=rlbench_put_money_in_safe $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/put_money_in_safe/taxpose_tc/precision_eval/lift data_root=/data wandb.group=rlbench_put_money_in_safe $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating preplace'
 echo "--------------------------------------------------------------------------------"
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/put_money_in_safe/taxpose_tc/precision_eval/preplace data_root=/data wandb.group=rlbench_put_money_in_safe $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/put_money_in_safe/taxpose_tc/precision_eval/preplace data_root=/data wandb.group=rlbench_put_money_in_safe $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating place'
 echo "--------------------------------------------------------------------------------"

diff --git a/configs/commands/rlbench/slide_block_to_target/taxpose_tc/precision_eval/precision_eval.sh b/configs/commands/rlbench/slide_block_to_target/taxpose_tc/precision_eval/precision_eval.sh
@@ -10,7 +10,6 @@ echo "--------------------------------------------------------------------------
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/slide_block_to_target/taxpose_tc/precision_eval/preslide data_root=/data wandb.group=rlbench_slide_block_to_target $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/slide_block_to_target/taxpose_tc/precision_eval/preslide data_root=/data wandb.group=rlbench_slide_block_to_target $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating postslide'
 echo "--------------------------------------------------------------------------------"

diff --git a/configs/commands/rlbench/stack_wine/taxpose_tc/precision_eval/precision_eval.sh b/configs/commands/rlbench/stack_wine/taxpose_tc/precision_eval/precision_eval.sh
@@ -10,28 +10,24 @@ echo "--------------------------------------------------------------------------
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/stack_wine/taxpose_tc/precision_eval/pregrasp data_root=/data wandb.group=rlbench_stack_wine $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/stack_wine/taxpose_tc/precision_eval/pregrasp data_root=/data wandb.group=rlbench_stack_wine $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating grasp'
 echo "--------------------------------------------------------------------------------"
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/stack_wine/taxpose_tc/precision_eval/grasp data_root=/data wandb.group=rlbench_stack_wine $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/stack_wine/taxpose_tc/precision_eval/grasp data_root=/data wandb.group=rlbench_stack_wine $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating lift'
 echo "--------------------------------------------------------------------------------"
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/stack_wine/taxpose_tc/precision_eval/lift data_root=/data wandb.group=rlbench_stack_wine $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/stack_wine/taxpose_tc/precision_eval/lift data_root=/data wandb.group=rlbench_stack_wine $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating preplace'
 echo "--------------------------------------------------------------------------------"
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/stack_wine/taxpose_tc/precision_eval/preplace data_root=/data wandb.group=rlbench_stack_wine $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/stack_wine/taxpose_tc/precision_eval/preplace data_root=/data wandb.group=rlbench_stack_wine $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating place'
 echo "--------------------------------------------------------------------------------"

diff --git a/configs/commands/rlbench/take_money_out_safe/taxpose_tc/precision_eval/precision_eval.sh b/configs/commands/rlbench/take_money_out_safe/taxpose_tc/precision_eval/precision_eval.sh
@@ -10,21 +10,18 @@ echo "--------------------------------------------------------------------------
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/take_money_out_safe/taxpose_tc/precision_eval/pregrasp data_root=/data wandb.group=rlbench_take_money_out_safe $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/take_money_out_safe/taxpose_tc/precision_eval/pregrasp data_root=/data wandb.group=rlbench_take_money_out_safe $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating grasp'
 echo "--------------------------------------------------------------------------------"
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/take_money_out_safe/taxpose_tc/precision_eval/grasp data_root=/data wandb.group=rlbench_take_money_out_safe $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/take_money_out_safe/taxpose_tc/precision_eval/grasp data_root=/data wandb.group=rlbench_take_money_out_safe $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating lift'
 echo "--------------------------------------------------------------------------------"
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/take_money_out_safe/taxpose_tc/precision_eval/lift data_root=/data wandb.group=rlbench_take_money_out_safe $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/take_money_out_safe/taxpose_tc/precision_eval/lift data_root=/data wandb.group=rlbench_take_money_out_safe $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating place'
 echo "--------------------------------------------------------------------------------"

diff --git a/...s/rlbench/take_umbrella_out_of_umbrella_stand/taxpose_tc/precision_eval/precision_eval.sh b/...s/rlbench/take_umbrella_out_of_umbrella_stand/taxpose_tc/precision_eval/precision_eval.sh
@@ -10,14 +10,12 @@ echo "--------------------------------------------------------------------------
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/take_umbrella_out_of_umbrella_stand/taxpose_tc/precision_eval/pregasp data_root=/data wandb.group=rlbench_take_umbrella_out_of_umbrella_stand $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/take_umbrella_out_of_umbrella_stand/taxpose_tc/precision_eval/pregasp data_root=/data wandb.group=rlbench_take_umbrella_out_of_umbrella_stand $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating grasp'
 echo "--------------------------------------------------------------------------------"
 echo "python scripts/eval_metrics.py --config-name commands/rlbench/take_umbrella_out_of_umbrella_stand/taxpose_tc/precision_eval/grasp data_root=/data wandb.group=rlbench_take_umbrella_out_of_umbrella_stand $@"
 
 python scripts/eval_metrics.py --config-name commands/rlbench/take_umbrella_out_of_umbrella_stand/taxpose_tc/precision_eval/grasp data_root=/data wandb.group=rlbench_take_umbrella_out_of_umbrella_stand $@
-
 echo "--------------------------------------------------------------------------------"
 echo 'Evaluating lift'
 echo "--------------------------------------------------------------------------------"

diff --git a/configs/dataset/real_world_mug/mug_place.yaml b/configs/dataset/real_world_mug/mug_place.yaml
@@ -42,8 +42,3 @@ val_dset:
     num_points: ${...train_dset.demo_dset.num_points}
     action_class: ${task.action_class}
     anchor_class: ${task.anchor_class}
-
-test_dset:
-  demo_dset:
-    dataset_indices: null
-    num_demo: null
diff --git a/configs/dataset/rlbench/_default.yaml b/configs/dataset/rlbench/_default.yaml
@@ -17,9 +17,9 @@ train_dset:
     teleport_initial_to_final: true
     with_symmetry: True
     occlusion_cfg: ???
-    num_points: 256
-    action_mode: "object"
-    anchor_mode: "single_object"
+    num_points: 512
+    action_mode: "gripper_and_object"
+    anchor_mode: "background_robot_removed"
 
 val_dset:
   demo_dset:

diff --git a/configs/dataset/rlbench/_default/_default.yaml b/configs/dataset/rlbench/_default/_default.yaml
@@ -22,9 +22,3 @@ val_dset:
     episodes:
       - 8
       # - 9
-
-test_dset:
-  demo_dset:
-    dataset_type: ${benchmark.name}
-    episodes:
-      # - 0
diff --git a/configs/dataset/rlbench/insert_onto_square_peg/_default.yaml b/configs/dataset/rlbench/insert_onto_square_peg/_default.yaml
@@ -1,10 +1,6 @@
 defaults:
 - ../_default@_here_
 - _self_
-test_dset:
-  demo_dset:
-    dataset_type: ${benchmark.name}
-    episodes: null
 train_dset:
   demo_dset:
     dataset_type: ${benchmark.name}

diff --git a/configs/dataset/rlbench/phone_on_base/_default.yaml b/configs/dataset/rlbench/phone_on_base/_default.yaml
@@ -1,10 +1,6 @@
 defaults:
 - ../_default@_here_
 - _self_
-test_dset:
-  demo_dset:
-    dataset_type: ${benchmark.name}
-    episodes: null
 train_dset:
   demo_dset:
     dataset_type: ${benchmark.name}

diff --git a/configs/dataset/rlbench/pick_and_lift/_default.yaml b/configs/dataset/rlbench/pick_and_lift/_default.yaml
@@ -1,10 +1,6 @@
 defaults:
 - ../_default@_here_
 - _self_
-test_dset:
-  demo_dset:
-    dataset_type: ${benchmark.name}
-    episodes: null
 train_dset:
   demo_dset:
     dataset_type: ${benchmark.name}
@@ -113,13 +109,13 @@ val_dset:
   demo_dset:
     dataset_type: ${benchmark.name}
     episodes:
-    - 0
-    - 1
-    - 2
-    - 3
-    - 4
-    - 5
-    - 6
-    - 7
-    - 8
-    - 9
+    - 100
+    - 101
+    - 102
+    - 103
+    - 104
+    - 105
+    - 106
+    - 107
+    - 108
+    - 109
diff --git a/configs/dataset/rlbench/pick_up_cup/_default.yaml b/configs/dataset/rlbench/pick_up_cup/_default.yaml
@@ -1,10 +1,6 @@
 defaults:
 - ../_default@_here_
 - _self_
-test_dset:
-  demo_dset:
-    dataset_type: ${benchmark.name}
-    episodes: null
 train_dset:
   demo_dset:
     dataset_type: ${benchmark.name}
@@ -113,13 +109,13 @@ val_dset:
   demo_dset:
     dataset_type: ${benchmark.name}
     episodes:
-    - 0
-    - 1
-    - 2
-    - 3
-    - 4
-    - 5
-    - 6
-    - 7
-    - 8
-    - 9
+    - 100
+    - 101
+    - 102
+    - 103
+    - 104
+    - 105
+    - 106
+    - 107
+    - 108
+    - 109
diff --git a/configs/dataset/rlbench/place_hanger_on_rack/_default.yaml b/configs/dataset/rlbench/place_hanger_on_rack/_default.yaml
@@ -1,10 +1,6 @@
 defaults:
 - ../_default@_here_
 - _self_
-test_dset:
-  demo_dset:
-    dataset_type: ${benchmark.name}
-    episodes: null
 train_dset:
   demo_dset:
     dataset_type: ${benchmark.name}

diff --git a/configs/dataset/rlbench/push_button/_default.yaml b/configs/dataset/rlbench/push_button/_default.yaml
@@ -1,10 +1,6 @@
 defaults:
 - ../_default@_here_
 - _self_
-test_dset:
-  demo_dset:
-    dataset_type: ${benchmark.name}
-    episodes: null
 train_dset:
   demo_dset:
     dataset_type: ${benchmark.name}
@@ -113,13 +109,13 @@ val_dset:
   demo_dset:
     dataset_type: ${benchmark.name}
     episodes:
-    - 0
-    - 1
-    - 2
-    - 3
-    - 4
-    - 5
-    - 6
-    - 7
-    - 8
-    - 9
+    - 100
+    - 101
+    - 102
+    - 103
+    - 104
+    - 105
+    - 106
+    - 107
+    - 108
+    - 109
diff --git a/configs/dataset/rlbench/put_knife_on_chopping_board/_default.yaml b/configs/dataset/rlbench/put_knife_on_chopping_board/_default.yaml
@@ -1,10 +1,6 @@
 defaults:
 - ../_default@_here_
 - _self_
-test_dset:
-  demo_dset:
-    dataset_type: ${benchmark.name}
-    episodes: null
 train_dset:
   demo_dset:
     dataset_type: ${benchmark.name}
@@ -113,13 +109,13 @@ val_dset:
   demo_dset:
     dataset_type: ${benchmark.name}
     episodes:
-    - 0
-    - 1
-    - 2
-    - 3
-    - 4
-    - 5
-    - 6
-    - 7
-    - 8
-    - 9
+    - 100
+    - 101
+    - 102
+    - 103
+    - 104
+    - 105
+    - 106
+    - 107
+    - 108
+    - 109