diff --git a/leaderboards/README.md b/leaderboards/README.md index a460c26..08af6cd 100644 --- a/leaderboards/README.md +++ b/leaderboards/README.md @@ -7,7 +7,7 @@ _Join the competition for low-level vision now!_
-_version_: v1.0.1.1015wip; _Timeliness_: Updated on 15th Oct. +_version_: v1.0.1.1015wip; _Timeliness_: Updated on 30th Oct.
@@ -38,19 +38,21 @@ About the partition of `dev` and `test` subsets, please see [our dataset release |**Model Name** | yes-or-no | what | how | distortion | others | in-context distortion | in-context others | overall | | - | - | - | - | - | - | -| - | -| -| idefics | 0.5618 | 0.4469 | 0.4402 | 0.4280 | 0.5417 | 0.4474 | 0.5633 | 0.4870 | -| instructblip_t5 | 0.6764 | 0.5996 | 0.5598 | 0.5623 | 0.6551 | 0.5822 | 0.6939 | 0.6147 | -| instructblip_vicuna | 0.7164 | 0.5265 | 0.4381 | 0.4864 | 0.6250 | 0.5559 | 0.6490 | 0.5672 | -| kosmos_2 | 0.5491 | 0.2965 | 0.3266 | 0.3755 | 0.4398 | 0.3849 | 0.3959 | 0.3993 | -| llama_adapter_v2 | 0.6618 | 0.5929 | 0.5213 | 0.5739 | 0.5625 | 0.6316 | 0.6490 | 0.5946 | -| llava_v1.5 | 0.6909 | 0.6327 | 0.5639 | 0.5525 | 0.6852 | 0.6086 | 0.7306 | **0.6314** (rank 1) | -| llava_v1 | 0.5400 | 0.5310 | 0.5538 | 0.4864 | 0.5463 | 0.5559 | 0.6327 | 0.5418 | -| minigpt4_13b | 0.5582 | 0.5022 | 0.4037 | 0.4202 | 0.4838 | 0.5197 | 0.6122 | 0.4903 | -| mplug_owl | 0.6600 | 0.5487 | 0.4402 | 0.5136 | 0.5509 | 0.5428 | 0.6571 | 0.5538 | -| otter_v1 | 0.5709 | 0.4071 | 0.3955 | 0.4222 | 0.4931 | 0.4408 | 0.5265 | 0.4635 | -| qwen_vl | 0.6309 | 0.5819 | 0.5639 | 0.5058 | 0.6273 | 0.5789 | 0.7388 | 0.5940 | -| shikra | 0.6564 | 0.4735 | 0.4909 | 0.4883 | 0.5949 | 0.5000 | 0.6408 | 0.5465 | -| visualglm | 0.6018 | 0.5420 | 0.4625 | 0.5175 | 0.5440 | 0.5362 | 0.5714 | 0.5378 | +| random guess | 0.5000 | 0.2786 | 0.3331 | 0.3789 | 0.3848 | 0.3828 | 0.3582 | 0.3780 | +| LLaVA-v1.5 (Vicuna-v1.5-7B) | 0.6636 | 0.5819 | 0.5051 | 0.4942 | 0.6574 | 0.5461 | 0.7061 | 0.5866 | +| LLaVA-v1.5 (Vicuna-v1.5-13B) | 0.6527 | 0.6438 | 0.5659 | 0.5603 | 0.6713 | 0.6118 | 0.6735 | 0.6214 | +| InternLM-XComposer (InternLM) | 0.6945 | 0.6527 | 0.6085 | 0.6167 | 0.7014 | 0.5691 | 0.7510 | 0.6535 | +| IDEFICS-Instruct (LLaMA-7B) | 0.5618 | 0.4469 | 0.4402 | 0.4280 | 0.5417 | 0.4474 | 0.5633 | 0.4870 | +| Qwen-VL (QwenLM) | 0.6309 | 0.5819 | 0.5639 | 0.5058 | 0.6273 | 0.5789 | 0.7388 | 0.5940 | +| Shikra (Vicuna-7B) | 0.6564 | 0.4735 | 0.4909 | 0.4883 | 0.5949 | 0.5000 | 0.6408 | 0.5465 | +| Otter-v1 (MPT-7B) | 0.5709 | 0.4071 | 0.3955 | 0.4222 | 0.4931 | 0.4408 | 0.5265 | 0.4635 | +| InstructBLIP (Flan-T5-XL) | 0.6764 | 0.5996 | 0.5598 | 0.5623 | 0.6551 | 0.5822 | 0.6939 | 0.6147 | +| InstructBLIP (Vicuna-7B) | 0.7164 | 0.5265 | 0.4381 | 0.4864 | 0.6250 | 0.5559 | 0.6490 | 0.5672 | +| VisualGLM-6B (GLM-6B) | 0.6018 | 0.5420 | 0.4625 | 0.5175 | 0.5440 | 0.5362 | 0.5714 | 0.5378 | +| mPLUG-Owl (LLaMA-7B) | 0.6600 | 0.5487 | 0.4402 | 0.5136 | 0.5509 | 0.5428 | 0.6571 | 0.5538 | +| LLaMA-Adapter-V2 | 0.6618 | 0.5929 | 0.5213 | 0.5739 | 0.5625 | 0.6316 | 0.6490 | 0.5946 | +| LLaVA-v1 (Vicuna-13B) | 0.5400 | 0.5310 | 0.5538 | 0.4864 | 0.5463 | 0.5559 | 0.6327 | 0.5418 | +| MiniGPT-4 (Vicuna-13B) | 0.5582 | 0.5022 | 0.4037 | 0.4202 | 0.4838 | 0.5197 | 0.6122 | 0.4903 |