diff --git a/README.md b/README.md index 6ff942b6..9ee8b0a8 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,66 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. - ``` -[2024-12-22 13:40:53,622 submission_checker.py:3167 INFO] Results=0, NoResults=0, Power Results=0 -[2024-12-22 13:40:53,623 submission_checker.py:3174 INFO] --- -[2024-12-22 13:40:53,623 submission_checker.py:3175 INFO] Closed Results=0, Closed Power Results=0 +[2025-01-01 22:47:10,667 submission_checker.py:3183 INFO] Results=43, NoResults=0, Power Results=0 +[2025-01-01 22:47:10,667 submission_checker.py:3190 INFO] --- +[2025-01-01 22:47:10,667 submission_checker.py:3191 INFO] Closed Results=8, Closed Power Results=0 -[2024-12-22 13:40:53,623 submission_checker.py:3180 INFO] Open Results=0, Open Power Results=0 +[2025-01-01 22:47:10,667 submission_checker.py:3196 INFO] Open Results=35, Open Power Results=0 -[2024-12-22 13:40:53,623 submission_checker.py:3185 INFO] Network Results=0, Network Power Results=0 +[2025-01-01 22:47:10,667 submission_checker.py:3201 INFO] Network Results=0, Network Power Results=0 -[2024-12-22 13:40:53,623 submission_checker.py:3190 INFO] --- -[2024-12-22 13:40:53,623 submission_checker.py:3192 INFO] Systems=0, Power Systems=0 -[2024-12-22 13:40:53,623 submission_checker.py:3196 INFO] Closed Systems=0, Closed Power Systems=0 -[2024-12-22 13:40:53,623 submission_checker.py:3201 INFO] Open Systems=0, Open Power Systems=0 -[2024-12-22 13:40:53,623 submission_checker.py:3206 INFO] Network Systems=0, Network Power Systems=0 -[2024-12-22 13:40:53,623 submission_checker.py:3211 INFO] --- -[2024-12-22 13:40:53,623 submission_checker.py:3216 INFO] SUMMARY: submission looks OK +[2025-01-01 22:47:10,667 submission_checker.py:3206 INFO] --- +[2025-01-01 22:47:10,667 submission_checker.py:3208 INFO] Systems=2, Power Systems=0 +[2025-01-01 22:47:10,667 submission_checker.py:3212 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-01 22:47:10,667 submission_checker.py:3217 INFO] Open Systems=2, Open Power Systems=0 +[2025-01-01 22:47:10,667 submission_checker.py:3222 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-01 22:47:10,667 submission_checker.py:3227 INFO] --- +[2025-01-01 22:47:10,667 submission_checker.py:3232 INFO] SUMMARY: submission looks OK INFO:root: ! call "postprocess" from /home/runner/CM/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|----------------|----------------|------------|--------------|--------------|------------|---------|---------------|------------|----------|------------|-------------------|-----------------------------|----------------------------|-----------------------------|--------------------------|-------------------------|------------|-------------|--------------------|---------|--------------|----------|-----------|------------|-------------|---------|---------------------| +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:----------------|:-------------|:---------------------------------------------------------------|:--------------------|:--------------------|:-------------|-------------:|:----------------------|------------------:|:-------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:-----------------------------------------------------------------------------------------------------------------------|:------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:-------------|:--------------------| +| 0 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | retinanet | retinanet | MultiStream | 5.63893 | mAP: 37.328 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/multistream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 1 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | retinanet | retinanet | Server | 1414.99 | mAP: 37.320 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/server | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 2 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | retinanet | retinanet | SingleStream | 1.73717 | mAP: 37.310 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/singlestream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 3 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | retinanet | retinanet | Offline | 1740.84 | mAP: 37.371 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/offline | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | +| 4 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | resnet50 | resnet | MultiStream | 0.502283 | acc: 76.064 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/multistream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 5 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | resnet50 | resnet | Server | 73744 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 6 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | resnet50 | resnet | SingleStream | 0.303938 | acc: 76.064 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/singlestream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 7 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | resnet50 | resnet | Offline | 88040.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | +| 8 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | bert-99.9 | bert-99.9 | SingleStream | 2.17514 | F1: 90.88109554940347 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99.9/singlestream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | fp16 | +| 9 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | bert-99.9 | bert-99.9 | Offline | 3339.26 | F1: 90.8832407068292 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99.9/offline | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp16 | +| 10 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | 3d-unet-99.9 | 3d-unet-99.9 | SingleStream | 429.287 | DICE: 0.86236 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9/singlestream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 11 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | 3d-unet-99.9 | 3d-unet-99.9 | Offline | 8.29797 | DICE: 0.86236 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9/offline | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | +| 12 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | bert-99 | bert-99 | SingleStream | 1.03373 | F1: 90.26682135974633 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 13 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | bert-99 | bert-99 | Offline | 8245.93 | F1: 90.15673510616978 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | +| 14 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | stable-diffusion-xl | stable-diffusion-xl | Server | 0.881818 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl/server | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 15 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | stable-diffusion-xl | stable-diffusion-xl | SingleStream | 1400.03 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl/singlestream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 16 | MLCommons | available | open | datacenter,edge | RTX4090x2 | RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config | stable-diffusion-xl | stable-diffusion-xl | Offline | 1.39663 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl/offline | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | +| 17 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | retinanet | retinanet | MultiStream | 11.6438 | mAP: 37.329 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/multistream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 18 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | retinanet | retinanet | Server | 637.175 | mAP: 37.341 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/server | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 19 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | retinanet | retinanet | SingleStream | 1.70916 | mAP: 37.326 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/singlestream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 20 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | retinanet | retinanet | Offline | 872.522 | mAP: 37.306 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/offline | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | +| 21 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | resnet50 | resnet | MultiStream | 0.798611 | acc: 76.064 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/multistream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 22 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | resnet50 | resnet | Server | 35357.8 | acc: 76.078 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 23 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | resnet50 | resnet | SingleStream | 0.300962 | acc: 76.064 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/singlestream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 24 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | resnet50 | resnet | Offline | 3633.97 | acc: 76.078 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | +| 25 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | bert-99.9 | bert-99.9 | Server | 1414.95 | F1: 90.89127264236201 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99.9/server | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Queries/s | fp16 | +| 26 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | bert-99.9 | bert-99.9 | SingleStream | 2.16956 | F1: 90.88109554940347 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99.9/singlestream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | fp16 | +| 27 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | bert-99.9 | bert-99.9 | Offline | 1176.59 | F1: 90.8832407068292 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99.9/offline | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp16 | +| 28 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | 3d-unet-99.9 | 3d-unet-99.9 | SingleStream | 437.373 | DICE: 0.86236 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9/singlestream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 29 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | 3d-unet-99.9 | 3d-unet-99.9 | Offline | 3.45061 | DICE: 0.86236 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9/offline | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | +| 30 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | bert-99 | bert-99 | Server | 3841.13 | F1: 90.25897829249658 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/server | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 31 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | bert-99 | bert-99 | SingleStream | 1.01571 | F1: 90.26682135974633 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 32 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | bert-99 | bert-99 | Offline | 4125.31 | F1: 90.15673510616978 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | +| 33 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | stable-diffusion-xl | stable-diffusion-xl | SingleStream | 1424.66 | nan | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl/singlestream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 34 | MLCommons | available | open | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.697739 | nan | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl/offline | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | +| 35 | MLCommons | available | closed | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | retinanet | retinanet | MultiStream | 10.958 | mAP: 37.344 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/multistream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 36 | MLCommons | available | closed | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | retinanet | retinanet | Server | 637.176 | mAP: 37.353 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/server | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 37 | MLCommons | available | closed | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | retinanet | retinanet | SingleStream | 1.74629 | mAP: 37.330 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/singlestream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 38 | MLCommons | available | closed | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | retinanet | retinanet | Offline | 862.711 | mAP: 37.335 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/offline | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | +| 39 | MLCommons | available | closed | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | resnet50 | resnet | MultiStream | 0.467708 | acc: 76.064 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/multistream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 40 | MLCommons | available | closed | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | resnet50 | resnet | Server | 35357.7 | acc: 76.078 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 41 | MLCommons | available | closed | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | resnet50 | resnet | SingleStream | 0.277197 | acc: 76.064 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/singlestream | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Latency (ms) | int8 | +| 42 | MLCommons | available | closed | datacenter,edge | RTX4090x1 | RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config | resnet50 | resnet | Offline | 43726.6 | acc: 76.078 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | \ No newline at end of file diff --git a/summary.csv b/summary.csv index b85f9e25..cb6de0de 100644 --- a/summary.csv +++ b/summary.csv @@ -1 +1,44 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,retinanet,retinanet,MultiStream,5.638927,mAP: 37.328,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/multistream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,retinanet,retinanet,Server,1414.99,mAP: 37.320,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/server,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Queries/s,"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,retinanet,retinanet,SingleStream,1.737171,mAP: 37.310,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/singlestream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,retinanet,retinanet,Offline,1740.84,mAP: 37.371,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/offline,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Samples/s,"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,resnet50,resnet,MultiStream,0.502283,acc: 76.064,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/multistream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,resnet50,resnet,Server,73744.0,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Queries/s,"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,resnet50,resnet,SingleStream,0.303938,acc: 76.064,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/singlestream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,resnet50,resnet,Offline,88040.1,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Samples/s,"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,bert-99.9,bert-99.9,SingleStream,2.175143,F1: 90.88109554940347,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99.9/singlestream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"fp16" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,bert-99.9,bert-99.9,Offline,3339.26,F1: 90.8832407068292,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99.9/offline,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Samples/s,"fp16" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,3d-unet-99.9,3d-unet-99.9,SingleStream,429.286663,DICE: 0.86236,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9/singlestream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,3d-unet-99.9,3d-unet-99.9,Offline,8.29797,DICE: 0.86236,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9/offline,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Samples/s,"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,bert-99,bert-99,SingleStream,1.033728,F1: 90.26682135974633,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,bert-99,bert-99,Offline,8245.93,F1: 90.15673510616978,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Samples/s,"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,stable-diffusion-xl,stable-diffusion-xl,Server,0.881818,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl/server,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Queries/s,"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,stable-diffusion-xl,stable-diffusion-xl,SingleStream,1400.027931,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl/singlestream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x2",RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config,stable-diffusion-xl,stable-diffusion-xl,Offline,1.39663,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl/offline,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Samples/s,"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,retinanet,retinanet,MultiStream,11.64376,mAP: 37.329,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/multistream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,retinanet,retinanet,Server,637.175,mAP: 37.341,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/server,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Queries/s,"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,retinanet,retinanet,SingleStream,1.709161,mAP: 37.326,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/singlestream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,retinanet,retinanet,Offline,872.522,mAP: 37.306,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/offline,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Samples/s,"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,resnet50,resnet,MultiStream,0.798611,acc: 76.064,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/multistream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,resnet50,resnet,Server,35357.8,acc: 76.078,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Queries/s,"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,resnet50,resnet,SingleStream,0.300962,acc: 76.064,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/singlestream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,resnet50,resnet,Offline,3633.97,acc: 76.078,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Samples/s,"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,bert-99.9,bert-99.9,Server,1414.95,F1: 90.89127264236201,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99.9/server,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Queries/s,"fp16" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,bert-99.9,bert-99.9,SingleStream,2.169561,F1: 90.88109554940347,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99.9/singlestream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"fp16" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,bert-99.9,bert-99.9,Offline,1176.59,F1: 90.8832407068292,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99.9/offline,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Samples/s,"fp16" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,3d-unet-99.9,3d-unet-99.9,SingleStream,437.373498,DICE: 0.86236,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9/singlestream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,3d-unet-99.9,3d-unet-99.9,Offline,3.45061,DICE: 0.86236,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9/offline,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Samples/s,"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,bert-99,bert-99,Server,3841.13,F1: 90.25897829249658,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/server,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Queries/s,"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,bert-99,bert-99,SingleStream,1.015707,F1: 90.26682135974633,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,bert-99,bert-99,Offline,4125.31,F1: 90.15673510616978,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Samples/s,"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,stable-diffusion-xl,stable-diffusion-xl,SingleStream,1424.658155,,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl/singlestream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,open,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,stable-diffusion-xl,stable-diffusion-xl,Offline,0.697739,,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl/offline,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Samples/s,"int8" +MLCommons,available,closed,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,retinanet,retinanet,MultiStream,10.957955,mAP: 37.344,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/multistream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,closed,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,retinanet,retinanet,Server,637.176,mAP: 37.353,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/server,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Queries/s,"int8" +MLCommons,available,closed,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,retinanet,retinanet,SingleStream,1.746292,mAP: 37.330,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/singlestream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,closed,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,retinanet,retinanet,Offline,862.711,mAP: 37.335,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/offline,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Samples/s,"int8" +MLCommons,available,closed,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,resnet50,resnet,MultiStream,0.467708,acc: 76.064,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/multistream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,closed,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,resnet50,resnet,Server,35357.7,acc: 76.078,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Queries/s,"int8" +MLCommons,available,closed,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,resnet50,resnet,SingleStream,0.277197,acc: 76.064,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/singlestream,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Latency (ms),"int8" +MLCommons,available,closed,"datacenter,edge","RTX4090x1",RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config,resnet50,resnet,Offline,43726.6,acc: 76.078,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)","",1,0,v5.0,0,False,Samples/s,"int8" diff --git a/summary.json b/summary.json index 0e0dcd23..2496e0af 100644 --- a/summary.json +++ b/summary.json @@ -1,3 +1,1292 @@ { - + "0":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"MultiStream", + "Result":5.638927, + "Accuracy":"mAP: 37.328", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/retinanet\/multistream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "1":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Server", + "Result":1414.99, + "Accuracy":"mAP: 37.320", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/retinanet\/server", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Queries\/s", + "weight_data_types":"int8" + }, + "2":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"SingleStream", + "Result":1.737171, + "Accuracy":"mAP: 37.310", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/retinanet\/singlestream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "3":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":1740.84, + "Accuracy":"mAP: 37.371", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/retinanet\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"int8" + }, + "4":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"MultiStream", + "Result":0.502283, + "Accuracy":"acc: 76.064", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/resnet50\/multistream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "5":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Server", + "Result":73744.0, + "Accuracy":"acc: 76.078", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/resnet50\/server", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Queries\/s", + "weight_data_types":"int8" + }, + "6":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"SingleStream", + "Result":0.303938, + "Accuracy":"acc: 76.064", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/resnet50\/singlestream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "7":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":88040.1, + "Accuracy":"acc: 76.078", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/resnet50\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"int8" + }, + "8":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"bert-99.9", + "MlperfModel":"bert-99.9", + "Scenario":"SingleStream", + "Result":2.175143, + "Accuracy":"F1: 90.88109554940347", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/bert-99.9\/singlestream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"fp16" + }, + "9":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"bert-99.9", + "MlperfModel":"bert-99.9", + "Scenario":"Offline", + "Result":3339.26, + "Accuracy":"F1: 90.8832407068292", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/bert-99.9\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp16" + }, + "10":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"3d-unet-99.9", + "MlperfModel":"3d-unet-99.9", + "Scenario":"SingleStream", + "Result":429.286663, + "Accuracy":"DICE: 0.86236", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/3d-unet-99.9\/singlestream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "11":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"3d-unet-99.9", + "MlperfModel":"3d-unet-99.9", + "Scenario":"Offline", + "Result":8.29797, + "Accuracy":"DICE: 0.86236", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/3d-unet-99.9\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"int8" + }, + "12":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", + "Scenario":"SingleStream", + "Result":1.033728, + "Accuracy":"F1: 90.26682135974633", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/bert-99\/singlestream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "13":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", + "Scenario":"Offline", + "Result":8245.93, + "Accuracy":"F1: 90.15673510616978", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/bert-99\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"int8" + }, + "14":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", + "Scenario":"Server", + "Result":0.881818, + "Accuracy":null, + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/stable-diffusion-xl\/server", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Queries\/s", + "weight_data_types":"int8" + }, + "15":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", + "Scenario":"SingleStream", + "Result":1400.027931, + "Accuracy":null, + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/stable-diffusion-xl\/singlestream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "16":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x2", + "Platform":"RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", + "Scenario":"Offline", + "Result":1.39663, + "Accuracy":null, + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config\/stable-diffusion-xl\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"int8" + }, + "17":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"MultiStream", + "Result":11.64376, + "Accuracy":"mAP: 37.329", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/retinanet\/multistream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "18":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Server", + "Result":637.175, + "Accuracy":"mAP: 37.341", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/retinanet\/server", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Queries\/s", + "weight_data_types":"int8" + }, + "19":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"SingleStream", + "Result":1.709161, + "Accuracy":"mAP: 37.326", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/retinanet\/singlestream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "20":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":872.522, + "Accuracy":"mAP: 37.306", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/retinanet\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"int8" + }, + "21":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"MultiStream", + "Result":0.798611, + "Accuracy":"acc: 76.064", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/resnet50\/multistream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "22":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Server", + "Result":35357.8, + "Accuracy":"acc: 76.078", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/resnet50\/server", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Queries\/s", + "weight_data_types":"int8" + }, + "23":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"SingleStream", + "Result":0.300962, + "Accuracy":"acc: 76.064", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/resnet50\/singlestream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "24":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":3633.97, + "Accuracy":"acc: 76.078", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/resnet50\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"int8" + }, + "25":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"bert-99.9", + "MlperfModel":"bert-99.9", + "Scenario":"Server", + "Result":1414.95, + "Accuracy":"F1: 90.89127264236201", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/bert-99.9\/server", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Queries\/s", + "weight_data_types":"fp16" + }, + "26":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"bert-99.9", + "MlperfModel":"bert-99.9", + "Scenario":"SingleStream", + "Result":2.169561, + "Accuracy":"F1: 90.88109554940347", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/bert-99.9\/singlestream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"fp16" + }, + "27":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"bert-99.9", + "MlperfModel":"bert-99.9", + "Scenario":"Offline", + "Result":1176.59, + "Accuracy":"F1: 90.8832407068292", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/bert-99.9\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp16" + }, + "28":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"3d-unet-99.9", + "MlperfModel":"3d-unet-99.9", + "Scenario":"SingleStream", + "Result":437.373498, + "Accuracy":"DICE: 0.86236", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/3d-unet-99.9\/singlestream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "29":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"3d-unet-99.9", + "MlperfModel":"3d-unet-99.9", + "Scenario":"Offline", + "Result":3.45061, + "Accuracy":"DICE: 0.86236", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/3d-unet-99.9\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"int8" + }, + "30":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", + "Scenario":"Server", + "Result":3841.13, + "Accuracy":"F1: 90.25897829249658", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/bert-99\/server", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Queries\/s", + "weight_data_types":"int8" + }, + "31":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", + "Scenario":"SingleStream", + "Result":1.015707, + "Accuracy":"F1: 90.26682135974633", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/bert-99\/singlestream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "32":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", + "Scenario":"Offline", + "Result":4125.31, + "Accuracy":"F1: 90.15673510616978", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/bert-99\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"int8" + }, + "33":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", + "Scenario":"SingleStream", + "Result":1424.658155, + "Accuracy":null, + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/stable-diffusion-xl\/singlestream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "34":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", + "Scenario":"Offline", + "Result":0.697739, + "Accuracy":null, + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"open\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/stable-diffusion-xl\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"int8" + }, + "35":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"closed", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"MultiStream", + "Result":10.957955, + "Accuracy":"mAP: 37.344", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"closed\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/retinanet\/multistream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "36":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"closed", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Server", + "Result":637.176, + "Accuracy":"mAP: 37.353", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"closed\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/retinanet\/server", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Queries\/s", + "weight_data_types":"int8" + }, + "37":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"closed", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"SingleStream", + "Result":1.746292, + "Accuracy":"mAP: 37.330", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"closed\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/retinanet\/singlestream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "38":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"closed", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":862.711, + "Accuracy":"mAP: 37.335", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"closed\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/retinanet\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"int8" + }, + "39":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"closed", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"MultiStream", + "Result":0.467708, + "Accuracy":"acc: 76.064", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"closed\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/resnet50\/multistream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "40":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"closed", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Server", + "Result":35357.7, + "Accuracy":"acc: 76.078", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"closed\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/resnet50\/server", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Queries\/s", + "weight_data_types":"int8" + }, + "41":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"closed", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"SingleStream", + "Result":0.277197, + "Accuracy":"acc: 76.064", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"closed\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/resnet50\/singlestream", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Latency (ms)", + "weight_data_types":"int8" + }, + "42":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"closed", + "SystemType":"datacenter,edge", + "SystemName":"RTX4090x1", + "Platform":"RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":43726.6, + "Accuracy":"acc: 76.078", + "number_of_nodes":1, + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":1, + "Location":"closed\/MLCommons\/results\/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config\/resnet50\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"int8" + } } \ No newline at end of file diff --git a/summary.xlsx b/summary.xlsx index c2164c22..fb8bb29d 100644 Binary files a/summary.xlsx and b/summary.xlsx differ diff --git a/summary_results.json b/summary_results.json index 79f51fa2..66f03fe9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -5,686 +5,290 @@ "Availability": "available", "Category": "closed", "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia-gpu-TensorRT-default_config", - "UsedModel": "3d-unet-99.9", - "Model": "3d-unet-99.9", - "Scenario": "Offline", - "Accuracy": "DICE: 0.86236", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/RTX4090x2-nvidia-gpu-TensorRT-default_config/3d-unet-99.9/offline", - "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "Notes": "Automated by MLCommons CM v3.2.6. ", - "compliance": 1, - "errors": 0, - "version": "v4.1", - "inferred": 0, - "has_power": false, - "weight_data_types": "int8", - "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x2-nvidia-gpu-TensorRT-default_config", - "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x2-nvidia-gpu-TensorRT-default_config", - "ColorKey": "availableMLCommons", - "Performance_Result": 8.29596, - "Performance_Units": "Samples/s" - }, - { - "ID": "4.1-0001", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "closed", - "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia-gpu-TensorRT-default_config", - "UsedModel": "3d-unet-99.9", - "Model": "3d-unet-99.9", - "Scenario": "SingleStream", - "Accuracy": "DICE: 0.86236", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/RTX4090x2-nvidia-gpu-TensorRT-default_config/3d-unet-99.9/singlestream", - "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "Notes": "Automated by MLCommons CM v3.2.6. ", - "compliance": 1, - "errors": 0, - "version": "v4.1", - "inferred": 0, - "has_power": false, - "weight_data_types": "int8", - "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x2-nvidia-gpu-TensorRT-default_config", - "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x2-nvidia-gpu-TensorRT-default_config", - "ColorKey": "availableMLCommons", - "Performance_Result": 431.158295, - "Performance_Units": "Latency (ms)" - }, - { - "ID": "4.1-0002", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "closed", - "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "UsedModel": "3d-unet-99", - "Model": "3d-unet-99", - "Scenario": "Offline", - "Accuracy": "DICE: 0.86236", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99/offline", - "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v4.1", - "inferred": 0, - "has_power": false, - "weight_data_types": "int8", - "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "ColorKey": "availableMLCommons", - "Performance_Result": 8.27882, - "Performance_Units": "Samples/s" - }, - { - "ID": "4.1-0002", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "closed", - "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "UsedModel": "3d-unet-99", - "Model": "3d-unet-99", - "Scenario": "SingleStream", - "Accuracy": "DICE: 0.86236", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99/singlestream", - "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v4.1", - "inferred": 0, - "has_power": false, - "weight_data_types": "int8", - "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "ColorKey": "availableMLCommons", - "Performance_Result": 431.283925, - "Performance_Units": "Latency (ms)" - }, - { - "ID": "4.1-0002", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "closed", - "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "UsedModel": "3d-unet-99.9", - "Model": "3d-unet-99.9", - "Scenario": "Offline", - "Accuracy": "DICE: 0.86236", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9/offline", - "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v4.1", - "inferred": 0, - "has_power": false, - "weight_data_types": "int8", - "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "ColorKey": "availableMLCommons", - "Performance_Result": 8.27882, - "Performance_Units": "Samples/s" - }, - { - "ID": "4.1-0002", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "closed", - "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "UsedModel": "3d-unet-99.9", - "Model": "3d-unet-99.9", - "Scenario": "SingleStream", - "Accuracy": "DICE: 0.86236", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9/singlestream", - "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v4.1", - "inferred": 0, - "has_power": false, - "weight_data_types": "int8", - "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "ColorKey": "availableMLCommons", - "Performance_Result": 431.283925, - "Performance_Units": "Latency (ms)" - }, - { - "ID": "4.1-0002", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "closed", - "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "UsedModel": "bert-99", - "Model": "bert-99", - "Scenario": "Offline", - "Accuracy": "F1: 90.15673510616978", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline", - "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v4.1", - "inferred": 0, - "has_power": false, - "weight_data_types": "int8", - "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "ColorKey": "availableMLCommons", - "Performance_Result": 8219.33, - "Performance_Units": "Samples/s" - }, - { - "ID": "4.1-0002", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "closed", - "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "UsedModel": "bert-99", - "Model": "bert-99", - "Scenario": "Server", - "Accuracy": "F1: 90.25897829249658", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/server", - "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v4.1", - "inferred": 0, - "has_power": false, - "weight_data_types": "int8", - "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "ColorKey": "availableMLCommons", - "Performance_Result": 6061.5, - "Performance_Units": "Queries/s" - }, - { - "ID": "4.1-0002", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "closed", - "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "UsedModel": "bert-99", - "Model": "bert-99", - "Scenario": "SingleStream", - "Accuracy": "F1: 90.26682135974633", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream", - "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v4.1", - "inferred": 0, - "has_power": false, - "weight_data_types": "int8", - "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "ColorKey": "availableMLCommons", - "Performance_Result": 1.039935, - "Performance_Units": "Latency (ms)" - }, - { - "ID": "4.1-0002", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "closed", - "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "UsedModel": "bert-99.9", - "Model": "bert-99.9", - "Scenario": "Offline", - "Accuracy": "F1: 90.8832407068292", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99.9/offline", - "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v4.1", - "inferred": 0, - "has_power": false, - "weight_data_types": "int8", - "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "ColorKey": "availableMLCommons", - "Performance_Result": 3314.54, - "Performance_Units": "Samples/s" - }, - { - "ID": "4.1-0002", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "closed", - "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "UsedModel": "bert-99.9", - "Model": "bert-99.9", - "Scenario": "Server", - "Accuracy": "F1: 90.89074704580993", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99.9/server", - "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v4.1", - "inferred": 0, - "has_power": false, - "weight_data_types": "int8", - "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.888629, - "Performance_Units": "Queries/s" - }, - { - "ID": "4.1-0002", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "closed", - "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "System": "RTX4090x1", + "Platform": "RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "UsedModel": "resnet50", "Model": "resnet", "Scenario": "MultiStream", "Accuracy": "acc: 76.064", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/multistream", + "a#": 1, + "Location": "closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/multistream", "Software": "TensorRT", "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "ColorKey": "availableMLCommons", - "Performance_Result": 1.017666, + "Performance_Result": 0.467708, "Performance_Units": "Latency (ms)" }, { - "ID": "4.1-0002", + "ID": "4.1-0001", "Submitter": "MLCommons", "Availability": "available", "Category": "closed", "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "System": "RTX4090x1", + "Platform": "RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "UsedModel": "resnet50", "Model": "resnet", "Scenario": "Offline", "Accuracy": "acc: 76.078", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/offline", + "a#": 1, + "Location": "closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/offline", "Software": "TensorRT", "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "ColorKey": "availableMLCommons", - "Performance_Result": 87803.5, + "Performance_Result": 43726.6, "Performance_Units": "Samples/s" }, { - "ID": "4.1-0002", + "ID": "4.1-0001", "Submitter": "MLCommons", "Availability": "available", "Category": "closed", "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "System": "RTX4090x1", + "Platform": "RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "UsedModel": "resnet50", "Model": "resnet", "Scenario": "Server", "Accuracy": "acc: 76.078", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/server", + "a#": 1, + "Location": "closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/server", "Software": "TensorRT", "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "ColorKey": "availableMLCommons", - "Performance_Result": 73743.9, + "Performance_Result": 35357.7, "Performance_Units": "Queries/s" }, { - "ID": "4.1-0002", + "ID": "4.1-0001", "Submitter": "MLCommons", "Availability": "available", "Category": "closed", "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "System": "RTX4090x1", + "Platform": "RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "UsedModel": "resnet50", "Model": "resnet", "Scenario": "SingleStream", "Accuracy": "acc: 76.064", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/singlestream", + "a#": 1, + "Location": "closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50/singlestream", "Software": "TensorRT", "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "ColorKey": "availableMLCommons", - "Performance_Result": 0.323576, + "Performance_Result": 0.277197, "Performance_Units": "Latency (ms)" }, { - "ID": "4.1-0003", + "ID": "4.1-0001", "Submitter": "MLCommons", "Availability": "available", "Category": "closed", "Suite": "datacenter,edge", - "System": "gh_ubuntu_x86", - "Platform": "gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "UsedModel": "resnet50", - "Model": "resnet", + "System": "RTX4090x1", + "Platform": "RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "UsedModel": "retinanet", + "Model": "retinanet", "Scenario": "MultiStream", - "Accuracy": "acc: 76.078", + "Accuracy": "mAP: 37.344", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/multistream", + "a#": 1, + "Location": "closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/multistream", "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "Notes": "Automated by MLCommons CM v3.2.6. ", + "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", + "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", + "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "ColorKey": "availableMLCommons", - "Performance_Result": 0.951987, + "Performance_Result": 10.957955, "Performance_Units": "Latency (ms)" }, { - "ID": "4.1-0003", + "ID": "4.1-0001", "Submitter": "MLCommons", "Availability": "available", "Category": "closed", "Suite": "datacenter,edge", - "System": "gh_ubuntu_x86", - "Platform": "gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "UsedModel": "resnet50", - "Model": "resnet", + "System": "RTX4090x1", + "Platform": "RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "UsedModel": "retinanet", + "Model": "retinanet", "Scenario": "Offline", - "Accuracy": "acc: 76.078", + "Accuracy": "mAP: 37.335", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline", + "a#": 1, + "Location": "closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/offline", "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "Notes": "Automated by MLCommons CM v3.2.6. ", + "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", + "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", + "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "ColorKey": "availableMLCommons", - "Performance_Result": 87471.2, + "Performance_Result": 862.711, "Performance_Units": "Samples/s" }, { - "ID": "4.1-0003", + "ID": "4.1-0001", "Submitter": "MLCommons", "Availability": "available", "Category": "closed", "Suite": "datacenter,edge", - "System": "gh_ubuntu_x86", - "Platform": "gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "UsedModel": "resnet50", - "Model": "resnet", + "System": "RTX4090x1", + "Platform": "RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "UsedModel": "retinanet", + "Model": "retinanet", "Scenario": "Server", - "Accuracy": "acc: 76.078", + "Accuracy": "mAP: 37.353", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server", + "a#": 1, + "Location": "closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/server", "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "Notes": "Automated by MLCommons CM v3.2.6. ", + "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", + "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", + "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "ColorKey": "availableMLCommons", - "Performance_Result": 73015.2, + "Performance_Result": 637.176, "Performance_Units": "Queries/s" }, { - "ID": "4.1-0003", + "ID": "4.1-0001", "Submitter": "MLCommons", "Availability": "available", "Category": "closed", "Suite": "datacenter,edge", - "System": "gh_ubuntu_x86", - "Platform": "gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "UsedModel": "resnet50", - "Model": "resnet", + "System": "RTX4090x1", + "Platform": "RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "UsedModel": "retinanet", + "Model": "retinanet", "Scenario": "SingleStream", - "Accuracy": "acc: 76.078", + "Accuracy": "mAP: 37.330", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/singlestream", + "a#": 1, + "Location": "closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet/singlestream", "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "Notes": "Automated by MLCommons CM v3.2.6. ", + "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", + "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", + "Unique ID (e.g. for Audit)": "datacenter,edge/closed/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "ColorKey": "availableMLCommons", - "Performance_Result": 0.342055, + "Performance_Result": 1.746292, "Performance_Units": "Latency (ms)" }, { - "ID": "4.1-0004", + "ID": "4.1-0002", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -707,7 +311,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -716,11 +320,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9", "ColorKey": "availableMLCommons", - "Performance_Result": 4.1389, + "Performance_Result": 3.45061, "Performance_Units": "Samples/s" }, { - "ID": "4.1-0004", + "ID": "4.1-0002", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -743,7 +347,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -752,11 +356,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9", "ColorKey": "availableMLCommons", - "Performance_Result": 437.219609, + "Performance_Result": 437.373498, "Performance_Units": "Latency (ms)" }, { - "ID": "4.1-0005", + "ID": "4.1-0003", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -779,7 +383,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -788,11 +392,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 4121.85, + "Performance_Result": 4125.31, "Performance_Units": "Samples/s" }, { - "ID": "4.1-0005", + "ID": "4.1-0003", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -815,7 +419,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -824,11 +428,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 3841.11, + "Performance_Result": 3841.13, "Performance_Units": "Queries/s" }, { - "ID": "4.1-0005", + "ID": "4.1-0003", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -851,7 +455,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -860,11 +464,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 1.034379, + "Performance_Result": 1.015707, "Performance_Units": "Latency (ms)" }, { - "ID": "4.1-0006", + "ID": "4.1-0004", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -874,7 +478,7 @@ "UsedModel": "bert-99.9", "Model": "bert-99.9", "Scenario": "Offline", - "Accuracy": "F1: 90.89170281131781", + "Accuracy": "F1: 90.8832407068292", "Nodes": 1, "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, @@ -887,7 +491,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "fp16", @@ -896,11 +500,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99.9", "ColorKey": "availableMLCommons", - "Performance_Result": 1673.21, + "Performance_Result": 1176.59, "Performance_Units": "Samples/s" }, { - "ID": "4.1-0006", + "ID": "4.1-0004", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -910,7 +514,7 @@ "UsedModel": "bert-99.9", "Model": "bert-99.9", "Scenario": "Server", - "Accuracy": "F1: 90.89074704580993", + "Accuracy": "F1: 90.89127264236201", "Nodes": 1, "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, @@ -923,7 +527,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "fp16", @@ -932,11 +536,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99.9", "ColorKey": "availableMLCommons", - "Performance_Result": 1414.96, + "Performance_Result": 1414.95, "Performance_Units": "Queries/s" }, { - "ID": "4.1-0007", + "ID": "4.1-0005", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -959,7 +563,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -968,11 +572,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 0.467159, + "Performance_Result": 0.798611, "Performance_Units": "Latency (ms)" }, { - "ID": "4.1-0007", + "ID": "4.1-0005", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -995,7 +599,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1004,11 +608,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 43806.6, + "Performance_Result": 3633.97, "Performance_Units": "Samples/s" }, { - "ID": "4.1-0007", + "ID": "4.1-0005", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1031,7 +635,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1040,11 +644,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 35357.7, + "Performance_Result": 35357.8, "Performance_Units": "Queries/s" }, { - "ID": "4.1-0007", + "ID": "4.1-0005", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1067,7 +671,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1076,11 +680,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 0.277551, + "Performance_Result": 0.300962, "Performance_Units": "Latency (ms)" }, { - "ID": "4.1-0008", + "ID": "4.1-0006", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1090,7 +694,7 @@ "UsedModel": "retinanet", "Model": "retinanet", "Scenario": "MultiStream", - "Accuracy": "mAP: 37.291", + "Accuracy": "mAP: 37.329", "Nodes": 1, "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, @@ -1103,7 +707,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1112,11 +716,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 10.925334, + "Performance_Result": 11.64376, "Performance_Units": "Latency (ms)" }, { - "ID": "4.1-0008", + "ID": "4.1-0006", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1126,7 +730,7 @@ "UsedModel": "retinanet", "Model": "retinanet", "Scenario": "Offline", - "Accuracy": "mAP: 37.350", + "Accuracy": "mAP: 37.306", "Nodes": 1, "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, @@ -1139,7 +743,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1148,11 +752,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 864.226, + "Performance_Result": 872.522, "Performance_Units": "Samples/s" }, { - "ID": "4.1-0008", + "ID": "4.1-0006", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1162,7 +766,7 @@ "UsedModel": "retinanet", "Model": "retinanet", "Scenario": "Server", - "Accuracy": "mAP: 37.358", + "Accuracy": "mAP: 37.341", "Nodes": 1, "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, @@ -1175,7 +779,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1184,11 +788,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 637.174, + "Performance_Result": 637.175, "Performance_Units": "Queries/s" }, { - "ID": "4.1-0008", + "ID": "4.1-0006", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1198,7 +802,7 @@ "UsedModel": "retinanet", "Model": "retinanet", "Scenario": "SingleStream", - "Accuracy": "mAP: 37.316", + "Accuracy": "mAP: 37.326", "Nodes": 1, "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, @@ -1211,7 +815,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1220,47 +824,83 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 1.745447, + "Performance_Result": 1.709161, "Performance_Units": "Latency (ms)" }, { - "ID": "4.1-0009", + "ID": "4.1-0007", "Submitter": "MLCommons", "Availability": "available", "Category": "open", "Suite": "datacenter,edge", - "System": "RTX4090x2", - "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "UsedModel": "3d-unet-99.9", - "Model": "3d-unet-99.9", + "System": "RTX4090x1", + "Platform": "RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "UsedModel": "stable-diffusion-xl", + "Model": "stable-diffusion-xl", "Scenario": "Offline", - "Accuracy": "DICE: 0.86236", + "Accuracy": "", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9/offline", + "a#": 1, + "Location": "open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl/offline", "Software": "TensorRT", "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", "p#": 1, - "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9", + "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl", "ColorKey": "availableMLCommons", - "Performance_Result": 8.32758, + "Performance_Result": 0.697739, "Performance_Units": "Samples/s" }, { - "ID": "4.1-0009", + "ID": "4.1-0007", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "datacenter,edge", + "System": "RTX4090x1", + "Platform": "RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "UsedModel": "stable-diffusion-xl", + "Model": "stable-diffusion-xl", + "Scenario": "SingleStream", + "Accuracy": "", + "Nodes": 1, + "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "Accelerator": "NVIDIA GeForce RTX 4090", + "a#": 1, + "Location": "open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl/singlestream", + "Software": "TensorRT", + "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "int8", + "p#": 1, + "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl", + "ColorKey": "availableMLCommons", + "Performance_Result": 1424.658155, + "Performance_Units": "Latency (ms)" + }, + { + "ID": "4.1-0008", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1269,7 +909,7 @@ "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", "UsedModel": "3d-unet-99.9", "Model": "3d-unet-99.9", - "Scenario": "SingleStream", + "Scenario": "Offline", "Accuracy": "DICE: 0.86236", "Nodes": 1, "Processor": "Intel(R) Xeon(R) w7-2495X", @@ -1277,13 +917,13 @@ "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", "a#": 2, - "Location": "open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9/singlestream", + "Location": "open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9/offline", "Software": "TensorRT", "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1292,47 +932,47 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9", "ColorKey": "availableMLCommons", - "Performance_Result": 429.320711, - "Performance_Units": "Latency (ms)" + "Performance_Result": 8.29797, + "Performance_Units": "Samples/s" }, { - "ID": "4.1-0010", + "ID": "4.1-0008", "Submitter": "MLCommons", "Availability": "available", "Category": "open", "Suite": "datacenter,edge", "System": "RTX4090x2", "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", - "UsedModel": "bert-99", - "Model": "bert-99", - "Scenario": "Offline", - "Accuracy": "F1: 90.15673510616978", + "UsedModel": "3d-unet-99.9", + "Model": "3d-unet-99.9", + "Scenario": "SingleStream", + "Accuracy": "DICE: 0.86236", "Nodes": 1, "Processor": "Intel(R) Xeon(R) w7-2495X", "host_processors_per_node": 1, "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", "a#": 2, - "Location": "open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline", + "Location": "open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9/singlestream", "Software": "TensorRT", "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, - "weight_data_types": "fp16", + "weight_data_types": "int8", "p#": 1, "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99", + "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/3d-unet-99.9", "ColorKey": "availableMLCommons", - "Performance_Result": 8296.67, - "Performance_Units": "Samples/s" + "Performance_Result": 429.286663, + "Performance_Units": "Latency (ms)" }, { - "ID": "4.1-0010", + "ID": "4.1-0009", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1341,21 +981,21 @@ "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", "UsedModel": "bert-99", "Model": "bert-99", - "Scenario": "Server", - "Accuracy": "F1: 90.25897829249658", + "Scenario": "Offline", + "Accuracy": "F1: 90.15673510616978", "Nodes": 1, "Processor": "Intel(R) Xeon(R) w7-2495X", "host_processors_per_node": 1, "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", "a#": 2, - "Location": "open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/server", + "Location": "open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline", "Software": "TensorRT", "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1364,11 +1004,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6061.51, - "Performance_Units": "Queries/s" + "Performance_Result": 8245.93, + "Performance_Units": "Samples/s" }, { - "ID": "4.1-0010", + "ID": "4.1-0009", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1391,20 +1031,20 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, - "weight_data_types": "fp16", + "weight_data_types": "int8", "p#": 1, "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 1.029183, + "Performance_Result": 1.033728, "Performance_Units": "Latency (ms)" }, { - "ID": "4.1-0011", + "ID": "4.1-0010", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1427,7 +1067,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "fp16", @@ -1436,11 +1076,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99.9", "ColorKey": "availableMLCommons", - "Performance_Result": 3247.32, + "Performance_Result": 3339.26, "Performance_Units": "Samples/s" }, { - "ID": "4.1-0012", + "ID": "4.1-0011", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1463,7 +1103,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1472,11 +1112,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.037738, + "Performance_Result": 0.502283, "Performance_Units": "Latency (ms)" }, { - "ID": "4.1-0012", + "ID": "4.1-0011", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1499,7 +1139,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1508,11 +1148,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 87916.0, + "Performance_Result": 88040.1, "Performance_Units": "Samples/s" }, { - "ID": "4.1-0012", + "ID": "4.1-0011", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1535,7 +1175,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1548,7 +1188,7 @@ "Performance_Units": "Queries/s" }, { - "ID": "4.1-0012", + "ID": "4.1-0011", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1571,7 +1211,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1580,11 +1220,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 0.598655, + "Performance_Result": 0.303938, "Performance_Units": "Latency (ms)" }, { - "ID": "4.1-0013", + "ID": "4.1-0012", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1594,7 +1234,7 @@ "UsedModel": "retinanet", "Model": "retinanet", "Scenario": "MultiStream", - "Accuracy": "mAP: 37.311", + "Accuracy": "mAP: 37.328", "Nodes": 1, "Processor": "Intel(R) Xeon(R) w7-2495X", "host_processors_per_node": 1, @@ -1607,7 +1247,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1616,11 +1256,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 6.227423, + "Performance_Result": 5.638927, "Performance_Units": "Latency (ms)" }, { - "ID": "4.1-0013", + "ID": "4.1-0012", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1630,7 +1270,7 @@ "UsedModel": "retinanet", "Model": "retinanet", "Scenario": "Offline", - "Accuracy": "mAP: 37.321", + "Accuracy": "mAP: 37.371", "Nodes": 1, "Processor": "Intel(R) Xeon(R) w7-2495X", "host_processors_per_node": 1, @@ -1643,7 +1283,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1652,11 +1292,11 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 1730.51, + "Performance_Result": 1740.84, "Performance_Units": "Samples/s" }, { - "ID": "4.1-0013", + "ID": "4.1-0012", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1666,7 +1306,7 @@ "UsedModel": "retinanet", "Model": "retinanet", "Scenario": "Server", - "Accuracy": "mAP: 37.326", + "Accuracy": "mAP: 37.320", "Nodes": 1, "Processor": "Intel(R) Xeon(R) w7-2495X", "host_processors_per_node": 1, @@ -1679,7 +1319,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1692,7 +1332,7 @@ "Performance_Units": "Queries/s" }, { - "ID": "4.1-0013", + "ID": "4.1-0012", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1702,7 +1342,7 @@ "UsedModel": "retinanet", "Model": "retinanet", "Scenario": "SingleStream", - "Accuracy": "mAP: 37.336", + "Accuracy": "mAP: 37.310", "Nodes": 1, "Processor": "Intel(R) Xeon(R) w7-2495X", "host_processors_per_node": 1, @@ -1715,7 +1355,7 @@ "Notes": "", "compliance": 1, "errors": 0, - "version": "v4.1", + "version": "v5.0", "inferred": 0, "has_power": false, "weight_data_types": "int8", @@ -1724,7 +1364,115 @@ "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 2.335437, + "Performance_Result": 1.737171, + "Performance_Units": "Latency (ms)" + }, + { + "ID": "4.1-0013", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "datacenter,edge", + "System": "RTX4090x2", + "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "UsedModel": "stable-diffusion-xl", + "Model": "stable-diffusion-xl", + "Scenario": "Offline", + "Accuracy": "", + "Nodes": 1, + "Processor": "Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "Accelerator": "NVIDIA GeForce RTX 4090", + "a#": 2, + "Location": "open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl/offline", + "Software": "TensorRT", + "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "int8", + "p#": 1, + "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl", + "ColorKey": "availableMLCommons", + "Performance_Result": 1.39663, + "Performance_Units": "Samples/s" + }, + { + "ID": "4.1-0013", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "datacenter,edge", + "System": "RTX4090x2", + "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "UsedModel": "stable-diffusion-xl", + "Model": "stable-diffusion-xl", + "Scenario": "Server", + "Accuracy": "", + "Nodes": 1, + "Processor": "Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "Accelerator": "NVIDIA GeForce RTX 4090", + "a#": 2, + "Location": "open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl/server", + "Software": "TensorRT", + "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "int8", + "p#": 1, + "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl", + "ColorKey": "availableMLCommons", + "Performance_Result": 0.881818, + "Performance_Units": "Queries/s" + }, + { + "ID": "4.1-0013", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "datacenter,edge", + "System": "RTX4090x2", + "Platform": "RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "UsedModel": "stable-diffusion-xl", + "Model": "stable-diffusion-xl", + "Scenario": "SingleStream", + "Accuracy": "", + "Nodes": 1, + "Processor": "Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "Accelerator": "NVIDIA GeForce RTX 4090", + "a#": 2, + "Location": "open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl/singlestream", + "Software": "TensorRT", + "operating_system": "Ubuntu 20.04 (linux-6.8.0-49-generic-glibc2.31)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "int8", + "p#": 1, + "Details": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config", + "Code": "https://github.com/mlcommons/submissions_inference_4.1/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "datacenter,edge/open/MLCommons/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/stable-diffusion-xl", + "ColorKey": "availableMLCommons", + "Performance_Result": 1400.027931, "Performance_Units": "Latency (ms)" } ] \ No newline at end of file