-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathmain.py
132 lines (91 loc) · 3.04 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
from catloader import get_cat_url_list, download_cats
from decorators import execute_in_threads, timer
from typing import List
@timer
def download_cats_single(cat_urls: List[str]):
return download_cats(cat_urls)
@timer
@execute_in_threads(max_threads=4, urls_per_thread=5)
def download_cats_4_5(cat_urls: List[str]):
return download_cats(cat_urls)
@timer
@execute_in_threads(max_threads=4, urls_per_thread=10)
def download_cats_4_10(cat_urls: List[str]):
download_cats(cat_urls)
@timer
@execute_in_threads(max_threads=4, urls_per_thread=25)
def download_cats_4_25(cat_urls: List[str]):
download_cats(cat_urls)
@timer
@execute_in_threads(max_threads=8, urls_per_thread=5)
def download_cats_8_5(cat_urls: List[str]):
download_cats(cat_urls)
@timer
@execute_in_threads(max_threads=8, urls_per_thread=10)
def download_cats_8_10(cat_urls: List[str]):
download_cats(cat_urls)
@timer
@execute_in_threads(max_threads=16, urls_per_thread=5)
def download_cats_16_5(cat_urls: List[str]):
download_cats(cat_urls)
@timer
@execute_in_threads(max_threads=50, urls_per_thread=2)
def download_cats_50_2(cat_urls: List[str]):
download_cats(cat_urls)
@timer
@execute_in_threads(max_threads=100, urls_per_thread=1)
def download_cats_100_1(cat_urls: List[str]):
download_cats(cat_urls)
print('---------------------------------')
print('--------- Single thread ---------')
print('---------------------------------')
urls = get_cat_url_list()
download_cats_single(cat_urls=urls)
print()
print('---------------------------------')
print('--- 4 threads, 5 urls/thread ----')
print('---------------------------------')
urls = get_cat_url_list()
download_cats_4_5(cat_urls=urls)
print()
print('---------------------------------')
print('--- 4 threads, 10 urls/thread ---')
print('---------------------------------')
urls = get_cat_url_list()
download_cats_4_10(cat_urls=urls)
print()
print('---------------------------------')
print('--- 4 threads, 25 urls/thread ---')
print('---------------------------------')
urls = get_cat_url_list()
download_cats_4_25(cat_urls=urls)
print()
print('---------------------------------')
print('--- 8 threads, 5 urls/thread ----')
print('---------------------------------')
urls = get_cat_url_list()
download_cats_8_5(cat_urls=urls)
print()
print('---------------------------------')
print('--- 8 threads, 10 urls/thread ---')
print('---------------------------------')
urls = get_cat_url_list()
download_cats_8_10(cat_urls=urls)
print()
print('---------------------------------')
print('--- 16 threads, 5 urls/thread ----')
print('---------------------------------')
urls = get_cat_url_list()
download_cats_16_5(cat_urls=urls)
print()
print('---------------------------------')
print('--- 50 threads, 2 urls/thread ----')
print('---------------------------------')
urls = get_cat_url_list()
download_cats_50_2(cat_urls=urls)
print()
print('---------------------------------')
print('--- 100 threads, 1 urls/thread ----')
print('---------------------------------')
urls = get_cat_url_list()
download_cats_100_1(cat_urls=urls)