-
Notifications
You must be signed in to change notification settings - Fork 5
/
utils.py
174 lines (157 loc) · 7.86 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
import sys
import os
import configargparse
from queue import Queue
def memoize(function):
memo = {}
def wrapper(*args):
if args in memo:
return memo[args]
else:
rv = function(*args)
memo[args] = rv
return rv
return wrapper
@memoize
def get_queues():
db_queue = Queue()
wh_queue = Queue()
process_queue = Queue()
stats_queue = Queue()
return (db_queue, wh_queue, process_queue, stats_queue)
@memoize
def get_args():
# Pre-check to see if the -cf or --config flag is used on the command line.
# If not, we'll use the env var or default value. This prevents layering of
# config files as well as a missing config.ini.
defaultconfigfiles = []
if '-cf' not in sys.argv and '--config' not in sys.argv:
defaultconfigfiles = [os.getenv('WHSRV_CONFIG', os.path.join(
os.path.dirname(__file__), './config/config.ini'))]
parser = configargparse.ArgParser(
default_config_files=defaultconfigfiles,
auto_env_var_prefix='WHSRV_')
parser.add_argument('-cf', '--config',
is_config_file=True, help='Set configuration file')
parser.add_argument('-H', '--host',
help='Set web server listening host. ' +
'default = 0.0.0.0',
default='0.0.0.0')
parser.add_argument('-P', '--port', type=int,
help='Set web server listening port. ' +
'default=5001', default=5001)
parser.add_argument('-cd', '--clear-db',
help='Clear the database tables, and recreate. ' +
'Does not delete the authorizations.',
action='store_true', default=False)
parser.add_argument('--db-name', help='Name of the database to be used.')
parser.add_argument('--db-user', help='Username for the database.')
parser.add_argument('--db-pass', help='Password for the database.')
parser.add_argument('--db-host', help='IP or hostname for the database.')
parser.add_argument(
'--db-port', help='Port for the database.', type=int, default=3306)
parser.add_argument('--db-threads',
help=('Number of db threads; increase if the db ' +
'queue falls behind.'),
type=int, default=2)
parser.add_argument('--httpd-threads',
help=('Number of httpd threads.'),
type=int, default=100)
parser.add_argument('-g', '--generate', help='Generate an authorization ' +
'token. An identifying string is required.')
ignore_list = parser.add_mutually_exclusive_group()
ignore_list.add_argument('-ig', '--ignore-pokemon',
action='append', default=[],
help=('List of Pokemon to ignore when ' +
'received.'))
parser.add_argument('-l', '--list', help='List authorization tokes.',
action='store_true', default=False)
parser.add_argument('-np', '--no-pokemon',
help='Pokemon will not be stored in the database.',
action='store_true', default=False)
parser.add_argument('-nk', '--no-pokestops',
help='Pokestops will not be stored in the database.',
action='store_true', default=False)
parser.add_argument('-ng', '--no-gyms',
help='Gyms will not be stored in the database.',
action='store_true', default=False)
parser.add_argument('-ngd', '--no-gymdetail',
help='Gym detail will not be stored in the database.',
action='store_true', default=False)
parser.add_argument('-nr', '--no-raids',
help='Raids will not be stored in the database.',
action='store_true', default=False)
parser.add_argument('-nw', '--no-weather',
help='Weather will not be stored in the database.',
action='store_true', default=False)
parser.add_argument('-rs', '--runtime-statistics',
help='Display usage statistics. Specified in ' +
'minutes', type=int, default=0)
parser.add_argument('-sh', '--safe-httpd',
help='Run a more conservative HTTPD service',
action='store_true', default=False)
parser.add_argument('-pd', '--purge-data',
help=('Clear Pokemon from database this many hours ' +
'after they disappear (0 to disable).'),
type=int, default=0)
parser.add_argument('-pi', '--pokemon-inserts',
help='Number of pokemon to commit ' +
' to the DB at once. Bulk inserts are faster than ' +
'singles.', default=1, type=int)
parser.add_argument('--process-threads',
help=('Number of main workers threads; ' +
'increase if the queue falls behind.'),
type=int, default=3)
parser.add_argument('-r', '--revoke', help='Revoke an authorization ' +
'token. An identifying string is required.')
parser.add_argument('-wh', '--webhook',
help='Define URL(s) to POST webhook information to.',
default=None, dest='webhooks', action='append')
parser.add_argument('-whr', '--wh-retries',
help=('Number of times to retry sending webhook ' +
'data on failure.'),
type=int, default=3)
parser.add_argument('-whbf', '--wh-backoff-factor',
help=('Factor (in seconds) by which the delay ' +
'until next retry will increase.'),
type=float, default=0.25)
parser.add_argument('--wh-threads',
help=('Number of webhook threads; increase if the ' +
'webhook queue falls behind.'),
type=int, default=1)
parser.add_argument('-whc', '--wh-concurrency',
help=('Async requests pool size.'), type=int,
default=25)
parser.add_argument('-wht', '--wh-timeout',
help='Timeout (in seconds) for webhook requests.',
type=float, default=1.0)
parser.add_argument('-whlfu', '--wh-lfu-size',
help='Webhook LFU cache max size.', type=int,
default=2500)
parser.add_argument('-whfi', '--wh-frame-interval',
help=('Minimum time (in ms) to wait before sending the'
+ ' next webhook data frame.'), type=int,
default=500)
verbosity = parser.add_mutually_exclusive_group()
verbosity.add_argument('-v', '--verbose',
help=('Show debug messages. ' +
'Optionally specify file to log to.'),
nargs='?', const='nofile', default=False,
metavar='filename.log')
parser.set_defaults(DEBUG=False)
args = parser.parse_args()
if None in (args.db_name, args.db_user,
args.db_pass, args.db_host):
parser.print_usage()
print(sys.argv[0] + ": DB info is not set correctly.")
exit(1)
return args
# Translate peewee model class attribute to database column name.
def peewee_attr_to_col(cls, field):
field_column = getattr(cls, field)
# Only try to do it on populated fields.
if field_column is not None:
field_column = field_column.db_column
else:
field_column = field
return field_column