forked from lucasfacchini/open-data-cnpj-import
-
Notifications
You must be signed in to change notification settings - Fork 1
/
mysql_import.py
59 lines (45 loc) · 1.58 KB
/
mysql_import.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
import sys
from parser.parsers import generate_parsers_from_files, EstabeleCsvParser
from parser.csv_reader import CsvReader
from parser.importer import SqlImport, MysqlImport
from tools.log import Log
DEFAULT_DIRECTORY = 'data/teste-extract'
def parse_args():
if len(sys.argv) < 5:
print('usage: mysql_import.py <host> <port> <user> <password> <database> <directory>')
exit()
args = {
'host': sys.argv[1],
'port': sys.argv[2],
'user': sys.argv[3],
'password': sys.argv[4],
'database': sys.argv[5]
}
if len(sys.argv) > 6:
args['directory'] = sys.argv[6]
else:
args['directory'] = DEFAULT_DIRECTORY
return args
args = parse_args()
log = Log()
sql = MysqlImport(args['host'], args['port'], args['user'], args['password'], args['database'], log)
#log.info('Creating schema')
#sql.run_script('schema/mysql/drop-tables.sql')
#sql.run_script('schema/mysql/create-tables.sql')
log.info('Analyzing files')
parsers = generate_parsers_from_files(args['directory'], log)
if len(parsers) > 0:
log.info('Found', len(parsers), 'files')
else:
log.info('No files found.')
log.info('Truncating tables')
""" Vamos excluir os dados das tabelas (tirando empresa e estabelecimento) antes de importar os novos dados """
for parser in parsers:
if not parser.TABLE in ['empresa', 'estabelecimento']:
sql.truncate_table(parser.TABLE)
count = 0
for parser in parsers:
log.info('Importing file', parser.get_name(), '-', count + 1, 'of', len(parsers))
sql.run(parser)
count += 1
sql.close()