forked from rovaughn/fastcsv
-
Notifications
You must be signed in to change notification settings - Fork 0
/
test.py
42 lines (36 loc) · 1.09 KB
/
test.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
import csv
import datetime
import itertools
import contextlib
import mmap
@contextlib.contextmanager
def csv_reader(filename):
with open('test.csv') as f:
data = mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ)
try:
def generator():
start = 0
while True:
eol = data.find(b'\n', start)
if eol == -1:
break
yield data[start:eol].split(b',')
start = eol + 1
yield generator()
finally:
data.close()
n = 1000000
with open('test.csv') as f:
r = csv.DictReader(f)
start = datetime.datetime.now()
for line in itertools.islice(r, n):
pass
end = datetime.datetime.now()
print('python csv\t%.0f ns / op' % (1e9 * (end - start).total_seconds() / n))
n = 1000000
with csv_reader('test.csv') as r:
start = datetime.datetime.now()
for line in itertools.islice(r, n):
pass
end = datetime.datetime.now()
print('python mmap-csv\t%.0f ns / op' % (1e9 * (end - start).total_seconds() / n))