-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathplayground.py
110 lines (78 loc) · 3.65 KB
/
playground.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
import json
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from datetime import datetime
from itertools import groupby
from operator import itemgetter
messages = []
grp_benchmark = itemgetter("benchmark","dim")
grp_instance = itemgetter("benchmark","instance")
result = []
new_info = True
data = None
#with open(r'experiment_data\1561177738\swarm_ea_1561177738.json') as json_file:
#with open(r'experiment_data\1561185389\swarm_ea_1561185389.json') as json_file:
# data = json.load(json_file)
# for d in data:
# print( d["benchmark"] , d["dim"], d["instance"])
#print(data[0])
#for dim_key, benchmark_group in groupby(data, grp_benchmark):
# print(dim_key, benchmark_group)
with open(r'D:\exp_data\1561769371\swarm_ea_1561769371.json') as json_file:
#with open(r'experiment_data\1561185389\swarm_ea_1561185389.json') as json_file:
chunk = pd.read_json(json_file,lines =True, chunksize = 2000 )
for data in chunk:
print('chunk')
data.drop(['alg_params','evals', 'algorithm', 'best_score', 'experiment_id',
'message_counter', 'message_id', 'params', 'worker_id'], axis = 1 ).to_csv('data.csv', index=False, header=False, mode='a')
#labels = ['ts','n','dim','instance','best_score','fopt']
#sns.set(style="white", palette="muted", color_codes=True)
#f, axes = plt.subplots(1, 6, sharex=True, sharey=True)
#sns.despine(left=True)
#df = pd.DataFrame.from_records(messages, columns=labels)
#df.set_index('ts', drop=False, inplace=True)
#group_by_dim = df.groupby("dim")
#for name, group in group_by_dim:
# #print(group.n.resample('120S').sum())
# print(group.agg( {'ts':[np.min, np.max]} ))
#print(group_by_dim.agg( {'ts':[np.min, np.max]} ))
#df2 = group_by_dim.resample('60S').agg( {'ts':[np.min, np.max] , 'n':np.sum })
#resample_step = {2:'20S', 3:'20S',5:'20S',10:'60S',20:'120S',40:'240S' }
#for name, group in df2:
# print (df2.get_group(group))
# print (name, group)
#sns.barplot(x=group.ts, y=group.n.sum, palette="rocket")
#for i, (name, group) in enumerate (df.groupby("dim")):
# group.set_index('ts', drop=False, inplace=True)
# print(name)
# resample = group.resample(resample_step[name]).agg( {'ts':[np.min, np.max] , 'n':np.sum })
# sns.barplot(x=resample.ts.amax, y=resample.n['sum'], color="b", ax=axes[i])
#for i, (name, group) in enumerate (df2.reset_index().groupby('dim')):
#sns.barplot(x=list(group.ts.amax), y=list(group.n.sum), palette="rocket")
#print([i for i in group.n['sum']] , list( range(1, len(group.n['sum']) + 1 )))
# print(resample)
#for i in group.ts.amax:
# print (i)
#sns.barplot(x=group.ts.amax, y=group.n['sum'], palette="rocket", ax=axes[count])
#plt.setp(axes, xticks=[])
#plt.tight_layout()
#plt.show()
#print(df2.ts.amin)
#df2.reset_index().plot(kind='bar', use_index=True, y='n')
# from rx import Observable, Observer
# from rx.subjects import Subject
# messages = Subject()
# valid_messages = messages.take(8)\
# .filter(lambda s: s[0] == 1 ) \
# .publish()
# valid_messages.subscribe(lambda s : print(s))
# valid_messages.buffer_with_count(2)\
# .subscribe(lambda s : print( s))
# valid_messages.connect()
# for s in [ (1, "ABB", 1) ,(1, "BCC", 2),(1, "DBA", 3),
# (1, "AAA", 4) ,(1, "AAA", 5),(1, "AAA", 6),
# (1, "AAA", 7) ,(1, "AAA", 8),(1, "AAA", 9),
# (3, "AAA", 10) ,(31, "AAA", 11) ] :
# messages.on_next(s)