Skip to content

Commit b0109fa

Browse files
author
reneehlozek
committed
fixed merge
2 parents 0d5c63b + 86c5141 commit b0109fa

31 files changed

Lines changed: 2640 additions & 1192 deletions

‎Plot_Table3.ipynb‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"metadata": {},
7+
"outputs": [],
8+
"source": [
9+
"import numpy as np\n",
10+
"import pandas as pd\n",
11+
"import pickle\n",
12+
"\n",
13+
"import matplotlib as mpl\n",
14+
"# print(mpl.rcParams.items)\n",
15+
"mpl.use('Agg')\n",
16+
"mpl.rcParams['text.usetex'] = False\n",
17+
"mpl.rcParams['mathtext.rm'] = 'serif'\n",
18+
"mpl.rcParams['font.family'] = 'serif'\n",
19+
"mpl.rcParams['font.serif'] = ['Times New Roman']\n",
20+
"# mpl.rcParams['font.family'] = ['Times New Roman']\n",
21+
"mpl.rcParams['axes.titlesize'] = 25\n",
22+
"mpl.rcParams['axes.labelsize'] = 20\n",
23+
"mpl.rcParams['xtick.labelsize'] = 15\n",
24+
"mpl.rcParams['ytick.labelsize'] = 15\n",
25+
"mpl.rcParams['savefig.dpi'] = 250\n",
26+
"mpl.rcParams['figure.dpi'] = 250\n",
27+
"mpl.rcParams['savefig.format'] = 'pdf'\n",
28+
"mpl.rcParams['savefig.bbox'] = 'tight'\n",
29+
"import matplotlib.pyplot as plt\n",
30+
"%matplotlib inline"
31+
]
32+
},
33+
{
34+
"cell_type": "code",
35+
"execution_count": null,
36+
"metadata": {},
37+
"outputs": [],
38+
"source": [
39+
"def truncate_colormap(cmap, minval=0.0, maxval=1.0, n=100):\n",
40+
" new_cmap = mpl.colors.LinearSegmentedColormap.from_list(\n",
41+
" 'trunc({n},{a:.2f},{b:.2f})'.format(n=cmap.name, a=minval, b=maxval),\n",
42+
" cmap(np.linspace(minval, maxval, n)))\n",
43+
" return new_cmap\n",
44+
"\n",
45+
"cmap = plt.get_cmap('hot_r')\n",
46+
"fave_cmap = truncate_colormap(cmap, 0.35, 1.0)"
47+
]
48+
},
49+
{
50+
"cell_type": "code",
51+
"execution_count": null,
52+
"metadata": {},
53+
"outputs": [],
54+
"source": [
55+
"metric_dictionary = {'TBDT':{'FoM': 1, 'LogLoss': 1, 'Brier': 1},\n",
56+
" 'TKNN':{'FoM': 7, 'LogLoss': 6, 'Brier': 7},\n",
57+
" 'TNB':{'FoM': 8, 'LogLoss': 9, 'Brier': 8},\n",
58+
" 'TNN':{'FoM': 5, 'LogLoss': 3, 'Brier': 3},\n",
59+
" 'TSVM':{'FoM': 3, 'LogLoss': 2, 'Brier': 2},\n",
60+
" 'WBDT':{'FoM': 2, 'LogLoss': 5, 'Brier': 4},\n",
61+
" 'WKNN':{'FoM': 9, 'LogLoss': 8, 'Brier': 9},\n",
62+
" 'WNB':{'FoM': 10, 'LogLoss': 10, 'Brier': 10},\n",
63+
" 'WNN':{'FoM': 6, 'LogLoss': 7, 'Brier': 6},\n",
64+
" 'WSVM':{'FoM': 4, 'LogLoss': 4, 'Brier': 5},\n",
65+
" }"
66+
]
67+
},
68+
{
69+
"cell_type": "code",
70+
"execution_count": null,
71+
"metadata": {},
72+
"outputs": [],
73+
"source": [
74+
"metric_dictionary['TBDT']"
75+
]
76+
},
77+
{
78+
"cell_type": "code",
79+
"execution_count": null,
80+
"metadata": {},
81+
"outputs": [],
82+
"source": [
83+
"symbols = {'TBDT':'o',\n",
84+
" 'TKNN':'d',\n",
85+
" 'TNB':'s',\n",
86+
" 'TNN':'*',\n",
87+
" 'TSVM':'^',\n",
88+
" 'WBDT':'o',\n",
89+
" 'WKNN':'d',\n",
90+
" 'WNB':'s',\n",
91+
" 'WNN':'*',\n",
92+
" 'WSVM':'^',\n",
93+
" }\n",
94+
"\n",
95+
"colors = {'TBDT':fave_cmap(0.05),\n",
96+
" 'TKNN':fave_cmap(0.3),\n",
97+
" 'TNB':fave_cmap(0.55),\n",
98+
" 'TNN':fave_cmap(0.8),\n",
99+
" 'TSVM':fave_cmap(1.0),\n",
100+
" 'WBDT':fave_cmap(0.05),\n",
101+
" 'WKNN':fave_cmap(0.3),\n",
102+
" 'WNB':fave_cmap(0.55),\n",
103+
" 'WNN':fave_cmap(0.75),\n",
104+
" 'WSVM':fave_cmap(1.0),\n",
105+
" }\n",
106+
"\n",
107+
"\n",
108+
"plt.figure()\n",
109+
"for key, value in metric_dictionary.items():\n",
110+
" val = []\n",
111+
" for k, v in value.items():\n",
112+
" val.append(v)\n",
113+
" if 'W' in key:\n",
114+
" plt.plot(val, label=key, marker=symbols[key], ls='--', color=colors[key])\n",
115+
" else:\n",
116+
" plt.plot(val, label=key, marker=symbols[key], color=colors[key])\n",
117+
"\n",
118+
"plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), prop={'size': 12})\n",
119+
"plt.xticks([0, 1, 2], ['FoM', 'LogLoss', 'Brier'])\n",
120+
"plt.yticks(np.arange(1, 11))\n",
121+
"plt.ylabel('Rank')\n",
122+
"\n",
123+
"#plt.savefig('Tables3_option1.pdf')"
124+
]
125+
},
126+
{
127+
"cell_type": "code",
128+
"execution_count": null,
129+
"metadata": {},
130+
"outputs": [],
131+
"source": [
132+
"\n",
133+
"colors = {'TBDT':fave_cmap(0.05),\n",
134+
" 'TKNN':fave_cmap(0.2375),\n",
135+
" 'TNB':fave_cmap(0.54),\n",
136+
" 'TNN':fave_cmap(0.712499999),\n",
137+
" 'TSVM':fave_cmap(1.0),\n",
138+
" 'WBDT':fave_cmap(0.05),\n",
139+
" 'WKNN':fave_cmap(0.2375),\n",
140+
" 'WNB':fave_cmap(0.54),\n",
141+
" 'WNN':fave_cmap(0.712499999),\n",
142+
" 'WSVM':fave_cmap(1.0),\n",
143+
" }\n",
144+
"\n",
145+
"plt.figure()\n",
146+
"for key, value in metric_dictionary.items():\n",
147+
" val = []\n",
148+
" for k, v in value.items():\n",
149+
" val.append(v)\n",
150+
" if 'W' in key:\n",
151+
" plt.plot(val, label=key, marker=symbols[key], ls='--', color=colors[key], lw=2, ms=7, alpha=0.3)\n",
152+
" else:\n",
153+
" plt.plot(val, label=key, marker=symbols[key], color=colors[key], lw=2, ms=7)\n",
154+
"\n",
155+
"plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), prop={'size': 12})\n",
156+
"plt.xticks([0, 1, 2], ['FoM', 'LogLoss', 'Brier'])\n",
157+
"plt.yticks(np.arange(1, 11))\n",
158+
"plt.ylabel('Rank')\n",
159+
"plt.gca().invert_yaxis()\n",
160+
"\n",
161+
"#plt.savefig('Tables3_option4.pdf')"
162+
]
163+
},
164+
{
165+
"cell_type": "code",
166+
"execution_count": null,
167+
"metadata": {},
168+
"outputs": [],
169+
"source": [
170+
"plt.figure()\n",
171+
"\n",
172+
"fom = []\n",
173+
"ll = []\n",
174+
"brier = []\n",
175+
"\n",
176+
"for key, value in metric_dictionary.items():\n",
177+
" fom.append(value['FoM'])\n",
178+
" ll.append(value['LogLoss'])\n",
179+
" brier.append(value['Brier'])\n",
180+
"\n",
181+
"plt.plot(fom, label='FoM', marker='o')\n",
182+
"plt.plot(ll, label='LogLoss', marker='D', alpha = 0.5)\n",
183+
"plt.plot(brier, label='Brier', marker='s', alpha=0.23)\n",
184+
"\n",
185+
"plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), prop={'size': 12})\n",
186+
"plt.xticks(np.arange(0, 10), list(metric_dictionary.keys()), rotation=45)\n",
187+
"plt.ylabel('Rank')\n",
188+
"plt.savefig('Tables3_option2.pdf')"
189+
]
190+
},
191+
{
192+
"cell_type": "code",
193+
"execution_count": null,
194+
"metadata": {},
195+
"outputs": [],
196+
"source": []
197+
}
198+
],
199+
"metadata": {
200+
"kernelspec": {
201+
"display_name": "Python 2",
202+
"language": "python",
203+
"name": "python2"
204+
},
205+
"language_info": {
206+
"codemirror_mode": {
207+
"name": "ipython",
208+
"version": 2
209+
},
210+
"file_extension": ".py",
211+
"mimetype": "text/x-python",
212+
"name": "python",
213+
"nbconvert_exporter": "python",
214+
"pygments_lexer": "ipython2",
215+
"version": "2.7.15"
216+
}
217+
},
218+
"nbformat": 4,
219+
"nbformat_minor": 2
220+
}

‎metrics_evaluation.ipynb‎

Lines changed: 33 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
"\n",
3131
"import numpy as np\n",
3232
"import pandas as pd\n",
33+
"from pycm import ConfusionMatrix\n",
3334
"\n",
3435
"import proclam\n",
3536
"from proclam import *"
@@ -186,7 +187,7 @@
186187
" data_info_dict['dirname'] = dirname + data_info_dict['label'] + '/'\n",
187188
" data_info_dict['classifications'] = ['%s/predicted_prob_%s.csv'%(name, name) for name in names]\n",
188189
" data_info_dict['truth_tables'] = ['%s/truth_table_%s.csv'%(name, name) for name in names]\n",
189-
" print(data_info_dict)\n",
190+
"# print(data_info_dict)\n",
190191
" return data_info_dict"
191192
]
192193
},
@@ -249,6 +250,13 @@
249250
"scrolled": true
250251
},
251252
"outputs": [],
253+
"source": []
254+
},
255+
{
256+
"cell_type": "code",
257+
"execution_count": null,
258+
"metadata": {},
259+
"outputs": [],
252260
"source": [
253261
"def read_class_pairs(pair, dataset, cc):#loc='', title=''):\n",
254262
" loc=dataset['dirname']\n",
@@ -362,12 +370,21 @@
362370
" for cc, pair in enumerate(dataset['class_pairs']):\n",
363371
" print(pair)\n",
364372
" probm, truthv = read_class_pairs(pair, dataset, cc)#loc=dataset['dirname'], title=dataset['label']+' '+dataset['names'][cc])\n",
365-
" for count, metric in enumerate(metricslist):\n",
366-
" D = getattr(proclam.metrics, metric)()\n",
367-
" hm = D.evaluate(probm, truthv)\n",
368-
" data[count][cc] = hm\n",
369-
" dataset['results'] = data\n",
370-
" metric_plot(dataset, metricslist, markerlist, colors)"
373+
"# plot_cm(probm, truthv, str(cc), loc='./sandbox/')\n",
374+
" det = proclam.metrics.util.prob_to_det(probm)\n",
375+
" cm = proclam.metrics.util.prob_to_cm(probm, truthv, per_class_norm=False, vb=False)\n",
376+
" rates = proclam.metrics.util.cm_to_rate(cm, vb=True)\n",
377+
"# print(rates)\n",
378+
" compare = ConfusionMatrix(truthv, det)\n",
379+
" printout = proclam.metrics.util.RateMatrix(compare.TPR, compare.FPR, compare.FNR, compare.TNR)\n",
380+
" print('for comparison: ' + str(printout))\n",
381+
"# for count, metric in enumerate(metricslist):\n",
382+
"# D = getattr(proclam.metrics, metric)()\n",
383+
"# hm = D.evaluate(probm, truthv)\n",
384+
"# data[count][cc] = hm\n",
385+
"# dataset['results'] = data\n",
386+
" \n",
387+
"# metric_plot(dataset, metricslist, markerlist, colors)"
371388
]
372389
},
373390
{
@@ -379,6 +396,13 @@
379396
"outputs": [],
380397
"source": []
381398
},
399+
{
400+
"cell_type": "code",
401+
"execution_count": null,
402+
"metadata": {},
403+
"outputs": [],
404+
"source": []
405+
},
382406
{
383407
"cell_type": "code",
384408
"execution_count": null,
@@ -390,9 +414,9 @@
390414
"metadata": {
391415
"anaconda-cloud": {},
392416
"kernelspec": {
393-
"display_name": "Python 3",
417+
"display_name": "proclam (Python 3)",
394418
"language": "python",
395-
"name": "python3"
419+
"name": "proclam_3"
396420
},
397421
"language_info": {
398422
"codemirror_mode": {

‎paper/authors.csv‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
Lastname,Firstname,Authorname,AuthorType,Affiliation,Contribution,Email
2+
Malz,Alex,A.I.~Malz,Contact,"German Centre of Cosmological Lensing, Ruhr-Universitaet Bochum, Universitaetsstra{\ss}e 150, 44801 Bochum, Germany","conceptualization, data curation, formal analysis, investigation, methodology, project administration, software, supervision, validation, visualization, writing - editing, writing - original draft",aimalz@nyu.edu
23
Malz,Alex,A.I.~Malz,Contact,"Center for Cosmology and Particle Physics, New York University, 726 Broadway, New York, NY 10004, USA","conceptualization, data curation, formal analysis, investigation, methodology, project administration, software, supervision, validation, visualization, writing - editing, writing - original draft",aimalz@nyu.edu
34
Malz,Alex,A.I.~Malz,Contact,"Department of Physics, New York University, 726 Broadway, New York, NY 10004, USA","conceptualization, data curation, formal analysis, investigation, methodology, project administration, software, supervision, validation, visualization, writing - editing, writing - original draft",aimalz@nyu.edu
45
Hlo\v{z}ek,Ren\'ee,R.~Hlo\v{z}ek,Contributor,"Department of Astronomy and Astrophysics, University of Toronto, 50 St. George St., Toronto, ON M5S 3H4, Canada","data curation, formal analysis, funding acquisition, investigation, project administration, software, supervision, validation, visualization, writing - editing, writing - original draft",hlozek@dunlap.utoronto.ca
56
Hlo\v{z}ek,Ren\'ee,R.~Hlo\v{z}ek,Contributor,"Dunlap Institute for Astronomy and Astrophysics, University of Toronto, 50 St. George St., Toronto, ON M5S 3H4, Canada","data curation, formal analysis, funding acquisition, investigation, project administration, software, supervision, validation, visualization, writing - editing, writing - original draft",hlozek@dunlap.utoronto.ca
67
Allam,Tarek,T.~Allam Jr,Contributor,"Mullard Space Science Laboratory, Department of Space and Climate Physics, University College London, Holmbury Hill Rd, Dorking RH5 6NT, UK","investigation, software, validation, writing - original draft",[email]
78
Bahmanyar,Anita,A.~Bahmanyar,Contributor,"Dunlap Institute for Astronomy and Astrophysics, University of Toronto, 50 St. George St., Toronto, ON M5S 3H4, Canada","formal analysis, investigation, methodology, software, writing - editing, writing - original draft",[email]
8-
Biswas,Rahul,R.~Biswas,Contributor,"The Oskar Klein Centre for Cosmoparticle Physics, Stockholm University, AlbaNova, Stockholm, SE-106 91, Sweden","conceptualization, methodology, software, writing - original draft",[email]
9+
Biswas,Rahul,R.~Biswas,Contributor,"The Oskar Klein Centre for Cosmoparticle Physics, Stockholm University, AlbaNova, Stockholm, SE-106 91, Sweden","conceptualization, methodology, software, supervision, writing - editing, writing - original draft",[email]
910
Dai,Mi,M.~Dai,Contributor,"Rutgers, the State University of New Jersey, 136 Frelinghuysen Road, Piscataway, NJ 08854 USA","writing - editing",[email]
1011
Galbany,Llu\'is,L.~Galbany,Contributor,"University of Pittsburgh, 300 Allen Hall, 3941 O'Hara St, Pittsburgh, PA 15260","writing - editing",[email]
1112
Ishida,Emille,E.E.O.~Ishida,Contributor,"Universit\'e Clermont Auvergne, CNRS/IN2P3, LPC, F-63000 Clermont-Ferrand, France","conceptualization, project administration, supervision, writing - editing",[email]
@@ -25,5 +26,5 @@ Narayan,Gautham,G.~Narayan,Contributor,"Space Telescope Science Institute, 3700
2526
Peiris,Hiranya,H.~Peiris,Contributor,"The Oskar Klein Centre for Cosmoparticle Physics, Stockholm University, AlbaNova, Stockholm, SE-106 91, Sweden","conceptualization, funding acquisition, supervision",[email]
2627
Peiris,Hiranya,H.~Peiris,Contributor,"Department of Physics and Astronomy, University College London, Gower Street, London, WC1E 6BT, UK","conceptualization, funding acquisition, supervision",[email]
2728
Peters,Christina~M.,C.M.~Peters,Contributor,"Dunlap Institute for Astronomy and Astrophysics, University of Toronto, 50 St. George St., Toronto, ON M5S 3H4, Canada","writing - editing",[email]
28-
Ponder,Kara,K.~Ponder,Contributor,"Berkeley Center for Cosmological Physics, Campbell Hall 341, University of California Berkeley, Berkeley, CA 94720, USA","writing - editing",[email]
29+
Ponder,Kara,K.~Ponder,Contributor,"Berkeley Center for Cosmological Physics, Campbell Hall 341, University of California Berkeley, Berkeley, CA 94720, USA","visualization, writing - editing",[email]
2930
Setzer,Christian,C.N.~Setzer,Contributor,"The Oskar Klein Centre for Cosmoparticle Physics, Stockholm University, AlbaNova, Stockholm, SE-106 91, Sweden","conceptualization, software",christian.setzer@fysik.su.se

‎paper/fig/Tables3_option4.png‎

137 KB
Loading

‎paper/fig/all_sim_cm.png‎

-21 KB
Loading

‎paper/fig/all_snphotcc_cm.png‎

1.33 KB
Loading

‎paper/fig/combined.png‎

-30.5 KB
Loading

‎paper/fig/examples.png‎

89 KB
Loading

0 commit comments

Comments
 (0)