-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathbatch_evaluation.m
executable file
·139 lines (114 loc) · 3.71 KB
/
batch_evaluation.m
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
%
% This function produces the DREAM4 scores for a batch of five networks
% (including he overall score for the sub-challenge)
%
% See go_all.m for an exmaple of how to call it.
%
% Gustavo A. Stolovitzky, Ph.D.
% Adj. Assoc Prof of Biomed Informatics, Columbia Univ
% Mngr, Func Genomics & Sys Biology, IBM Research
% P.O.Box 218 Office : (914) 945-1292
% Yorktown Heights, NY 10598 Fax : (914) 945-4217
% http://www.research.ibm.com/people/g/gustavo
% http://domino.research.ibm.com/comm/research_projects.nsf/pages/fungen.index.html
% gustavo@us.ibm.com
%
% Robert Prill, Ph.D.
% Postdoctoral Researcher
% Computational Biology Center, IBM Research
% P.O.Box 218
% Yorktown Heights, NY 10598
% Office : 914-945-1377
% http://domino.research.ibm.com/comm/research_people.nsf/pages/rjprill.index.html
% rjprill@us.ibm.com
%
function [ AUROC, ...
AUPR, ...
P_AUROC, ...
P_AUPR, ...
AUROC_PVAL, ...
AUPR_PVAL, ...
AUROC_SCORE, ...
AUPR_SCORE, ...
SCORE] = batch_evaluation(sub_challenge_number)
%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% PARAMS
%% pick a number:
%% 1 for size 10
%% 2 for size 100
%% 3 for size 100 multifactorial
ii = sub_challenge_number;
%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% PATHS
% where the gold standards are located
% GOLDDIRS = {'../INPUT/gold_standards/10/' ...
% '../INPUT/gold_standards/100/' ...
% '../INPUT/gold_standards/100_multifactorial/'};
GOLDDIRS = {'..\HIDI_Script\HiDi_Script\INPUT\gold_standards\10\' ...
'..\HIDI_Script\HiDi_Script\INPUT\gold_standards\100\' ...
'..\HIDI_Script\HiDi_Script\INPUT\gold_standards\100_multifactorial\'};
%% where the predictions are located
% INDIRS = {'../INPUT/my_predictions/10/',...
% '../INPUT/my_predictions/100/',...
% '../INPUT/my_predictions/100_multifactorial/'};
INDIRS = {'..\HIDI_Script\HiDi_Script\INPUT\my_predictions\10\',...
'..\HIDI_Script\HiDi_Script\INPUT\my_predictions\100\',...
'..\HIDI_Script\HiDi_Script\INPUT\my_predictions\100_multifactorial\'};
%% where the prob densities are located
PDFDIR = '..\HIDI_Script\HiDi_Script\INPUT\probability_densities\';
PDF_ROOTS = {'pdf_size10_', 'pdf_size100_', 'pdf_multifactorial_'};
%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
INDIR = INDIRS{ii};
GOLDDIR = GOLDDIRS{ii};
PDF_ROOT = PDF_ROOTS{ii};
files = directory_list(INDIR);
%% for each file
for fi = 1:length(files)
file = [ INDIR files{fi} ];
disp([ ' ' files{fi} ])
%% figure out the name of the network
if ii<3
network_name = figure_out_network_name(file);
else
network_name = figure_out_network_name(file,3);
end
NAMES{fi} = strrep(network_name,'_','\_');
%% figure out the name of the gold standard
goldfile = [ GOLDDIR 'DREAM4_GoldStandard_InSilico_' network_name '.tsv' ];
%% load gold standard
gold_data = load_dream_network(goldfile);
%% load predictions
test_data = load_dream_network(file);
%% fetch the probability density function
pdffile = [ PDFDIR PDF_ROOT num2str(fi) ];
pdf_data = load(pdffile);
%% calculate performance metrics
[aupr auroc prec rec tpr fpr p_auroc p_aupr] = DREAM4_Challenge2_Evaluation(test_data, gold_data, pdf_data);
%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% remember
AUPR(fi) = aupr;
AUROC(fi) = auroc;
P_AUROC(fi) = p_auroc;
P_AUPR(fi) = p_aupr;
PREC{fi} = prec;
REC{fi} = rec;
TPR{fi} = tpr;
FPR{fi} = fpr;
end
%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% overall scores
AUROC_SCORE = -mean(log10(P_AUROC));
AUPR_SCORE = -mean(log10(P_AUPR));
AUROC_PVAL = 10.^-AUROC_SCORE;
AUPR_PVAL = 10.^-AUPR_SCORE;
SCORE = 1/2 * (AUROC_SCORE + AUPR_SCORE);
%% show something on screen
% AUROC
% AUPR
% P_AUROC
% P_AUPR
% AUROC_PVAL
% AUPR_PVAL
% AUROC_SCORE
% AUPR_SCORE
% SCORE