/* This is much modified version of the ROC code from perf. * Basically I pulled out everything that wasn't needed for the ROC calculations. * See the licensing info at the end of the file. */ #include "perf_roc.h" #include #include #include #define MAX_ITEMS 500000 #define debug 0 int total_true_0, total_true_1; int no_item; float true[MAX_ITEMS]; float pred[MAX_ITEMS]; float fraction[MAX_ITEMS]; //for the fractional cases to handle ties void quicksort( int p, int r ); void loadDataFromFiles( FILE *f_in, FILE *f_out ) { /***** LOAD THE DATA FILES OR READ DATA FROM STANDARD INPUT *****/ no_item = 0; while (1) { if( f_in == NULL) { /* Input is from standard input */ if ((scanf("%f %f", &true[no_item], &pred[no_item])) == EOF) break; } else if( f_out == NULL) { /* Read input from one file */ if ((fscanf(f_in, "%f %f", &true[no_item], &pred[no_item])) == EOF) break; } else { /* Else read input from two files */ fscanf(f_in, "%f", &true[no_item]); fscanf(f_out, "%f", &pred[no_item]); if (feof(f_in)) { if (!feof(f_out)) fprintf(stderr, "Warning: True values file is longer.\n"); break; } if (feof(f_out)) { fprintf(stderr, "Warning: Predicted values file is longer.\n"); break; } } no_item++; if ( no_item >= MAX_ITEMS ) { printf ("Aborting. Exceeded %d items.\n", MAX_ITEMS); exit(1); } } /* End of loading loop */ } int loadDataFromArrays( float *true_in, float *pred_in, unsigned int count ) { if( count > MAX_ITEMS ) return 1; no_item = count; for( unsigned int index = 0; index < count; ++index ) { true[index] = true_in[index]; pred[index] = pred_in[index]; } return 0; } void start_computations() { /***** HERE WE START THE COMPUTATIONS *****/ /* Note that some metrics (RMSE, Norm, Cross-Entropy, Slac-Q, Calibration) assume that the predicted values (and maybe also the targets) are scaled [0-1]. */ int targs_all_same = 0; double mean_true = 0.0; double mean_pred = 0.0; double max_true = -9e99; double min_true = +9e99; double assumed_true = 1.0; for( int item=0; item max_true ) max_true = true[item]; } mean_true = mean_true / ((double) no_item); mean_pred = mean_pred / ((double) no_item); if( debug ) { printf("%d pats read. mean_true %6.4lf. mean_pred %6.4lf\n", no_item, mean_true, mean_pred); fflush(stdout); } for( int item=0; item min_true) && (true[item] < max_true) ) { fprintf(stderr, "Error: More than 2 unique target values detected. Exiting.\n"); exit(-1); } } if ( min_true == max_true ) { targs_all_same = 1; if ( min_true <= 0.0 ) assumed_true = 0.0; else assumed_true = 1.0; fprintf(stderr, "Warning: All targets in block have same value %8.4lf. Assuming targets are class %d\n", min_true, ((int) assumed_true)); } total_true_0 = 0; total_true_1 = 0; for( int item=0; item-1; item--) { tt+= fraction[item]; tf-= fraction[item]; ft+= 1 - fraction[item]; ff-= 1 - fraction[item]; sens = ((double) tt) / ((double) (tt+tf)); spec = ((double) ff) / ((double) (ft+ff)); tpf = sens; fpf = 1.0 - spec; roc_area+= 0.5*(tpf+tpf_prev)*(fpf-fpf_prev); tpf_prev = tpf; fpf_prev = fpf; } return roc_area; } /* partition is used by quicksort */ int partition( int p, int r ) { int i, j; float x; x = pred[p]; i = p - 1; j = r + 1; while (1) { do j--; while (!(pred[j] <= x)); do i++; while (!(pred[i] >= x)); if (i < j) { float tempf = pred[i]; pred[i] = pred[j]; pred[j] = tempf; tempf = true[i]; true[i] = true[j]; true[j] = tempf; } else return(j); } } /* vanilla quicksort */ void quicksort( int p, int r ) { int q; if (p < r) { q = partition (p,r); quicksort (p,q); quicksort (q+1,r); } } /* Version 5.12 KDDCUP-2004 July 12, 2004 */ /* COPYRIGHT INFO perf is available at http://www.cs.cornell.edu/~caruana/ Author: Rich Caruana caruana@cs.cornell.edu Cornell University Department of Computer Science 4157 Upson Hall Ithaca, NY 14853 USA LICENSING This program is granted free of charge for research and education purposes. You must obtain a license from the author to use it for commercial purposes. Please acknowledge perf in any scientific results and reports where perf was used. Many of the performance metrics perf calculates are not always defined the same way by different communities. By citing perf you make it clear exactly what metrics you are reporting. We do not yet have a report describing perf, so the best way to acknowledge using to code is: R. Caruana, The PERF Performance Evaluation Code, http://www.cs.cornell.edu/~caruana/perf The software must not be modified and distributed without prior permission of the author. By using perf you agree to the licensing terms. NO WARRANTY BECAUSE THIS PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. */