#!/usr/bin/perl

#  This software was produced by NIST, an agency of the
#  U.S. government, and by statute is not subject to copyright in the
#  United States.  Recipients of this software assume all
#  responsibilities associated with its operation, modification and
#  maintenance.

#  Usage: Usage: evalLLf.pl [-d] file1 ..., -d to get precise  details

# Evaluation script: calculates truePositives, falseNegatives, falsePositives, trueNegatives
# of low level features (pan, tilt and zoom)

# 1) Read truth data for features $fn(n=1-3) into a hash %td (key = "fn_shotId", value="0|1")
# 2) Counts true data for each $fn in %td in %relT (key = fn, value=Pn)
# 3) Counts false data for each $fn in %td in %falT (key = fn, value=Qn)
# 4) Read submission data for features fn into a hash %sd for shotIds defined in %td (key = fn_shotId, value="1")
# 5) Counts true positive (number of true data for each fn in %sd and %td) in %truePos (key = fn, value=Rn)
# 6) Counts true negative (number of false data for each fn in %td and doesn't exist in %sd) in @tn (index=fn, Sn)
# 7) Counts false positive (number of false data for each fn in %td and exist and true in %sd) in %falsePos (key = fn, value=Tn)
# 8) Counts false negative (number of true data for each fn in %td and doesn't exist in %sd) in @fn (index=fn, values=Vn)
# 9) Calculate precision and recall for each feature
#
#    foreach f in %relT
#      precision = $truePos{$fn}/($truePos{$fn}+$falsePos{$fn});
#      recall = $truePos{$fn}/$relT{$fn};
#
# tzveta, September 20, 2005

use warnings;
use strict;

@ARGV > 0 or die "Usage: evalLLf.pl [-d] file1 ...\n";

my $flag="-t"; # flag set to default "-t" for table sumary output when script is run without "-d" option

if ($ARGV[0] eq "-d"){
    # if detailed output is requested (script run with "-d" option)
    $flag = shift(@ARGV); # change default flag to "-d" and remove its value from argument list
};

my %td; # truth data hash
my %sd; # submit data hash (only existing data in %td is considered)
my %relT; # count true data in %td per feature
my %falT; # count false data in %td per feature
my %truePos; # count true data in %sd per feature (data with same key in %td must have value 1(true))
my %falsePos; # count false true data in %sd per feature (data with same key in %td must have value 0(false))

my $key;

############ CHANGE THIS PATH ACCORDING YOUR llf.truth FILE LOCATION #############
my $truthData = "/home/barney1/trecvid/5/llf/llftruth/tv5.llf.truth";

open (TRUTHDATA, "$truthData") || die "cannot open $truthData for reading: $!";
# Loop thru truth data (llf.truth) file to:
# Fill truth data hash %td according llf.truth file. Feature number (fN, N=1-3) concatenated with shotId is used for key.
# Count number of true shots ( value 1 in %td ) per feature in hash table %relT. Feature number (1-3) is used as key.
# Count number of false shots ( value 0 in %td ) per feature in hash table %falT. Feature number (1-3) is used as key.

while (<TRUTHDATA>){
  if ( my ($f, $shot1, $shot2, $value) = /^(\d+)\s+ shot(\d+)_(\d+)\s+(\d+)\s*$/){
    my $shotId = "shot".$shot1."_".$shot2;
    my $k1 = $f."_".$shotId; # key
    $td{$k1}=$value; # for each key put the corresponding value ("0" or "1")
    if ($value eq 1){ # if true
      # initialize true data counter in %td to "0" (used only to get rid on undefined warning in the output file)
      if (!defined($relT{$f})){$relT{$f}=0}
      $relT{$f}++; # count number of true shots per feature in truth data
    }
    else {
      if (!defined($falT{$f})){$falT{$f}=0} # initialize false data counter in %td to "0"
      $falT{$f}++;  # count number of false shots per feature in truth data
    }
  }
}
close (TRUTHDATA) || die "can't close $truthData $!";

my $fN;

# Loop thru submission files
for my $fname (@ARGV){
  open (IN, $fname) || die "cannot open $fname for reading: $!";
  # when detailed output is requested (script run with option -d )
  # print total shots in truth data per feature
  # print total TRUE shots in truth data per feature
  if ($flag eq "-d"){
      print("Distribution of data:\n---------------------\n");
      foreach $key (sort(keys (%relT))) { # for each feature
        print("total feature ", $key, ":\t", $relT{$key}+$falT{$key},"\n"); # how much is the total data per feature
	print("relevant feature ", $key, ":\t", $relT{$key},"\n");    # how much is the total true data per feature
      }
  }
  my($sysId,$id,$priority);

  # Read each line in current file and extract various pieces of info
  # needed for the evaluation
  while (my $line = <IN>) {

    # Parse out the sysId
    if ($line =~ /sysId\s*=\s*"([^"]+)/) {
        $sysId = $1;
    }

    # Parse out the priority
    if ($line =~ /\s*priority="([^"]+)"/) {
      $priority = $1;
      $id=$sysId." "."priority"." ".$priority; # concatenate sysId with priority
      $fname = substr($fname,rindex($fname,"/")+1,100); # extract the file name from the path
      if ($flag eq "-d"){print ("\nFile: ",$fname,"\t","Run: ",$id,"\n")}
      $fN=0;
      %sd=(); # submit data hash is made empty for each new run
      %truePos=(); # number of true positive data per feature is made empty for each new run
      %falsePos=(); # number of false positive data per feature is made empty for each new run
    }

    if ($line =~ /fNum\s*=\s*"([^"]+)"/){$fN = $1}

    # Parse out the shot id
    if ($line =~ /shotId\s*=\s*"([^"]+)"/){
      my $shot = $1;
      my $k2 = $fN."_".$shot; # same format of key as in %td
      # if jugged eg. exist in %td and doesn't exist in %sd (this to avoid duplicates)
      if ((defined($td{$k2})) && (!defined($sd{$k2}))){
	$sd{$k2}=1; # fill with value 1, the key in the submit data hash %sd
        if ($td{$k2} eq 1){ # if hits eg. has value "1" also in %td
	  if (!defined($truePos{$fN})){$truePos{$fN}=0} # get rid of the undefined warning
          $truePos{$fN}++;  # increment the true submit data per feature counter
        }
        if ($td{$k2} eq 0){ # false positive eg. has value "0" in %td
	  if (!defined($falsePos{$fN})){$falsePos{$fN}=0} # get rid of the undefined warning
          $falsePos{$fN}++;  # increment the false positive submit data per feature counter
        }
      }
      else {} #print($fN,"\t",$shot," ignored\n\n")}
    }

    # When a result is complete calculate precision and recall and print them out
    if ($line =~ /\/videoCameraMotionDetectionRunResult/ ){
      my (@tn,@fn,@prec,@rec);
      my ($avgPr, $avgRec) = 0;
      foreach $key (sort(keys (%relT))) { # for each feature
        #set false positive counter to zero when there aren't false positive shots in the result
        if (!defined($falsePos{$key})){$falsePos{$key}=0}
        $prec[$key]=$truePos{$key}/($truePos{$key}+$falsePos{$key}); # precision (per feature)
        $rec[$key]=$truePos{$key}/$relT{$key}; #recall (per feature)
        if ($flag eq "-d"){
	  printf("\n\nPrecision for feature %1d : %5.3f\n",$key, $prec[$key]);
	  printf("Recall for feature %1d : %5.3f\n\n",$key, $rec[$key]);
        }
        $avgPr += $prec[$key]; # $truePos{$key}/($truePos{$key}+$falsePos{$key});  # sum(precision(fN)), N=3
        $avgRec += $rec[$key]; # $truePos{$key}/$relT{$key};  # sum(recall(fN)), N=3
        my $search=$key; # number of feature (1 .. 3)
        if ($flag eq "-d"){print("TRUE POSITIVE: (", $truePos{$key}, ")\n---------------------\n")}
       	foreach $key (sort (keys (%td))) {
	  if (substr($key,0,1) eq $search){ # extract the number of feature from the key
            # if true in %td and exist and true in %sd print the shot name
	    if (($flag eq "-d") && ($td{$key} eq 1) &&  (defined($sd{$key}))){print(substr($key,2,20),"\n")}
          }
	}
        if ($flag eq "-d"){print("\nTRUE NEGATIVE:\t")}
        $tn[$search]=0; #initializing true negative's counter
        foreach $key (sort(keys (%td))) {
	  if (substr($key,0,1) eq $search){
	    # count true negative shots ( false in %td and doesn't exist in %sd )
	    if (($td{$key} eq 0) && (!defined($sd{$key}))){$tn[$search]++}
          }
	}

        if ($flag eq "-d"){
	  print("(",$tn[$search],")\n----------------------\n");
          foreach $key (sort(keys (%td))) {
	    if (substr($key,0,1) eq $search){
	      # print true negative ( false in %td and doesn't exist in %sd ) shot name
	      if (($td{$key} eq 0) && (!defined($sd{$key}))){print(substr($key,2,20),"\n")}
	    }
	  }
       	  print("\nFALSE POSITIVE: (", $falsePos{$key}, ")\n---------------------\n");
          foreach $key (sort(keys (%td))) {
	    if (substr($key,0,1) eq $search){
	      # print false positive ( false in %td and exist and true in %sd ) shot name
	      if (($td{$key} eq 0) && (defined($sd{$key}))){print(substr($key,2,20),"\n")}
	    }
          }
          print("\nFALSE NEGATIVE:\t");
        }

        $fn[$search]=0; #initializing false negative's counter

	foreach $key (sort(keys (%td))) {
	  if (substr($key,0,1) eq $search){
            # count false negative shots ( true in %td and doesn't exist in %sd )
	    if (($td{$key} eq 1) && (!defined($sd{$key}))){$fn[$search]++}
	  }
	}

        if ($flag eq "-d"){
          print("(",$fn[$search],")\n----------------------\n");
	  foreach $key (sort(keys (%td))) {
	    if (substr($key,0,1) eq $search){
	      # print false negative ( true in %td and doesn't exist in %sd ) shot name
	      if (($td{$key} eq 1) && (!defined($sd{$key}))){print(substr($key,2,20),"\n")}
	    }
          }
	}
      }
      $avgPr /= 3;  # mean precision: sum(precision(fN)/N), N=3
      $avgRec /= 3; # mean recall sum(recall(fN)/N), N=3
      if ($flag eq "-d"){
        printf("\n\nMEAN PRECISION: %5.3f\n", $avgPr);
        printf("MEAN RACALL: %5.3f\n\n", $avgRec);
      }

# When a result is complete and no detailed results were requested print them out in default (table) output

      if ($flag eq "-t"){
	my $old = select STDOUT;
        $= = 70; # changing the default page length (60 lines) to 70 lines
        select $old;
        write;

format STDOUT =
@<<<<<<<<<<<@ @### @### @### @### @.### @.### @### @### @### @### @.### @.### @### @### @### @### @.### @.### @.### @.###
$sysId,$priority,$truePos{1},$tn[1],$falsePos{1},$fn[1],$prec[1],$rec[1],$truePos{2},$tn[2],$falsePos{2},$fn[2],$prec[2],$rec[2],$truePos{3},$tn[3],$falsePos{3},$fn[3],$prec[3],$rec[3],$avgPr,$avgRec
.
      }
    }
  }
  close(IN) || die "can't close $fname: $!";
}

format STDOUT_TOP =
             |         P A N *  ( f1 )       |         T I L T *  ( f2 )     |         Z O O M *  ( f3 )     |   MEAN
============= ==== ==== ==== ==== ===== ===== ==== ==== ==== ==== ===== ===== ==== ==== ==== ==== ===== ===== ===== =====
sysId         true true fal. fal. prec. recal true true fal. fal. prec. recal true true fal. fal. prec. recal PREC. RECAL
priority     | +    -    +    -              | +    -    +    -              | +    -    +    -              |
============= ==== ==== ==== ==== ===== ===== ==== ==== ==== ==== ===== ===== ==== ==== ==== ==== ===== ===== ===== =====
.