| AnnotationDiffer.java |
1 /*
2 * Copyright (c) 1998-2005, The University of Sheffield.
3 *
4 * This file is part of GATE (see http://gate.ac.uk/), and is free
5 * software, licenced under the GNU Library General Public License,
6 * Version 2, June 1991 (in the distribution as file licence.html,
7 * and also available at http://gate.ac.uk/gate/licence.html).
8 *
9 * Valentin Tablan 28/01/2003
10 *
11 * $Id: AnnotationDiffer.java,v 1.17 2005/07/13 10:37:35 valyt Exp $
12 *
13 */
14 package gate.util;
15
16 import java.util.*;
17
18 import gate.Annotation;
19
20 /**
21 * This class provides the logic used by the Annotation Diff tool. It starts
22 * with two collections of annotation objects, one of key annotations
23 * (representing the gold standard) and one of response annotations
24 * (representing the system's responses). It will then pair the keys and
25 * responses in a way that maximises the score. Each key - response pair gets a
26 * score of {@link #CORRECT} (2), {@link #PARTIALLY_CORRECT} (1) or
27 * {@link #WRONG} (0)depending on whether the two annotations match are
28 * overlapping or completely unmatched. Each pairing also has a type of
29 * {@link #CORRECT_TYPE}, {@link #PARTIALLY_CORRECT_TYPE},
30 * {@link #SPURIOUS_TYPE} or {@link #MISSING_TYPE} further detailing the type of
31 * error for the wrong matches (<i>missing</i> being the keys that weren't
32 * matched to a response while <i>spurious</i> are the responses that were
33 * over-generated and are not matching any key.
34 *
35 * Precision, recall and f-measure are also calculated.
36 */
37 public class AnnotationDiffer {
38
39
40 /**
41 * Interface representing a pairing between a key annotation and a response
42 * one.
43 */
44 public static interface Pairing{
45 /**
46 * Gets the key annotation of the pairing. Can be <tt>null</tt> (for
47 * spurious matches).
48 * @return an {@link Annotation} object.
49 */
50 public Annotation getKey();
51
52 /**
53 * Gets the response annotation of the pairing. Can be <tt>null</tt> (for
54 * missing matches).
55 * @return an {@link Annotation} object.
56 */
57 public Annotation getResponse();
58
59 /**
60 * Gets the type of the pairing, one of {@link #CORRECT_TYPE},
61 * {@link #PARTIALLY_CORRECT_TYPE}, {@link #SPURIOUS_TYPE} or
62 * {@link #MISSING_TYPE},
63 * @return an <tt>int</tt> value.
64 */
65 public int getType();
66 }
67
68 /**
69 * Computes a diff between two collections of annotations.
70 * @param key the colelction of key annotations.
71 * @param response the collection of response annotations.
72 * @return a list of {@link Pairing} objects representing the pairing set
73 * that results in the best score.
74 */
75 public List calculateDiff(Collection key, Collection response){
76 //initialise data structures
77 if(key == null || key.size() == 0)
78 keyList = new ArrayList();
79 else
80 keyList = new ArrayList(key);
81
82 if(response == null || response.size() == 0)
83 responseList = new ArrayList();
84 else
85 responseList = new ArrayList(response);
86
87 keyChoices = new ArrayList(keyList.size());
88 keyChoices.addAll(Collections.nCopies(keyList.size(), null));
89 responseChoices = new ArrayList(responseList.size());
90 responseChoices.addAll(Collections.nCopies(responseList.size(), null));
91
92 possibleChoices = new ArrayList();
93
94 //1) try all possible pairings
95 for(int i = 0; i < keyList.size(); i++){
96 for(int j =0; j < responseList.size(); j++){
97 Annotation keyAnn = (Annotation)keyList.get(i);
98 Annotation resAnn = (Annotation)responseList.get(j);
99 PairingImpl choice = null;
100 if(keyAnn.coextensive(resAnn)){
101 //we have full overlap -> CORRECT or WRONG
102 if(keyAnn.isCompatible(resAnn, significantFeaturesSet)){
103 //we have a full match
104 choice = new PairingImpl(i, j, CORRECT);
105 }else{
106 //the two annotations are coextensive but don't match
107 //we have a missmatch
108 choice = new PairingImpl(i, j, WRONG);
109 }
110 }else if(keyAnn.overlaps(resAnn)){
111 //we have partial overlap -> PARTIALLY_CORRECT or WRONG
112 if(keyAnn.isPartiallyCompatible(resAnn, significantFeaturesSet)){
113 choice = new PairingImpl(i, j, PARTIALLY_CORRECT);
114 }else{
115 choice = new PairingImpl(i, j, WRONG);
116 }
117 }
118
119 // // >>>>>>
120 // if(significantFeaturesSet == null){
121 // //full comaptibility required
122 // if(keyAnn.isCompatible(resAnn)){
123 // choice = new PairingImpl(i, j, CORRECT);
124 // }else if(keyAnn.isPartiallyCompatible(resAnn)){
125 // choice = new PairingImpl(i, j, PARTIALLY_CORRECT);
126 // }
127 // }else{
128 // //compatibility tests restricted to a set of features
129 // if(keyAnn.isCompatible(resAnn, significantFeaturesSet)){
130 // choice = new PairingImpl(i, j, CORRECT);
131 // }else if(keyAnn.isPartiallyCompatible(resAnn, significantFeaturesSet)){
132 // choice = new PairingImpl(i, j, PARTIALLY_CORRECT);
133 // }
134 // }
135 // // <<<<<<
136
137 //add the new choice if any
138 if (choice != null) {
139 addPairing(choice, i, keyChoices);
140 addPairing(choice, j, responseChoices);
141 possibleChoices.add(choice);
142 }
143 }//for j
144 }//for i
145
146 //2) from all possible pairings, find the maximal set that also
147 //maximises the total score
148 Collections.sort(possibleChoices, new PairingScoreComparator());
149 Collections.reverse(possibleChoices);
150 finalChoices = new ArrayList();
151 correctMatches = 0;
152 partiallyCorrectMatches = 0;
153 missing = 0;
154 spurious = 0;
155
156 while(!possibleChoices.isEmpty()){
157 PairingImpl bestChoice = (PairingImpl)possibleChoices.remove(0);
158 bestChoice.consume();
159 finalChoices.add(bestChoice);
160 switch(bestChoice.type){
161 case CORRECT:{
162 if(correctAnnotations == null) correctAnnotations = new HashSet();
163 correctAnnotations.add(bestChoice.getResponse());
164 correctMatches++;
165 break;
166 }
167 case PARTIALLY_CORRECT:{
168 if(partiallyCorrectAnnotations == null) partiallyCorrectAnnotations = new HashSet();
169 partiallyCorrectAnnotations.add(bestChoice.getResponse());
170 partiallyCorrectMatches++;
171 break;
172 }
173 case WRONG:{
174 if(bestChoice.getKey() != null){
175 //we have a missed key
176 if(missingAnnotations == null) missingAnnotations = new HashSet();
177 missingAnnotations.add(bestChoice.getKey());
178 missing ++;
179 }
180 if(bestChoice.getResponse() != null){
181 //we have a spurious response
182 if(spuriousAnnotations == null) spuriousAnnotations = new HashSet();
183 spuriousAnnotations.add(bestChoice.getResponse());
184 spurious ++;
185 }
186 break;
187 }
188 default:{
189 throw new GateRuntimeException("Invalid pairing type: " +
190 bestChoice.type);
191 }
192 }
193 }
194 //add choices for the incorrect matches (MISSED, SPURIOUS)
195 //get the unmatched keys
196 for(int i = 0; i < keyChoices.size(); i++){
197 List aList = (List)keyChoices.get(i);
198 if(aList == null || aList.isEmpty()){
199 if(missingAnnotations == null) missingAnnotations = new HashSet();
200 missingAnnotations.add((Annotation)(keyList.get(i)));
201 finalChoices.add(new PairingImpl(i, -1, WRONG));
202 missing ++;
203 }
204 }
205
206 //get the unmatched responses
207 for(int i = 0; i < responseChoices.size(); i++){
208 List aList = (List)responseChoices.get(i);
209 if(aList == null || aList.isEmpty()){
210 if(spuriousAnnotations == null) spuriousAnnotations = new HashSet();
211 spuriousAnnotations.add((Annotation)(responseList.get(i)));
212 finalChoices.add(new PairingImpl(-1, i, WRONG));
213 spurious ++;
214 }
215 }
216
217 return finalChoices;
218 }
219
220 /**
221 * Gets the strict precision (the ratio of correct responses out of all the
222 * provided responses).
223 * @return a <tt>double</tt> value.
224 */
225 public double getPrecisionStrict(){
226 if(responseList.size() == 0) {
227 return 1.0;
228 }
229 return correctMatches/(double)responseList.size();
230 }
231
232 /**
233 * Gets the strict recall (the ratio of key matched to a response out of all
234 * the keys).
235 * @return a <tt>double</tt> value.
236 */
237 public double getRecallStrict(){
238 if(keyList.size() == 0) {
239 return 1.0;
240 }
241 return correctMatches/(double)keyList.size();
242 }
243
244 /**
245 * Gets the lenient precision (where the partial matches are considered as
246 * correct).
247 * @return a <tt>double</tt> value.
248 */
249 public double getPrecisionLenient(){
250 if(responseList.size() == 0) {
251 return 1.0;
252 }
253 return ((double)correctMatches + partiallyCorrectMatches) / (double)responseList.size();
254 }
255
256 /**
257 * Gets the average of the strict and lenient precision values.
258 * @return a <tt>double</tt> value.
259 */
260 public double getPrecisionAverage() {
261 return ((double)getPrecisionLenient() + getPrecisionStrict()) / (double)(2.0);
262 }
263
264 /**
265 * Gets the lenient recall (where the partial matches are considered as
266 * correct).
267 * @return a <tt>double</tt> value.
268 */
269 public double getRecallLenient(){
270 if(keyList.size() == 0) {
271 return 1.0;
272 }
273 return ((double)correctMatches + partiallyCorrectMatches) / (double)keyList.size();
274 }
275
276 /**
277 * Gets the average of the strict and lenient recall values.
278 * @return a <tt>double</tt> value.
279 */
280 public double getRecallAverage() {
281 return ((double) getRecallLenient() + getRecallStrict()) / (double)(2.0);
282 }
283
284 /**
285 * Gets the strict F-Measure (the harmonic weighted mean of the strict
286 * precision and the strict recall) using the provided parameter as relative
287 * weight.
288 * @param beta The relative weight of precision and recall. A value of 1
289 * gives equal weights to precision and recall. A value of 0 takes the recall
290 * value completely out of the equation.
291 * @return a <tt>double</tt>value.
292 */
293 public double getFMeasureStrict(double beta){
294 double precision = getPrecisionStrict();
295 double recall = getRecallStrict();
296 double betaSq = beta * beta;
297 double answer = (double)(((double)(betaSq + 1) * precision * recall ) / (double)(betaSq * precision + recall));
298 if(Double.isNaN(answer)) answer = 0.0;
299 return answer;
300 }
301
302 /**
303 * Gets the lenient F-Measure (F-Measure where the lenient precision and
304 * recall values are used) using the provided parameter as relative weight.
305 * @param beta The relative weight of precision and recall. A value of 1
306 * gives equal weights to precision and recall. A value of 0 takes the recall
307 * value completely out of the equation.
308 * @return a <tt>double</tt>value.
309 */
310 public double getFMeasureLenient(double beta){
311 double precision = getPrecisionLenient();
312 double recall = getRecallLenient();
313 double betaSq = beta * beta;
314 double answer = (double)(((double)(betaSq + 1) * precision * recall) / ((double)betaSq * precision + recall));
315 if(Double.isNaN(answer)) answer = 0.0;
316 return answer;
317 }
318
319 /**
320 * Gets the average of strict and lenient F-Measure values.
321 * @param beta The relative weight of precision and recall. A value of 1
322 * gives equal weights to precision and recall. A value of 0 takes the recall
323 * value completely out of the equation.
324 * @return a <tt>double</tt>value.
325 */
326 public double getFMeasureAverage(double beta) {
327 double answer = ((double)getFMeasureLenient(beta) + (double)getFMeasureStrict(beta)) / (double)(2.0);
328 return answer;
329 }
330
331 /**
332 * Gets the number of correct matches.
333 * @return an <tt>int<tt> value.
334 */
335 public int getCorrectMatches(){
336 return correctMatches;
337 }
338
339 /**
340 * Gets the number of partially correct matches.
341 * @return an <tt>int<tt> value.
342 */
343 public int getPartiallyCorrectMatches(){
344 return partiallyCorrectMatches;
345 }
346
347 /**
348 * Gets the number of pairings of type {@link #MISSING_TYPE}.
349 * @return an <tt>int<tt> value.
350 */
351 public int getMissing(){
352 return missing;
353 }
354
355 /**
356 * Gets the number of pairings of type {@link #SPURIOUS_TYPE}.
357 * @return an <tt>int<tt> value.
358 */
359 public int getSpurious(){
360 return spurious;
361 }
362
363 /**
364 * Gets the number of pairings of type {@link #SPURIOUS_TYPE}.
365 * @return an <tt>int<tt> value.
366 */
367 public int getFalsePositivesStrict(){
368 return responseList.size() - correctMatches;
369 }
370
371 /**
372 * Gets the number of responses that aren't either correct or partially
373 * correct.
374 * @return an <tt>int<tt> value.
375 */
376 public int getFalsePositivesLenient(){
377 return responseList.size() - correctMatches - partiallyCorrectMatches;
378 }
379
380 /**
381 * Gets the number of keys provided.
382 * @return an <tt>int<tt> value.
383 */
384 public int getKeysCount() {
385 return keyList.size();
386 }
387
388 /**
389 * Gets the number of responses provided.
390 * @return an <tt>int<tt> value.
391 */
392 public int getResponsesCount() {
393 return responseList.size();
394 }
395
396 /**
397 * Prints to System.out the pairings that are not correct.
398 */
399 public void printMissmatches(){
400 //get the partial correct matches
401 Iterator iter = finalChoices.iterator();
402 while(iter.hasNext()){
403 PairingImpl aChoice = (PairingImpl)iter.next();
404 switch(aChoice.type){
405 case PARTIALLY_CORRECT:{
406 System.out.println("Missmatch (partially correct):");
407 System.out.println("Key: " + keyList.get(aChoice.keyIndex).toString());
408 System.out.println("Response: " + responseList.get(aChoice.responseIndex).toString());
409 break;
410 }
411 }
412 }
413
414 //get the unmatched keys
415 for(int i = 0; i < keyChoices.size(); i++){
416 List aList = (List)keyChoices.get(i);
417 if(aList == null || aList.isEmpty()){
418 System.out.println("Missed Key: " + keyList.get(i).toString());
419 }
420 }
421
422 //get the unmatched responses
423 for(int i = 0; i < responseChoices.size(); i++){
424 List aList = (List)responseChoices.get(i);
425 if(aList == null || aList.isEmpty()){
426 System.out.println("Spurious Response: " + responseList.get(i).toString());
427 }
428 }
429 }
430
431
432
433 /**
434 * Performs some basic checks over the internal data structures from the last
435 * run.
436 * @throws Exception
437 */
438 void sanityCheck()throws Exception{
439 //all keys and responses should have at most one choice left
440 Iterator iter =keyChoices.iterator();
441 while(iter.hasNext()){
442 List choices = (List)iter.next();
443 if(choices != null){
444 if(choices.size() > 1){
445 throw new Exception("Multiple choices found!");
446 }else if(!choices.isEmpty()){
447 //size must be 1
448 PairingImpl aChoice = (PairingImpl)choices.get(0);
449 //the SAME choice should be found for the associated response
450 List otherChoices = (List)responseChoices.get(aChoice.responseIndex);
451 if(otherChoices == null ||
452 otherChoices.size() != 1 ||
453 otherChoices.get(0) != aChoice){
454 throw new Exception("Reciprocity error!");
455 }
456 }
457 }
458 }
459
460 iter =responseChoices.iterator();
461 while(iter.hasNext()){
462 List choices = (List)iter.next();
463 if(choices != null){
464 if(choices.size() > 1){
465 throw new Exception("Multiple choices found!");
466 }else if(!choices.isEmpty()){
467 //size must be 1
468 PairingImpl aChoice = (PairingImpl)choices.get(0);
469 //the SAME choice should be found for the associated response
470 List otherChoices = (List)keyChoices.get(aChoice.keyIndex);
471 if(otherChoices == null){
472 throw new Exception("Reciprocity error : null!");
473 }else if(otherChoices.size() != 1){
474 throw new Exception("Reciprocity error: not 1!");
475 }else if(otherChoices.get(0) != aChoice){
476 throw new Exception("Reciprocity error: different!");
477 }
478 }
479 }
480 }
481 }
482
483 /**
484 * Adds a new pairing to the internal data structures.
485 * @param pairing the pairing to be added
486 * @param index the index in the list of pairings
487 * @param listOfPairings the list of {@link Pairing}s where the
488 * pairing should be added
489 */
490 protected void addPairing(PairingImpl pairing, int index, List listOfPairings){
491 List existingChoices = (List)listOfPairings.get(index);
492 if(existingChoices == null){
493 existingChoices = new ArrayList();
494 listOfPairings.set(index, existingChoices);
495 }
496 existingChoices.add(pairing);
497 }
498
499 /**
500 * Gets the set of features considered significant for the matching algorithm.
501 * @return a Set.
502 */
503 public java.util.Set getSignificantFeaturesSet() {
504 return significantFeaturesSet;
505 }
506
507 /**
508 * Set the set of features considered significant for the matching algorithm.
509 * A <tt>null</tt> value means that all features are significant, an empty
510 * set value means that no features are significant while a set of String
511 * values specifies that only features with names included in the set are
512 * significant.
513 * @param significantFeaturesSet a Set of String values or <tt>null<tt>.
514 */
515 public void setSignificantFeaturesSet(java.util.Set significantFeaturesSet) {
516 this.significantFeaturesSet = significantFeaturesSet;
517 }
518
519 /**
520 * Represents a pairing of a key annotation with a response annotation and
521 * the associated score for that pairing.
522 */
523 public class PairingImpl implements Pairing{
524 PairingImpl(int keyIndex, int responseIndex, int type) {
525 this.keyIndex = keyIndex;
526 this.responseIndex = responseIndex;
527 this.type = type;
528 scoreCalculated = false;
529 }
530
531 public int getScore(){
532 if(scoreCalculated) return score;
533 else{
534 calculateScore();
535 return score;
536 }
537 }
538
539 public Annotation getKey(){
540 return keyIndex == -1 ? null : (Annotation)keyList.get(keyIndex);
541 }
542
543 public Annotation getResponse(){
544 return responseIndex == -1 ? null :
545 (Annotation)responseList.get(responseIndex);
546 }
547
548 public int getType(){
549 return type;
550 }
551
552 /**
553 * Removes all mutually exclusive OTHER choices possible from
554 * the data structures.
555 * <tt>this</tt> gets removed from {@link #possibleChoices} as well.
556 */
557 public void consume(){
558 possibleChoices.remove(this);
559 List sameKeyChoices = (List)keyChoices.get(keyIndex);
560 sameKeyChoices.remove(this);
561 possibleChoices.removeAll(sameKeyChoices);
562
563 List sameResponseChoices = (List)responseChoices.get(responseIndex);
564 sameResponseChoices.remove(this);
565 possibleChoices.removeAll(sameResponseChoices);
566
567 Iterator iter = new ArrayList(sameKeyChoices).iterator();
568 while(iter.hasNext()){
569 ((PairingImpl)iter.next()).remove();
570 }
571 iter = new ArrayList(sameResponseChoices).iterator();
572 while(iter.hasNext()){
573 ((PairingImpl)iter.next()).remove();
574 }
575 sameKeyChoices.add(this);
576 sameResponseChoices.add(this);
577 }
578
579 /**
580 * Removes this choice from the two lists it belongs to
581 */
582 protected void remove(){
583 List fromKey = (List)keyChoices.get(keyIndex);
584 fromKey.remove(this);
585 List fromResponse = (List)responseChoices.get(responseIndex);
586 fromResponse.remove(this);
587 }
588
589 /**
590 * Calculates the score for this choice as:
591 * type - sum of all the types of all OTHER mutually exclusive choices
592 */
593 void calculateScore(){
594 //this needs to be a set so we don't count conflicts twice
595 Set conflictSet = new HashSet();
596 //add all the choices from the same response annotation
597 conflictSet.addAll((List)responseChoices.get(responseIndex));
598 //add all the choices from the same key annotation
599 conflictSet.addAll((List)keyChoices.get(keyIndex));
600 //remove this choice from the conflict set
601 conflictSet.remove(this);
602 score = type;
603 Iterator conflictIter = conflictSet.iterator();
604 while(conflictIter.hasNext()) score -= ((PairingImpl)conflictIter.next()).type;
605 scoreCalculated = true;
606 }
607
608 int keyIndex;
609 int responseIndex;
610 int type;
611 int score;
612 boolean scoreCalculated;
613 }
614
615 /**
616 * Compares two pairings:
617 * the better score is preferred;
618 * for the same score the better type is preferred (exact matches are
619 * preffered to partial ones).
620 */
621 protected static class PairingScoreComparator implements Comparator{
622 /**
623 * Compares two choices:
624 * the better score is preferred;
625 * for the same score the better type is preferred (exact matches are
626 * preffered to partial ones).
627 * @return a positive value if the first pairing is better than the second,
628 * zero if they score the same or negative otherwise.
629 */
630
631 public int compare(Object o1, Object o2){
632 PairingImpl first = (PairingImpl)o1;
633 PairingImpl second = (PairingImpl)o2;
634 //compare by score
635 int res = first.getScore() - second.getScore();
636 //compare by type
637 if(res == 0) res = first.getType() - second.getType();
638 //compare by completeness (a wrong match with both key and response
639 //is "better" than one with only key or response
640 if(res == 0){
641 res = (first.getKey() == null ? 0 : 1) +
642 (first.getResponse() == null ? 0 : 1) +
643 (second.getKey() == null ? 0 : -1) +
644 (second.getResponse() == null ? 0 : -1);
645 }
646 return res;
647 }
648 }
649
650 /**
651 * Compares two choices based on start offset of key (or response
652 * if key not present) and type if offsets are equal.
653 */
654 public static class PairingOffsetComparator implements Comparator{
655 /**
656 * Compares two choices based on start offset of key (or response
657 * if key not present) and type if offsets are equal.
658 */
659 public int compare(Object o1, Object o2){
660 Pairing first = (Pairing)o1;
661 Pairing second = (Pairing)o2;
662 Annotation key1 = first.getKey();
663 Annotation key2 = second.getKey();
664 Annotation res1 = first.getResponse();
665 Annotation res2 = second.getResponse();
666 Long start1 = key1 == null ? null : key1.getStartNode().getOffset();
667 if(start1 == null) start1 = res1.getStartNode().getOffset();
668 Long start2 = key2 == null ? null : key2.getStartNode().getOffset();
669 if(start2 == null) start2 = res2.getStartNode().getOffset();
670 int res = start1.compareTo(start2);
671 if(res == 0){
672 //compare by type
673 res = second.getType() - first.getType();
674 }
675
676 //
677 //
678 //
679 // //choices with keys are smaller than ones without
680 // if(key1 == null && key2 != null) return 1;
681 // if(key1 != null && key2 == null) return -1;
682 // if(key1 == null){
683 // //both keys are null
684 // res = res1.getStartNode().getOffset().
685 // compareTo(res2.getStartNode().getOffset());
686 // if(res == 0) res = res1.getEndNode().getOffset().
687 // compareTo(res2.getEndNode().getOffset());
688 // if(res == 0) res = second.getType() - first.getType();
689 // }else{
690 // //both keys are present
691 // res = key1.getStartNode().getOffset().compareTo(
692 // key2.getStartNode().getOffset());
693 //
694 // if(res == 0){
695 // //choices with responses are smaller than ones without
696 // if(res1 == null && res2 != null) return 1;
697 // if(res1 != null && res2 == null) return -1;
698 // if(res1 != null){
699 // res = res1.getStartNode().getOffset().
700 // compareTo(res2.getStartNode().getOffset());
701 // }
702 // if(res == 0)res = key1.getEndNode().getOffset().compareTo(
703 // key2.getEndNode().getOffset());
704 // if(res == 0 && res1 != null){
705 // res = res1.getEndNode().getOffset().
706 // compareTo(res2.getEndNode().getOffset());
707 // }
708 // if(res == 0) res = second.getType() - first.getType();
709 // }
710 // }
711 return res;
712 }
713
714 }
715
716 /**
717 * A method that returns specific type of annotations
718 * @param type
719 * @return a {@link Set} of {@link Pairing}s.
720 */
721 public Set getAnnotationsOfType(int type) {
722 switch(type) {
723 case CORRECT_TYPE:
724 return (correctAnnotations == null)? new HashSet() : correctAnnotations;
725 case PARTIALLY_CORRECT_TYPE:
726 return (partiallyCorrectAnnotations == null) ? new HashSet() : partiallyCorrectAnnotations;
727 case SPURIOUS_TYPE:
728 return (spuriousAnnotations == null) ? new HashSet() : spuriousAnnotations;
729 case MISSING_TYPE:
730 return (missingAnnotations == null) ? new HashSet() : missingAnnotations;
731 default:
732 return new HashSet();
733 }
734 }
735
736
737 public HashSet correctAnnotations, partiallyCorrectAnnotations,
738 missingAnnotations, spuriousAnnotations;
739
740
741 /** Type for correct pairings (when the key and response match completely)*/
742 public static final int CORRECT_TYPE = 1;
743
744 /**
745 * Type for partially correct pairings (when the key and response match
746 * in type and significant features but the spans are just overlapping and
747 * not identical.
748 */
749 public static final int PARTIALLY_CORRECT_TYPE = 2;
750
751 /**
752 * Type for spurious pairings (where the response is not matching any key).
753 */
754 public static final int SPURIOUS_TYPE = 3;
755
756 /**
757 * Type for missing pairings (where the key was not matched to a response).
758 */
759 public static final int MISSING_TYPE = 4;
760
761 /**
762 * Score for a correct pairing.
763 */
764 public static final int CORRECT = 2;
765
766 /**
767 * Score for a partially correct pairing.
768 */
769 public static final int PARTIALLY_CORRECT = 1;
770
771 /**
772 * Score for a wrong (missing or spurious) pairing.
773 */
774 public static final int WRONG = 0;
775
776 /**
777 * The set of significant features used for matching.
778 */
779 private java.util.Set significantFeaturesSet;
780
781 /**
782 * The number of correct matches.
783 */
784 protected int correctMatches;
785
786 /**
787 * The number of partially correct matches.
788 */
789 protected int partiallyCorrectMatches;
790
791 /**
792 * The number of missing matches.
793 */
794 protected int missing;
795
796 /**
797 * The number of spurious matches.
798 */
799 protected int spurious;
800
801 /**
802 * A list with all the key annotations
803 */
804 protected List keyList;
805
806 /**
807 * A list with all the response annotations
808 */
809 protected List responseList;
810
811 /**
812 * A list of lists representing all possible choices for each key
813 */
814 protected List keyChoices;
815
816 /**
817 * A list of lists representing all possible choices for each response
818 */
819 protected List responseChoices;
820
821 /**
822 * All the posible choices are added to this list for easy iteration.
823 */
824 protected List possibleChoices;
825
826 /**
827 * A list with the choices selected for the best result.
828 */
829 protected List finalChoices;
830
831 }
832