
No correct answer (no “gold data”).
100 patients
|
60% agreement (45 + 15)
But as Yes is more common in both raters, some agreement might occur by chance.
100 patients
|
|
60% agreement (45 + 15 = 25 + 35)
Yes is more common in both raters if the left table. In the right table, No is more common for rater B.
κ = (po - pe) / (1 - pe)
100 patients
|
|
100 patients
|
|
100 patients
|
|
Pilot phase: 4 annotators, 900 sentences
Additional funding: 16 annotators, 200K sentences (about 3 annotators per sentence)
Calculating Krippendorff’s α took 6 hours
Devel::NYTProf identified the culprit:
the _build_coincidence method.
sub _build_coincidence($self) { my %coinc; my @s = $self->vals; for my $v (@s) { for my $v_ (@s) { $coinc{$v}{$v_} = sum(map { my $unit = $_; my @k = keys %$unit; sum(0, map { my $i = $_; scalar grep $unit->{$_} eq $v_, grep $i ne $_, @k } grep $unit->{$_} eq $v, @k ) / (@k - 1) } @{ $self->units }); } } return \%coinc }
Five nested loops (for, for, map, map, grep).
sub _build_coincidence($self) { my %coinc; my @s = $self->vals; for my $v (@s) { # 5 for my $v_ (@s) { # 4 $coinc{$v}{$v_} = sum(map { # 3 my $unit = $_; my @k = keys %$unit; sum(0, # 2 map { # 2 my $i = $_; scalar grep $unit->{$_} eq $v_, # 1 grep $i ne $_, @k # 1 } grep $unit->{$_} eq $v, @k # 2 ) / (@k - 1) } @{ $self->units }); } } return \%coinc }
sub _build_coincidence($self) { my %coinc; my @vals = $self->vals; # Initialize coincidence counts for my $v (@vals) { for my $v_ (@vals) { $coinc{$v}{$v_} = 0; } } # Iterate over each unit for my $unit (@{ $self->units }) { my @keys = keys %$unit; my $unit_count = @keys; # Count occurrences of each value in the current unit my %value_count; $value_count{ $unit->{$_} }++ for @keys; # Calculate coincidences based on the value counts for my $v (@vals) { for my $v_ (@vals) { if (exists $value_count{$v} && exists $value_count{$v_}) { my $coinc_count = 0; # Count pairs of keys that match the values for my $key1 (@keys) { for my $key2 (@keys) { next if $key1 eq $key2; # Skip same keys if ($unit->{$key1} eq $v && $unit->{$key2} eq $v_) { $coinc_count++; } } } # Update the coincidence count $coinc{$v}{$v_} += $coinc_count / ($unit_count - 1) if $unit_count > 1; } } } } return \%coinc }
$value_count{ $unit->{$_} }++ for @keys; # ... if (exists $value_count{$v} && exists $value_count{$v_}) {
sub _build_coincidence($self) { my @vals = $self->vals; my %coinc; @{ $coinc{$_} }{@vals} = (0) x @vals for @vals; for my $unit (@{ $self->units }) { my %is_value; @is_value{ values %$unit } = (); my @values = keys %is_value; my @keys = keys %$unit; for my $v (@values) { for my $v_ (@values) { my $coinc_count = 0; for my $key1 (@keys) { for my $key2 (@keys) { next if $key1 eq $key2; ++$coinc_count if $unit->{$key1} eq $v && $unit->{$key2} eq $v_; } } $coinc{$v}{$v_} += $coinc_count / (@keys - 1); } } } return \%coinc }
@{ $coinc{$_} }{@vals} = (0) x @vals for @vals;
# instead of
for my $v (@vals) {
for my $v_ (@vals) {
$coinc{$v}{$v_} = 0;
}
}

| LPW | 2026 |
