use strict;

my %prev;
my %word;

#### read the corpus and record vocabulary and word co-occurrences ####

while (<>) {
    chomp;

    s/[\.\,\:\;\-\"\'\!\?\(\)\*\+\/\\\<\>\]\[\#\{\}\=\_]/ $1 /g;
    s/  / /g;

    my @w = split " ", $_;

    ## record the context "beginning of paragraph" ##

    my $prev = "#";
    $word{"#"}++;

    ## read the line and record vocabulary and word co-occurrences ##

    for ( my $i=0; $i<@w; $i++ ) {

	if ( $w[$i] =~ /[A-Za-z0-9]/ ) {

	    $w[$i] =~ tr/[A-Z]/[a-z]/;

	    $word{$w[$i]}++;
	    $prev{$w[$i]}{$prev}++;

	    $prev = $w[$i];
	}
    }
}

#### create a subset of the vocabulary with frequent words only ####

my %wordSelected;
foreach my $w (sort keys %word) {

    ##parameter## threshold of frequency for selecting words
    if ( $word{$w} > 1000 ) {
	$wordSelected{$w} = 1;
    }
}

#### print relation (name) ####

print "\@relation 'previous-occurrence'\n";

#### print attributes (columns) ####

my $row = "";
foreach my $w (sort keys %wordSelected) { $row .= "$w, " }
$row =~ s/, $//g;
print "\@attribute 'word' {$row}\n";

foreach my $w (sort keys %wordSelected) {
    print "\@attribute '$w' real\n";
}

#### print objects (rows) ####

print "\@data\n";

foreach my $w (keys %wordSelected) {

    print "$w,";

    my $total;
    foreach my $ww (keys %{$prev{$w}} ) { $total += $prev{$w}{$ww} }
    
    my $row = "";
    foreach my $ww (sort keys %wordSelected) {

	## non-empty values of word co-occurrence ##
	if ( $prev{$w}{$ww} ) {
	    my $n = $prev{$w}{$ww} / $total;
	    $row .= "$n,";
	}
	## empty values of word co-occurrence ##
	else { $row .= "0," }
    }
    $row =~ s/,$//g;
    print "$row\n";
}
