#!/bin/bash

# This script takes a corpus and a universe as input and computes phrase sampling features.
# Copyright 05/03/2014 Germán Sanchis-Trilles & Luis A. Leiva.
# Dual licensed under the MIT and GPL Version 2 licenses:
# http://opensource.org/licenses/mit-license.php
# http://opensource.org/licenses/GPL-2.0

HELP_MSG="\n
The script takes a corpus and a universe as input and computes phrase sampling features.        \n
Text files are asumed to be in Unix format (important for e.g. line breaks).                    \n
                                                                                                \n
Usage: $0 -c corpus -u universe [-w nwords_universe] [-a voc_universe] [-r] [-y] [-v]           \n
  \t Options in [] are optional. Options -w and -a are encouraged, as computing them is costly. \n
Input:                                                                                          \n
  \t -c corpus: Plain text file containing the target domain corpus.                            \n
  \t -u universe: Plain text file containing the language data considered as universe.          \n
  \t\t If -w and -a are given, then you can use '-u null' or '-u false' or '-u whatever'.       \n
  \t -w nwords_universe: Number of running words in the universe.                               \n
  \t -a voc_universe: Vocabulary of the universe. Each line entry must be structured as         \n
  \t\t <word count> tuples, with space as separator.                                            \n
  \t -r: Print file header (feature column names).                                              \n    
  \t -y: Do not delete temporary files. Useful for debugging.                                   \n
  \t -v: Activate verbose mode (set -x). Useful for debugging.                                  \n
  \t -h: Show this help message and exit.                                                       \n
Output:                                                                                         \n
  \t <nwords sdchar oovs-ratio lprob> tuples. One for each line according to the input corpus.  \n"

me=${BASH_SOURCE[0]}

nmoptions=$(cat $me | awk '/^exit$/{ exit }{ print $0 }' | grep "++moptions" | wc -l | awk '{ print $1 - 1 }')
moptions=0
cmd=("$@")
for ((i=0; i<${#cmd[@]}; i++)); do
    case ${cmd[$i]} in
      "-c") corpus=${cmd[$((++i))]}; ((++moptions));;
      "-u") universe=${cmd[$((++i))]}; ((++moptions));;
      "-w") totalwords=${cmd[$((++i))]};;
      "-a") univoc=${cmd[$((++i))]};;
      "-r") printheader="true";;
      "-y") deletetemp="false";;
      "-v") set -x;;
      *)    echo -e $HELP_MSG; exit 1;;
    esac
done

# Verify given options.
if [ $moptions -lt $nmoptions ]; then 
  echo -e $HELP_MSG
  exit 1
fi

# Set temp dir.
if [ -z "$(which tmpdir)" ]; then 
  if [ -d $HOME/tmp ]; then
    TMPPREF="$HOME/tmp"
  else 
    TMPPREF="/tmp"
  fi
else
  TMPPREF=$(tmpdir)
fi

tmpdir=`mktemp -d $TMPPREF/conftmp.XXXXXXXXXXX`
if [ "$deletetemp" != "false" ]; then
  trap "rm -rf $tmpdir" EXIT
else
  echo "Temp dir $tmpdir will NOT be deleted" >&2
  trap "echo 'To delete temp dir do: rm -rf '$tmpdir >&2" EXIT
fi

# Set temp filenames.
tmp_fea_file="$tmpdir/fea.txt"
tmp_voc_file="$tmpdir/voc.txt"
tmp_wrd_file="$tmpdir/wrd.txt"
tmp_chr_file="$tmpdir/chr.txt"
tmp_lex_file="$tmpdir/lex.txt"

# If the total number of words in universe is not provided, compute it.
if [ -z "$totalwords" ]; then
  echo "Computing universe words..." >&2
  if [ -z "$universe" ] || [ ! -f $universe ]; then 
    echo "Error! Universe file not not found."
    exit 1
  fi
  totalwords=$(wc -w $universe | awk '{ print $1 }')
fi

# If the voc of universe is not available, compute it.
if [ -z "$univoc" ]; then
  echo "Computing universe vocabulary..." >&2
  if [ ! -f $universe ]; then 
    echo "Error! Universe file not not found."
    exit 1
  fi
  cat $universe | \
    # one word per line, strip empty lines
    tr ' ' '\n' | grep -v "^$" | \
    # establish (unordered) dictionary
    awk '{ dict[$1]++ } END { for (i in dict) { print i,dict[i] } }' | \
    # order dictionary and produce final vocabulary file
    sort -n -k 2 > $tmp_voc_file
  univoc=$tmp_voc_file
fi

echo "Computing word counts per phrase..." >&2
cat $corpus | awk '{ print NF }' > $tmp_wrd_file

echo "Computing phrase features..." >&2
cat $corpus | awk '{
  nchar = 0
  nchar2 = 0
  for (w = 1; w <= NF; ++w) {
    nc = split($w, wcs, "")
    nchar += nc
    nchar2 += nc*nc
  }
  if (NF <= 1) charvar = 0                               # sdvar is not defined if sentence has only 1 word
  else charvar = sqrt( (nchar2-nchar*nchar/NF)/(NF-1) )  # compute sdvar otherwise
  print charvar
}' > $tmp_chr_file

# Compute lexical data: out-of-vocabulary ratio and phrase probability.
if [ -z "$LEXDATA" ]; then
  awk -v N=$totalwords \
  'BEGIN {
    minw = 1E100
  }
  {
    if (FNR == NR) {
      dict[$1] = $2
      if ($2 < minw) minw = $2
    }
    else {
      if (NF == 0) print 0,0
      else {
        noovs = 0
        probs = 0
        for (i = 1; i <= NF; ++i) {
          if ( !($i in dict) ) { 
            noovs++                             # if word is OOV, accumulate count and
            probs += log(minw/NF) - log(N)      # smooth prob with min voc count, penalized by NF
          }
          else probs += log(dict[$i]) - log(N)  # if word is in dict, accumulate lprob
        }
        print noovs/NF, probs/NF
      }
    }
  }' $univoc $corpus > $tmp_lex_file
  LEXDATA=$tmp_lex_file
fi

# Create features file, optionally with header: NW, SDChr, OOV, LPRob
if [ "$printheader" == "true" ]; then 
  echo "NW SDChr OOV LProb" > $tmp_fea_file
fi
paste -d" " $tmp_wrd_file $tmp_chr_file $LEXDATA >> $tmp_fea_file

# Finally print results.
cat $tmp_fea_file
