#!/bin/zsh
set -eux
pv tok2014tweets| cut -f5 |python make_counts.py > twcounts
pv 2004,7-05_nyt_tok|python make_counts.py > nytcounts

cat twcounts|tabawk '$2>100' | tabsort -rgk2 > twcounts.sort
cat nytcounts|tabawk '$2>100' | tabsort -rgk2 > nytcounts.sort

# head -2000 twcounts.sort > twcounts.trimmed1
# head -2000 nytcounts.sort > nytcounts.trimmed1

python finalize.py

