|
|
@@ -14,11 +14,11 @@ DATADIR=data
|
|
|
mkdir -p "${RESULTDIR}"
|
|
|
mkdir -p "${DATADIR}"
|
|
|
|
|
|
-if [ ! -f "${DATADIR}/text9" ]
|
|
|
+if [ ! -f "${DATADIR}/fil9" ]
|
|
|
then
|
|
|
wget -c http://mattmahoney.net/dc/enwik9.zip -P "${DATADIR}"
|
|
|
unzip "${DATADIR}/enwik9.zip" -d "${DATADIR}"
|
|
|
- perl wikifil.pl "${DATADIR}/enwik9" > "${DATADIR}"/text9
|
|
|
+ perl wikifil.pl "${DATADIR}/enwik9" > "${DATADIR}"/fil9
|
|
|
fi
|
|
|
|
|
|
if [ ! -f "${DATADIR}/rw/rw.txt" ]
|
|
|
@@ -29,12 +29,12 @@ fi
|
|
|
|
|
|
make
|
|
|
|
|
|
-./fasttext skipgram -input "${DATADIR}"/text9 -output "${RESULTDIR}"/text9 -lr 0.025 -dim 100 \
|
|
|
+./fasttext skipgram -input "${DATADIR}"/fil9 -output "${RESULTDIR}"/fil9 -lr 0.025 -dim 100 \
|
|
|
-ws 5 -epoch 1 -minCount 5 -neg 5 -loss ns -bucket 2000000 \
|
|
|
-minn 3 -maxn 6 -thread 4 -t 1e-4 -lrUpdateRate 100
|
|
|
|
|
|
cut -f 1,2 "${DATADIR}"/rw/rw.txt | awk '{print tolower($0)}' | tr '\t' '\n' > "${DATADIR}"/queries.txt
|
|
|
|
|
|
-cat "${DATADIR}"/queries.txt | ./fasttext print-vectors "${RESULTDIR}"/text9.bin > "${RESULTDIR}"/vectors.txt
|
|
|
+cat "${DATADIR}"/queries.txt | ./fasttext print-vectors "${RESULTDIR}"/fil9.bin > "${RESULTDIR}"/vectors.txt
|
|
|
|
|
|
python eval.py -m "${RESULTDIR}"/vectors.txt -d "${DATADIR}"/rw/rw.txt
|