dictionary.h
Go to the documentation of this file.
#ifndef FASTTEXT_DICTIONARY_H
#define FASTTEXT_DICTIONARY_H
#include <vector>
#include <string>
#include <istream>
#include <ostream>
#include <random>
#include <memory>
#include <unordered_map>
#include "args.h"
#include "real.h"
namespace fasttext {
typedef int32_t id_type;
enum class entry_type : int8_t {word=0, label=1};
struct entry {
std::string word;
int64_t count;
entry_type type;
std::vector<int32_t> subwords;
};
class Dictionary {
private:
static const int32_t MAX_VOCAB_SIZE = 30000000;
static const int32_t MAX_LINE_SIZE = 1024;
int32_t find(const std::string&) const;
void initTableDiscard();
void initNgrams();
std::shared_ptr<Args> args_;
std::vector<int32_t> word2int_;
std::vector<entry> words_;
std::vector<real> pdiscard_;
int32_t size_;
int32_t nwords_;
int32_t nlabels_;
int64_t ntokens_;
int64_t pruneidx_size_ = -1;
std::unordered_map<int32_t, int32_t> pruneidx_;
void addWordNgrams(
std::vector<int32_t>& line,
const std::vector<int32_t>& hashes,
int32_t n) const;
public:
static const std::string EOS;
static const std::string BOW;
static const std::string EOW;
explicit Dictionary(std::shared_ptr<Args>);
int32_t nwords() const;
int32_t nlabels() const;
int64_t ntokens() const;
int32_t getId(const std::string&) const;
entry_type getType(int32_t) const;
entry_type getType(const std::string&) const;
bool discard(int32_t, real) const;
std::string getWord(int32_t) const;
const std::vector<int32_t>& getSubwords(int32_t) const;
const std::vector<int32_t> getSubwords(const std::string&) const;
void computeSubwords(const std::string&, std::vector<int32_t>&) const;
void computeSubwords(
const std::string&,
std::vector<int32_t>&,
std::vector<std::string>&) const;
void getSubwords(
const std::string&,
std::vector<int32_t>&,
std::vector<std::string>&) const;
uint32_t hash(const std::string& str) const;
void add(const std::string&);
bool readWord(std::istream&, std::string&) const;
void readFromFile(std::istream&);
std::string getLabel(int32_t) const;
void save(std::ostream&) const;
void load(std::istream&);
std::vector<int64_t> getCounts(entry_type) const;
int32_t getLine(std::istream&, std::vector<int32_t>&, std::vector<int32_t>&,
std::vector<int32_t>&, std::minstd_rand&) const;
int32_t getLine(std::istream&, std::vector<int32_t>&,
std::vector<int32_t>&, std::minstd_rand&) const;
void threshold(int64_t, int64_t);
void prune(std::vector<int32_t>&);
};
}
#endif
fasttext::Dictionary::pruneidx_
std::unordered_map< int32_t, int32_t > pruneidx_
Definition: dictionary.h:56
fasttext::Dictionary
Definition: dictionary.h:36
fasttext::entry::word
std::string word
Definition: dictionary.h:30
fasttext::Dictionary::pdiscard_
std::vector< real > pdiscard_
Definition: dictionary.h:49
fasttext::Dictionary::EOW
static const std::string EOW
Definition: dictionary.h:66
fasttext::Dictionary::nwords_
int32_t nwords_
Definition: dictionary.h:51
fasttext
Definition: args.cc:17
fasttext::Dictionary::BOW
static const std::string BOW
Definition: dictionary.h:65
fasttext::Dictionary::nlabels_
int32_t nlabels_
Definition: dictionary.h:52
fasttext::entry::type
entry_type type
Definition: dictionary.h:32
fasttext::id_type
int32_t id_type
Definition: dictionary.h:26
real.h
fasttext::Dictionary::args_
std::shared_ptr< Args > args_
Definition: dictionary.h:45
fasttext::Dictionary::word2int_
std::vector< int32_t > word2int_
Definition: dictionary.h:46
fasttext::entry::subwords
std::vector< int32_t > subwords
Definition: dictionary.h:33
fasttext::entry
Definition: dictionary.h:29
fasttext::entry_type::word
fasttext::entry_type
entry_type
Definition: dictionary.h:27
fasttext::entry_type::label
fasttext::entry::count
int64_t count
Definition: dictionary.h:31
args.h
fasttext::real
float real
Definition: real.h:15
fasttext::Dictionary::ntokens_
int64_t ntokens_
Definition: dictionary.h:53
fasttext::Dictionary::size_
int32_t size_
Definition: dictionary.h:50
fasttext::Dictionary::EOS
static const std::string EOS
Definition: dictionary.h:64
fasttext::Dictionary::words_
std::vector< entry > words_
Definition: dictionary.h:47