productquantizer.h
Go to the documentation of this file.
#ifndef FASTTEXT_PRODUCT_QUANTIZER_H
#define FASTTEXT_PRODUCT_QUANTIZER_H
#include <cstring>
#include <istream>
#include <ostream>
#include <vector>
#include <random>
#include "real.h"
#include "vector.h"
namespace fasttext {
class ProductQuantizer {
private:
const int32_t nbits_ = 8;
const int32_t ksub_ = 1 << nbits_;
const int32_t max_points_per_cluster_ = 256;
const int32_t max_points_ = max_points_per_cluster_ * ksub_;
const int32_t seed_ = 1234;
const int32_t niter_ = 25;
const real eps_ = 1e-7;
int32_t dim_;
int32_t nsubq_;
int32_t dsub_;
int32_t lastdsub_;
std::vector<real> centroids_;
std::minstd_rand rng;
public:
ProductQuantizer() {}
ProductQuantizer(int32_t, int32_t);
real* get_centroids (int32_t, uint8_t);
const real* get_centroids(int32_t, uint8_t) const;
real assign_centroid(const real*, const real*, uint8_t*, int32_t) const;
void Estep(const real*, const real*, uint8_t*, int32_t, int32_t) const;
void MStep(const real*, real*, const uint8_t*, int32_t, int32_t);
void kmeans(const real*, real*, int32_t, int32_t);
void train(int, const real*);
real mulcode(const Vector&, const uint8_t*, int32_t, real) const;
void addcode(Vector&, const uint8_t*, int32_t, real) const;
void compute_code(const real*, uint8_t*) const;
void compute_codes(const real*, uint8_t*, int32_t) const;
void save(std::ostream&);
void load(std::istream&);
};
}
#endif
fasttext::ProductQuantizer::mulcode
real mulcode(const Vector &, const uint8_t *, int32_t, real) const
Definition: productquantizer.cc:149
fasttext::ProductQuantizer::MStep
void MStep(const real *, real *, const uint8_t *, int32_t, int32_t)
Definition: productquantizer.cc:67
fasttext::ProductQuantizer::nsubq_
int32_t nsubq_
Definition: productquantizer.h:35
fasttext::ProductQuantizer::kmeans
void kmeans(const real *, real *, int32_t, int32_t)
Definition: productquantizer.cc:113
fasttext::ProductQuantizer::save
void save(std::ostream &)
Definition: productquantizer.cc:192
fasttext::ProductQuantizer::assign_centroid
real assign_centroid(const real *, const real *, uint8_t *, int32_t) const
Definition: productquantizer.cc:43
fasttext::ProductQuantizer::get_centroids
real * get_centroids(int32_t, uint8_t)
Definition: productquantizer.cc:38
fasttext::ProductQuantizer::seed_
const int32_t seed_
Definition: productquantizer.h:30
fasttext::ProductQuantizer
Definition: productquantizer.h:24
fasttext
Definition: args.cc:17
fasttext::Vector
Definition: vector.h:23
fasttext::ProductQuantizer::niter_
const int32_t niter_
Definition: productquantizer.h:31
fasttext::ProductQuantizer::dsub_
int32_t dsub_
Definition: productquantizer.h:36
fasttext::ProductQuantizer::addcode
void addcode(Vector &, const uint8_t *, int32_t, real) const
Definition: productquantizer.cc:164
fasttext::ProductQuantizer::eps_
const real eps_
Definition: productquantizer.h:32
fasttext::ProductQuantizer::rng
std::minstd_rand rng
Definition: productquantizer.h:41
real.h
fasttext::ProductQuantizer::centroids_
std::vector< real > centroids_
Definition: productquantizer.h:39
fasttext::ProductQuantizer::lastdsub_
int32_t lastdsub_
Definition: productquantizer.h:37
fasttext::ProductQuantizer::compute_codes
void compute_codes(const real *, uint8_t *, int32_t) const
Definition: productquantizer.cc:185
fasttext::ProductQuantizer::dim_
int32_t dim_
Definition: productquantizer.h:34
fasttext::ProductQuantizer::train
void train(int, const real *)
Definition: productquantizer.cc:128
fasttext::ProductQuantizer::compute_code
void compute_code(const real *, uint8_t *) const
Definition: productquantizer.cc:177
fasttext::ProductQuantizer::max_points_
const int32_t max_points_
Definition: productquantizer.h:29
fasttext::ProductQuantizer::ksub_
const int32_t ksub_
Definition: productquantizer.h:27
fasttext::ProductQuantizer::nbits_
const int32_t nbits_
Definition: productquantizer.h:26
fasttext::ProductQuantizer::max_points_per_cluster_
const int32_t max_points_per_cluster_
Definition: productquantizer.h:28
vector.h
fasttext::real
float real
Definition: real.h:15
fasttext::ProductQuantizer::ProductQuantizer
ProductQuantizer()
Definition: productquantizer.h:44
fasttext::ProductQuantizer::Estep
void Estep(const real *, const real *, uint8_t *, int32_t, int32_t) const
Definition: productquantizer.cc:59
fasttext::ProductQuantizer::load
void load(std::istream &)
Definition: productquantizer.cc:200