add tensor permute

traph/include/traph/core/tensor.h

@@ -42,7 +42,7 @@ namespace traph
         virtual void neg_() = 0;
         virtual idx_type offset() const = 0;
 		virtual layout_type order() const = 0;
-        // virtual std::shared_ptr<TensorInterface> permute(const DimVector& dims) const = 0;
+        virtual std::shared_ptr<TensorInterface> permute(const DimVector& dims) const = 0;
         virtual PlatformType platform() const = 0;
         virtual void pow_(f32 exp) = 0;
         virtual void reshape_(const DimVector& dims) = 0;
@@ -99,6 +99,7 @@ namespace traph
         virtual void neg_() = 0;
         virtual idx_type offset() const = 0;
 		virtual layout_type order() const = 0;
+        virtual std::shared_ptr<TensorInterface> permute(const DimVector& dims) const = 0;
         virtual PlatformType platform() const = 0;
         virtual void pow_(f32 exp) = 0;
         virtual T reduce(std::function<T(T,T)> f) const = 0;

traph/include/traph/nn/layers/linear.h

@@ -18,9 +18,9 @@ namespace traph
         {
             _in_features = in_features;
             _out_features = out_features;
-            _weight = randn<f32>({out_features, in_features}, true);
+            _weight = ones<f32>({out_features, in_features}, true);
             if(bias)
-                _bias = randn<f32>({out_features}, true);
+                _bias = ones<f32>({out_features}, true);
             
             register_parameter("weight", _weight);
             register_parameter("bias", _bias);

traph/include/traph/tensor/byte_tensor.h

@@ -74,6 +74,7 @@ namespace traph
         virtual void neg_() override;
         virtual idx_type offset() const override;
 		virtual layout_type order() const override;
+        virtual std::shared_ptr<TensorInterface> permute(const DimVector& dims) const override;
 		virtual PlatformType platform() const override;
         virtual void pow_(f32 exp) override;
 		virtual u8 reduce(std::function<u8(u8, u8)> f) const override;

traph/include/traph/tensor/char_tensor.h

@@ -72,6 +72,7 @@ namespace traph
         virtual void neg_() override;
         virtual idx_type offset() const override;
 		virtual layout_type order() const override;
+        virtual std::shared_ptr<TensorInterface> permute(const DimVector& dims) const override;
 		virtual PlatformType platform() const override;
         virtual void pow_(f32 exp) override;
 		virtual i8 reduce(std::function<i8(i8, i8)> f) const override;

traph/include/traph/tensor/double_tensor.h

@@ -72,6 +72,7 @@ namespace traph
         virtual void neg_() override;
         virtual idx_type offset() const override;
 		virtual layout_type order() const override;
+        virtual std::shared_ptr<TensorInterface> permute(const DimVector& dims) const override;
 		virtual PlatformType platform() const override;
         virtual void pow_(f32 exp) override;
 		virtual f64 reduce(std::function<f64(f64, f64)> f) const override;

traph/include/traph/tensor/float_tensor.h

@@ -73,6 +73,7 @@ namespace traph
         virtual void neg_() override;
         virtual idx_type offset() const override;
 		virtual layout_type order() const override;
+        virtual std::shared_ptr<TensorInterface> permute(const DimVector& dims) const override;
 		virtual PlatformType platform() const override;
         virtual void pow_(f32 exp) override;
 		virtual f32 reduce(std::function<f32(f32, f32)> f) const override;

traph/include/traph/tensor/int_tensor.h

@@ -72,6 +72,7 @@ namespace traph
         virtual void neg_() override;
         virtual idx_type offset() const override;
 		virtual layout_type order() const override;
+        virtual std::shared_ptr<TensorInterface> permute(const DimVector& dims) const override;
 		virtual PlatformType platform() const override;
         virtual void pow_(f32 exp) override;
 		virtual i32 reduce(std::function<i32(i32, i32)> f) const override;

traph/include/traph/tensor/long_tensor.h

@@ -72,6 +72,7 @@ namespace traph
         virtual void neg_() override;
 		virtual idx_type offset() const override;
 		virtual layout_type order() const override;
+        virtual std::shared_ptr<TensorInterface> permute(const DimVector& dims) const override;
 		virtual PlatformType platform() const override;
         virtual void pow_(f32 exp) override;
 		virtual i64 reduce(std::function<i64(i64, i64)> f) const override;

traph/include/traph/tensor/short_tensor.h

@@ -72,6 +72,7 @@ namespace traph
         virtual void neg_() override;
         virtual idx_type offset() const override;
 		virtual layout_type order() const override;
+        virtual std::shared_ptr<TensorInterface> permute(const DimVector& dims) const override;
 		virtual PlatformType platform() const override;
         virtual void pow_(f32 exp) override;
 		virtual i16 reduce(std::function<i16(i16, i16)> f) const override;

traph/include/traph/tensor/tensor.h

@@ -76,6 +76,7 @@ namespace traph
         virtual void neg_() override;
         virtual idx_type offset() const override;
 		virtual layout_type order() const override;
+        virtual std::shared_ptr<TensorInterface> permute(const DimVector& dims) const override;
         virtual PlatformType platform() const override;
         virtual void pow_(f32 exp) override;
         virtual T reduce(std::function<T(T,T)> f) const override;
@@ -105,6 +106,25 @@ namespace traph
 
     // TODO: macros
     // apply apply2 reduce...
+
+#define TENSOR_APPLY(TYPE, TENSOR, CODE)
+	{
+        std::function<void(idx_type, idx_type>)> apply_impl =
+        [&](idx_type dim, idx_type idx){
+            idx_type dim_size = TENSOR->_dimensions.size();
+            
+            for(idx_type i = 0; i < TENSOR->_dimensions[dim]; ++i)
+            {
+                if(dim_idx == dim_size - 1)
+                    CODE
+                else
+                    apply_impl(dim_idx + 1, idx, f);
+                idx += TENSOR->_strides[dim];
+            }
+        };
+
+        apply_impl(0, _offset);
+	}
 }
 
 #endif // !TRAPH_TENSOR

traph/source/demo/main.cpp

@@ -21,7 +21,7 @@ public:
 
 	MyModel()
 		:linear1(new Linear(32, 2, false))
-		// linear2(new Linear(512, 2, false))
+		// linear2(new Linear(16, 2, false))
 		// linear3(new Linear(256, 128, false))
 	{
 		add_module("linear1", linear1);
@@ -100,7 +100,7 @@ int main()
 
 	std::cout << "Start Training..." << std::endl;
 
-	for (int epoch = 0; epoch < 1000; ++epoch)
+	for (int epoch = 0; epoch < 100; ++epoch)
 	{
 		float loss100 = 0.f;
 

traph/source/tensor/byte_tensor.cpp

@@ -385,6 +385,40 @@ namespace traph
 
     layout_type Tensor<u8>::order() const { return _order; }
 
+    std::shared_ptr<TensorInterface> Tensor<u8>::permute(const DimVector& dims) const
+    {
+        // check dims
+        if(dims.size() != _strides.size())
+            throw std::runtime_error("permute dimension must have the same size");
+        std::vector<int> check_vec(dims.size(), 0);
+        for(int i = 0; i < dims.size();++i)
+            if(dims[i] >= 0 && dims[i] < dims.size())
+                check_vec[dims[i]] = 1;
+            else
+                throw std::runtime_error("permute dimension must in ndimension range");
+        
+        for(int i = 0; i < check_vec.size();++i)
+        {
+            if(check_vec[i] != 1)
+                throw std::runtime_error("permute dimension error");
+        }
+        // permute
+        std::shared_ptr<Tensor<u8>> result(new Tensor<u8>);
+        result->_rep = _rep;
+        result->_dimensions = _dimensions;
+        result->_offset = _offset;
+        result->_strides = _strides;
+        result->_order = _order;
+
+        for(int i=0; i<dims.size(); ++i)
+        {
+            result->_dimensions[i] = _dimensions[dims[i]];
+            result->_strides[i] = _strides[dims[i]];
+        }
+
+        return result;
+    }
+
     PlatformType Tensor<u8>::platform() const { return PlatformType::CPU; }
 
     void Tensor<u8>::pow_(f32 exp)
@@ -581,8 +615,15 @@ namespace traph
 
     std::shared_ptr<TensorInterface> Tensor<u8>::transpose(idx_type dim0, idx_type dim1)
     {
-        std::shared_ptr<TensorInterface> result= this->clone();
+        std::shared_ptr<Tensor<u8>> result(new Tensor<u8>);
+        result->_rep = _rep;
+        result->_dimensions = _dimensions;
+        result->_offset = _offset;
+        result->_strides = _strides;
+        result->_order = _order;
+
         result->transpose_(dim0, dim1);
+
         return result;
     }
 }

traph/source/tensor/char_tensor.cpp

@@ -385,6 +385,40 @@ namespace traph
 
     layout_type Tensor<i8>::order() const { return _order; }
 
+    std::shared_ptr<TensorInterface> Tensor<i8>::permute(const DimVector& dims) const
+    {
+        // check dims
+        if(dims.size() != _strides.size())
+            throw std::runtime_error("permute dimension must have the same size");
+        std::vector<int> check_vec(dims.size(), 0);
+        for(int i = 0; i < dims.size();++i)
+            if(dims[i] >= 0 && dims[i] < dims.size())
+                check_vec[dims[i]] = 1;
+            else
+                throw std::runtime_error("permute dimension must in ndimension range");
+        
+        for(int i = 0; i < check_vec.size();++i)
+        {
+            if(check_vec[i] != 1)
+                throw std::runtime_error("permute dimension error");
+        }
+        // permute
+        std::shared_ptr<Tensor<i8>> result(new Tensor<i8>);
+        result->_rep = _rep;
+        result->_dimensions = _dimensions;
+        result->_offset = _offset;
+        result->_strides = _strides;
+        result->_order = _order;
+
+        for(int i=0; i<dims.size(); ++i)
+        {
+            result->_dimensions[i] = _dimensions[dims[i]];
+            result->_strides[i] = _strides[dims[i]];
+        }
+
+        return result;
+    }
+
     PlatformType Tensor<i8>::platform() const { return PlatformType::CPU; }
 
     void Tensor<i8>::pow_(f32 exp)
@@ -581,8 +615,15 @@ namespace traph
 
     std::shared_ptr<TensorInterface> Tensor<i8>::transpose(idx_type dim0, idx_type dim1)
     {
-        std::shared_ptr<TensorInterface> result= this->clone();
+        std::shared_ptr<Tensor<i8>> result(new Tensor<i8>);
+        result->_rep = _rep;
+        result->_dimensions = _dimensions;
+        result->_offset = _offset;
+        result->_strides = _strides;
+        result->_order = _order;
+
         result->transpose_(dim0, dim1);
+
         return result;
     }
 }

traph/source/tensor/double_tensor.cpp

@@ -386,6 +386,40 @@ namespace traph
 
     layout_type Tensor<f64>::order() const { return _order; }
 
+    std::shared_ptr<TensorInterface> Tensor<f64>::permute(const DimVector& dims) const
+    {
+        // check dims
+        if(dims.size() != _strides.size())
+            throw std::runtime_error("permute dimension must have the same size");
+        std::vector<int> check_vec(dims.size(), 0);
+        for(int i = 0; i < dims.size();++i)
+            if(dims[i] >= 0 && dims[i] < dims.size())
+                check_vec[dims[i]] = 1;
+            else
+                throw std::runtime_error("permute dimension must in ndimension range");
+        
+        for(int i = 0; i < check_vec.size();++i)
+        {
+            if(check_vec[i] != 1)
+                throw std::runtime_error("permute dimension error");
+        }
+        // permute
+        std::shared_ptr<Tensor<f64>> result(new Tensor<f64>);
+        result->_rep = _rep;
+        result->_dimensions = _dimensions;
+        result->_offset = _offset;
+        result->_strides = _strides;
+        result->_order = _order;
+
+        for(int i=0; i<dims.size(); ++i)
+        {
+            result->_dimensions[i] = _dimensions[dims[i]];
+            result->_strides[i] = _strides[dims[i]];
+        }
+
+        return result;
+    }
+
     PlatformType Tensor<f64>::platform() const { return PlatformType::CPU; }
 
     void Tensor<f64>::pow_(f32 exp)
@@ -581,8 +615,15 @@ namespace traph
 
     std::shared_ptr<TensorInterface> Tensor<f64>::transpose(idx_type dim0, idx_type dim1)
     {
-        std::shared_ptr<TensorInterface> result= this->clone();
+        std::shared_ptr<Tensor<f64>> result(new Tensor<f64>);
+        result->_rep = _rep;
+        result->_dimensions = _dimensions;
+        result->_offset = _offset;
+        result->_strides = _strides;
+        result->_order = _order;
+
         result->transpose_(dim0, dim1);
+
         return result;
     }
 }

traph/source/tensor/float_tensor.cpp

@@ -388,6 +388,40 @@ namespace traph
 
     layout_type Tensor<f32>::order() const { return _order; }
 
+    std::shared_ptr<TensorInterface> Tensor<f32>::permute(const DimVector& dims) const
+    {
+        // check dims
+        if(dims.size() != _strides.size())
+            throw std::runtime_error("permute dimension must have the same size");
+        std::vector<int> check_vec(dims.size(), 0);
+        for(int i = 0; i < dims.size();++i)
+            if(dims[i] >= 0 && dims[i] < dims.size())
+                check_vec[dims[i]] = 1;
+            else
+                throw std::runtime_error("permute dimension must in ndimension range");
+        
+        for(int i = 0; i < check_vec.size();++i)
+        {
+            if(check_vec[i] != 1)
+                throw std::runtime_error("permute dimension error");
+        }
+        // permute
+        std::shared_ptr<Tensor<f32>> result(new Tensor<f32>);
+        result->_rep = _rep;
+        result->_dimensions = _dimensions;
+        result->_offset = _offset;
+        result->_strides = _strides;
+        result->_order = _order;
+
+        for(int i=0; i<dims.size(); ++i)
+        {
+            result->_dimensions[i] = _dimensions[dims[i]];
+            result->_strides[i] = _strides[dims[i]];
+        }
+
+        return result;
+    }
+
     PlatformType Tensor<f32>::platform() const { return PlatformType::CPU; }
 
     void Tensor<f32>::pow_(f32 exp)
@@ -583,8 +617,15 @@ namespace traph
 
     std::shared_ptr<TensorInterface> Tensor<f32>::transpose(idx_type dim0, idx_type dim1)
     {
-        std::shared_ptr<TensorInterface> result= this->clone();
+        std::shared_ptr<Tensor<f32>> result(new Tensor<f32>);
+        result->_rep = _rep;
+        result->_dimensions = _dimensions;
+        result->_offset = _offset;
+        result->_strides = _strides;
+        result->_order = _order;
+
         result->transpose_(dim0, dim1);
+
         return result;
     }
 }

traph/source/tensor/int_tensor.cpp

@@ -386,6 +386,40 @@ namespace traph
 
     layout_type Tensor<i32>::order() const { return _order; }
 
+    std::shared_ptr<TensorInterface> Tensor<i32>::permute(const DimVector& dims) const
+    {
+        // check dims
+        if(dims.size() != _strides.size())
+            throw std::runtime_error("permute dimension must have the same size");
+        std::vector<int> check_vec(dims.size(), 0);
+        for(int i = 0; i < dims.size();++i)
+            if(dims[i] >= 0 && dims[i] < dims.size())
+                check_vec[dims[i]] = 1;
+            else
+                throw std::runtime_error("permute dimension must in ndimension range");
+        
+        for(int i = 0; i < check_vec.size();++i)
+        {
+            if(check_vec[i] != 1)
+                throw std::runtime_error("permute dimension error");
+        }
+        // permute
+        std::shared_ptr<Tensor<i32>> result(new Tensor<i32>);
+        result->_rep = _rep;
+        result->_dimensions = _dimensions;
+        result->_offset = _offset;
+        result->_strides = _strides;
+        result->_order = _order;
+
+        for(int i=0; i<dims.size(); ++i)
+        {
+            result->_dimensions[i] = _dimensions[dims[i]];
+            result->_strides[i] = _strides[dims[i]];
+        }
+
+        return result;
+    }
+
     PlatformType Tensor<i32>::platform() const { return PlatformType::CPU; }
 
     void Tensor<i32>::pow_(f32 exp)
@@ -582,8 +616,15 @@ namespace traph
 
     std::shared_ptr<TensorInterface> Tensor<i32>::transpose(idx_type dim0, idx_type dim1)
     {
-        std::shared_ptr<TensorInterface> result= this->clone();
+        std::shared_ptr<Tensor<i32>> result(new Tensor<i32>);
+        result->_rep = _rep;
+        result->_dimensions = _dimensions;
+        result->_offset = _offset;
+        result->_strides = _strides;
+        result->_order = _order;
+
         result->transpose_(dim0, dim1);
+
         return result;
     }
 }

traph/source/tensor/long_tensor.cpp

@@ -386,6 +386,40 @@ namespace traph
 
     layout_type Tensor<i64>::order() const { return _order; }
 
+    std::shared_ptr<TensorInterface> Tensor<i64>::permute(const DimVector& dims) const
+    {
+        // check dims
+        if(dims.size() != _strides.size())
+            throw std::runtime_error("permute dimension must have the same size");
+        std::vector<int> check_vec(dims.size(), 0);
+        for(int i = 0; i < dims.size();++i)
+            if(dims[i] >= 0 && dims[i] < dims.size())
+                check_vec[dims[i]] = 1;
+            else
+                throw std::runtime_error("permute dimension must in ndimension range");
+        
+        for(int i = 0; i < check_vec.size();++i)
+        {
+            if(check_vec[i] != 1)
+                throw std::runtime_error("permute dimension error");
+        }
+        // permute
+        std::shared_ptr<Tensor<i64>> result(new Tensor<i64>);
+        result->_rep = _rep;
+        result->_dimensions = _dimensions;
+        result->_offset = _offset;
+        result->_strides = _strides;
+        result->_order = _order;
+
+        for(int i=0; i<dims.size(); ++i)
+        {
+            result->_dimensions[i] = _dimensions[dims[i]];
+            result->_strides[i] = _strides[dims[i]];
+        }
+
+        return result;
+    }
+
     PlatformType Tensor<i64>::platform() const { return PlatformType::CPU; }
 
     void Tensor<i64>::pow_(f32 exp)
@@ -582,8 +616,15 @@ namespace traph
 
     std::shared_ptr<TensorInterface> Tensor<i64>::transpose(idx_type dim0, idx_type dim1)
     {
-        std::shared_ptr<TensorInterface> result= this->clone();
+        std::shared_ptr<Tensor<i64>> result(new Tensor<i64>);
+        result->_rep = _rep;
+        result->_dimensions = _dimensions;
+        result->_offset = _offset;
+        result->_strides = _strides;
+        result->_order = _order;
+
         result->transpose_(dim0, dim1);
+
         return result;
     }
 }

traph/source/tensor/short_tensor.cpp

@@ -386,6 +386,40 @@ namespace traph
 
     layout_type Tensor<i16>::order() const { return _order; }
 
+    std::shared_ptr<TensorInterface> Tensor<i16>::permute(const DimVector& dims) const
+    {
+        // check dims
+        if(dims.size() != _strides.size())
+            throw std::runtime_error("permute dimension must have the same size");
+        std::vector<int> check_vec(dims.size(), 0);
+        for(int i = 0; i < dims.size();++i)
+            if(dims[i] >= 0 && dims[i] < dims.size())
+                check_vec[dims[i]] = 1;
+            else
+                throw std::runtime_error("permute dimension must in ndimension range");
+        
+        for(int i = 0; i < check_vec.size();++i)
+        {
+            if(check_vec[i] != 1)
+                throw std::runtime_error("permute dimension error");
+        }
+        // permute
+        std::shared_ptr<Tensor<i16>> result(new Tensor<i16>);
+        result->_rep = _rep;
+        result->_dimensions = _dimensions;
+        result->_offset = _offset;
+        result->_strides = _strides;
+        result->_order = _order;
+
+        for(int i=0; i<dims.size(); ++i)
+        {
+            result->_dimensions[i] = _dimensions[dims[i]];
+            result->_strides[i] = _strides[dims[i]];
+        }
+
+        return result;
+    }
+
     PlatformType Tensor<i16>::platform() const { return PlatformType::CPU; }
 
     void Tensor<i16>::pow_(f32 exp)
@@ -582,8 +616,15 @@ namespace traph
 
     std::shared_ptr<TensorInterface> Tensor<i16>::transpose(idx_type dim0, idx_type dim1)
     {
-        std::shared_ptr<TensorInterface> result= this->clone();
+        std::shared_ptr<Tensor<i16>> result(new Tensor<i16>);
+        result->_rep = _rep;
+        result->_dimensions = _dimensions;
+        result->_offset = _offset;
+        result->_strides = _strides;
+        result->_order = _order;
+
         result->transpose_(dim0, dim1);
+
         return result;
     }
 }

traph/source/tensor/tensor.cpp

@@ -157,6 +157,11 @@ namespace traph
     template<typename T>
     layout_type Tensor<T>::order() const { throw std::runtime_error("No implement"); }
     template<typename T>
+    std::shared_ptr<TensorInterface> Tensor<T>::permute(const DimVector& dims) const
+    {
+        throw std::runtime_error("No implement");
+    }
+    template<typename T>
     PlatformType Tensor<T>::platform() const { throw std::runtime_error("No implement"); }
     template<typename T>
     void Tensor<T>::pow_(f32 exp)