Absolute paths in -update scripts

tools/caffe-update

@@ -2,9 +2,10 @@
 
 set -e
 
-tools=$(dirname ${0})
-src=${tools}/../src
-third_party=${tools}/../third_party
+root=$(cd $(dirname ${0})/..; pwd)
+src=${root}/src
+tools=${root}/tools
+third_party=${root}/third_party
 
 identifier=caffe
 repository=https://github.com/BVLC/${identifier}.git
@@ -20,4 +21,4 @@ else
 fi
 
 echo "Generate '../src/caffe.js'"
-${tools}/../node_modules/protobufjs/bin/pbjs -t static-module -w closure -r caffe -o ${src}/caffe.js ${third_party}/caffe/src/caffe/proto/caffe.proto
+${root}/node_modules/protobufjs/bin/pbjs -t static-module -w closure -r caffe -o ${src}/caffe.js ${third_party}/${identifier}/src/caffe/proto/caffe.proto

tools/caffe2-update

@@ -2,9 +2,11 @@
 
 set -e
 
-tools=$(dirname ${0})
-src=${tools}/../src
-third_party=${tools}/../third_party
+root=$(cd $(dirname ${0})/..; pwd)
+build=${root}/build
+src=${root}/src
+tools=${root}/tools
+third_party=${root}/third_party
 
 identifier=pytorch
 repository=https://github.com/pytorch/pytorch.git
@@ -30,7 +32,7 @@ brew "gflags"
 brew "glog"
 EOS
 
-export PYTHONUSERBASE=${third_party}/${identifier}/build/pypi
+export PYTHONUSERBASE=${build}/third_party/pypi/${identifier}
 export PATH=$PATH:${PYTHONUSERBASE}/bin
 
 pip install --user future leveldb numpy protobuf pydot python-gflags pyyaml scikit-image setuptools six

tools/coreml-converter

@@ -2,8 +2,10 @@
 
 set -e
 
-tools=$(dirname ${0})
-third_party=${tools}/../third_party
+root=$(cd $(dirname ${0})/..; pwd)
+build=${root}/build
+tools=${root}/tools
+third_party=${root}/third_party
 
 identifier=coremltools
 repository=https://github.com/apple/${identifier}.git
@@ -18,8 +20,9 @@ else
     git -C "${third_party}" clone --recursive ${repository}
 fi
 
-export PYTHONUSERBASE=${third_party}/${identifier}/build/pypi
+export PYTHONUSERBASE=${build}/third_party/pypi/${identifier}
 export PATH=$PATH:${PYTHONUSERBASE}/bin
+
 pip install --quiet --user onnx
 pip install --quiet --user sklearn
 pip install --quiet --user ${third_party}/${identifier}

tools/coreml-update

@@ -2,9 +2,10 @@
 
 set -e
 
-tools=$(dirname ${0})
-src=${tools}/../src
-third_party=${tools}/../third_party
+root=$(cd $(dirname ${0})/..; pwd)
+src=${root}/src
+tools=${root}/tools
+third_party=${root}/third_party
 
 identifier=coremltools
 repository=https://github.com/apple/${identifier}.git
@@ -20,4 +21,4 @@ else
 fi
 
 echo "Generate '../src/coreml.js'"
-${tools}/../node_modules/protobufjs/bin/pbjs -t static-module -w closure -r coreml -o ${src}/coreml.js ${third_party}/coremltools/mlmodel/format/Model.proto
+${root}/node_modules/protobufjs/bin/pbjs -t static-module -w closure -r coreml -o ${src}/coreml.js ${third_party}/${identifier}/mlmodel/format/Model.proto

tools/keras-update

@@ -2,8 +2,9 @@
 
 set -e
 
-tools=$(dirname ${0})
-third_party=${tools}/../third_party
+root=$(cd $(dirname ${0})/..; pwd)
+tools=${root}/tools
+third_party=${root}/third_party
 
 identifier=keras
 repository=https://github.com/keras-team/${identifier}.git

tools/mxnet-update

@@ -2,9 +2,10 @@
 
 set -e
 
-tools=$(dirname ${0})
-src=${tools}/../src
-third_party=${tools}/../third_party
+root=$(cd $(dirname ${0})/..; pwd)
+src=${root}/src
+tools=${root}/tools
+third_party=${root}/third_party
 
 identifier=mxnet
 repository=https://github.com/apache/incubator-${identifier}.git

tools/onnx-converter

@@ -2,8 +2,10 @@
 
 set -e
 
-tools=$(dirname ${0})
-third_party=${tools}/../third_party
+root=$(cd $(dirname ${0})/..; pwd)
+build=${root}/build
+tools=${root}/tools
+third_party=${root}/third_party
 
 identifier=onnxmltools
 repository=https://github.com/onnx/${identifier}.git
@@ -18,8 +20,9 @@ else
     git -C "${third_party}" clone --recursive ${repository}
 fi
 
-export PYTHONUSERBASE=${third_party}/${identifier}/build/pypi
+export PYTHONUSERBASE=${build}/third_party/pypi/${identifier}
 export PATH=$PATH:${PYTHONUSERBASE}/bin
+
 pip install --quiet --user ${third_party}/${identifier}
 
 python ${tools}/onnx-converter.py $@

tools/onnx-metadata.py

@@ -74,51 +74,6 @@ categories = {
     'Crop': 'Data',
 
     'Gather': 'Transform',
-
-    # 'Gemm': '',
-    # 'MatMul': '',
-    # 'Hardmax':
-    # 'Log':
-    # 'Max':
-    # 'Div': 'Basic',
-    # 'Ceil': 'Basic',
-    # 'Exp': 'Basic',
-    # 'Floor': 'Basic',
-    # 'Sqrt': 'Basic',
-    # 'Sub': 'Basic',
-    # 'Sum': 'Basic',
-    # 'Min': 'Basic',
-    # 'Mul': 'Basic',
-    # 'Neg': 'Basic',
-    # 'Abs': 'Basic',
-    # 'Add': 'Basic',
-    # 'Pow': 'Basic',
-    # 'ArgMax':
-    # 'ArgMin':
-    # 'Cast':
-    # 'Clip':
-    # 'DepthToSpace':
-    # 'Mean': 
-    # 'Pad':
-    # 'RandomNormal':
-    # 'RandomNormalLike':
-    # 'RandomUniform':
-    # 'RandomUniformLike':
-    # 'Reciprocal':
-    # 'ReduceL1':
-    # 'ReduceL2':
-    # 'ReduceLogSum':
-    # 'ReduceLogSumExp':
-    # 'ReduceMax':
-    # 'ReduceMean':
-    # 'ReduceMin':
-    # 'ReduceProd':
-    # 'ReduceSum':
-    # 'ReduceSumSquare':
-    # 'SpaceToDepth':
-    # 'Squeeze':
-    # 'Tile':
-    # 'Gather':
 }
 
 def generate_json_attr_type(type):
@@ -135,7 +90,7 @@ def generate_json_attr_default_value(attr_value):
     if attr_value.HasField('i'):
         return attr_value.i
     if attr_value.HasField('s'):
-        return attr_value.s
+        return attr_value.s.decode('utf8')
     if attr_value.HasField('f'):
         return attr_value.f
     return None
@@ -235,5 +190,6 @@ def generate_json(schemas, json_file):
             fout.write('\n')
 
 if __name__ == '__main__':
-    schemas = sorted(defs.get_all_schemas_with_history(), key=lambda schema: schema.name)
+    schemas = defs.get_all_schemas_with_history()
+    schemas = sorted(schemas, key=lambda schema: schema.name)
     generate_json(schemas, '../src/onnx-metadata.json')

tools/onnx-update

@@ -2,9 +2,11 @@
 
 set -e
 
-tools=$(dirname ${0})
-src=${tools}/../src
-third_party=${tools}/../third_party
+root=$(cd $(dirname ${0})/..; pwd)
+build=${root}/build
+src=${root}/src
+third_party=${root}/third_party
+tools=${root}/tools
 
 identifier=onnx
 repository=https://github.com/onnx/${identifier}.git
@@ -19,16 +21,16 @@ else
     git -C "${third_party}" clone --recursive ${repository}
 fi
 
-echo "Build ONNX"
+echo "Build"
 export ONNX_ML=1
 export ONNX_NAMESPACE=onnx
 pushd "${third_party}/${identifier}" > /dev/null
-python setup.py build --build-lib build/lib
+python setup.py build --build-lib ${build}/third_party/lib/${identifier}
 popd > /dev/null
 
 echo "Generate '../src/onnx-metadata.json'"
 pushd ${tools} > /dev/null
-export PYTHONPATH=../third_party/${identifier}/build/lib
+export PYTHONPATH=${build}/third_party/lib/${identifier}
 python onnx-metadata.py
 popd > /dev/null
 

tools/tf-metadata.py

@@ -125,7 +125,7 @@ def convert_attr_value(attr_value):
         attr_value_list = attr_value.list
         if len(attr_value_list.s) > 0:
             for s in attr_value_list.s:
-                list.append(s)
+                list.append(s.decode('utf8'))
         if len(attr_value_list.i) > 0:
             for i in attr_value_list.i:
                 list.append(i)
@@ -141,7 +141,7 @@ def convert_attr_value(attr_value):
                     raise Exception()
         return list
     if attr_value.HasField('s'):
-        return attr_value.s
+        return attr_value.s.decode('utf8')
     if attr_value.HasField('i'):
         return attr_value.i
     if attr_value.HasField('f'):

tools/tf-update

@@ -2,9 +2,10 @@
 
 set -e
 
-tools=$(dirname ${0})
-src=${tools}/../src
-third_party=${tools}/../third_party
+root=$(cd $(dirname ${0})/..; pwd)
+src=${root}/src
+tools=${root}/tools
+third_party=${root}/third_party
 
 identifier=tensorflow
 repository=https://github.com/tensorflow/${identifier}.git
@@ -20,7 +21,7 @@ else
 fi
 
 echo "Generate '../src/tf.js'"
-${tools}/../node_modules/protobufjs/bin/pbjs -t static-module -w closure -r tf -o ${src}/tf.js \
+${root}/node_modules/protobufjs/bin/pbjs -t static-module -w closure -r tf -o ${src}/tf.js \
     ${third_party}/${identifier}/tensorflow/core/protobuf/saved_model.proto \
     ${third_party}/${identifier}/tensorflow/core/protobuf/meta_graph.proto \
     ${third_party}/${identifier}/tensorflow/core/protobuf/saver.proto \
@@ -36,7 +37,7 @@ ${tools}/../node_modules/protobufjs/bin/pbjs -t static-module -w closure -r tf -
     ${third_party}/${identifier}/tensorflow/core/framework/resource_handle.proto \
 
 echo "Generate '../src/tf-metadata.json'"
-protoc --proto_path "${third_party}/${identifier}" tensorflow/core/framework/attr_value.proto --python_out=${tools}
+protoc --proto_path ${third_party}/${identifier} tensorflow/core/framework/attr_value.proto --python_out=${tools}
 protoc --proto_path ${third_party}/${identifier} tensorflow/core/framework/tensor.proto --python_out=${tools}
 protoc --proto_path ${third_party}/${identifier} tensorflow/core/framework/types.proto --python_out=${tools}
 protoc --proto_path ${third_party}/${identifier} tensorflow/core/framework/tensor_shape.proto --python_out=${tools}

tools/tflite-update

@@ -2,9 +2,10 @@
 
 set -e
 
-tools=$(dirname ${0})
-src=${tools}/../src
-third_party=${tools}/../third_party
+root=$(cd $(dirname ${0})/..; pwd)
+src=${root}/src
+tools=${root}/tools
+third_party=${root}/third_party
 
 mkdir -p ${third_party}