Update OpenVINO support (#191)

src/openvino-metadata.json

@@ -561,7 +561,9 @@
   {
     "name": "Split",
     "schema": {
-      "attributes": [],
+      "attributes": [
+        { "name": "axis", "type": "int32" }
+      ],
       "category": "Tensor",
       "description": "**Short description**: *Split* layer splits the input into several output groups. Group sizes are denoted by the number and the size of output ports.\n**Detailed description**: [Reference](http://caffe.berkeleyvision.org/tutorial/layers/split.html)\n**Parameters**: *None*\n**Mathematical Formulation**\nSplits input blob among children. For example, blob is *BxC+CxHxW* and there are two children. Then, output blob is *BxCxHxW*.\n**Example**\n\n```html\n<layer ... type=\"Split\" ... >\n    <input> ... </input>\n    <output> ... </output>\n</layer>\n```",
       "inputs": null,
@@ -610,7 +612,7 @@
           "description": " *operation* is the simple mathematical operation to be performed over inputs. For example, *operation* equal *mul* means that input blobs are multiplied.",
           "name": "operation",
           "option": "required",
-          "type": ""
+          "type": "string"
         }
       ],
       "description": "**Short description**: *Eltwise* layer performs element-wise operation, which is specified in parameters, over given inputs.\n**Parameters**: *Eltwise* layer parameters should be specified in the `elementwise_data` node, which is placed as a child of the layer node.\n**Mathematical Formulation** *Eltwise* accepts 2 inputs of any number of dimensions - from 1 to 4, however, it is required for both of them to have absolutely same dimensions. The produced blob is also of the same dimension as each of its parents\n*Eltwise* does the following with the input blobs:\n\\f[\no_{i} = f(b_{i}^{1}, b_{i}^{2})\n\\f]\nwhere \\f$b_{i}^{1}\\f$ - first blob \\f$i\\f$-th element, \\f$b_{i}^{2}\\f$ - second blob \\f$i\\f$-th element, \\f$o_{i}\\f$ - output blob \\f$i\\f$-th element, \\f$f(a, b)\\f$ - is a function that performs an operation over its two arguments \\f$a, b\\f$.\n*   For *sum* operation, \\f$f(a, b)\\f$ is defined as\n    \\f[\n    f(a,b) = a + b\n    \\f]\n*   For *mul* operation, \\f$f(a, b)\\f$ is defined as\n    \\f[\n    f(a,b) = a * b\n    \\f]\n*   For *max* operation, \\f$f(a, b)\\f$ is defined as\n    \\f[\n    f(a,b) = \\left\\{\\begin{array}{ll}\n\t\ta \\quad \\mbox{if } a \\geq b \\\\\n\t\tb \\quad \\mbox{if } b > a\n\t\\end{array}\\right. \\f]\n**Example**\n\n```html\n<layer ... type=\"Eltwise\" ... >\n    <elementwise_data operation=\"sum\"/>\n    <input> ... </input>\n    <output> ... </output>\n</layer>\n```",
@@ -1487,5 +1489,47 @@
         { "name": "pad_mode" }
       ]
     }
+  },
+  {
+    "name": "GRUCell",
+    "schema": {
+      "category": "Layer"
+    }
+  },
+  {
+    "name": "LSTMCell",
+    "schema": {
+      "category": "Layer"
+    }
+  },
+  {
+    "name": "MaxPool",
+    "schema": {
+      "category": "Pool"
+    }
+  },
+  {
+    "name": "Transpose",
+    "schema": {
+      "category": "Transform"
+    }
+  },
+  {
+    "name": "Squeeze",
+    "schema": {
+      "category": "Transform"
+    }
+  },
+  {
+    "name": "Unsqueeze",
+    "schema": {
+      "category": "Transform"
+    }
+  },
+  {
+    "name": "Gather",
+    "schema": {
+      "category": "Transform"
+    }
   }
 ]

src/openvino.js

@@ -73,7 +73,11 @@ openvino.ModelFactory = class {
                     throw new openvino.Error("File format is not OpenVINO IR.");
                 }
                 const net = openvino.XmlReader.read(xmlDoc.documentElement);
-                return new openvino.Model(metadata, net, bin);
+                const model = new openvino.Model(metadata, net, bin);
+                if (net.disconnectedLayers) {
+                    host.exception(new openvino.Error('Graph contains not connected layers ' + JSON.stringify(net.disconnectedLayers) + '.'));
+                }
+                return model;
             }
             catch (error) {
                 host.exception(error, false);
@@ -114,7 +118,7 @@ openvino.Graph = class {
         this._outputs = [];
         this._arguments = {};
 
-        for (const layer of net.layers) {
+        for (const layer of this._const(net.layers, net.edges)) {
             const inputs = layer.inputs.map((input) => this._argument(layer.id, layer.precision, input, net.edges));
             const outputs = layer.outputs.map((output) => this._argument(layer.id, output.precision || layer.precision, output, null));
             switch (layer.type) {
@@ -169,8 +173,7 @@ openvino.Graph = class {
             }
         }
         if (nodesWithNonExistentInputs.size !== 0){
-            const layerNames = Array.from(nodesWithNonExistentInputs).map((node) => node.name);
-            throw new openvino.Error('Graph contains not connected layers ' + JSON.stringify(layerNames) + '.');
+            net.disconnectedLayers = Array.from(nodesWithNonExistentInputs).map((node) => node.name);
         }
     }
 
@@ -204,16 +207,16 @@ openvino.Graph = class {
     }
 
     _replaceTensorIteratorWithSubgraph(metadata, bin, layers, edges) {
-        const tiNodes = layers.filter((node) => node.type === 'TensorIterator');
-        for (const singleTensorIteratorNode of tiNodes) {
-            const singleTensorIteratorNodeId = singleTensorIteratorNode.id;
+        const tensorIteratorLayers = layers.filter((node) => node.type === 'TensorIterator');
+        for (const tensorIteratorLayer of tensorIteratorLayers) {
+            const singleTensorIteratorNodeId = tensorIteratorLayer.id;
             const tiNode = this._nodes.find((n) => n._id === singleTensorIteratorNodeId);
-            const iteratorLayers = singleTensorIteratorNode.body.layers;
-            const iteratorEdgeMap = singleTensorIteratorNode.body.edges;
-            const iteratorBackEdgesMap = singleTensorIteratorNode.back_edges;
+            const iteratorLayers = tensorIteratorLayer.body.layers;
+            const iteratorEdgeMap = tensorIteratorLayer.body.edges;
+            const iteratorBackEdgesMap = tensorIteratorLayer.back_edges;
             const iteratorAllEdges = Object.assign({}, iteratorEdgeMap, iteratorBackEdgesMap);
-            const mappingForNestedIR = singleTensorIteratorNode.port_map;
-            for (const nestedLayer of iteratorLayers) {
+            const mappingForNestedIR = tensorIteratorLayer.port_map;
+            for (const nestedLayer of this._const(iteratorLayers, iteratorAllEdges, iteratorBackEdgesMap)) {
                 const inputs = nestedLayer.inputs.map((input) => this._argument(nestedLayer.id, nestedLayer.precision, input, iteratorAllEdges));
                 const outputs = nestedLayer.outputs.map((output) => this._argument(nestedLayer.id, nestedLayer.precision || output.precision, output, null));
                 let nestedNode = new openvino.Node(this, metadata, bin, nestedLayer, inputs, outputs);
@@ -319,9 +322,100 @@ openvino.Graph = class {
                 }
             }
 
-            this._nodes = this._nodes.filter((node) => node.id !== singleTensorIteratorNode.id);
+            this._nodes = this._nodes.filter((node) => node.id !== tensorIteratorLayer.id);
         }
     }
+
+    _const(layers, edges, back_edges) {
+        let results = [];
+        back_edges = back_edges || {};
+        layers = layers.slice();
+        for (const layer of layers) {
+            if (layer.type === 'Const' && layer.inputs.length === 0 && layer.outputs.length === 1 &&
+                layer.blobs.length === 0 && layer.data && layer.data.length > 3) {
+                const data = {};
+                for (const attribute of layer.data) {
+                    data[attribute.name] = attribute.value;
+                }
+                if (data['element_type'] && data['offset'] && data['size']) {
+                    const element_type = data['element_type'];
+                    let precision = null;
+                    switch (element_type) {
+                        case 'f16': precision = 'FP16'; break;
+                        case 'f32': precision = 'FP32'; break;
+                        default: precision = element_type.toUpperCase();
+                    }
+                    const shape = data['shape'] ? data['shape'].split(',').map((dim) => parseInt(dim.trim(), 10)) : null;
+                    layer.data = [];
+                    layer.blobs.push({ name: 'custom', precision: precision, offset: data['offset'], size: data['size'], shape: shape });
+                }
+            }
+            if (layer.type === 'Const' && layer.blobs.length === 1 && !layer.blobs[0].shape &&
+                layer.inputs.length === 0 && layer.outputs.length === 1 && layer.outputs[0].dims) {
+                layer.blobs[0].shape = layer.outputs[0].dims;
+            }
+        }
+
+        let constMap = new Map();
+        for (let layer of layers) {
+            if (layer.type === 'Const' && layer.inputs.length === 0 && layer.outputs.length === 1) {
+                const from = layer.id + ':' + layer.outputs[0].id;
+                constMap.set(from, { layer: layer, counter: 0 });
+            }
+        }
+        for (let to of Object.keys(edges)) {
+            const from = edges[to];
+            if (constMap.has(from)) {
+                constMap.get(from).counter++;
+            }
+        }
+        if (back_edges) {
+            for (let to of Object.keys(back_edges)) {
+                const from = back_edges[to];
+                if (constMap.has(from)) {
+                    constMap.get(from).counter++;
+                }
+            }
+        }
+        for (const pair of constMap) {
+            if (pair[1].counter !== 1) {
+                constMap.delete(pair[0]);
+            }
+        }
+        for (const layer of layers) {
+            if (layer.blobs.length === 0) {
+                for (let i = layer.inputs.length - 1; i > 0; i--) {
+                    const input = layer.inputs[i];
+                    const to = layer.id + ':' + input.id;
+                    const from = edges[to] || back_edges[to];
+                    if (!constMap.has(from)) {
+                        break;
+                    }
+                    const constLayer = constMap.get(from).layer;
+                    const blob = constLayer.blobs[0];
+                    blob.id = constLayer.name || constLayer.id;
+                    blob.kind = 'Const';
+                    layer.blobs.push(blob);
+                    layer.inputs.splice(i, 1);
+                    constMap.get(from).layer = null;
+                    constMap.get(from).delete = true;
+                }
+            }
+        }
+
+        while (layers.length > 0) {
+            const layer = layers.shift();
+            if (layer.type === 'Const' && layer.inputs.length === 0 && layer.outputs.length === 1) {
+                const from = layer.id + ':' + layer.outputs[0].id;
+                if (constMap.has(from) && constMap.get(from).delete) {
+                    continue;
+                }
+            }
+            results.push(layer);
+        }
+
+        return results;
+    }
 };
 
 openvino.Node = class {
@@ -359,23 +453,25 @@ openvino.Node = class {
             const offset = blob.offset;
             const size = blob.size;
             const data = (bin && (offset + size) <= bin.length) ? bin.slice(offset, offset + size) : null;
-            let shape = null;
-            const blobPrecision = blob.precision || precision;
+            let dimensions = blob.shape || null;
+            const kind = blob.kind || 'Blob';
+            const id = blob.id || '';
+            const dataType = blob.precision || precision;
             const precisionMap = {
                 'FP16': 2, 'FP32': 4,
                 'I8': 1, 'I16': 2, 'I32': 4, 'I64': 8,
                 'U8': 1, 'U16': 2, 'U32': 4, 'U64': 8
             };
-            if (precisionMap[blobPrecision]) {
-                let itemSize = precisionMap[blobPrecision];
+            const itemSize = precisionMap[dataType];
+            if (itemSize) {
                 switch (this._type + ':' + name) {
                     case 'FullyConnected:weights': {
                         const outSize = parseInt(attributes['out-size'], 10);
-                        shape = [ size / (outSize * itemSize), outSize ];
+                        dimensions = [ size / (outSize * itemSize), outSize ];
                         break;
                     }
                     case 'FullyConnected:biases': {
-                        shape = [ parseInt(attributes['out-size'], 10) ];
+                        dimensions = [ parseInt(attributes['out-size'], 10) ];
                         break;
                     }
                     case 'Convolution:weights':
@@ -386,7 +482,7 @@ openvino.Node = class {
                             [ parseInt(attributes['kernel-x'], 10), parseInt(attributes['kernel-y'], 10) ] :
                             attributes['kernel'].split(',').map((v) => parseInt(v.trim(), 10));
                         const n = parseInt(attributes['output'], 10);
-                        shape = [ Math.floor(c / group), n ].concat(kernel);
+                        dimensions = [ Math.floor(c / group), n ].concat(kernel);
                         break;
                     }
                     case 'ScaleShift:weights': 
@@ -394,7 +490,7 @@ openvino.Node = class {
                     case 'Convolution:biases':
                     case 'Normalize:weights':
                     case 'PReLU:weights': {
-                        shape = [ Math.floor(size / itemSize) ];
+                        dimensions = [ Math.floor(size / itemSize) ];
                         break;
                     }
                     case 'Const:custom': {
@@ -403,17 +499,15 @@ openvino.Node = class {
                             this._outputs[0].arguments[0].type &&
                             this._outputs[0].arguments[0].type.shape &&
                             this._outputs[0].arguments[0].type.shape.dimensions) {
-                            shape = this._outputs[0].arguments[0].type.shape.dimensions;
+                            dimensions = this._outputs[0].arguments[0].type.shape.dimensions;
                         }
                         break;
                     }
                 }
             }
-            if (shape) {
-                shape = new openvino.TensorShape(shape);
-            }
+            const shape = dimensions ? new openvino.TensorShape(dimensions) : null;
             this._initializers.push(new openvino.Parameter(name, [
-                new openvino.Argument('', null, new openvino.Tensor(blobPrecision, shape, data))
+                new openvino.Argument(id, null, new openvino.Tensor(dataType, shape, data, kind))
             ]));
         }
     }
@@ -474,9 +568,9 @@ openvino.Parameter = class {
 openvino.Argument = class {
 
     constructor(id, type, initializer) {
-        if (typeof id !== 'string') {
-            // throw new openvino.Error("Invalid argument identifier '" + JSON.stringify(id) + "'.");
-        }
+        // if (typeof id !== 'string') {
+        //     throw new openvino.Error("Invalid argument identifier '" + JSON.stringify(id) + "'.");
+        // }
         this._id = id;
         this._type = type || null;
         this._initializer = initializer || null;
@@ -611,17 +705,18 @@ openvino.Attribute = class {
 
 openvino.Tensor = class {
 
-    constructor(precision, shape, data) {
+    constructor(precision, shape, data, kind) {
         this._data = data;
         this._type = new openvino.TensorType(precision, shape);
+        this._kind = kind;
     }
 
-    get type() {
-        return this._type;
+    get kind() {
+        return this._kind;
     }
 
-    get kind() {
-        return 'Blob';
+    get type() {
+        return this._type;
     }
 
     get state() {
@@ -644,7 +739,7 @@ openvino.Tensor = class {
         }
         context.limit = 10000;
         const value = this._decode(context, 0);
-        return JSON.stringify(value, null, 4);
+        return openvino.Tensor._stringify(value, '', '    ');
     }
 
     _context() {
@@ -751,6 +846,32 @@ openvino.Tensor = class {
         }
         return results;
     }
+
+    static _stringify(value, indentation, indent) {
+        if (Array.isArray(value)) {
+            let result = [];
+            result.push(indentation + '[');
+            const items = value.map((item) => openvino.Tensor._stringify(item, indentation + indent, indent));
+            if (items.length > 0) {
+                result.push(items.join(',\n'));
+            }
+            result.push(indentation + ']');
+            return result.join('\n');
+        }
+        if (typeof value == 'string') {
+            return indentation + value;
+        }
+        if (value == Infinity) {
+            return indentation + 'Infinity';
+        }
+        if (value == -Infinity) {
+            return indentation + '-Infinity';
+        }
+        if (isNaN(value)) {
+            return indentation + 'NaN';
+        }
+        return indentation + value.toString();
+    }
 };
 
 openvino.TensorType = class {

test/models.json

@@ -3827,39 +3827,55 @@
   },
   {
      "type":   "openvino",
-     "target": "2019_R7_netron_issue372.xml",
-     "source": "https://github.com/lutzroeder/netron/files/3846426/ir.xml.zip[ir.xml]",
+     "target": "netron_issue_372.xml",
+     "source": "https://github.com/lutzroeder/netron/files/4447843/netron_issue_372.zip[netron_issue_372.xml]",
      "format": "OpenVINO IR",
      "link":   "https://github.com/lutzroeder/netron/issues/372"
    },
    {
     "type":   "openvino",
-    "target": "2019_with_pad_simple.xml",
+    "target": "with_pad_simple.xml",
     "source": "https://github.com/lutzroeder/netron/files/2871673/with_pad_simple.xml.zip[with_pad_simple.xml]",
     "format": "OpenVINO IR",
     "link":   "https://download.01.org/openvinotoolkit"
   },
   {
     "type":   "openvino",
-    "target": "2019_with_nd_conv_simple.xml",
+    "target": "with_nd_conv_simple.xml",
     "source": "https://github.com/lutzroeder/netron/files/2871696/with_nd_conv_simple.xml.zip[with_nd_conv_simple.xml]",
     "format": "OpenVINO IR",
     "link":   "https://download.01.org/openvinotoolkit"
   },
   {
     "type":   "openvino",
-    "target": "2019_with_gather.xml",
+    "target": "with_gather.xml",
     "source": "https://github.com/lutzroeder/netron/files/2871676/with_gather.xml.zip[with_gather.xml]",
     "format": "OpenVINO IR",
     "link":   "https://download.01.org/openvinotoolkit"
   },
   {
     "type":   "openvino",
-    "target": "2019_ti_with_large_body.xml",
+    "target": "ti_with_large_body.xml",
     "source": "https://github.com/lutzroeder/netron/files/2871678/ti_with_large_body.xml.zip[ti_with_large_body.xml]",
     "format": "OpenVINO IR",
     "link":   "https://download.01.org/openvinotoolkit"
   },
+  {
+    "type":   "openvino",
+    "target": "text-recognition-0012.xml,text-recognition-0012.bin",
+    "source": "https://download.01.org/opencv/2020/openvinotoolkit/2020.2/open_model_zoo/models_bin/3/text-recognition-0012/FP16-INT8/text-recognition-0012.xml,https://download.01.org/opencv/2020/openvinotoolkit/2020.2/open_model_zoo/models_bin/3/text-recognition-0012/FP16-INT8/text-recognition-0012.bin",
+    "error":  "Graph contains not connected layers [\"shadow/LSTMLayers/encoder/stack_bidirectional_rnn/cell_0/concat\",\"shadow/LSTMLayers/encoder/stack_bidirectional_rnn/cell_1/concat\",\"shadow/LSTMLayers/decoder/stack_bidirectional_rnn/cell_0/concat\",\"shadow/LSTMLayers/decoder/stack_bidirectional_rnn/cell_1/concat/fq_input_0\",\"shadow/LSTMLayers/decoder/stack_bidirectional_rnn/cell_1/concat/fq_input_1\"].",
+    "format": "OpenVINO IR",
+    "link":   "https://download.01.org/opencv/2020/openvinotoolkit/2020.2/open_model_zoo/models_bin/3/text-recognition-0012/FP16-INT8"
+  },
+  {
+    "type":   "openvino",
+    "target": "text-spotting-0002-recognizer-decoder.xml,text-spotting-0002-recognizer-decoder.bin",
+    "source": "https://download.01.org/opencv/2020/openvinotoolkit/2020.2/open_model_zoo/models_bin/3/text-spotting-0002-recognizer-decoder/FP32/text-spotting-0002-recognizer-decoder.xml,https://download.01.org/opencv/2020/openvinotoolkit/2020.2/open_model_zoo/models_bin/3/text-spotting-0002-recognizer-decoder/FP32/text-spotting-0002-recognizer-decoder.bin",
+    "error":  "Graph contains not connected layers [\"hidden\",\"77/SqueezeNumDirections/0\"].",
+    "format": "OpenVINO IR",
+    "link":   "https://download.01.org/opencv/2020/openvinotoolkit/2020.2/open_model_zoo/models_bin/3/text-spotting-0002-recognizer-decoder/FP32"
+  },
   {
     "type":   "paddle",
     "target": "PyramidBox_WiderFace.tar.gz",