Add DLC test files (#640)

Makefile

@@ -24,6 +24,7 @@ update: install
 	@./tools/circle sync schema metadata
 	@./tools/cntk sync schema
 	@./tools/coreml sync schema
+	@./tools/dlc schema
 	@./tools/dnn schema
 	@./tools/mnn sync schema
 	@./tools/mslite sync schema metadata

source/dlc-schema.js

@@ -0,0 +1,29 @@
+var $root = flatbuffers.get('dlc');
+
+$root.dlc = $root.dlc || {};
+
+$root.dlc.NetDefinition = class NetDefinition {
+
+    static decode(/* reader, position */) {
+        const $ = new $root.dlc.NetDefinition();
+        return $;
+    }
+};
+
+$root.dlc.NetParameter = class NetParameter {
+
+    static decode(reader, position) {
+        const $ = new $root.dlc.NetParameter();
+        $.params = reader.tableArray(position, 4, $root.dlc.Parameter.decode);
+        return $;
+    }
+};
+
+$root.dlc.Parameter = class Parameter {
+
+    static decode(reader, position) {
+        const $ = new $root.dlc.Parameter();
+        $.name = reader.string_(position, 4, null);
+        return $;
+    }
+};

source/dlc.js

@@ -1,20 +1,155 @@
 
 var dlc = dlc || {};
+var text = text || require('./text');
 
 dlc.ModelFactory = class {
 
     match(context) {
+        return dlc.Container.open(context);
+    }
+
+    open(context, match) {
+        return context.require('./dlc-schema').then(() => {
+            dlc.schema = flatbuffers.get('dlc').dlc;
+            const container = match;
+            return new dlc.Model(container);
+        });
+    }
+};
+
+dlc.Model = class {
+
+    constructor(container) {
+        if (container.metadata.size > 0) {
+            const converter = container.metadata.get('converter-command');
+            if (converter) {
+                const source = converter.split(' ').shift().trim();
+                if (source.length > 0) {
+                    this._source = source;
+                    const version = container.metadata.get('converter-version');
+                    if (version) {
+                        this._source = this._source + ' v' + version;
+                    }
+                }
+            }
+        }
+        container.model;
+        container.params;
+        this._graphs = [];
+    }
+
+    get format() {
+        return 'DLC';
+    }
+
+    get source() {
+        return this._source;
+    }
+
+    get graphs() {
+        return this._graphs;
+    }
+};
+
+dlc.Container = class {
+
+    static open(context) {
         const entries = context.entries('zip');
-        if (entries.has('model')) {
-            return 'dlc';
+        if (entries.size > 0) {
+            const model = entries.get('model');
+            const params = entries.get('model.params');
+            if (model || params) {
+                return new dlc.Container(model, params, entries.get('dlc.metadata'));
+            }
+        }
+        const stream = context.stream;
+        switch (dlc.Container._idenfitier(stream)) {
+            case 'NETD':
+                return new dlc.Container(stream, null, null);
+            case 'NETP':
+                return new dlc.Container(null, stream, null);
+            default:
+                break;
         }
-        return undefined;
+        return null;
     }
 
-    open(/* context */) {
-        return Promise.resolve().then(() => {
-            throw new dlc.Error("File contains undocumented DLC data.");
-        });
+    constructor(model, params, metadata) {
+        this._model = model || null;
+        this._params = params || null;
+        this._metadata = metadata || new Uint8Array(0);
+    }
+
+    get model() {
+        if (this._model && this._model.peek) {
+            const stream = this._model;
+            const reader = this._open(stream, 'NETD');
+            stream.seek(0);
+            this._model = dlc.schema.NetDefinition.decode(reader, reader.root);
+            throw new dlc.Error("File contains undocumented '" + reader.identifier + "' data.");
+        }
+        return this._model;
+    }
+
+    get params() {
+        if (this._params && this._params.peek) {
+            const stream = this._params;
+            const reader = this._open(stream, 'NETP');
+            stream.seek(0);
+            this._params = dlc.schema.NetParameter.decode(reader, reader.root);
+            throw new dlc.Error("File contains undocumented '" + reader.identifier + "' data.");
+        }
+        return this._params;
+    }
+
+    get metadata() {
+        if (this._metadata && this._metadata.peek) {
+            const reader = text.Reader.open(this._metadata);
+            const metadata = new Map();
+            for (;;) {
+                const line = reader.read();
+                if (line === undefined) {
+                    break;
+                }
+                const index = line.indexOf('=');
+                if (index === -1) {
+                    break;
+                }
+                const key = line.substring(0, index);
+                const value = line.substring(index + 1);
+                metadata.set(key, value);
+            }
+            this._metadata = metadata;
+        }
+        return this._metadata;
+    }
+
+    _open(stream, identifier) {
+        const signature = [ 0xD5, 0x0A, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 ];
+        if (signature.length <= stream.length && stream.peek(signature.length).every((value, index) => value === signature[index])) {
+            stream.read(8);
+        }
+        const buffer = stream.read();
+        const reader = flatbuffers.BinaryReader.open(buffer);
+        if (identifier != reader.identifier) {
+            throw new dlc.Error("File contains undocumented '" + reader.identifier + "' data.");
+        }
+        return reader;
+    }
+
+    static _idenfitier(stream) {
+        const signature = [ 0xD5, 0x0A, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 ];
+        if (stream.length > 16 && stream.peek(signature.length).every((value, index) => value === signature[index])) {
+            const buffer = stream.peek(16).slice(8, 16);
+            const reader = flatbuffers.BinaryReader.open(buffer);
+            return reader.identifier;
+        }
+        else if (stream.length > 8) {
+            const buffer = stream.peek(8);
+            const reader = flatbuffers.BinaryReader.open(buffer);
+            return reader.identifier;
+        }
+        return '';
     }
 };
 

source/view.js

@@ -1583,7 +1583,7 @@ view.ModelFactoryService = class {
         this.register('./darknet', [ '.cfg', '.model', '.txt', '.weights' ]);
         this.register('./weka', [ '.model' ]);
         this.register('./rknn', [ '.rknn', '.onnx' ]);
-        this.register('./dlc', [ '.dlc' ]);
+        this.register('./dlc', [ '.dlc', 'model', '.params' ]);
         this.register('./armnn', [ '.armnn', '.json' ]);
         this.register('./mnn', ['.mnn']);
         this.register('./ncnn', [ '.param', '.bin', '.cfg.ncnn', '.weights.ncnn', '.ncnnmodel' ]);

test/models.json

@@ -1931,11 +1931,39 @@
     "format":   "Deeplearning4j",
     "link":     "https://github.com/eclipse/deeplearning4j"
   },
+  {
+    "type":     "dlc",
+    "target":   "dmonitoring_model_q/model,dmonitoring_model_q/model.params",
+    "source":   "https://github.com/lutzroeder/netron/files/8767267/dmonitoring_model_q.zip[model,model.params]",
+    "error":    "File contains undocumented 'NETD' data in 'model'.",
+    "link":     "https://github.com/lutzroeder/netron/issues/640"
+  },
+  {
+    "type":     "dlc",
+    "target":   "dmonitoring_model_q.dlc",
+    "source":   "https://github.com/lutzroeder/netron/files/8767255/dmonitoring_model_q.dlc.zip[dmonitoring_model_q.dlc]",
+    "error":    "File contains undocumented 'NETD' data in 'dmonitoring_model_q.dlc'.",
+    "link":     "https://github.com/lutzroeder/netron/issues/640"
+  },
   {
     "type":     "dlc",
     "target":   "inception_v3_quantized.dlc",
     "source":   "https://github.com/lutzroeder/netron/files/5615377/inception_v3_quantized.dlc.zip[inception_v3_quantized.dlc]",
-    "error":    "File contains undocumented DLC data in 'inception_v3_quantized.dlc'.",
+    "error":    "File contains undocumented 'NETD' data in 'inception_v3_quantized.dlc'.",
+    "link":     "https://github.com/lutzroeder/netron/issues/640"
+  },
+  {
+    "type":     "dlc",
+    "target":   "model_front_mibokeh_video.dlc",
+    "source":   "https://github.com/lutzroeder/netron/files/8767521/model_front_mibokeh_video.dlc.zip[model_front_mibokeh_video.dlc]",
+    "error":    "File contains undocumented 'NETD' data in 'model_front_mibokeh_video.dlc'.",
+    "link":     "https://github.com/lutzroeder/netron/issues/640"
+  },
+  {
+    "type":     "dlc",
+    "target":   "mobile_mosaic_hta/model.params",
+    "source":   "https://github.com/lutzroeder/netron/files/8767531/mobile_mosaic_hta.zip[model.params]",
+    "error":    "File contains undocumented 'NETP' data in 'model.params'.",
     "link":     "https://github.com/lutzroeder/netron/issues/640"
   },
   {

tools/dlc

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+pushd $(cd $(dirname ${0})/..; pwd) > /dev/null
+
+schema() {
+    echo "dlc schema"
+    [[ $(grep -U $'\x0D' ./source/dlc-schema.js) ]] && crlf=1
+    node ./tools/flatc.js --root dlc --out ./source/dlc-schema.js ./tools/dlc.fbs
+    if [[ -n ${crlf} ]]; then
+        unix2dos --quiet --newfile ./source/dlc-schema.js ./source/dlc-schema.js
+    fi
+}
+
+while [ "$#" != 0 ]; do
+    command="$1" && shift
+    case "${command}" in
+        "schema") schema;;
+    esac
+done

tools/dlc.fbs

@@ -0,0 +1,12 @@
+namespace dlc;
+
+table NetDefinition {
+}
+
+table NetParameter {
+    params:[Parameter];
+}
+
+table Parameter {
+    name:string;
+}