Flux prototype (#334)

setup.py

@@ -94,6 +94,7 @@ setuptools.setup(
             'coreml.js', 'coreml-metadata.json', 'coreml-proto.js',
             'darknet.js', 'darknet-metadata.json',
             'dl4j.js', 'dl4j-metadata.json',
+            'flux.js', 'flux-metadata.json', 'bson.js',
             'keras.js', 'keras-metadata.json', 'hdf5.js',
             'mlnet.js', 'mlnet-metadata.json',
             'mxnet.js', 'mxnet-metadata.json',

src/bson.js

@@ -0,0 +1,161 @@
+/* jshint esversion: 6 */
+/* eslint "indent": [ "error", 4, { "SwitchCase": 1 } ] */
+
+// Experimental BSON JavaScript reader
+
+var bson = {};
+var long = long || { Long: require('long') };
+
+// http://bsonspec.org/spec.html
+bson.Reader = class {
+
+    constructor(buffer) {
+        this._asciiDecoder = new TextDecoder('ascii');
+        this._utf8Decoder = new TextDecoder('utf-8');
+        this._buffer = buffer;
+        this._position = 0;
+        this._view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+    }
+
+    read() {
+        return this.document();
+    }
+
+    document(isArray) {
+        let start = this._position;
+        let size = this.int32();
+        if (size < 5 || start + size > this._buffer.length || this._buffer[start + size - 1] != 0x00) {
+            throw new bson.Reader('Invalid BSON size.');
+        }
+        let element = isArray ? [] : {};
+        let index = 0;
+        for (;;) {
+            var type = this.byte();
+            if (type == 0x00) {
+                break;
+            }
+            let key = this.cstring();
+            let value = null;
+            switch (type) {
+                case 0x01:
+                    value = this.double();
+                    break
+                case 0x02:
+                    value = this.string();
+                    break;
+                case 0x03:
+                    value = this.document(false);
+                    break;
+                case 0x04:
+                    value = this.document(true);
+                    break;
+                case 0x05:
+                    value = this.binary();
+                    break;
+                case 0x08:
+                    value = this.boolean();
+                    break;
+                case 0x0A:
+                    value = null;
+                    break;
+                case 0x10:
+                    value = this.int32();
+                    break;
+                case 0x11:
+                    value = this.uint64();
+                    break;
+                case 0x12:
+                    value = this.int64();
+                    break;
+                default:
+                    throw new bson.Error("Unknown value type '" + type + "'.");    
+            }
+            if (isArray)  {
+                if (index !== parseInt(key, 10)) {
+                    throw new bson.Error("Invalid array index '" + key + "'.");    
+                }
+                element.push(value);
+                index++;
+            }
+            else {
+                element[key] = value;
+            }
+        }
+        return element;
+    }
+
+    cstring() {
+        var end = this._buffer.indexOf(0x00, this._position);
+        let value = this._asciiDecoder.decode(this._buffer.subarray(this._position, end));
+        this._position = end + 1;
+        return value;
+    }
+
+    string() {
+        let end = this.int32() + this._position - 1;
+        let value = this._utf8Decoder.decode(this._buffer.subarray(this._position, end));
+        this._position = end;
+        if (this.byte() != '0x00') {
+            throw new bson.Error('String missing terminal 0.');
+        }
+        return value;
+    }
+
+    binary() {
+        let size = this.int32();
+        let subtype = this.byte();
+        let data = this._buffer.subarray(this._position, this._position + size);
+        this._position += size;
+        switch (subtype) {
+            case 0x00:
+                return data;
+            default:
+                throw new bson.Error("Unknown binary subtype '" + subtype + "'.");
+        }
+    }
+    
+    boolean()  {
+        let value = this.byte();
+        switch (value) {
+            case 0x00: return false;
+            case 0x01: return true;
+            default: throw new bson.Error("Invalid boolean value '" + value + "'.");
+        }
+    }
+
+    byte() {
+        return this._buffer[this._position++];
+    }
+
+    int32() {
+        let value = this._view.getInt32(this._position, true);
+        this._position += 4;
+        return value;
+    }
+
+    int64() {
+        let low = this._view.getUint32(this._position, true);
+        let hi = this._view.getUint32(this._position + 4, true);
+        this._position += 8;
+        return new long.Long(low, hi, false).toNumber();
+    }
+
+    uint64() {
+        let low = this._view.getUint32(this._position, true);
+        let hi = this._view.getUint32(this._position + 4, true);
+        this._position += 8;
+        return new long.Long(low, hi, true).toNumber();
+    }
+}
+
+bson.Error = class extends Error {
+
+    constructor(message) {
+        super(message);
+        this.name = 'BSON Error';
+    }
+}
+
+if (typeof module !== 'undefined' && typeof module.exports === 'object') {
+    module.exports.Reader = bson.Reader; 
+}

src/flux-metadata.json

@@ -0,0 +1,2 @@
+[
+]

src/flux.js

@@ -0,0 +1,153 @@
+/* jshint esversion: 6 */
+/* eslint "indent": [ "error", 4, { "SwitchCase": 1 } ] */
+
+// Experimental
+
+var flux = flux || {};
+var marked = marked || require('marked');
+
+flux.ModelFactory = class {
+
+    match(context) {
+        let identifier = context.identifier; 
+        let extension = identifier.split('.').pop().toLowerCase();
+        if (extension === 'bson') {
+            return true;
+        }
+        return false;
+    }
+
+    open(context, host) {
+        return host.require('./bson').then((bson) => {
+            let model = null;
+            let identifier = context.identifier;
+            try {
+                let reader = new bson.Reader(context.buffer);
+                let root = reader.read();
+                root = flux.ModelFactory._backref(root, root);
+                model = root.model;
+                if (!model) {
+                    throw new flux.Error('File does not contain Flux model.');
+                }
+            }
+            catch (error) {
+                let message = error && error.message ? error.message : error.toString();
+                message = message.endsWith('.') ? message.substring(0, message.length - 1) : message;
+                throw new flux.Error(message + " in '" + identifier + "'.");
+            }
+
+            return flux.Metadata.open(host).then((metadata) => {
+                let identifier = context.identifier;
+                try {
+                    return new flux.Model(metadata, model);
+                }
+                catch (error) {
+                    let message = error && error.message ? error.message : error.toString();
+                    message = message.endsWith('.') ? message.substring(0, message.length - 1) : message;
+                    throw new flux.Error(message + " in '" + identifier + "'.");
+                }
+            });
+        });
+    }
+
+    static _backref(obj, root) {
+        if (Array.isArray(obj)) {
+            for (let i = 0; i < obj.length; i++) {
+                obj[i] = flux.ModelFactory._backref(obj[i], root);
+            }
+        }
+        else if (obj === Object(obj)) {
+            if (obj.tag == 'backref' && obj.ref) {
+                if (!root._backrefs[obj.ref - 1]) {
+                    throw new flux.Error("Invalid backref '" + obj.ref + "'.");
+                }
+                obj = root._backrefs[obj.ref - 1];
+            }
+            for (let key of Object.keys(obj)) {
+                if (obj !== root || key !== '_backrefs') {
+                    obj[key] = flux.ModelFactory._backref(obj[key], root);
+                }
+            }
+        }
+        return obj;
+    }
+}
+
+flux.Model = class {
+
+    constructor(/* root */) {
+        // debugger;
+        this._format = 'Flux';
+        this._graphs = [];
+    }
+
+    get format() {
+        return this._format;
+    }
+
+    get graphs() {
+        return this._graphs;
+    }
+}
+
+flux.Metadata = class {
+
+    static open(host) {
+        if (flux.Metadata._metadata) {
+            return Promise.resolve(flux.Metadata._metadata);
+        }
+        return host.request(null, 'flux-metadata.json', 'utf-8').then((data) => {
+            flux.Metadata._metadata = new flux.Metadata(data);
+            return flux.Metadata._metadata;
+        }).catch(() => {
+            flux.Metadata._metadata = new flux.Metadata(null);
+            return flux.Metadata._metadatas;
+        });
+    }
+
+    constructor(data) {
+        this._map = {};
+        this._attributeCache = {};
+        if (data) {
+            let items = JSON.parse(data);
+            if (items) {
+                for (let item of items) {
+                    if (item.name && item.schema) {
+                        this._map[item.name] = item.schema;
+                    }
+                }
+            }
+        }
+    }
+
+    getSchema(operator) {
+        return this._map[operator] || null;
+    }
+
+    getAttributeSchema(operator, name) {
+        let map = this._attributeCache[operator];
+        if (!map) {
+            map = {};
+            let schema = this.getSchema(operator);
+            if (schema && schema.attributes && schema.attributes.length > 0) {
+                for (let attribute of schema.attributes) {
+                    map[attribute.name] = attribute;
+                }
+            }
+            this._attributeCache[operator] = map;
+        }
+        return map[name] || null;
+    }
+};
+
+flux.Error = class extends Error {
+
+    constructor(message) {
+        super(message);
+        this.name = 'Flux Error';
+    }
+};
+
+if (module && module.exports) {
+    module.exports.ModelFactory = flux.ModelFactory;
+}

src/view.js

@@ -1146,6 +1146,7 @@ view.ModelFactoryService = class {
         this.register('./darknet', [ '.cfg' ]);
         this.register('./paddle', [ '.paddle', '__model__' ]);
         this.register('./ncnn', [ '.param', '.bin', '.cfg.ncnn', '.weights.ncnn' ]);
+        this.register('./flux', [ '.bson' ]);
         this.register('./dl4j', [ '.zip' ]);
         this.register('./mlnet', [ '.zip' ]);
     }

test/models.json

@@ -1919,6 +1919,13 @@
     "format": "Deeplearning4j",
     "link":   "https://github.com/eclipse/deeplearning4j"
   },
+  {
+    "type":   "flux",
+    "target": "helloworld.bson",
+    "source": "https://github.com/lutzroeder/netron/files/3638212/helloworld.zip[helloworld.bson]",
+    "format": "Flux",
+    "link":   "https://github.com/lutzroeder/netron/issues/334"
+  },
   {
     "type":   "keras",
     "target": "babi_rnn.h5",