Add host file stream (#497)

source/electron.js

@@ -292,7 +292,7 @@ host.ElectronHost = class {
                     reject(new Error("The file '" + file + "' size (" + stats.size.toString() + ") for encoding '" + encoding + "' is greater than 2 GB."));
                 }
                 else {
-                    reject(new Error("The file '" + file + "' size (" + stats.size.toString() + ") is greater than 2 GB."));
+                    resolve(new host.ElectronHost.FileStream(pathname, stats.size));
                 }
             });
         });
@@ -512,12 +512,9 @@ host.ElectronHost.BinaryStream = class {
         return this._length;
     }
 
-    create(buffer) {
-        return new host.ElectronHost.BinaryStream(buffer);
-    }
-
     stream(length) {
-        return this.create(this.read(length));
+        const buffer = this.read(length);
+        return new host.ElectronHost.BinaryStream(buffer.slice(0));
     }
 
     seek(position) {
@@ -556,6 +553,69 @@ host.ElectronHost.BinaryStream = class {
     }
 };
 
+host.ElectronHost.FileStream = class {
+
+    constructor(file, length) {
+        this._file = file;
+        this._length = length;
+        this._position = 0;
+    }
+
+    get position() {
+        return this._position;
+    }
+
+    get length() {
+        return this._length;
+    }
+
+    stream(length) {
+        const buffer = this.read(length);
+        return new host.ElectronHost.BinaryStream(buffer);
+    }
+
+    seek(position) {
+        this._position = position >= 0 ? position : this._length + position;
+    }
+
+    skip(offset) {
+        this._position += offset;
+    }
+
+    peek(length) {
+        length = length !== undefined ? length : this._length - this._position;
+        const position = this._position;
+        this.skip(length);
+        this.seek(position);
+        const descriptor = fs.openSync(this._file, 'r');
+        const buffer = new Uint8Array(length);
+        fs.readSync(descriptor, buffer, 0, length, position);
+        fs.closeSync(descriptor);
+        return buffer;
+    }
+
+    read(length) {
+        length = length !== undefined ? length : this._length - this._position;
+        const position = this._position;
+        this.skip(length);
+        const descriptor = fs.openSync(this._file, 'r');
+        const buffer = new Uint8Array(length);
+        fs.readSync(descriptor, buffer, 0, length, position);
+        fs.closeSync(descriptor);
+        return buffer;
+    }
+
+    byte() {
+        const position = this._position;
+        this.skip(1);
+        const descriptor = fs.openSync(this._file, 'r');
+        const buffer = new Uint8Array(1);
+        fs.readSync(descriptor, buffer, 0, 1, position);
+        fs.closeSync(descriptor);
+        return buffer[0];
+    }
+};
+
 host.ElectronHost.ElectonContext = class {
 
     constructor(host, folder, identifier, stream) {

source/gzip.js

@@ -62,10 +62,11 @@ gzip.Entry = class {
         if ((flags & 1) != 0) { // CRC16x
             stream.skip(2);
         }
-        this._stream = stream.stream(stream.length - stream.position - 8);
+        const compressedStream = stream.stream(stream.length - stream.position - 8);
         reader = new gzip.BinaryReader(stream.read(8));
         reader.uint32(); // CRC32
-        this._size = reader.uint32();
+        const length = reader.uint32();
+        this._stream = new gzip.InflaterStream(compressedStream, length);
     }
 
     get name() {
@@ -73,29 +74,93 @@ gzip.Entry = class {
     }
 
     get stream() {
-        if (this._size !== undefined) {
-            const compressedData = this._stream.read();
-            let buffer = null;
+        return this._stream;
+    }
+
+    get data() {
+        return this.stream.peek();
+    }
+};
+
+gzip.InflaterStream = class {
+
+    constructor(stream, length) {
+        this._stream = stream;
+        this._position = 0;
+        this._length = length;
+    }
+
+    get position() {
+        return this._position;
+    }
+
+    get length() {
+        return this._length;
+    }
+
+    inflate() {
+        if (this._buffer === undefined) {
+            const compressed = this._stream.peek();
             if (typeof process === 'object' && typeof process.versions == 'object' && typeof process.versions.node !== 'undefined') {
-                buffer = require('zlib').inflateRawSync(compressedData);
+                this._buffer = require('zlib').inflateRawSync(compressed);
             }
             else if (typeof pako !== 'undefined') {
-                buffer = pako.inflateRaw(compressedData);
+                this._buffer = pako.inflateRaw(compressed);
             }
             else {
-                buffer = new require('./zip').Inflater().inflateRaw(compressedData);
+                this._buffer = new require('./zip').Inflater().inflateRaw(compressed);
             }
-            if (this._size != buffer.length) {
+            if (this._buffer.length !== this._length) {
                 throw new gzip.Error('Invalid size.');
             }
-            this._stream = this._stream.create(buffer);
-            delete this._size;
+            delete this._stream;
         }
-        return this._stream;
     }
 
-    get data() {
-        return this.stream.peek();
+    seek(position) {
+        if (this._buffer === undefined) {
+            this.inflate();
+        }
+        this._position = position >= 0 ? position : this._length + position;
+    }
+
+    skip(offset) {
+        if (this._buffer === undefined) {
+            this.inflate();
+        }
+        this._position += offset;
+    }
+
+    stream(length) {
+        return new gzip.BinaryReader(this.read(length));
+    }
+
+    peek(length) {
+        const position = this._position;
+        length = length !== undefined ? length : this._length - this._position;
+        this.skip(length);
+        const end = this._position;
+        this.seek(position);
+        if (position === 0 && length === this._length) {
+            return this._buffer;
+        }
+        return this._buffer.subarray(position, end);
+    }
+
+    read(length) {
+        const position = this._position;
+        length = length !== undefined ? length : this._length - this._position;
+        this.skip(length);
+        if (position === 0 && length === this._length) {
+            return this._buffer;
+        }
+        return this._buffer.subarray(position, this._position);
+    }
+
+    byte() {
+        const position = this._position;
+        this.skip(1);
+        return this._buffer[position];
     }
 };
 

source/index.js

@@ -825,12 +825,9 @@ host.BrowserHost.BinaryStream = class {
         return this._length;
     }
 
-    create(buffer) {
-        return new host.BrowserHost.BinaryStream(buffer);
-    }
-
     stream(length) {
-        return this.create(this.read(length));
+        const buffer = this.read(length);
+        return new host.BrowserHost.BinaryStream(buffer.slice(0));
     }
 
     seek(position) {

source/pytorch.js

@@ -2290,7 +2290,7 @@ pytorch.Container = class {
         const stream = context.stream;
         const signature = [ 0x80, undefined, 0x8a, 0x0a, 0x6c, 0xfc, 0x9c, 0x46, 0xf9, 0x20, 0x6a, 0xa8, 0x50, 0x19 ];
         if (signature.length <= stream.length && stream.peek(signature.length).every((value, index) => signature[index] === undefined || signature[index] === value)) {
-            return new pytorch.Container.Pickle(stream.peek(), pickle, exception);
+            return new pytorch.Container.Pickle(stream, pickle, exception);
         }
         if (context.entries('tar').some((entry) => entry.name == 'pickle')) {
             return new pytorch.Container.Tar(context.entries('tar'), pickle, exception);
@@ -2431,8 +2431,8 @@ pytorch.Container.Tar = class {
 
 pytorch.Container.Pickle = class {
 
-    constructor(buffer, pickle, exception) {
-        this._buffer = buffer;
+    constructor(stream, pickle, exception) {
+        this._stream = stream;
         this._pickle = pickle;
         this._exceptionCallback = exception;
     }
@@ -2457,14 +2457,14 @@ pytorch.Container.Pickle = class {
     }
 
     _unpickle() {
-        if (!this._buffer) {
+        if (!this._stream) {
             return;
         }
 
         const execution = new pytorch.Execution(null, null, this._exceptionCallback);
-        const unpickler = new this._pickle.Unpickler(this._buffer);
+        const unpickler = new this._pickle.Unpickler(this._stream.peek());
 
-        this._buffer = null;
+        this._stream = null;
         this._pickle = null;
         this._exceptionCallback = null;
 
@@ -3769,7 +3769,7 @@ pytorch.nnapi.SerializedModel = class {
                             operand.value = reader.int32();
                             break;
                         case 'int32*':
-                            operand.value = reader.bytes(value.source_length);
+                            operand.value = reader.read(value.source_length);
                             break;
                         default:
                             throw new pytorch.Error("Unsupported NNAPI operand type '" + operand.type.toString() + "'.");
@@ -3833,7 +3833,7 @@ pytorch.nnapi.SerializedModel.BinaryReader = class {
         }
     }
 
-    bytes(length) {
+    read(length) {
         const position = this._position;
         this.skip(length);
         return this._buffer.subarray(position, this._position);

source/zip.js

@@ -146,7 +146,7 @@ zip.Entry = class {
             }
             case 8: {
                 // Deflate
-                this._size = size;
+                this._stream = new zip.InflaterStream(this._stream, size);
                 break;
             }
             default:
@@ -160,15 +160,6 @@ zip.Entry = class {
     }
 
     get stream() {
-        if (this._size !== undefined) {
-            const compressedData = this._stream.peek();
-            const buffer = new zip.Inflater().inflateRaw(compressedData);
-            if (this._size != buffer.length) {
-                throw new zip.Error('Invalid uncompressed size.');
-            }
-            this._stream = this._stream.create(buffer);
-            delete this._size;
-        }
         return this._stream;
     }
 
@@ -502,7 +493,76 @@ zip.BitReader = class {
         this.data -= length;
         return tree.symbol[sum + current];
     }
+};
+
+zip.InflaterStream = class {
+
+    constructor(stream, length) {
+        this._stream = stream;
+        this._position = 0;
+        this._length = length;
+    }
+
+    get position() {
+        return this._position;
+    }
+
+    get length() {
+        return this._length;
+    }
+
+    inflate() {
+        if (this._buffer === undefined) {
+            const compressed = this._stream.peek();
+            this._buffer = new zip.Inflater().inflateRaw(compressed);
+            if (this._length != this._buffer.length) {
+                throw new zip.Error('Invalid uncompressed size.');
+            }
+            delete this._stream;
+        }
+    }
+
+    seek(position) {
+        if (this._buffer === undefined) {
+            this.inflate();
+        }
+        this._position = position >= 0 ? position : this._length + position;
+    }
 
+    skip(offset) {
+        if (this._buffer === undefined) {
+            this.inflate();
+        }
+        this._position += offset;
+    }
+
+    peek(length) {
+        const position = this._position;
+        length = length !== undefined ? length : this._length - position;
+        this.skip(length);
+        const end = this._position;
+        this.seek(position);
+        if (position === 0 && length === this._length) {
+            return this._buffer;
+        }
+        return this._buffer.subarray(position, end);
+    }
+
+    read(length) {
+        const position = this._position;
+        length = length !== undefined ? length : this._length - position;
+        this.skip(length);
+        if (position === 0 && length === this._length) {
+            return this._buffer;
+        }
+        return this._buffer.subarray(position, this._position);
+    }
+
+    byte() {
+        const position = this._position;
+        this.skip(1);
+        return this._buffer[position];
+    }
 };
 
 zip.BinaryReader = class {

test/models.js

@@ -148,12 +148,9 @@ class TestBinaryStream {
         return this._length;
     }
 
-    create(buffer) {
-        return new TestBinaryStream(buffer);
-    }
-
     stream(length) {
-        return this.create(this.read(length));
+        const buffer = this.read(length);
+        return new TestBinaryStream(buffer.slice(0));
     }
 
     seek(position) {