ncnn.js 41 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932
  1. import * as base from './base.js';
  2. const ncnn = {};
  3. // https://github.com/Tencent/ncnn/wiki/param-and-model-file-structure
  4. // https://github.com/Tencent/ncnn/wiki/operation-param-weight-table
  5. // https://github.com/Tencent/ncnn/wiki/operators
  6. ncnn.ModelFactory = class {
  7. match(context) {
  8. const identifier = context.identifier.toLowerCase();
  9. if (identifier.endsWith('.param.bin') || identifier.endsWith('.ncnnmodel')) {
  10. const stream = context.stream;
  11. if (stream.length >= 4) {
  12. const buffer = stream.peek(4);
  13. const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
  14. const signature = view.getUint32(0, true);
  15. if (signature === 0x007685dd) {
  16. context.type = 'ncnn.model.bin';
  17. }
  18. }
  19. } else if (identifier.endsWith('.param') || identifier.endsWith('.cfg.ncnn')) {
  20. const reader = context.read('text', 0x10000);
  21. const type = identifier.endsWith('.pnnx.param') ? 'pnnx.model' : 'ncnn.model';
  22. if (reader) {
  23. try {
  24. const signature = reader.read('\n');
  25. if (signature !== undefined) {
  26. if (signature.trim() === '7767517') {
  27. context.type = type;
  28. return;
  29. }
  30. const header = signature.trim().split(' ');
  31. if (header.length === 2 && header.every((value) => value >>> 0 === parseFloat(value))) {
  32. context.type = type;
  33. }
  34. }
  35. } catch {
  36. // continue regardless of error
  37. }
  38. }
  39. } else if (identifier.endsWith('.ncnn.bin')) {
  40. context.type = 'ncnn.weights';
  41. } else if (identifier.endsWith('.pnnx.bin')) {
  42. const entries = context.peek('zip');
  43. if (entries && entries.size > 0) {
  44. context.type = 'pnnx.weights';
  45. context.target = entries;
  46. }
  47. } else if (identifier.endsWith('.bin') || identifier.endsWith('.weights.ncnn')) {
  48. const stream = context.stream;
  49. const length = stream.length;
  50. if (length > 4) {
  51. const buffer = stream.peek(4);
  52. const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
  53. const signature = view.getUint32(0, true);
  54. if (signature === 0x00000000) {
  55. const size = Math.min(stream.length, 1024) & 0xFFFC;
  56. const buffer = stream.peek(size);
  57. const length = size >> 2;
  58. const array = new Float32Array(buffer.buffer, buffer.byteOffset, length);
  59. const values = Array.from(array).slice(1);
  60. if (values.every((value) => !Number.isNaN(value) && Number.isFinite(value) && value > -20.0 && value < 20.0)) {
  61. context.type = 'ncnn.weights';
  62. }
  63. } else {
  64. const buffer = stream.peek(Math.min(0x20000, length));
  65. for (let i = 0; i < buffer.length - 4; i++) {
  66. const signature = (buffer[i] | buffer[i + 1] << 8 | buffer[i + 2] << 16 | buffer [i + 3] << 24) >>> 0;
  67. if (signature === 0xdeadbeef) { // Core ML
  68. return;
  69. }
  70. if (signature === 0x01306b47 || signature === 0x000d4b38 || signature === 0x0002c056) {
  71. context.type = 'ncnn.weights';
  72. return;
  73. }
  74. }
  75. }
  76. }
  77. }
  78. }
  79. filter(context, type) {
  80. return (context.type !== 'ncnn.model' && context.type !== 'ncnn.model.bin') || type !== 'ncnn.weights';
  81. }
  82. async open(context) {
  83. const metadata = await context.metadata('ncnn-metadata.json');
  84. const identifier = context.identifier.toLowerCase();
  85. const format = context.type.split('.').shift();
  86. switch (context.type) {
  87. case 'pnnx.model':
  88. case 'ncnn.model': {
  89. let file = null;
  90. if (identifier.endsWith('.param')) {
  91. file = context.identifier.replace(/\.param$/, '.bin');
  92. } else if (identifier.endsWith('.cfg.ncnn')) {
  93. file = context.identifier.replace(/\.cfg\.ncnn$/, '.weights.ncnn');
  94. }
  95. let content = null;
  96. try {
  97. content = await context.fetch(file);
  98. } catch {
  99. // continue regardless of error
  100. }
  101. const param = context.read('text');
  102. const reader = new ncnn.TextParamReader(param);
  103. const blobs = new ncnn.BlobReader(content);
  104. return new ncnn.Model(metadata, format, reader, blobs);
  105. }
  106. case 'ncnn.model.bin': {
  107. const bin = `${context.identifier.substring(0, context.identifier.length - 10)}.bin`;
  108. let content = null;
  109. try {
  110. content = await context.fetch(bin);
  111. } catch {
  112. // continue regardless of error
  113. }
  114. const param = context.stream.peek();
  115. const reader = new ncnn.BinaryParamReader(param);
  116. const blobs = new ncnn.BlobReader(content);
  117. return new ncnn.Model(metadata, format, reader, blobs);
  118. }
  119. case 'pnnx.weights':
  120. case 'ncnn.weights': {
  121. let file = null;
  122. if (identifier.endsWith('.bin')) {
  123. file = context.identifier.replace(/\.bin$/, '.param');
  124. } else if (identifier.endsWith('.weights.ncnn')) {
  125. file = context.identifier.replace(/\.weights\.ncnn$/, '.cfg.ncnn');
  126. }
  127. let reader = null;
  128. try {
  129. const content = await context.fetch(file);
  130. const param = content.read('text');
  131. reader = new ncnn.TextParamReader(param);
  132. } catch {
  133. const content = await context.fetch(`${file}.bin`);
  134. const param = content.stream.peek();
  135. reader = new ncnn.BinaryParamReader(param);
  136. }
  137. const blobs = new ncnn.BlobReader(context);
  138. return new ncnn.Model(metadata, format, reader, blobs);
  139. }
  140. default: {
  141. throw new ncnn.Error(`Unsupported ncnn format '${context.type}'.`);
  142. }
  143. }
  144. }
  145. };
  146. ncnn.Model = class {
  147. constructor(metadata, format, param, blobs) {
  148. this.format = format === 'pnnx' ? 'PNNX' : 'ncnn';
  149. this.graphs = [new ncnn.Graph(metadata, format, param, blobs)];
  150. }
  151. };
  152. ncnn.Graph = class {
  153. constructor(metadata, format, param, blobs) {
  154. this.inputs = [];
  155. this.outputs = [];
  156. this.nodes = [];
  157. const layers = param.layers;
  158. const values = new Map();
  159. values.map = (name, type, tensor) => {
  160. if (name.length === 0 && tensor) {
  161. return new ncnn.Value(name, type, tensor);
  162. }
  163. if (!values.has(name)) {
  164. values.set(name, new ncnn.Value(name, type || null, tensor || null));
  165. } else if (tensor || (type && !type.equals(values.get(name).type))) {
  166. throw new ncnn.Error(`Duplicate value '${name}'.`);
  167. }
  168. return values.get(name);
  169. };
  170. for (const layer of layers) {
  171. const params = layer.params;
  172. if (params && params.size > 0) {
  173. for (const [key, list] of params) {
  174. if (key === '30' && Array.isArray(list)) {
  175. const value = list.map((item) => parseInt(item, 10));
  176. for (const output of layer.outputs || []) {
  177. if (value.length > 0 && value[0] <= value.length - 1) {
  178. const shape = new Array(value.shift());
  179. for (let i = 0; i < shape.length; i++) {
  180. shape[i] = value.shift();
  181. }
  182. const type = new ncnn.TensorType('float32', new ncnn.TensorShape(shape));
  183. values.map(output, type);
  184. }
  185. params.delete(key);
  186. }
  187. }
  188. }
  189. }
  190. }
  191. for (const layer of layers) {
  192. if (layer.type === 'Input' || layer.type === 16) {
  193. const dimensions = Array.from(layer.params.values()).map((value) => isNaN(parseInt(value, 10)) ? value : parseInt(value, 10));
  194. const shape = new ncnn.TensorShape(dimensions);
  195. const type = new ncnn.TensorType('float32', shape);
  196. const input = new ncnn.Argument(layer.name, layer.outputs.map((output) => values.map(output, type)));
  197. this.inputs.push(input);
  198. } else {
  199. const node = new ncnn.Node(metadata, format, blobs, layer, values);
  200. this.nodes.push(node);
  201. }
  202. }
  203. blobs.validate();
  204. }
  205. };
  206. ncnn.Argument = class {
  207. constructor(name, value, type, visible) {
  208. this.name = name;
  209. this.value = value;
  210. this.type = type;
  211. this.visible = visible !== false;
  212. }
  213. };
  214. ncnn.Value = class {
  215. constructor(name, type, initializer) {
  216. if (typeof name !== 'string') {
  217. throw new ncnn.Error(`Invalid value identifier '${JSON.stringify(name)}'.`);
  218. }
  219. this.name = name;
  220. this.type = initializer ? initializer.type : type;
  221. this.initializer = initializer || null;
  222. this.quantization = initializer ? initializer.quantization : null;
  223. }
  224. };
  225. ncnn.Node = class {
  226. constructor(metadata, format, blobs, layer, values) {
  227. this.inputs = [];
  228. this.outputs = [];
  229. this.attributes = [];
  230. this.chain = [];
  231. this.name = layer.name || '';
  232. this.type = { ...metadata.type(layer.type) };
  233. delete this.type.identifier;
  234. const params = layer.params;
  235. const inputs = layer.inputs || [];
  236. let inputIndex = 0;
  237. if (this.type && Array.isArray(this.type.inputs)) {
  238. for (const input of this.type.inputs) {
  239. if (inputIndex < inputs.length || input.optional === false) {
  240. const count = (input.type === 'Tensor[]') ? (inputs.length - inputIndex) : 1;
  241. const list = inputs.slice(inputIndex, inputIndex + count).filter((id) => id !== '' || input.option !== 'optional').map((id) => values.map(id));
  242. const argument = new ncnn.Argument(input.name, list);
  243. this.inputs.push(argument);
  244. inputIndex += count;
  245. }
  246. }
  247. }
  248. this.inputs.push(...inputs.slice(inputIndex).map((input, index) => {
  249. const name = ((inputIndex + index) === 0) ? 'input' : (inputIndex + index).toString();
  250. return new ncnn.Argument(name, [values.map(input)]);
  251. }));
  252. const outputs = layer.outputs || [];
  253. let outputIndex = 0;
  254. if (this.type && Array.isArray(this.type.outputs)) {
  255. for (const output of this.type.outputs) {
  256. if (outputIndex < outputs.length || output.option !== 'optional') {
  257. const count = (output.type === 'Tensor[]') ? (outputs.length - outputIndex) : 1;
  258. const list = outputs.slice(outputIndex, outputIndex + count).map((id) => values.map(id));
  259. const argument = new ncnn.Argument(output.name, list);
  260. this.outputs.push(argument);
  261. outputIndex += count;
  262. }
  263. }
  264. }
  265. this.outputs.push(...outputs.slice(outputIndex).map((output, index) => {
  266. const name = ((outputIndex + index) === 0) ? 'output' : (outputIndex + index).toString();
  267. return new ncnn.Argument(name, [values.map(output)]);
  268. }));
  269. blobs.weight = (name, shape, code) => {
  270. const blob = blobs.load(shape, code || 0);
  271. const dataType = blob ? (blob.dataType || '?') : (code || 0).toString();
  272. const data = blob ? blob.data : null;
  273. const quantization = blob && blob.quantization ? blob.quantization : null;
  274. const type = new ncnn.TensorType(dataType, new ncnn.TensorShape(shape));
  275. const tensor = new ncnn.Tensor(type, data, quantization);
  276. const argument = new ncnn.Argument(name, [values.map('', null, tensor)]);
  277. this.inputs.push(argument);
  278. };
  279. switch (this.type.name) {
  280. case 'BatchNorm': {
  281. const channels = parseInt(params.get('0') || 0, 10);
  282. blobs.weight('slope', [channels], 1);
  283. blobs.weight('mean', [channels], 1);
  284. blobs.weight('variance', [channels], 1);
  285. blobs.weight('bias', [channels], 1);
  286. break;
  287. }
  288. case 'InnerProduct': {
  289. const num_output = parseInt(params.get('0') || 0, 10);
  290. const bias_term = parseInt(params.get('1') || 0, 10);
  291. const weight_data_size = parseInt(params.get('2') || 0, 10);
  292. const int8_scale_term = parseInt(params.get('8') || 0, 10);
  293. const activation_type = parseInt(params.get('9') || 0, 10);
  294. blobs.weight('weight', [num_output, weight_data_size / num_output]);
  295. if (bias_term) {
  296. blobs.weight('bias', [num_output], 1);
  297. }
  298. if (int8_scale_term) {
  299. blobs.weight('weight_scales', [num_output], 1);
  300. blobs.weight('bottom_scales', [1], 1);
  301. }
  302. const activation_names = ['', 'ReLU', 'Leaky ReLU', 'Clip', 'Sigmoid', 'Mish', 'HardSwish'];
  303. if (activation_type > 0 && activation_type < activation_names.length) {
  304. const layer = { type: activation_names[activation_type] };
  305. this.chain.push(new ncnn.Node(metadata, format, blobs, layer, values));
  306. }
  307. params.delete('2');
  308. break;
  309. }
  310. case 'Bias': {
  311. const bias_data_size = parseInt(params.get('0') || 0, 10);
  312. blobs.weight('bias', [bias_data_size], 1);
  313. break;
  314. }
  315. case 'Embed': {
  316. const num_output = parseInt(params.get('0') || 0, 10);
  317. const weight_data_size = parseInt(params.get('3') || 0, 10);
  318. blobs.weight('weight', [weight_data_size / num_output, num_output]);
  319. if (parseInt(params.get('2') || 0, 10) === 1) {
  320. blobs.weight('bias', [num_output], 1);
  321. }
  322. params.get('3');
  323. break;
  324. }
  325. case 'Convolution':
  326. case 'ConvolutionDepthWise':
  327. case 'Deconvolution':
  328. case 'DeconvolutionDepthWise': {
  329. const num_output = parseInt(params.get('0') || 0, 10);
  330. const kernel_w = parseInt(params.get('1') || 0, 10);
  331. const kernel_h = parseInt(params.get('11') || kernel_w, 10);
  332. const weight_data_size = parseInt(params.get('6') || 0, 10);
  333. blobs.weight('weight', [num_output, weight_data_size / (num_output * kernel_w * kernel_h), kernel_h, kernel_w]);
  334. if (parseInt(params.get('5') || 0, 10) === 1) {
  335. blobs.weight('bias', [num_output], 1);
  336. }
  337. const int8_scale_term = parseInt(params.get('8') || 0, 10);
  338. if (this.type.name === 'Convolution') {
  339. if (int8_scale_term) {
  340. blobs.weight('weight_scales', [num_output], 1);
  341. blobs.weight('bottom_scales', [1], 1);
  342. }
  343. if (int8_scale_term > 100) {
  344. blobs.weight('top_scales', [1], 1);
  345. }
  346. } else if (this.type.name === 'ConvolutionDepthWise') {
  347. const group = parseInt(params.get('7') || 1, 10);
  348. if (int8_scale_term === 1 || int8_scale_term === 101) {
  349. blobs.weight('weight_scales', [group], 1);
  350. blobs.weight('bottom_scales', [1], 1);
  351. } else if (int8_scale_term === 2 || int8_scale_term === 102) {
  352. blobs.weight('weight_scales', [1], 1);
  353. blobs.weight('bottom_scales', [1], 1);
  354. }
  355. if (int8_scale_term > 100) {
  356. blobs.weight('top_scales', [1], 1);
  357. }
  358. }
  359. params.delete('6');
  360. const activation_names = ['', 'ReLU', 'LeakyReLU', 'Clip', 'Sigmoid', 'Mish', 'HardSwish'];
  361. const activation_type = parseInt(params.get('9') || 0, 10);
  362. if (activation_type > 0 && activation_type < activation_names.length) {
  363. const layer = { type: activation_names[activation_type] };
  364. this.chain.push(new ncnn.Node(metadata, format, blobs, layer, values));
  365. }
  366. break;
  367. }
  368. case 'Convolution1D':
  369. case 'ConvolutionDepthWise1D': {
  370. const num_output = parseInt(params.get('0') || 0, 10);
  371. const kernel_w = parseInt(params.get('1') || 0, 10);
  372. const dynamic_weight = parseInt(params.get('19') || 0, 10);
  373. if (!dynamic_weight) {
  374. const weight_data_size = parseInt(params.get('6') || 0, 10);
  375. blobs.weight('weight', [num_output, weight_data_size / (num_output * kernel_w), kernel_w]);
  376. if (parseInt(params.get('5') || 0, 10) === 1) {
  377. blobs.weight('bias', [num_output], 1);
  378. }
  379. params.delete('6');
  380. }
  381. params.delete('19');
  382. const activation_names = ['', 'ReLU', 'LeakyReLU', 'Clip', 'Sigmoid', 'Mish', 'HardSwish'];
  383. const activation_type = parseInt(params.get('9') || 0, 10);
  384. if (activation_type > 0 && activation_type < activation_names.length) {
  385. const layer = { type: activation_names[activation_type] };
  386. const node = new ncnn.Node(metadata, format, blobs, layer, values);
  387. this.chain.push(node);
  388. }
  389. break;
  390. }
  391. case 'Deconvolution1D': {
  392. const activation_names = ['', 'ReLU', 'LeakyReLU', 'Clip', 'Sigmoid', 'Mish', 'HardSwish'];
  393. const activation_type = parseInt(params.get('9') || 0, 10);
  394. if (activation_type > 0 && activation_type < activation_names.length) {
  395. const layer = { type: activation_names[activation_type] };
  396. const node = new ncnn.Node(metadata, format, blobs, layer, values);
  397. this.chain.push(node);
  398. }
  399. const num_output = parseInt(params.get('0') || 0, 10);
  400. const kernel_w = parseInt(params.get('1') || 0, 10);
  401. const dynamic_weight = parseInt(params.get('28') || 0, 10);
  402. if (!dynamic_weight) {
  403. const weight_data_size = parseInt(params.get('6') || 0, 10);
  404. blobs.weight('weight', [num_output, weight_data_size / (num_output * kernel_w), kernel_w]);
  405. if (parseInt(params.get('5') || 0, 10) === 1) {
  406. blobs.weight('bias', [num_output], 1);
  407. }
  408. params.delete('6');
  409. }
  410. params.delete('28');
  411. break;
  412. }
  413. case 'Convolution3D':
  414. case 'ConvolutionDepthWise3D': {
  415. const num_output = parseInt(params.get('0') || 0, 10);
  416. const kernel_w = parseInt(params.get('1') || 0, 10);
  417. const kernel_h = parseInt(params.get('11') || kernel_w, 10);
  418. const kernel_d = parseInt(params.get('21') || kernel_w, 10);
  419. const weight_data_size = parseInt(params.get('6') || 0, 10);
  420. blobs.weight('weight', [num_output, weight_data_size / (num_output * kernel_w * kernel_h * kernel_d), kernel_d, kernel_h, kernel_w]);
  421. if (parseInt(params.get('5') || 0, 10) === 1) {
  422. blobs.weight('bias', [num_output], 1);
  423. }
  424. params.delete('6');
  425. const activation_names = ['', 'ReLU', 'LeakyReLU', 'Clip', 'Sigmoid', 'Mish', 'HardSwish'];
  426. const activation_type = parseInt(params.get('9') || 0, 10);
  427. if (activation_type > 0 && activation_type < activation_names.length) {
  428. const layer = { type: activation_names[activation_type] };
  429. this.chain.push(new ncnn.Node(metadata, format, blobs, layer, values));
  430. }
  431. break;
  432. }
  433. case 'Quantize': {
  434. const scale_data_size = parseInt(params.get('0') || 1, 10);
  435. blobs.weight('scale', [scale_data_size], 1);
  436. break;
  437. }
  438. case 'Dequantize': {
  439. const scale_data_size = parseInt(params.get('0') || 1, 10);
  440. const bias_data_size = parseInt(params.get('1') || 0, 10);
  441. blobs.weight('scale', [scale_data_size], 1);
  442. blobs.weight('bias', [bias_data_size], 1);
  443. break;
  444. }
  445. case 'Requantize': {
  446. const scale_in_data_size = parseInt(params.get('0') || 1, 10);
  447. const scale_out_data_size = parseInt(params.get('1') || 1, 10);
  448. const bias_data_size = parseInt(params.get('2') || 0, 10);
  449. blobs.weight('scale_in', [scale_in_data_size], 1);
  450. blobs.weight('scale_out', [scale_out_data_size], 1);
  451. blobs.weight('bias', [bias_data_size], 1);
  452. break;
  453. }
  454. case 'InstanceNorm': {
  455. const affine = parseInt(params.get('2') || 1, 10);
  456. if (affine === 1) {
  457. const channels = parseInt(params.get('0') || 0, 10);
  458. blobs.weight('gamma', [channels], 1);
  459. blobs.weight('beta', [channels], 1);
  460. }
  461. break;
  462. }
  463. case 'Scale': {
  464. const scale_data_size = parseInt(params.get('0') || 0, 10);
  465. if (scale_data_size !== -233) {
  466. blobs.weight('scale', [scale_data_size], 1);
  467. if (params.get('1') === '1') {
  468. blobs.weight('bias', [scale_data_size], 1);
  469. }
  470. }
  471. break;
  472. }
  473. case 'Normalize': {
  474. const scale_data_size = parseInt(params.get('3') || 0, 10);
  475. blobs.weight('scale', [scale_data_size], 1);
  476. break;
  477. }
  478. case 'PReLU': {
  479. const num_slope = parseInt(params.get('0') || 0, 10);
  480. blobs.weight('slope', [num_slope], 1);
  481. break;
  482. }
  483. case 'Padding': {
  484. const per_channel_pad_data_size = parseInt(params.get('6') || 0, 10);
  485. blobs.weight('per_channel_pad_data', [per_channel_pad_data_size], 1);
  486. break;
  487. }
  488. case 'MemoryData': {
  489. const w = parseInt(params.get('0') || 0, 10);
  490. const h = parseInt(params.get('1') || 0, 10);
  491. const d = parseInt(params.get('11') || 0, 10);
  492. const c = parseInt(params.get('2') || 0, 10);
  493. const load_type = parseInt(params.get('21') || 1, 10);
  494. /* eslint-disable no-negated-condition */
  495. if (d !== 0) {
  496. blobs.weight('data', [w, h, d, c], load_type);
  497. } else if (c !== 0) {
  498. blobs.weight('data', [w, h, c], load_type);
  499. } else if (h !== 0) {
  500. blobs.weight('data', [w, h], load_type);
  501. } else if (w !== 0) {
  502. blobs.weight('data', [w], load_type);
  503. } else {
  504. blobs.weight('data', [1], load_type);
  505. }
  506. /* eslint-enable no-negated-condition */
  507. break;
  508. }
  509. case 'GroupNorm': {
  510. const affine = parseInt(params.get('3') || 1, 10);
  511. if (affine === 1) {
  512. const channels = parseInt(params.get('1') || 0, 10);
  513. blobs.weight('gamma', [channels], 1);
  514. blobs.weight('beta', [channels], 1);
  515. }
  516. break;
  517. }
  518. case 'LayerNorm': {
  519. const channels = parseInt(params.get('0') || 0, 10);
  520. blobs.weight('gamma', [channels], 1);
  521. blobs.weight('beta', [channels], 1);
  522. break;
  523. }
  524. case 'RNN': {
  525. const num_output = parseInt(params.get('0') || 0, 10);
  526. const weight_data_size = parseInt(params.get('1') || 0, 10);
  527. const direction = parseInt(params.get('2') || 0, 10);
  528. const num_directions = direction === 2 ? 2 : 1;
  529. blobs.weight('weight_xc', [num_directions, num_output, weight_data_size / num_directions / num_output]);
  530. blobs.weight('bias_c', [num_directions, num_output]);
  531. blobs.weight('weight_hc', [num_directions, num_output, num_output]);
  532. params.delete('1');
  533. break;
  534. }
  535. case 'LSTM': {
  536. const num_output = parseInt(params.get('0') || 0, 10);
  537. const weight_data_size = parseInt(params.get('1') || 0, 10);
  538. const direction = parseInt(params.get('2') || 0, 10);
  539. const num_directions = direction === 2 ? 2 : 1;
  540. blobs.weight('weight_xc', [num_directions, 4, num_output, weight_data_size / num_directions / num_output / 4]);
  541. blobs.weight('bias_c', [num_directions, 4, num_output]);
  542. blobs.weight('weight_hc', [num_directions, 4, num_output, num_output]);
  543. params.delete('1');
  544. break;
  545. }
  546. case 'GRU': {
  547. const num_output = parseInt(params.get('0') || 0, 10);
  548. const weight_data_size = parseInt(params.get('1') || 0, 10);
  549. const direction = parseInt(params.get('2') || 0, 10);
  550. const num_directions = direction === 2 ? 2 : 1;
  551. blobs.weight('weight_xc', [num_directions, 3, num_output, weight_data_size / num_directions / num_output / 3]);
  552. blobs.weight('bias_c', [num_directions, 4, num_output]);
  553. blobs.weight('weight_hc', [num_directions, 3, num_output, num_output]);
  554. params.delete('1');
  555. break;
  556. }
  557. case 'MultiHeadAttention': {
  558. const embed_dim = parseInt(params.get('0') || 0, 10);
  559. // const num_head = parseInt(params.get('1') || 0, 10);
  560. // const weight_data_size = parseInt(params.get('2') || 0, 10);
  561. blobs.weight('weight_q', [embed_dim, embed_dim]);
  562. blobs.weight('bias_q', [embed_dim], 1);
  563. blobs.weight('weight_k', [embed_dim, embed_dim]);
  564. blobs.weight('bias_k', [embed_dim], 1);
  565. blobs.weight('weight_v', [embed_dim, embed_dim]);
  566. blobs.weight('bias_v', [embed_dim], 1);
  567. blobs.weight('weight_out', [embed_dim, embed_dim]);
  568. blobs.weight('bias_out', [embed_dim], 1);
  569. params.delete('2');
  570. break;
  571. }
  572. case 'Gemm': {
  573. const transA = parseInt(params.get('2') || 0, 10);
  574. const transB = parseInt(params.get('3') || 0, 10);
  575. const constantA = parseInt(params.get('4') || 0, 10);
  576. const constantB = parseInt(params.get('5') || 0, 10);
  577. const constantC = parseInt(params.get('6') || 0, 10);
  578. const M = parseInt(params.get('7') || 0, 10);
  579. const N = parseInt(params.get('8') || 0, 10);
  580. const K = parseInt(params.get('9') || 0, 10);
  581. const constant_broadcast_type_C = parseInt(params.get('10') || 0, 10);
  582. if (constantA === 1) {
  583. blobs.weight('A', transA === 0 ? [K, M] : [M, K]);
  584. }
  585. if (constantB === 1) {
  586. blobs.weight('B', transB === 1 ? [N, K] : [K, N]);
  587. }
  588. if (constantC === 1 && constant_broadcast_type_C !== -1) {
  589. let shape = null;
  590. switch (constant_broadcast_type_C) {
  591. case 0: shape = [1]; break;
  592. case 1: shape = [M]; break;
  593. case 2: shape = [1, M]; break;
  594. case 3: shape = [N, M]; break;
  595. case 4: shape = [N, 1]; break;
  596. default: break;
  597. }
  598. if (shape) {
  599. blobs.weight('C', shape);
  600. }
  601. }
  602. break;
  603. }
  604. default: {
  605. break;
  606. }
  607. }
  608. if (params && params.size > 0) {
  609. const attributes = this.type && Array.isArray(this.type.attributes) ? this.type.attributes : [];
  610. for (const [index, obj] of params) {
  611. const metadata = attributes[index];
  612. let name = index;
  613. let value = obj;
  614. let type = '';
  615. let visible = true;
  616. if (metadata) {
  617. name = metadata.name;
  618. type = metadata.type ? metadata.type : type;
  619. switch (type) {
  620. case 'int32': {
  621. value = parseInt(obj, 10);
  622. break;
  623. }
  624. case 'float32': {
  625. value = parseFloat(obj);
  626. break;
  627. }
  628. case 'float32[]': {
  629. value = obj.map((v) => parseFloat(v));
  630. break;
  631. }
  632. default: {
  633. value = type ? ncnn.Utility.value(obj, type) : obj;
  634. break;
  635. }
  636. }
  637. if (metadata && metadata.visible === false) {
  638. visible = false;
  639. } else if (metadata.default !== undefined) {
  640. if (value === metadata.default || (value && value.toString() === metadata.default.toString())) {
  641. visible = false;
  642. }
  643. }
  644. }
  645. const argument = new ncnn.Argument(name, value, type, visible);
  646. this.attributes.push(argument);
  647. }
  648. }
  649. }
  650. };
  651. ncnn.Tensor = class {
  652. constructor(type, values, quantization) {
  653. this.type = type;
  654. this.values = values;
  655. this.quantization = quantization;
  656. }
  657. };
  658. ncnn.TensorType = class {
  659. constructor(dataType, shape) {
  660. this.dataType = dataType || '?';
  661. this.shape = shape;
  662. }
  663. equals(obj) {
  664. return obj && this.dataType === obj.dataType && this.shape && this.shape.equals(obj.shape);
  665. }
  666. toString() {
  667. return this.dataType + this.shape.toString();
  668. }
  669. };
  670. ncnn.TensorShape = class {
  671. constructor(dimensions) {
  672. this.dimensions = dimensions;
  673. }
  674. equals(obj) {
  675. return obj && Array.isArray(obj.dimensions) &&
  676. Array.isArray(this.dimensions) && this.dimensions.length === obj.dimensions.length
  677. && obj.dimensions.every((value, index) => this.dimensions[index] === value);
  678. }
  679. toString() {
  680. return this.dimensions ? (`[${this.dimensions.map((dimension) => dimension ? dimension.toString() : '?').join(',')}]`) : '';
  681. }
  682. };
  683. ncnn.Utility = class {
  684. static value(value, type) {
  685. ncnn.Utility._enum = ncnn.Utility._enum || new Map([
  686. ['BinaryOpType', ['Add', 'Sub', 'Mul', 'Div', 'Max', 'Min', 'Pow', 'RSub', 'RDiv']],
  687. ['CastOpType', ['Auto', 'Float32', 'Float16', 'Int8', 'BFloat16']],
  688. ['EltwiseType', ['Prod', 'Sum', 'Max']],
  689. ['PaddingType', ['Constant', 'Replicate', 'Reflect']],
  690. ['PoolingType', ['Max', 'Average']],
  691. ['InterpResizeType', ['', 'Nearest', 'Bilinear', 'Bicubic']],
  692. ['PermuteOrderType', ['WH WHC WHDC', 'HW HWC HWDC', 'WCH WDHC', 'CWH DWHC', 'HCW HDWC', 'CHW DHWC', 'WHCD', 'HWCD', 'WCHD', 'CWHD', 'HCWD', 'CHWD', 'WDCH', 'DWCH', 'WCDH', 'CWDH', 'DCWH', 'CDWH', 'HDCW', 'DHCW', 'HCDW', 'CHDW', 'DCHW', 'CDHW']],
  693. ['ReductionOpType', ['Sum', 'ASum', 'SumSq', 'Mean', 'Max', 'Min', 'Prod', 'L1', 'L2', 'LogSum', 'LogSumExp']],
  694. ['UnaryOpType', ['Abs', 'Neg', 'Floor', 'Ceil', 'Square', 'Sqrt', 'Rsq', 'Exp', 'Log', 'Sin', 'Cos', 'Tan', 'ASin', 'ACos', 'ATan', 'Reciprocal', 'Tanh']]
  695. ]);
  696. if (ncnn.Utility._enum.has(type) && typeof value === 'string') {
  697. const index = parseInt(value, 10);
  698. const list = ncnn.Utility._enum.get(type);
  699. if (Number.isInteger(index) && index < list.length) {
  700. return list[index];
  701. }
  702. }
  703. return value;
  704. }
  705. };
  706. ncnn.TextParamReader = class {
  707. constructor(reader) {
  708. const lines = [];
  709. for (let line = reader.read('\n'); line !== undefined; line = reader.read('\n')) {
  710. line = line.trim();
  711. lines.push(line);
  712. }
  713. const signature = lines.shift();
  714. const header = (signature === '7767517' ? lines.shift() : signature).split(' ');
  715. if (header.length !== 2 || !header.every((value) => value >>> 0 === parseFloat(value))) {
  716. throw new ncnn.Error('Invalid header.');
  717. }
  718. this.layers = [];
  719. while (lines.length > 0) {
  720. const line = lines.shift();
  721. if (line.length > 0) {
  722. const columns = line.split(' ').filter((s) => s.length !== 0);
  723. const type = columns.shift();
  724. const name = columns.shift();
  725. const inputCount = parseInt(columns.shift(), 10);
  726. const outputCount = parseInt(columns.shift(), 10);
  727. const inputs = columns.splice(0, inputCount);
  728. const outputs = columns.splice(0, outputCount);
  729. const params = new Map();
  730. let index = 0;
  731. for (const column of columns) {
  732. const parts = column.split('=');
  733. if (parts.length > 2) {
  734. throw new ncnn.Error(`Invalid attribute '${column}'.`);
  735. }
  736. let key = (parts.length === 2) ? parts[0].trim() : index.toString();
  737. let value = (parts.length === 2) ? parts[1].trim() : parts[0].trim();
  738. const keyInt = parseInt(key, 10);
  739. if (keyInt < 0) {
  740. value = value.split(',').map((v) => v.trim());
  741. value.shift();
  742. key = (-(keyInt + 23300)).toString();
  743. }
  744. params.set(key, value);
  745. index++;
  746. }
  747. this.layers.push({ type, name, inputs, outputs, params });
  748. }
  749. }
  750. }
  751. };
  752. ncnn.BinaryParamReader = class {
  753. constructor(buffer) {
  754. const reader = base.BinaryReader.open(buffer);
  755. if (reader.int32() !== 0x007685dd) {
  756. throw new ncnn.Error('Invalid signature.');
  757. }
  758. const layerCount = reader.int32();
  759. /* const blobCount = */ reader.int32();
  760. this.layers = [];
  761. for (let i = 0; i < layerCount; i++) {
  762. const type = reader.int32();
  763. const name = i.toString();
  764. const inputs = new Array(reader.int32());
  765. const outputs = new Array(reader.int32());
  766. for (let j = 0; j < inputs.length; j++) {
  767. inputs[j] = reader.int32().toString();
  768. }
  769. for (let j = 0; j < outputs.length; j++) {
  770. outputs[j] = reader.int32().toString();
  771. }
  772. const params = new Map();
  773. let id = reader.int32();
  774. while (id !== -233) {
  775. const isArray = id <= -23300;
  776. if (isArray) {
  777. id = -id - 23300;
  778. }
  779. const key = id.toString();
  780. if (isArray) {
  781. const length = reader.int32();
  782. const values = [];
  783. for (let i = 0; i < length; i++) {
  784. values.push(reader.int32());
  785. }
  786. params.set(key, values);
  787. } else {
  788. const value = reader.int32();
  789. params.set(key, value);
  790. }
  791. id = reader.int32();
  792. }
  793. this.layers.push({ type, name, inputs, outputs, params });
  794. }
  795. }
  796. };
  797. ncnn.BlobReader = class {
  798. constructor(context) {
  799. if (context) {
  800. this._identifier = context.identifier;
  801. if (this._identifier.toLowerCase().endsWith('.pnnx.bin')) {
  802. this._entires = context.peek('zip');
  803. } else {
  804. this._buffer = context.stream.peek();
  805. this._position = 0;
  806. }
  807. }
  808. }
  809. skip(length) {
  810. this._position += length;
  811. if (this._position > this._buffer.length) {
  812. throw new ncnn.Error('Unexpected end of file.');
  813. }
  814. }
  815. read(length) {
  816. const position = this._position;
  817. this.skip(length);
  818. return this._buffer.subarray(position, this._position);
  819. }
  820. align(size) {
  821. const remainder = this._position % size;
  822. if (remainder !== 0) {
  823. this.skip(size - remainder);
  824. }
  825. }
  826. load(shape, type) {
  827. if (!this._buffer) {
  828. return null;
  829. }
  830. const size = shape.reduce((a, b) => a * b, 1);
  831. if (type === 0) {
  832. const buffer = this.read(4);
  833. const [f0, f1, f2, f3] = buffer;
  834. const flag = f0 | f1 << 8 | f2 << 16 | f3 << 24;
  835. // https://github.com/Tencent/ncnn/blob/master/src/modelbin.cpp
  836. if (flag === 0x01306B47) { // float16
  837. const data = this.read(size * 2);
  838. this.align(4);
  839. return { dataType: 'float16', data };
  840. } else if (flag === 0x000D4B38) { // int8
  841. const data = this.read(size);
  842. this.align(4);
  843. return { dataType: 'int8', data };
  844. } else if (flag === 0x00000001) { // qint8
  845. // this.skip(size + 1024);
  846. // data = null;
  847. // return { dataType: 'qint8', data };
  848. throw new ncnn.Error("Unsupported weight type '0x00000001'.");
  849. } else if (flag === 0x0002C056) {
  850. // size * sizeof(float) - raw data with extra scaling
  851. throw new ncnn.Error("Unsupported weight type '0x0002C056'.");
  852. } else if (flag === 0x00000000) { // float32
  853. const data = this.read(size * 4);
  854. return { dataType: 'float32', data };
  855. } else {
  856. const size = shape.reduce((a, b) => a * b, 1);
  857. const buffer = this.read(1024);
  858. const quantization = {
  859. type: 'lookup',
  860. value: Array.from(new Float32Array(buffer.buffer, buffer.bufferOffset, buffer.length / 4))
  861. };
  862. const data = this.read(size);
  863. this.align(4);
  864. return { dataType: 'uint8', data, quantization };
  865. }
  866. } else if (type === 1) {
  867. const data = this.read(size * 4);
  868. return { dataType: 'float32', data };
  869. }
  870. throw new ncnn.Error(`Load type '${type}' not supported.`);
  871. }
  872. validate() {
  873. const files = [
  874. ['encoder_jit_trace-pnnx.ncnn.bin', 139191256]
  875. ];
  876. if (this._buffer && this._buffer.length !== this._position &&
  877. !this._identifier.toLowerCase().endsWith('.pnnx.bin') &&
  878. !files.find((file) => file[0] === this._identifier && file[1] === this._buffer.length)) {
  879. throw new ncnn.Error('Invalid weights data size.');
  880. }
  881. }
  882. };
  883. ncnn.Error = class extends Error {
  884. constructor(message) {
  885. super(message);
  886. this.name = 'Error loading ncnn model.';
  887. }
  888. };
  889. export const ModelFactory = ncnn.ModelFactory;