coreml.js 70 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724
  1. var coreml = coreml || {};
  2. var json = json || require('./json');
  3. var protobuf = protobuf || require('./protobuf');
  4. var base = base || require('./base');
  5. coreml.ModelFactory = class {
  6. match(context) {
  7. const stream = context.stream;
  8. const identifier = context.identifier.toLowerCase();
  9. const extension = identifier.split('.').pop().toLowerCase();
  10. const tags = context.tags('pb');
  11. if (tags.get(1) === 0 && tags.get(2) === 2) {
  12. if (extension === 'pb') {
  13. const tags = context.tags('pb+');
  14. const keys = Object.keys(tags).map((key) => parseInt(key, 10));
  15. const match = (key) =>
  16. (key >= 200 && key < 220) ||
  17. (key >= 300 && key < 320) ||
  18. (key >= 400 && key < 420) ||
  19. (key >= 500 && key < 520) ||
  20. (key >= 550 && key < 560) ||
  21. (key >= 600 && key < 620) ||
  22. (key === 900) ||
  23. (key >= 2000 && key < 2010) ||
  24. (key === 3000);
  25. if (!keys.some((key) => match(key))) {
  26. return null;
  27. }
  28. }
  29. return 'coreml.pb';
  30. }
  31. if (identifier === 'manifest.json') {
  32. const obj = context.open('json');
  33. if (obj && obj.rootModelIdentifier && obj.itemInfoEntries) {
  34. const entries = Object.keys(obj.itemInfoEntries).map((key) => obj.itemInfoEntries[key]);
  35. if (entries.filter((entry) => entry.path.toLowerCase().endsWith('.mlmodel').length === 1)){
  36. return 'coreml.manifest';
  37. }
  38. }
  39. }
  40. if (identifier === 'metadata.json') {
  41. const obj = context.open('json');
  42. if (obj && obj.rootModelIdentifier && obj.itemInfoEntries) {
  43. return 'coreml.metadata';
  44. }
  45. }
  46. if (identifier === 'featuredescriptions.json') {
  47. const obj = context.open('json');
  48. if (obj && (obj.Inputs || obj.Outputs)) {
  49. return 'coreml.featuredescriptions';
  50. }
  51. }
  52. if (extension === 'bin' && stream.length > 16) {
  53. const buffer = stream.peek(Math.min(256, stream.length));
  54. for (let i = 0; i < buffer.length - 4; i++) {
  55. const signature = (buffer[i] | buffer[i + 1] << 8 | buffer[i + 2] << 16 | buffer [i + 3] << 24) >>> 0;
  56. if (signature === 0xdeadbeef) {
  57. return 'coreml.weights';
  58. }
  59. }
  60. }
  61. return undefined;
  62. }
  63. open(context, match) {
  64. return context.require('./coreml-proto').then(() => {
  65. return coreml.Metadata.open(context).then((metadata) => {
  66. const openModel = (stream, context, path, format) => {
  67. let model = null;
  68. try {
  69. coreml.proto = protobuf.get('coreml').CoreML.Specification;
  70. const reader = protobuf.BinaryReader.open(stream);
  71. model = coreml.proto.Model.decode(reader);
  72. }
  73. catch (error) {
  74. const message = error && error.message ? error.message : error.toString();
  75. throw new coreml.Error('File format is not coreml.Model (' + message.replace(/\.$/, '') + ').');
  76. }
  77. const weightPaths = new Set();
  78. const walkProgram = (program) => {
  79. for (const entry of Object.entries(program.functions)) {
  80. const func = entry[1];
  81. for (const entry of Object.entries(func.block_specializations)) {
  82. const block = entry[1];
  83. for (const operation of block.operations) {
  84. for (const entry of Object.entries(operation.attributes)) {
  85. const value = entry[1];
  86. if (value.blobFileValue && value.blobFileValue.fileName) {
  87. weightPaths.add(value.blobFileValue.fileName);
  88. }
  89. }
  90. }
  91. }
  92. }
  93. };
  94. const walkModel = (model) => {
  95. if (model.mlProgram) {
  96. walkProgram(model.mlProgram);
  97. }
  98. if (model.pipeline && model.pipeline.models) {
  99. for (const node of model.pipeline.models) {
  100. walkModel(node);
  101. }
  102. }
  103. if (model.pipelineClassifier && model.pipelineClassifier.pipeline && model.pipelineClassifier.pipeline.models) {
  104. for (const node of model.pipelineClassifier.pipeline.models) {
  105. walkModel(node);
  106. }
  107. }
  108. if (model.pipelineRegressor && model.pipelineRegressor.pipeline && model.pipelineRegressor.pipeline.models) {
  109. for (const node of model.pipelineRegressor.pipeline.models) {
  110. walkModel(node);
  111. }
  112. }
  113. };
  114. walkModel(model);
  115. if (weightPaths.size > 0) {
  116. const items = path.split('/');
  117. items.pop();
  118. const folder = items.join('/');
  119. const keys = Array.from(weightPaths);
  120. const paths = keys.map((path) => {
  121. const items = path.split('/');
  122. if (items[0] === '@model_path') {
  123. items[0] = folder;
  124. }
  125. return items.join('/');
  126. });
  127. const promises = paths.map((path) => context.request(path, null));
  128. return Promise.all(promises).then((streams) => {
  129. const weights = new Map();
  130. for (let i = 0; i < keys.length; i++) {
  131. weights.set(keys[i], streams[i]);
  132. }
  133. return new coreml.Model(metadata, format, model, weights);
  134. }).catch((/* err */) => {
  135. return new coreml.Model(metadata, format, model, new Map());
  136. });
  137. }
  138. return new coreml.Model(metadata, format, model, new Map());
  139. };
  140. const openManifest = (obj, context, path) => {
  141. const entries = Object.keys(obj.itemInfoEntries).map((key) => obj.itemInfoEntries[key]);
  142. const entry = entries.filter((entry) => entry.path.toLowerCase().endsWith('.mlmodel'))[0];
  143. const file = path + 'Data/' + entry.path;
  144. return context.request(file, null).then((stream) => {
  145. return openModel(stream, context, file, 'Core ML Package');
  146. });
  147. };
  148. const openManifestStream = (context, path) => {
  149. return context.request(path + 'Manifest.json', null).then((stream) => {
  150. const reader = json.TextReader.open(stream);
  151. const obj = reader.read();
  152. return openManifest(obj, context, path);
  153. });
  154. };
  155. switch (match) {
  156. case 'coreml.pb': {
  157. return openModel(context.stream, context, context.identifier);
  158. }
  159. case 'coreml.manifest': {
  160. const obj = context.open('json');
  161. return openManifest(obj, context, '');
  162. }
  163. case 'coreml.featuredescriptions':
  164. case 'coreml.metadata': {
  165. return openManifestStream(context, '../../');
  166. }
  167. case 'coreml.weights': {
  168. return openManifestStream(context, '../../../');
  169. }
  170. default: {
  171. throw new coreml.Error("Unsupported Core ML format '" + match + "'.");
  172. }
  173. }
  174. });
  175. });
  176. }
  177. };
  178. coreml.Model = class {
  179. constructor(metadata, format, model, weights) {
  180. this._format = (format || 'Core ML') + ' v' + model.specificationVersion.toString();
  181. this._metadata = [];
  182. this._graphs = [ new coreml.Graph(metadata, model, weights) ];
  183. if (model.description && model.description.metadata) {
  184. const properties = model.description.metadata;
  185. if (properties.versionString) {
  186. this._version = properties.versionString;
  187. }
  188. if (properties.shortDescription) {
  189. this._description = properties.shortDescription;
  190. }
  191. if (properties.author) {
  192. this._metadata.push({ name: 'author', value: properties.author });
  193. }
  194. if (properties.license) {
  195. this._metadata.push({ name: 'license', value: properties.license });
  196. }
  197. if (metadata.userDefined && Object.keys(properties.userDefined).length > 0) {
  198. /* empty */
  199. }
  200. }
  201. }
  202. get format() {
  203. return this._format;
  204. }
  205. get version() {
  206. return this._version || null;
  207. }
  208. get description() {
  209. return this._description || null;
  210. }
  211. get metadata() {
  212. return this._metadata;
  213. }
  214. get graphs() {
  215. return this._graphs;
  216. }
  217. };
  218. coreml.Graph = class {
  219. constructor(metadata, model, weights) {
  220. this._metadata = metadata;
  221. this._description = model.description;
  222. this._groups = false;
  223. this._inputs = [];
  224. this._outputs = [];
  225. this._nodes = [];
  226. if (this._description) {
  227. this._inputs = this._description.input.map((input) => {
  228. const argument = new coreml.Argument(input.name, coreml.Utility.featureType(input.type), input.shortDescription, null);
  229. return new coreml.Parameter(input.name, true, [ argument ]);
  230. });
  231. this._outputs = this._description.output.map((output) => {
  232. const argument = new coreml.Argument(output.name, coreml.Utility.featureType(output.type), output.shortDescription, null);
  233. return new coreml.Parameter(output.name, true, [ argument ]);
  234. });
  235. }
  236. this._type = this._loadModel(model, {}, '', weights);
  237. }
  238. get name() {
  239. return '';
  240. }
  241. get type() {
  242. return this._type;
  243. }
  244. get inputs() {
  245. return this._inputs;
  246. }
  247. get outputs() {
  248. return this._outputs;
  249. }
  250. get nodes() {
  251. return this._nodes;
  252. }
  253. get groups() {
  254. return this._groups;
  255. }
  256. _updateOutput(name, newName) {
  257. for (const node of this._nodes) {
  258. for (const output of node.outputs) {
  259. for (const argument of output.arguments) {
  260. if (argument.name === name) {
  261. argument.name = newName;
  262. }
  263. }
  264. }
  265. }
  266. return newName;
  267. }
  268. _updateClassifierOutput(group, classifier) {
  269. let labelProbabilityLayerName = classifier.labelProbabilityLayerName;
  270. if (!labelProbabilityLayerName && this._nodes.length > 0) {
  271. const node = this._nodes.slice(-1).pop();
  272. if (node && node.outputs.length == 1 && node.outputs[0].arguments.length == 1) {
  273. labelProbabilityLayerName = node.outputs[0].arguments[0].name;
  274. }
  275. }
  276. let predictedFeatureName = this._description.predictedFeatureName;
  277. let predictedProbabilitiesName = this._description.predictedProbabilitiesName;
  278. if ((predictedFeatureName || predictedProbabilitiesName) && labelProbabilityLayerName && classifier.ClassLabels) {
  279. predictedFeatureName = predictedFeatureName ? predictedFeatureName : '?';
  280. predictedProbabilitiesName = predictedProbabilitiesName ? predictedProbabilitiesName : '?';
  281. const labelProbabilityInput = this._updateOutput(labelProbabilityLayerName, labelProbabilityLayerName + ':labelProbabilityLayerName');
  282. const type = classifier.ClassLabels;
  283. const inputs = [
  284. new coreml.Parameter('input', true, [ new coreml.Argument(labelProbabilityInput) ])
  285. ];
  286. const outputs = [
  287. new coreml.Parameter('probabilities', true, [ new coreml.Argument(predictedProbabilitiesName) ]),
  288. new coreml.Parameter('feature', true, [ new coreml.Argument(predictedFeatureName) ])
  289. ];
  290. const node = new coreml.Node(this._metadata, this._group, type, null, '', classifier[type], inputs, outputs);
  291. this._nodes.push(node);
  292. }
  293. }
  294. _updatePreprocessing(scope, group, preprocessing) {
  295. if (preprocessing && preprocessing.length > 0) {
  296. const preprocessingInput = this._description.input[0].name;
  297. const inputNodes = [];
  298. for (const node of this._nodes) {
  299. if (node.inputs.some((input) => input.arguments.some((arg) => arg.name == preprocessingInput))) {
  300. inputNodes.push(node);
  301. }
  302. }
  303. let preprocessorOutput = preprocessingInput;
  304. let preprocessorIndex = 0;
  305. for (const p of preprocessing) {
  306. const input = p.featureName ? p.featureName : preprocessorOutput;
  307. preprocessorOutput = preprocessingInput + ':' + preprocessorIndex.toString();
  308. this._createNode(scope, group, p.preprocessor, null, '', p[p.preprocessor], [ input ], [ preprocessorOutput ]);
  309. preprocessorIndex++;
  310. }
  311. for (const node of inputNodes) {
  312. for (const input of node.inputs) {
  313. for (const arg of input.arguments) {
  314. if (arg.name === preprocessingInput) {
  315. arg.name = preprocessorOutput;
  316. }
  317. }
  318. }
  319. }
  320. }
  321. }
  322. _loadModel(model, scope, group, weights) {
  323. this._groups = this._groups | (group.length > 0 ? true : false);
  324. const description = model && model.description && model.description.metadata && model.description.metadata.shortDescription ? model.description.metadata.shortDescription : '';
  325. switch (model.Type) {
  326. case 'neuralNetworkClassifier': {
  327. const neuralNetworkClassifier = model.neuralNetworkClassifier;
  328. for (const layer of neuralNetworkClassifier.layers) {
  329. this._createNode(scope, group, layer.layer, layer.name, group === '' ? '' : description, layer[layer.layer], layer.input, layer.output);
  330. }
  331. this._updateClassifierOutput(group, neuralNetworkClassifier);
  332. this._updatePreprocessing(scope, group, neuralNetworkClassifier.preprocessing);
  333. return 'Neural Network Classifier';
  334. }
  335. case 'neuralNetwork': {
  336. const neuralNetwork = model.neuralNetwork;
  337. for (const layer of neuralNetwork.layers) {
  338. this._createNode(scope, group, layer.layer, layer.name, group === '' ? '' : description, layer[layer.layer], layer.input, layer.output);
  339. }
  340. this._updatePreprocessing(scope, group, neuralNetwork.preprocessing);
  341. return 'Neural Network';
  342. }
  343. case 'neuralNetworkRegressor': {
  344. const neuralNetworkRegressor = model.neuralNetworkRegressor;
  345. for (const layer of neuralNetworkRegressor.layers) {
  346. this._createNode(scope, group, layer.layer, layer.name, description, layer[layer.layer], layer.input, layer.output);
  347. }
  348. this._updatePreprocessing(scope, group, neuralNetworkRegressor);
  349. return 'Neural Network Regressor';
  350. }
  351. case 'pipeline': {
  352. for (let i = 0; i < model.pipeline.models.length; i++) {
  353. this._loadModel(model.pipeline.models[i], scope, (group ? (group + '/') : '') + 'pipeline[' + i.toString() + ']');
  354. }
  355. return 'Pipeline';
  356. }
  357. case 'pipelineClassifier': {
  358. for (let i = 0; i < model.pipelineClassifier.pipeline.models.length; i++) {
  359. this._loadModel(model.pipelineClassifier.pipeline.models[i], scope, (group ? (group + '/') : '') + 'pipelineClassifier[' + i.toString() + ']');
  360. }
  361. return 'Pipeline Classifier';
  362. }
  363. case 'pipelineRegressor': {
  364. for (let i = 0; i < model.pipelineRegressor.pipeline.models.length; i++) {
  365. this._loadModel(model.pipelineRegressor.pipeline.models[i], scope, (group ? (group + '/') : '') + 'pipelineRegressor[' + i.toString() + ']');
  366. }
  367. return 'Pipeline Regressor';
  368. }
  369. case 'glmClassifier': {
  370. this._createNode(scope, group, 'glmClassifier', null, description,
  371. {
  372. classEncoding: model.glmClassifier.classEncoding,
  373. offset: model.glmClassifier.offset,
  374. weights: model.glmClassifier.weights
  375. },
  376. [ model.description.input[0].name ],
  377. [ model.description.predictedProbabilitiesName ]);
  378. this._updateClassifierOutput(group, model.glmClassifier);
  379. return 'Generalized Linear Classifier';
  380. }
  381. case 'glmRegressor': {
  382. this._createNode(scope, group, 'glmRegressor', null, description,
  383. model.glmRegressor,
  384. [ model.description.input[0].name ],
  385. [ model.description.output[0].name ]);
  386. return 'Generalized Linear Regressor';
  387. }
  388. case 'treeEnsembleClassifier': {
  389. this._createNode(scope, group, 'treeEnsembleClassifier', null, description,
  390. model.treeEnsembleClassifier.treeEnsemble,
  391. [ model.description.input[0].name ],
  392. [ model.description.output[0].name ]);
  393. this._updateClassifierOutput(group, model.treeEnsembleClassifier);
  394. return 'Tree Ensemble Classifier';
  395. }
  396. case 'treeEnsembleRegressor': {
  397. this._createNode(scope, group, 'treeEnsembleRegressor', null, description,
  398. model.treeEnsembleRegressor.treeEnsemble,
  399. [ model.description.input[0].name ],
  400. [ model.description.output[0].name ]);
  401. return 'Tree Ensemble Regressor';
  402. }
  403. case 'supportVectorClassifier': {
  404. this._createNode(scope, group, 'supportVectorClassifier', null, description,
  405. {
  406. coefficients: model.supportVectorClassifier.coefficients,
  407. denseSupportVectors: model.supportVectorClassifier.denseSupportVectors,
  408. kernel: model.supportVectorClassifier.kernel,
  409. numberOfSupportVectorsPerClass: model.supportVectorClassifier.numberOfSupportVectorsPerClass,
  410. probA: model.supportVectorClassifier.probA,
  411. probB: model.supportVectorClassifier.probB,
  412. rho: model.supportVectorClassifier.rho,
  413. supportVectors: model.supportVectorClassifier.supportVectors
  414. },
  415. [ model.description.input[0].name ],
  416. [ model.description.output[0].name ]);
  417. this._updateClassifierOutput(group, model.supportVectorClassifier);
  418. return 'Support Vector Classifier';
  419. }
  420. case 'supportVectorRegressor': {
  421. this._createNode(scope, group, 'supportVectorRegressor', null, description,
  422. {
  423. coefficients: model.supportVectorRegressor.coefficients,
  424. kernel: model.supportVectorRegressor.kernel,
  425. rho: model.supportVectorRegressor.rho,
  426. supportVectors: model.supportVectorRegressor.supportVectors
  427. },
  428. [ model.description.input[0].name ],
  429. [ model.description.output[0].name ]);
  430. return 'Support Vector Regressor';
  431. }
  432. case 'oneHotEncoder': {
  433. const categoryType = model.oneHotEncoder.CategoryType;
  434. const oneHotEncoderParams = { outputSparse: model.oneHotEncoder.outputSparse };
  435. oneHotEncoderParams[categoryType] = model.oneHotEncoder[categoryType];
  436. this._createNode(scope, group, 'oneHotEncoder', null, description,
  437. oneHotEncoderParams,
  438. [ model.description.input[0].name ],
  439. [ model.description.output[0].name ]);
  440. return 'One Hot Encoder';
  441. }
  442. case 'imputer': {
  443. const imputedValue = model.imputer.ImputedValue;
  444. const replaceValue = model.imputer.ReplaceValue;
  445. const imputerParams = {};
  446. imputerParams[imputedValue] = model.imputer[imputedValue];
  447. imputerParams[replaceValue] = model.imputer[replaceValue];
  448. this._createNode(scope, group, 'oneHotEncoder', null, description,
  449. imputerParams,
  450. [ model.description.input[0].name ],
  451. [ model.description.output[0].name ]);
  452. return 'Imputer';
  453. }
  454. case 'featureVectorizer': {
  455. this._createNode(scope, group, 'featureVectorizer', null, description,
  456. model.featureVectorizer,
  457. coreml.Graph._formatFeatureDescriptionList(model.description.input),
  458. [ model.description.output[0].name ]);
  459. return 'Feature Vectorizer';
  460. }
  461. case 'dictVectorizer': {
  462. this._createNode(scope, group, 'dictVectorizer', null, description,
  463. model.dictVectorizer,
  464. [ model.description.input[0].name ],
  465. [ model.description.output[0].name ]);
  466. return 'Dictionary Vectorizer';
  467. }
  468. case 'scaler': {
  469. this._createNode(scope, group, 'scaler', null, description,
  470. model.scaler,
  471. [ model.description.input[0].name ],
  472. [ model.description.output[0].name ]);
  473. return 'Scaler';
  474. }
  475. case 'categoricalMapping': {
  476. this._createNode(scope, group, 'categoricalMapping', null, description,
  477. model.categoricalMapping,
  478. [ model.description.input[0].name ],
  479. [ model.description.output[0].name ]);
  480. return 'Categorical Mapping';
  481. }
  482. case 'normalizer': {
  483. this._createNode(scope, group, 'normalizer', null, description,
  484. model.normalizer,
  485. [ model.description.input[0].name ],
  486. [ model.description.output[0].name ]);
  487. return 'Normalizer';
  488. }
  489. case 'arrayFeatureExtractor': {
  490. this._createNode(scope, group, 'arrayFeatureExtractor', null, description,
  491. { extractIndex: model.arrayFeatureExtractor.extractIndex },
  492. [ model.description.input[0].name ],
  493. [ model.description.output[0].name ]);
  494. return 'Array Feature Extractor';
  495. }
  496. case 'nonMaximumSuppression': {
  497. const nonMaximumSuppressionParams = {
  498. pickTop: model.nonMaximumSuppression.pickTop,
  499. stringClassLabels: model.nonMaximumSuppression.stringClassLabels,
  500. iouThreshold: model.nonMaximumSuppression.iouThreshold,
  501. confidenceThreshold: model.nonMaximumSuppression.confidenceThreshold
  502. };
  503. this._createNode(scope, group, 'nonMaximumSuppression', null, description,
  504. nonMaximumSuppressionParams,
  505. [
  506. model.nonMaximumSuppression.confidenceInputFeatureName,
  507. model.nonMaximumSuppression.coordinatesInputFeatureName,
  508. model.nonMaximumSuppression.iouThresholdInputFeatureName,
  509. model.nonMaximumSuppression.confidenceThresholdInputFeatureName,
  510. ],
  511. [
  512. model.nonMaximumSuppression.confidenceOutputFeatureName,
  513. model.nonMaximumSuppression.coordinatesOutputFeatureName
  514. ]);
  515. return 'Non Maximum Suppression';
  516. }
  517. case 'wordTagger': {
  518. this._createNode(scope, group, 'wordTagger', null, description,
  519. model.wordTagger,
  520. [ model.description.input[0].name ],
  521. [
  522. model.wordTagger.tokensOutputFeatureName,
  523. model.wordTagger.tokenTagsOutputFeatureName,
  524. model.wordTagger.tokenLocationsOutputFeatureName,
  525. model.wordTagger.tokenLengthsOutputFeatureName
  526. ]);
  527. return 'Word Tagger';
  528. }
  529. case 'textClassifier': {
  530. this._createNode(scope, group, 'textClassifier', null, description,
  531. model.textClassifier,
  532. [ model.description.input[0].name ],
  533. [ model.description.output[0].name ]);
  534. return 'Text Classifier';
  535. }
  536. case 'visionFeaturePrint': {
  537. const visionFeaturePrintParams = {
  538. scene: model.visionFeaturePrint.scene
  539. };
  540. this._createNode(scope, group, 'visionFeaturePrint', null, description,
  541. visionFeaturePrintParams,
  542. [ model.description.input[0].name ],
  543. [ model.description.output[0].name ]);
  544. return 'Vision Feature Print';
  545. }
  546. case 'soundAnalysisPreprocessing': {
  547. this._createNode(scope, group, 'soundAnalysisPreprocessing', null, description,
  548. model.soundAnalysisPreprocessing,
  549. [ model.description.input[0].name ],
  550. [ model.description.output[0].name ]);
  551. return 'Sound Analysis Preprocessing';
  552. }
  553. case 'kNearestNeighborsClassifier': {
  554. this._createNode(scope, group, 'kNearestNeighborsClassifier', null, description,
  555. model.kNearestNeighborsClassifier,
  556. [ model.description.input[0].name ],
  557. [ model.description.output[0].name ]);
  558. this._updateClassifierOutput(group, model.kNearestNeighborsClassifier);
  559. return 'Nearest Neighbors Classifier';
  560. }
  561. case 'itemSimilarityRecommender': {
  562. this._createNode(scope, group, 'itemSimilarityRecommender', null, description,
  563. {
  564. itemStringIds: model.itemSimilarityRecommender.itemStringIds.vector,
  565. itemItemSimilarities: model.itemSimilarityRecommender.itemItemSimilarities
  566. },
  567. model.description.input.map((feature) => feature.name),
  568. model.description.output.map((feature) => feature.name));
  569. return 'Item Similarity Recommender';
  570. }
  571. case 'audioFeaturePrint': {
  572. this._createNode(scope, group, 'audioFeaturePrint', null, description,
  573. model.audioFeaturePrint,
  574. [ model.description.input[0].name ],
  575. [ model.description.output[0].name ]);
  576. return 'Audio Feature Print';
  577. }
  578. case 'linkedModel': {
  579. this._createNode(scope, group, 'linkedModel', null, description,
  580. model.linkedModel.linkedModelFile,
  581. [ model.description.input[0].name ],
  582. [ model.description.output[0].name ]);
  583. return 'Linked Model';
  584. }
  585. case 'customModel': {
  586. this._createNode(scope, group, 'customModel', null, description,
  587. { className: model.customModel.className, parameters: model.customModel.parameters },
  588. [ model.description.input[0].name ],
  589. [ model.description.output[0].name ]);
  590. return 'customModel';
  591. }
  592. case 'mlProgram': {
  593. return this._loadProgram(model.mlProgram, scope, group, weights);
  594. }
  595. default: {
  596. throw new coreml.Error("Unsupported model type '" + JSON.stringify(Object.keys(model)) + "'.");
  597. }
  598. }
  599. }
  600. _loadProgram(program, scope, group, weights) {
  601. // TODO: need to handle functions other than main?
  602. const main = program.functions.main;
  603. // TODO: need to handle more than one block specialization?
  604. const block = main.block_specializations.CoreML5 || main.block_specializations.CoreML6;
  605. const convertValue = (value) => {
  606. switch (value.value) {
  607. case 'immediateValue': {
  608. const tensor = value.immediateValue.tensor;
  609. let values = null;
  610. switch (tensor.value) {
  611. case 'ints':
  612. values = tensor.ints.values;
  613. break;
  614. case 'strings':
  615. values = tensor.strings.values;
  616. break;
  617. case 'bools':
  618. values = tensor.bools.values;
  619. break;
  620. case 'floats':
  621. values = tensor.floats.values;
  622. break;
  623. case 'bytes':
  624. values = tensor.bytes.values;
  625. break;
  626. default:
  627. throw new coreml.Error("Unsupported tensor value '" + tensor.value + "'.");
  628. }
  629. return values;
  630. }
  631. case 'blobFileValue': {
  632. const type = coreml.Utility.valueType(value.type);
  633. const blob = value.blobFileValue;
  634. const offset = blob.offset.toNumber();
  635. const file = blob.fileName;
  636. let data = null;
  637. const stream = weights.get(file);
  638. if (stream) {
  639. stream.seek(offset);
  640. const buffer = stream.read(32);
  641. const reader = new base.BinaryReader(buffer);
  642. const signature = reader.uint32();
  643. if (signature == 0xdeadbeef) {
  644. reader.uint32(); // dataType
  645. const size = reader.uint64();
  646. stream.seek(reader.uint64());
  647. const length = (type.shape.dimensions || []).reduce((a, b) => a * b, 1);
  648. switch (type.dataType) {
  649. case 'float32': {
  650. const buffer = stream.read(size);
  651. data = new Float32Array(buffer.buffer, buffer.byteOffset, length).slice();
  652. break;
  653. }
  654. case 'float16': {
  655. data = stream.read(size);
  656. break;
  657. }
  658. case 'uint8': {
  659. data = stream.read(size);
  660. break;
  661. }
  662. default:
  663. throw new coreml.Error("Unsupported blob data type '" + type.dataType + "'.");
  664. }
  665. }
  666. }
  667. return new coreml.Tensor('Blob', type, data);
  668. }
  669. default: {
  670. throw new coreml.Error("Unsupported value '" + value.value + "'.");
  671. }
  672. }
  673. };
  674. const args = new Map();
  675. const arg = (name) => {
  676. if (!args.has(name)) {
  677. args.set(name, { name: name, to: [], from: [] });
  678. }
  679. return args.get(name);
  680. };
  681. const operations = block.operations.map((op) => {
  682. const operation = {
  683. type: op.type,
  684. attributes: {}
  685. };
  686. for (const entry of Object.entries(op.attributes)) {
  687. const key = entry[0];
  688. const value = entry[1];
  689. operation.attributes[key] = convertValue(value);
  690. }
  691. operation.inputs = Object.entries(op.inputs).map((entry) => {
  692. const key = entry[0];
  693. const input = entry[1];
  694. const args = input.arguments.map((argument) => {
  695. if (argument.name) {
  696. const value = arg(argument.name);
  697. value.to.push(operation);
  698. return value;
  699. }
  700. return { value: argument.value };
  701. });
  702. return {
  703. name: key,
  704. arguments: args
  705. };
  706. });
  707. operation.outputs = op.outputs.map((output) => {
  708. const value = arg(output.name);
  709. value.type = coreml.Utility.valueType(output.type);
  710. value.from.push(operation);
  711. return {
  712. name: 'output',
  713. arguments: [ value ]
  714. };
  715. });
  716. return operation;
  717. });
  718. for (const op of operations) {
  719. if (op.type === 'const' && op.inputs.length === 0 &&
  720. op.outputs.length === 1 && op.outputs[0].arguments.length === 1) {
  721. const argument = op.outputs[0].arguments[0];
  722. if (op.attributes && op.attributes.val) {
  723. const type = argument.type;
  724. const data = op.attributes.val;
  725. if (data instanceof Uint8Array && data.length === 2 &&
  726. type.dataType === 'float16' && type.shape.dimensions.length === 0) {
  727. const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
  728. argument.value = view.getFloat16(0, true);
  729. }
  730. else {
  731. argument.value = data;
  732. }
  733. argument.const = true;
  734. op.delete = true;
  735. }
  736. }
  737. }
  738. for (const op of operations) {
  739. for (const input of op.inputs) {
  740. if (input.arguments.length > 1 && input.arguments.some((argument) => argument.const)) {
  741. if (input.arguments.every((argument) => argument.value instanceof coreml.Tensor)) {
  742. continue;
  743. }
  744. for (const argument of input.arguments) {
  745. for (const from of argument.from) {
  746. from.delete = false;
  747. }
  748. delete argument.value;
  749. }
  750. }
  751. }
  752. }
  753. for (const op of operations) {
  754. if (op.delete) {
  755. continue;
  756. }
  757. op.inputs = op.inputs.filter((input) => {
  758. if (input.arguments.every((argument) => argument.value === undefined || argument.value instanceof coreml.Tensor)) {
  759. return true;
  760. }
  761. if (input.arguments.length === 1) {
  762. const argument = input.arguments[0];
  763. op.attributes[input.name] = argument.value;
  764. return false;
  765. }
  766. op.attributes[input.name] = input.arguments.map((argument) => argument.value[0]);
  767. return false;
  768. });
  769. }
  770. const tensors = new Map();
  771. const tensor = (arg) => {
  772. if (!tensors.has(arg.name)) {
  773. tensors.set(arg.name, new coreml.Argument(arg.name, arg.type, null, arg.value));
  774. }
  775. return tensors.get(arg.name);
  776. };
  777. for (const op of operations) {
  778. if (op.delete) {
  779. continue;
  780. }
  781. op.inputs = op.inputs.map((input) => new coreml.Parameter(input.name, true, input.arguments.map((argument) => tensor(argument))));
  782. op.outputs = op.outputs.map((output) => new coreml.Parameter(output.name, true, output.arguments.map((argument) => tensor(argument))));
  783. }
  784. for (const op of operations.filter((op) => !op.delete)) {
  785. const type = 'program:' + op.type;
  786. const metadata = this._metadata.type(type);
  787. if (metadata && Array.isArray(metadata.inputs)) {
  788. let index = 1;
  789. const map = new Map(metadata.inputs.map((input) => [ input.name, index++ ]));
  790. op.inputs.sort((a, b) => (map.get(a.name) || map.size) - (map.get(b.name) || map.size));
  791. }
  792. const node = new coreml.Node(this._metadata, group, type, null, null, op.attributes, op.inputs, op.outputs);
  793. this._nodes.push(node);
  794. }
  795. return 'ML Program';
  796. }
  797. _createNode(scope, group, type, name, description, data, inputs, outputs, outputTypes) {
  798. inputs = inputs.map((input) => scope[input] ? scope[input].argument : input);
  799. outputs = outputs.map((output) => {
  800. if (scope[output]) {
  801. scope[output].counter++;
  802. const next = output + '\n' + scope[output].counter.toString(); // custom argument id
  803. scope[output].argument = next;
  804. return next;
  805. }
  806. scope[output] = {
  807. argument: output,
  808. counter: 0
  809. };
  810. return output;
  811. });
  812. const initializers = [];
  813. const attributes = {};
  814. if (data) {
  815. const map = this._initialize(type, data, initializers);
  816. for (const key of Object.keys(data)) {
  817. if (map[key]) {
  818. continue;
  819. }
  820. attributes[key] = data[key];
  821. }
  822. }
  823. const inputParameters = this._metadata.getInputs(type, inputs).map((input) => {
  824. return new coreml.Parameter(input.name, true, input.arguments.map((argument) => {
  825. return new coreml.Argument(argument.name, argument.type, null, null);
  826. }));
  827. });
  828. inputParameters.push(...initializers);
  829. const outputParameters = outputs.map((output, index) => {
  830. const name = this._metadata.getOutputName(type, index);
  831. const outputType = outputTypes ? outputTypes[index] : null;
  832. return new coreml.Parameter(name, true, [ new coreml.Argument(output, outputType, null, null) ]);
  833. });
  834. const node = new coreml.Node(this._metadata, group, type, name, description, attributes, inputParameters, outputParameters);
  835. this._nodes.push(node);
  836. return node;
  837. }
  838. _initializer(type, initializers, kind, name, shape, data) {
  839. let dataType = '?';
  840. let quantization = null;
  841. let values = null;
  842. if (data) {
  843. if (data.floatValue && data.floatValue.length > 0) {
  844. values = data.floatValue;
  845. dataType = 'float32';
  846. }
  847. else if (data.float16Value && data.float16Value.length > 0) {
  848. values = data.float16Value; // byte[]
  849. dataType = 'float16';
  850. }
  851. else if (data.rawValue && data.rawValue.length > 0) {
  852. if (data.quantization) {
  853. values = data.rawValue;
  854. dataType = 'uint' + data.quantization.numberOfBits.toString();
  855. }
  856. else {
  857. shape = [];
  858. }
  859. }
  860. quantization = data.quantization || null;
  861. }
  862. const tensorType = new coreml.TensorType(dataType, new coreml.TensorShape(shape));
  863. const tensor = new coreml.Tensor(kind, tensorType, values, quantization);
  864. const argument = new coreml.Argument('', null, null, tensor);
  865. const visible = this._metadata.visible(type, name);
  866. initializers.push(new coreml.Parameter(name, visible, [ argument ]));
  867. }
  868. _initialize(type, data, initializers) {
  869. switch (type) {
  870. case 'convolution': {
  871. const weightsShape = [ data.outputChannels, data.kernelChannels, data.kernelSize[0], data.kernelSize[1] ];
  872. if (data.isDeconvolution) {
  873. weightsShape[0] = data.kernelChannels;
  874. weightsShape[1] = Math.floor(data.outputChannels / (data.nGroups != 0 ? data.nGroups : 1));
  875. }
  876. this._initializer(type, initializers, 'Weights', 'weights', weightsShape, data.weights);
  877. if (data.hasBias) {
  878. this._initializer(type, initializers, 'Weights', 'bias', [ data.outputChannels ], data.bias);
  879. }
  880. return { 'weights': true, 'bias': data.hasBias };
  881. }
  882. case 'innerProduct':
  883. this._initializer(type, initializers, 'Weights', 'weights', [ data.outputChannels, data.inputChannels ], data.weights);
  884. if (data.hasBias) {
  885. this._initializer(type, initializers, 'Weights', 'bias', [ data.outputChannels ], data.bias);
  886. }
  887. return { 'weights': true, 'bias': data.hasBias };
  888. case 'batchnorm':
  889. this._initializer(type, initializers, 'Weights', 'gamma', [ data.channels ], data.gamma);
  890. this._initializer(type, initializers, 'Weights', 'beta', [ data.channels ], data.beta);
  891. if (data.mean) {
  892. this._initializer(type, initializers, 'Weights', 'mean', [ data.channels ], data.mean);
  893. }
  894. if (data.variance) {
  895. this._initializer(type, initializers, 'Weights', 'variance', [ data.channels ], data.variance);
  896. }
  897. return { 'gamma': true, 'beta': true, 'mean': true, 'variance': true };
  898. case 'embedding':
  899. this._initializer(type, initializers, 'Weights', 'weights', [ data.inputDim, data.outputChannels ], data.weights);
  900. return { 'weights': true };
  901. case 'loadConstant':
  902. case 'loadConstantND':
  903. this._initializer(type, initializers, 'Weights', 'data', data.shape, data.data);
  904. return { 'data': true };
  905. case 'scale':
  906. this._initializer(type, initializers, 'Weights', 'scale', data.shapeScale, data.scale);
  907. if (data.hasBias) {
  908. this._initializer(type, initializers, 'Weights', 'bias', data.shapeBias, data.bias);
  909. }
  910. return { 'scale': true, 'bias': data.hasBias };
  911. case 'bias':
  912. this._initializer(type, initializers, 'Weights', 'bias', data.shape, data.bias);
  913. return { 'bias': true };
  914. case 'simpleRecurrent':
  915. this._initializer(type, initializers, 'Weights', 'weights', [ data.outputVectorSize, data.inputVectorSize ], data.weightMatrix);
  916. this._initializer(type, initializers, 'Weights', 'recurrent', [ data.outputVectorSize, data.inputVectorSize ], data.recursionMatrix);
  917. if (data.hasBiasVectors) {
  918. this._initializer(type, initializers, 'Weights', 'bias', [ data.outputVectorSize ], data.biasVector);
  919. }
  920. return { 'weightMatrix': true, 'recursionMatrix': true, 'biasVector': data.hasBiasVectors };
  921. case 'gru': {
  922. const recursionMatrixShape = [ data.outputVectorSize, data.outputVectorSize ];
  923. const weightMatrixShape = [ data.outputVectorSize, data.inputVectorSize ];
  924. const biasVectorShape = [ data.outputVectorSize ];
  925. this._initializer(type, initializers, 'Weights', 'updateGateWeightMatrix', weightMatrixShape, data.updateGateWeightMatrix);
  926. this._initializer(type, initializers, 'Weights', 'resetGateWeightMatrix', weightMatrixShape, data.resetGateWeightMatrix);
  927. this._initializer(type, initializers, 'Weights', 'outputGateWeightMatrix', weightMatrixShape, data.outputGateWeightMatrix);
  928. this._initializer(type, initializers, 'Weights', 'updateGateRecursionMatrix', recursionMatrixShape, data.updateGateRecursionMatrix);
  929. this._initializer(type, initializers, 'Weights', 'resetGateRecursionMatrix', recursionMatrixShape, data.resetGateRecursionMatrix);
  930. this._initializer(type, initializers, 'Weights', 'outputGateRecursionMatrix', recursionMatrixShape, data.outputGateRecursionMatrix);
  931. if (data.hasBiasVectors) {
  932. this._initializer(type, initializers, 'Weights', 'updateGateBiasVector', biasVectorShape, data.updateGateBiasVector);
  933. this._initializer(type, initializers, 'Weights', 'resetGateBiasVector', biasVectorShape, data.resetGateBiasVector);
  934. this._initializer(type, initializers, 'Weights', 'outputGateBiasVector', biasVectorShape, data.outputGateBiasVector);
  935. }
  936. return {
  937. 'updateGateWeightMatrix': true, 'resetGateWeightMatrix': true, 'outputGateWeightMatrix': true,
  938. 'updateGateRecursionMatrix': true, 'resetGateRecursionMatrix': true, 'outputGateRecursionMatrix': true,
  939. 'updateGateBiasVector': data.hasBiasVectors, 'resetGateBiasVector': data.hasBiasVectors, 'outputGateBiasVector': data.hasBiasVectors
  940. };
  941. }
  942. case 'uniDirectionalLSTM':
  943. case 'biDirectionalLSTM': {
  944. const count = (type == 'uniDirectionalLSTM') ? 1 : 2;
  945. const matrixShape = [ data.outputVectorSize, data.inputVectorSize ];
  946. const vectorShape = [ data.outputVectorSize ];
  947. for (let i = 0; i < count; i++) {
  948. const weights = count == 1 ? data.weightParams : data.weightParams[i];
  949. const suffix = (i == 0) ? '' : '_rev';
  950. this._initializer(type, initializers, 'Weights', 'inputGateWeightMatrix' + suffix, matrixShape, weights.inputGateWeightMatrix);
  951. this._initializer(type, initializers, 'Weights', 'forgetGateWeightMatrix' + suffix, matrixShape, weights.forgetGateWeightMatrix);
  952. this._initializer(type, initializers, 'Weights', 'blockInputWeightMatrix' + suffix, matrixShape, weights.blockInputWeightMatrix);
  953. this._initializer(type, initializers, 'Weights', 'outputGateWeightMatrix' + suffix, matrixShape, weights.outputGateWeightMatrix);
  954. this._initializer(type, initializers, 'Weights', 'inputGateRecursionMatrix' + suffix, matrixShape, weights.inputGateRecursionMatrix);
  955. this._initializer(type, initializers, 'Weights', 'forgetGateRecursionMatrix' + suffix, matrixShape,weights.forgetGateRecursionMatrix);
  956. this._initializer(type, initializers, 'Weights', 'blockInputRecursionMatrix' + suffix, matrixShape, weights.blockInputRecursionMatrix);
  957. this._initializer(type, initializers, 'Weights', 'outputGateRecursionMatrix' + suffix, matrixShape, weights.outputGateRecursionMatrix);
  958. if (data.params.hasBiasVectors) {
  959. this._initializer(type, initializers, 'Weights', 'inputGateBiasVector' + suffix, vectorShape, weights.inputGateBiasVector);
  960. this._initializer(type, initializers, 'Weights', 'forgetGateBiasVector' + suffix, vectorShape, weights.forgetGateBiasVector);
  961. this._initializer(type, initializers, 'Weights', 'blockInputBiasVector' + suffix, vectorShape, weights.blockInputBiasVector);
  962. this._initializer(type, initializers, 'Weights', 'outputGateBiasVector' + suffix, vectorShape, weights.outputGateBiasVector);
  963. }
  964. if (data.params.hasPeepholeVectors) {
  965. this._initializer(type, initializers, 'Weights', 'inputGatePeepholeVector' + suffix, vectorShape, weights.inputGatePeepholeVector);
  966. this._initializer(type, initializers, 'Weights', 'forgetGatePeepholeVector' + suffix, vectorShape, weights.forgetGatePeepholeVector);
  967. this._initializer(type, initializers, 'Weights', 'outputGatePeepholeVector' + suffix, vectorShape, weights.outputGatePeepholeVector);
  968. }
  969. }
  970. return { 'weightParams': true };
  971. }
  972. case 'dictVectorizer':
  973. data.stringToIndex = this._convertVector(data.stringToIndex);
  974. return {};
  975. case 'wordTagger':
  976. data.modelParameterData = Array.from(data.modelParameterData);
  977. data.stringTags = this._convertVector(data.stringTags);
  978. return { tokensOutputFeatureName: true, tokenTagsOutputFeatureName: true, tokenLengthsOutputFeatureName: true, tokenLocationsOutputFeatureName: true };
  979. case 'textClassifier':
  980. data.modelParameterData = Array.from(data.modelParameterData);
  981. data.stringClassLabels = this._convertVector(data.stringClassLabels);
  982. return {};
  983. case 'nonMaximumSuppression':
  984. data.stringClassLabels = this._convertVector(data.stringClassLabels);
  985. return {};
  986. default:
  987. return {};
  988. }
  989. }
  990. _convertVector(value) {
  991. if (value && Object.keys(value).length == 1 && value.vector) {
  992. return value.vector;
  993. }
  994. return value;
  995. }
  996. static _formatFeatureDescriptionList(list) {
  997. return list.map((item) => item.name);
  998. }
  999. };
  1000. coreml.Parameter = class {
  1001. constructor(name, visible, args) {
  1002. this._name = name;
  1003. this._visible = visible;
  1004. this._arguments = args;
  1005. }
  1006. get name() {
  1007. return this._name;
  1008. }
  1009. get visible() {
  1010. return this._visible;
  1011. }
  1012. get arguments() {
  1013. return this._arguments;
  1014. }
  1015. };
  1016. coreml.Argument = class {
  1017. constructor(name, type, description, initializer) {
  1018. if (typeof name !== 'string') {
  1019. throw new coreml.Error("Invalid argument identifier '" + JSON.stringify(name) + "'.");
  1020. }
  1021. this._name = name;
  1022. this._type = type;
  1023. this._description = description || null;
  1024. this._initializer = initializer || null;
  1025. }
  1026. get name() {
  1027. return this._name;
  1028. }
  1029. set name(value) {
  1030. this._name = value;
  1031. }
  1032. get type() {
  1033. if (this._initializer) {
  1034. return this._initializer.type;
  1035. }
  1036. return this._type;
  1037. }
  1038. get description() {
  1039. return this._description;
  1040. }
  1041. get quantization() {
  1042. if (this._initializer) {
  1043. return this._initializer.quantization;
  1044. }
  1045. return null;
  1046. }
  1047. get initializer() {
  1048. return this._initializer;
  1049. }
  1050. };
  1051. coreml.Node = class {
  1052. constructor(metadata, group, type, name, description, attributes, inputs, outputs) {
  1053. if (!type) {
  1054. throw new Error('Undefined node type.');
  1055. }
  1056. if (group) {
  1057. this._group = group;
  1058. }
  1059. this._type = Object.assign({}, metadata.type(type) || { name: type });
  1060. this._type.name = type.split(':').pop();
  1061. this._name = name || '';
  1062. this._description = description || '';
  1063. this._inputs = inputs;
  1064. this._outputs = outputs;
  1065. this._attributes = [];
  1066. if (attributes) {
  1067. for (const key of Object.keys(attributes)) {
  1068. const schema = metadata.attribute(type, key);
  1069. const value = attributes[key];
  1070. const attribute = new coreml.Attribute(schema, key, value);
  1071. this._attributes.push(attribute);
  1072. }
  1073. }
  1074. }
  1075. get type() {
  1076. return this._type;
  1077. }
  1078. get name() {
  1079. return this._name;
  1080. }
  1081. get description() {
  1082. return this._description;
  1083. }
  1084. get metadata() {
  1085. return this._metadata;
  1086. }
  1087. get group() {
  1088. return this._group ? this._group : null;
  1089. }
  1090. get inputs() {
  1091. return this._inputs;
  1092. }
  1093. get outputs() {
  1094. return this._outputs;
  1095. }
  1096. get attributes() {
  1097. return this._attributes;
  1098. }
  1099. };
  1100. coreml.Attribute = class {
  1101. constructor(metadata, name, value) {
  1102. this._name = name;
  1103. this._value = value;
  1104. if (this._value instanceof coreml.Tensor) {
  1105. this._type = 'tensor';
  1106. }
  1107. if (metadata) {
  1108. if (metadata.type) {
  1109. this._type = metadata.type;
  1110. }
  1111. if (this._type && coreml.proto) {
  1112. this._value = coreml.Utility.enum(this._type, this._value);
  1113. }
  1114. if (Object.prototype.hasOwnProperty.call(metadata, 'visible') && !metadata.visible) {
  1115. this._visible = false;
  1116. }
  1117. else if (Object.prototype.hasOwnProperty.call(metadata, 'default')) {
  1118. if (Array.isArray(value)) {
  1119. value = value.map((item) => item.toNumber());
  1120. }
  1121. if (JSON.stringify(metadata.default) == JSON.stringify(value)) {
  1122. this._visible = false;
  1123. }
  1124. }
  1125. }
  1126. }
  1127. get name() {
  1128. return this._name;
  1129. }
  1130. get type() {
  1131. return this._type;
  1132. }
  1133. get value() {
  1134. return this._value;
  1135. }
  1136. get visible() {
  1137. return this._visible == false ? false : true;
  1138. }
  1139. };
  1140. coreml.Tensor = class {
  1141. constructor(kind, type, data, quantization) {
  1142. this._kind = kind;
  1143. this._type = type;
  1144. this._data = data;
  1145. this._quantization = quantization;
  1146. }
  1147. get kind() {
  1148. return this._kind;
  1149. }
  1150. get type() {
  1151. return this._type;
  1152. }
  1153. get quantization() {
  1154. if (this._quantization) {
  1155. if (this._quantization.lookupTableQuantization &&
  1156. this._quantization.lookupTableQuantization.floatValue &&
  1157. this._quantization.lookupTableQuantization.floatValue.length > 0) {
  1158. const map = [];
  1159. for (const key of Object.keys(this._quantization.lookupTableQuantization.floatValue)) {
  1160. map.push(key.toString() + ' = ' + this._quantization.lookupTableQuantization.floatValue[key].toString());
  1161. }
  1162. return map.join('; ');
  1163. }
  1164. return '?';
  1165. }
  1166. return null;
  1167. }
  1168. get state() {
  1169. return this._context().state;
  1170. }
  1171. get value() {
  1172. const context = this._context();
  1173. if (context.state) {
  1174. return null;
  1175. }
  1176. context.limit = Number.MAX_SAFE_INTEGER;
  1177. return this._decode(context, 0);
  1178. }
  1179. toString() {
  1180. const context = this._context();
  1181. if (context.state) {
  1182. return '';
  1183. }
  1184. context.limit = 10000;
  1185. const value = this._decode(context, 0);
  1186. return JSON.stringify(value, null, 4);
  1187. }
  1188. _context() {
  1189. const context = {};
  1190. context.state = null;
  1191. context.index = 0;
  1192. context.count = 0;
  1193. context.dataType = this._type.dataType;
  1194. context.dimensions = this._type.shape.dimensions;
  1195. if (!this._data) {
  1196. context.state = 'Tensor data is empty.';
  1197. return context;
  1198. }
  1199. switch (context.dataType) {
  1200. case 'float32':
  1201. context.data = this._data;
  1202. break;
  1203. case 'float16':
  1204. context.data = new DataView(this._data.buffer, this._data.byteOffset, this._data.byteLength);
  1205. break;
  1206. case 'uint8':
  1207. context.data = new DataView(this._data.buffer, this._data.byteOffset, this._data.byteLength);
  1208. break;
  1209. default:
  1210. if (this._quantization) {
  1211. context.dataType = 'quantization';
  1212. context.bits = this._quantization.numberOfBits.toNumber();
  1213. context.data = new DataView(this._data.buffer, this._data.byteOffset, this._data.byteLength);
  1214. }
  1215. else {
  1216. context.state = 'Tensor data type is not implemented.';
  1217. }
  1218. break;
  1219. }
  1220. return context;
  1221. }
  1222. _decode(context, dimension) {
  1223. const results = [];
  1224. const size = context.dimensions[dimension];
  1225. if (dimension == context.dimensions.length - 1) {
  1226. for (let i = 0; i < size; i++) {
  1227. if (context.count > context.limit) {
  1228. results.push('...');
  1229. return results;
  1230. }
  1231. switch (context.dataType) {
  1232. case 'float32':
  1233. results.push(this._data[context.index]);
  1234. context.index++;
  1235. break;
  1236. case 'float16':
  1237. results.push(context.data.getFloat16(context.index, true));
  1238. context.index += 2;
  1239. break;
  1240. case 'uint8':
  1241. results.push(context.data.getUint8(context.index, true));
  1242. context.index += 1;
  1243. break;
  1244. case 'quantization':
  1245. results.push(context.data.getBits(context.index, context.bits));
  1246. context.index++;
  1247. break;
  1248. default:
  1249. break;
  1250. }
  1251. context.count++;
  1252. }
  1253. }
  1254. else {
  1255. for (let j = 0; j < size; j++) {
  1256. if (context.count > context.limit) {
  1257. results.push('...');
  1258. return results;
  1259. }
  1260. results.push(this._decode(context, dimension + 1));
  1261. }
  1262. }
  1263. return results;
  1264. }
  1265. };
  1266. coreml.TensorType = class {
  1267. constructor(dataType, shape) {
  1268. this._dataType = dataType;
  1269. this._shape = shape || new coreml.TensorShape([]);
  1270. }
  1271. get dataType() {
  1272. return this._dataType;
  1273. }
  1274. get shape() {
  1275. return this._shape;
  1276. }
  1277. toString() {
  1278. return this.dataType + this._shape.toString();
  1279. }
  1280. };
  1281. coreml.TensorShape = class {
  1282. constructor(dimensions) {
  1283. this._dimensions = dimensions;
  1284. }
  1285. get dimensions() {
  1286. return this._dimensions;
  1287. }
  1288. toString() {
  1289. if (!this._dimensions || this._dimensions.length == 0) {
  1290. return '';
  1291. }
  1292. return '[' + this._dimensions.map((dimension) => dimension.toString()).join(',') + ']';
  1293. }
  1294. };
  1295. coreml.ListType = class {
  1296. constructor(elementType) {
  1297. this._elementType = elementType;
  1298. }
  1299. toString() {
  1300. return 'list<' + this._elementType.toString() + '>';
  1301. }
  1302. };
  1303. coreml.MapType = class {
  1304. constructor(keyType, valueType) {
  1305. this._keyType = keyType;
  1306. this._valueType = valueType;
  1307. }
  1308. get keyType() {
  1309. return this._keyType;
  1310. }
  1311. get valueType() {
  1312. return this._valueType;
  1313. }
  1314. toString() {
  1315. return 'map<' + this._keyType + ',' + this._valueType.toString() + '>';
  1316. }
  1317. };
  1318. coreml.SequenceType = class {
  1319. constructor(type) {
  1320. this._type = type;
  1321. }
  1322. get type() {
  1323. return this._type;
  1324. }
  1325. toString() {
  1326. return 'sequence<' + this._type + '>';
  1327. }
  1328. };
  1329. coreml.ImageType = class {
  1330. constructor(colorSpace, width, height) {
  1331. this._width = width;
  1332. this._height = height;
  1333. switch (colorSpace) {
  1334. case coreml.proto.ImageFeatureType.ColorSpace.GRAYSCALE:
  1335. this._colorSpace = 'grayscale';
  1336. break;
  1337. case coreml.proto.ImageFeatureType.ColorSpace.RGB:
  1338. this._colorSpace = 'RGB';
  1339. break;
  1340. case coreml.proto.ImageFeatureType.ColorSpace.BGR:
  1341. this._colorSpace = 'BGR';
  1342. break;
  1343. case coreml.proto.ImageFeatureType.ColorSpace.GRAYSCALE_FLOAT16:
  1344. this._colorSpace = 'grayscale:float16';
  1345. break;
  1346. default:
  1347. throw new coreml.Error("Unsupported image color space '" + colorSpace + "'.");
  1348. }
  1349. }
  1350. toString() {
  1351. return 'image<' + this._colorSpace + ',' + this._width. toString() + 'x' + this._height.toString() + '>';
  1352. }
  1353. };
  1354. coreml.OptionalType = class {
  1355. constructor(type) {
  1356. this._type = type;
  1357. }
  1358. get type() {
  1359. return this._type;
  1360. }
  1361. toString() {
  1362. return 'optional<' + this._type.toString() + '>';
  1363. }
  1364. };
  1365. coreml.Utility = class {
  1366. static enum(name, value) {
  1367. let type = coreml.proto;
  1368. const parts = name.split('.');
  1369. while (type && parts.length > 0) {
  1370. type = type[parts.shift()];
  1371. }
  1372. if (type) {
  1373. coreml.Utility._enumKeyMap = coreml.Utility._enumKeyMap || new Map();
  1374. if (!coreml.Utility._enumKeyMap.has(name)) {
  1375. const map = new Map(Object.entries(type).map((pair) => [ pair[1], pair[0] ]));
  1376. coreml.Utility._enumKeyMap.set(name, map);
  1377. }
  1378. const map = coreml.Utility._enumKeyMap.get(name);
  1379. if (map.has(value)) {
  1380. return map.get(value);
  1381. }
  1382. }
  1383. return value;
  1384. }
  1385. static featureType(type) {
  1386. let result = '?';
  1387. if (type) {
  1388. switch (type.Type) {
  1389. case 'multiArrayType': {
  1390. let shape = new coreml.TensorShape([]);
  1391. if (type.multiArrayType.shape && type.multiArrayType.shape.length > 0) {
  1392. shape = new coreml.TensorShape(type.multiArrayType.shape);
  1393. }
  1394. let dataType = '?';
  1395. const ArrayDataType = coreml.proto.ArrayFeatureType.ArrayDataType;
  1396. switch (type.multiArrayType.dataType) {
  1397. case ArrayDataType.INVALID_ARRAY_DATA_TYPE:
  1398. dataType = '?';
  1399. break;
  1400. case ArrayDataType.FLOAT16:
  1401. dataType = 'float16';
  1402. break;
  1403. case ArrayDataType.FLOAT32:
  1404. dataType = 'float32';
  1405. break;
  1406. case ArrayDataType.DOUBLE:
  1407. dataType = 'float64';
  1408. break;
  1409. case ArrayDataType.INT32:
  1410. dataType = 'int32';
  1411. break;
  1412. default:
  1413. throw new coreml.Error("Unsupported array data type '" + type.multiArrayType.dataType + "'.");
  1414. }
  1415. result = new coreml.TensorType(dataType, shape);
  1416. break;
  1417. }
  1418. case 'stringType': {
  1419. result = new coreml.TensorType('string');
  1420. break;
  1421. }
  1422. case 'doubleType': {
  1423. result = new coreml.TensorType('float64');
  1424. break;
  1425. }
  1426. case 'int64Type': {
  1427. result = new coreml.TensorType('int64');
  1428. break;
  1429. }
  1430. case 'dictionaryType': {
  1431. result = new coreml.MapType(type.dictionaryType.KeyType.replace('KeyType', ''), 'float64');
  1432. break;
  1433. }
  1434. case 'sequenceType': {
  1435. result = new coreml.SequenceType(coreml.Utility.featureType(type[type.Type]));
  1436. break;
  1437. }
  1438. case 'imageType': {
  1439. result = new coreml.ImageType(type.imageType.colorSpace, type.imageType.width, type.imageType.height);
  1440. break;
  1441. }
  1442. default: {
  1443. throw new coreml.Error("Unsupported feature type '" + type.Type + "'.");
  1444. }
  1445. }
  1446. if (type.isOptional) {
  1447. result = new coreml.OptionalType(result);
  1448. }
  1449. }
  1450. return result;
  1451. }
  1452. static tensorType(type) {
  1453. if (!coreml.Utility._dataTypes) {
  1454. coreml.Utility._dataTypes = new Map();
  1455. const DataType = coreml.proto.MILSpec.DataType;
  1456. for (const pair of Object.entries(DataType)) {
  1457. if (pair[0] === 'UNUSED_TYPE') {
  1458. continue;
  1459. }
  1460. const name = pair[0] === 'bool' ? 'boolean' : pair[0].toLowerCase();
  1461. coreml.Utility._dataTypes.set(pair[1], name);
  1462. }
  1463. }
  1464. const shape = (type.dimensions.map(dim => dim.constant ? dim.constant.size : '?'));
  1465. const dataType = coreml.Utility._dataTypes.get(type.dataType);
  1466. if (dataType === null) {
  1467. throw new coreml.Error("Unsupported data type '" + type.dataType + "'.");
  1468. }
  1469. return new coreml.TensorType(dataType, new coreml.TensorShape(shape));
  1470. }
  1471. static valueType(type) {
  1472. switch (type.type) {
  1473. case 'tensorType':
  1474. return coreml.Utility.tensorType(type.tensorType);
  1475. case 'listType':
  1476. return new coreml.ListType(coreml.Utility.valueType(type.listType.type));
  1477. default:
  1478. throw new coreml.Error("Unsupported value type '" + type.type + "'.");
  1479. }
  1480. }
  1481. };
  1482. coreml.Metadata = class {
  1483. static open(context) {
  1484. if (coreml.Metadata._metadata) {
  1485. return Promise.resolve(coreml.Metadata._metadata);
  1486. }
  1487. return context.request('coreml-metadata.json', 'utf-8', null).then((data) => {
  1488. coreml.Metadata._metadata = new coreml.Metadata(data);
  1489. return coreml.Metadata._metadata;
  1490. }).catch(() => {
  1491. coreml.Metadata._metadata = new coreml.Metadata(null);
  1492. return coreml.Metadata._metadata;
  1493. });
  1494. }
  1495. constructor(data) {
  1496. this._map = new Map();
  1497. this._attributeCache = new Map();
  1498. this._inputCache = new Map();
  1499. if (data) {
  1500. const metadata = JSON.parse(data);
  1501. this._map = new Map(metadata.map((item) => [ item.name, item ]));
  1502. }
  1503. }
  1504. type(name) {
  1505. return this._map.get(name);
  1506. }
  1507. attribute(type, name) {
  1508. const key = type + ':' + name;
  1509. if (!this._attributeCache.has(key)) {
  1510. this._attributeCache.set(key, null);
  1511. const metadata = this.type(type);
  1512. if (metadata && Array.isArray(metadata.attributes) && metadata.attributes.length > 0) {
  1513. for (const attribute of metadata.attributes) {
  1514. this._attributeCache.set(type + ':' + attribute.name, attribute);
  1515. }
  1516. }
  1517. }
  1518. return this._attributeCache.get(key);
  1519. }
  1520. visible(type, name) {
  1521. const key = type + ':' + name;
  1522. if (!this._inputCache.has(key)) {
  1523. this._inputCache.set(key, null);
  1524. const metadata = this.type(type);
  1525. if (metadata && Array.isArray(metadata.inputs) && metadata.inputs.length > 0) {
  1526. for (const input of metadata.inputs) {
  1527. this._inputCache.set(type + ':' + input.name, input);
  1528. }
  1529. }
  1530. }
  1531. const input = this._inputCache.get(key);
  1532. if (input) {
  1533. return input.visible === false ? false : true;
  1534. }
  1535. return true;
  1536. }
  1537. getInputs(type, inputs) {
  1538. const results = [];
  1539. const schema = this._map.get(type);
  1540. let index = 0;
  1541. while (index < inputs.length) {
  1542. const result = { arguments: [] };
  1543. let count = 1;
  1544. let name = null;
  1545. if (schema && schema.inputs) {
  1546. if (index < schema.inputs.length) {
  1547. const input = schema.inputs[index];
  1548. name = input.name;
  1549. if (schema.inputs[index].option == 'variadic') {
  1550. count = inputs.length - index;
  1551. }
  1552. }
  1553. }
  1554. else if (index == 0) {
  1555. name = 'input';
  1556. }
  1557. result.name = name ? name : '(' + index.toString() + ')';
  1558. const array = inputs.slice(index, index + count);
  1559. for (let j = 0; j < array.length; j++) {
  1560. result.arguments.push({ name: array[j] });
  1561. }
  1562. index += count;
  1563. results.push(result);
  1564. }
  1565. return results;
  1566. }
  1567. getOutputName(type, index) {
  1568. const schema = this._map.get(type);
  1569. if (schema) {
  1570. const outputs = schema.outputs;
  1571. if (outputs && index < outputs.length) {
  1572. const output = outputs[index];
  1573. if (output) {
  1574. const name = output.name;
  1575. if (name) {
  1576. return name;
  1577. }
  1578. }
  1579. }
  1580. }
  1581. if (index == 0) {
  1582. return 'output';
  1583. }
  1584. return '(' + index.toString() + ')';
  1585. }
  1586. };
  1587. coreml.Error = class extends Error {
  1588. constructor(message) {
  1589. super(message);
  1590. this.name = 'Error loading Core ML model.';
  1591. }
  1592. };
  1593. if (typeof module !== 'undefined' && typeof module.exports === 'object') {
  1594. module.exports.ModelFactory = coreml.ModelFactory;
  1595. }