mlnet.js 76 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497
  1. // Experimental
  2. var mlnet = mlnet || {};
  3. var base = base || require('./base');
  4. var zip = zip || require('./zip');
  5. mlnet.ModelFactory = class {
  6. match(context) {
  7. const entries = context.entries('zip');
  8. if (entries.size > 0) {
  9. const root = new Set([ 'TransformerChain', 'Predictor']);
  10. if (Array.from(entries.keys()).some((name) => root.has(name.split('\\').shift().split('/').shift()))) {
  11. return 'mlnet';
  12. }
  13. }
  14. return undefined;
  15. }
  16. open(context) {
  17. return context.metadata('mlnet-metadata.json').then((metadata) => {
  18. const entries = context.entries('zip');
  19. const reader = new mlnet.ModelReader(entries);
  20. return new mlnet.Model(metadata, reader);
  21. });
  22. }
  23. };
  24. mlnet.Model = class {
  25. constructor(metadata, reader) {
  26. this._format = "ML.NET";
  27. if (reader.version && reader.version.length > 0) {
  28. this._format += ' v' + reader.version;
  29. }
  30. this._graphs = [];
  31. this._graphs.push(new mlnet.Graph(metadata, reader));
  32. }
  33. get format() {
  34. return this._format;
  35. }
  36. get graphs() {
  37. return this._graphs;
  38. }
  39. };
  40. mlnet.Graph = class {
  41. constructor(metadata, reader) {
  42. this._inputs = [];
  43. this._outputs = [];
  44. this._nodes = [];
  45. this._groups = false;
  46. if (reader.schema && reader.schema.inputs) {
  47. for (const input of reader.schema.inputs) {
  48. this._inputs.push(new mlnet.Parameter(input.name, [
  49. new mlnet.Argument(input.name, new mlnet.TensorType(input.type))
  50. ]));
  51. }
  52. }
  53. const scope = new Map();
  54. if (reader.dataLoaderModel) {
  55. this._loadTransformer(metadata, scope, '', reader.dataLoaderModel);
  56. }
  57. if (reader.predictor) {
  58. this._loadTransformer(metadata, scope, '', reader.predictor);
  59. }
  60. if (reader.transformerChain) {
  61. this._loadTransformer(metadata, scope, '', reader.transformerChain);
  62. }
  63. }
  64. _loadTransformer(metadata, scope, group, transformer) {
  65. switch (transformer.__type__) {
  66. case 'TransformerChain':
  67. case 'Text':
  68. this._loadChain(metadata, scope, transformer.__name__, transformer.chain);
  69. break;
  70. default:
  71. this._createNode(metadata, scope, group, transformer);
  72. break;
  73. }
  74. }
  75. _loadChain(metadata, scope, name, chain) {
  76. this._groups = true;
  77. const group = name.split('/').splice(1).join('/');
  78. for (const childTransformer of chain) {
  79. this._loadTransformer(metadata, scope, group, childTransformer);
  80. }
  81. }
  82. _createNode(metadata, scope, group, transformer) {
  83. if (transformer.inputs && transformer.outputs) {
  84. for (const input of transformer.inputs) {
  85. input.name = scope[input.name] ? scope[input.name].argument : input.name;
  86. }
  87. for (const output of transformer.outputs) {
  88. if (scope[output.name]) {
  89. scope[output.name].counter++;
  90. const next = output.name + '\n' + scope[output.name].counter.toString(); // custom argument id
  91. scope[output.name].argument = next;
  92. output.name = next;
  93. }
  94. else {
  95. scope[output.name] = {
  96. argument: output.name,
  97. counter: 0
  98. };
  99. }
  100. }
  101. }
  102. this._nodes.push(new mlnet.Node(metadata, group, transformer));
  103. }
  104. get groups() {
  105. return this._groups;
  106. }
  107. get inputs() {
  108. return this._inputs;
  109. }
  110. get outputs() {
  111. return this._outputs;
  112. }
  113. get nodes() {
  114. return this._nodes;
  115. }
  116. };
  117. mlnet.Parameter = class {
  118. constructor(name, args) {
  119. this._name = name;
  120. this._arguments = args;
  121. }
  122. get name() {
  123. return this._name;
  124. }
  125. get visible() {
  126. return true;
  127. }
  128. get arguments() {
  129. return this._arguments;
  130. }
  131. };
  132. mlnet.Argument = class {
  133. constructor(name, type) {
  134. if (typeof name !== 'string') {
  135. throw new mlnet.Error("Invalid argument identifier '" + JSON.stringify(name) + "'.");
  136. }
  137. this._name = name;
  138. this._type = type;
  139. }
  140. get name() {
  141. return this._name;
  142. }
  143. get type() {
  144. return this._type;
  145. }
  146. };
  147. mlnet.Node = class {
  148. constructor(metadata, group, transformer) {
  149. this._metadata = metadata;
  150. this._group = group;
  151. this._name = transformer.__name__;
  152. this._inputs = [];
  153. this._outputs = [];
  154. this._attributes = [];
  155. const type = transformer.__type__;
  156. this._type = metadata.type(type) || { name: type };
  157. if (transformer.inputs) {
  158. let i = 0;
  159. for (const input of transformer.inputs) {
  160. this._inputs.push(new mlnet.Parameter(i.toString(), [
  161. new mlnet.Argument(input.name)
  162. ]));
  163. i++;
  164. }
  165. }
  166. if (transformer.outputs) {
  167. let i = 0;
  168. for (const output of transformer.outputs) {
  169. this._outputs.push(new mlnet.Parameter(i.toString(), [
  170. new mlnet.Argument(output.name)
  171. ]));
  172. i++;
  173. }
  174. }
  175. for (const key of Object.keys(transformer).filter((key) => !key.startsWith('_') && key !== 'inputs' && key !== 'outputs')) {
  176. const schema = metadata.attribute(type, this._name);
  177. this._attributes.push(new mlnet.Attribute(schema, key, transformer[key]));
  178. }
  179. }
  180. get group() {
  181. return this._group;
  182. }
  183. get type() {
  184. return this._type;
  185. }
  186. get name() {
  187. return this._name;
  188. }
  189. get inputs() {
  190. return this._inputs;
  191. }
  192. get outputs() {
  193. return this._outputs;
  194. }
  195. get attributes() {
  196. return this._attributes;
  197. }
  198. };
  199. mlnet.Attribute = class {
  200. constructor(schema, name, value) {
  201. this._name = name;
  202. this._value = value;
  203. if (schema) {
  204. if (schema.type) {
  205. this._type = schema.type;
  206. }
  207. if (this._type) {
  208. let type = mlnet;
  209. const id = this._type.split('.');
  210. while (type && id.length > 0) {
  211. type = type[id.shift()];
  212. }
  213. if (type) {
  214. mlnet.Attribute._reverseMap = mlnet.Attribute._reverseMap || {};
  215. let reverse = mlnet.Attribute._reverseMap[this._type];
  216. if (!reverse) {
  217. reverse = {};
  218. for (const key of Object.keys(type)) {
  219. reverse[type[key.toString()]] = key;
  220. }
  221. mlnet.Attribute._reverseMap[this._type] = reverse;
  222. }
  223. if (Object.prototype.hasOwnProperty.call(reverse, this._value)) {
  224. this._value = reverse[this._value];
  225. }
  226. }
  227. }
  228. }
  229. }
  230. get type() {
  231. return this._type;
  232. }
  233. get name() {
  234. return this._name;
  235. }
  236. get value() {
  237. return this._value;
  238. }
  239. get visible() {
  240. return true;
  241. }
  242. };
  243. mlnet.TensorType = class {
  244. constructor(codec) {
  245. mlnet.TensorType._map = mlnet.TensorType._map || new Map([
  246. [ 'Byte', 'uint8' ],
  247. [ 'Boolean', 'boolean' ],
  248. [ 'Single', 'float32' ],
  249. [ 'Double', 'float64' ],
  250. [ 'UInt32', 'uint32' ],
  251. [ 'TextSpan', 'string' ]
  252. ]);
  253. this._dataType = '?';
  254. this._shape = new mlnet.TensorShape(null);
  255. if (mlnet.TensorType._map.has(codec.name)) {
  256. this._dataType = mlnet.TensorType._map.get(codec.name);
  257. }
  258. else if (codec.name == 'VBuffer') {
  259. if (mlnet.TensorType._map.has(codec.itemType.name)) {
  260. this._dataType = mlnet.TensorType._map.get(codec.itemType.name);
  261. }
  262. else {
  263. throw new mlnet.Error("Unsupported data type '" + codec.itemType.name + "'.");
  264. }
  265. this._shape = new mlnet.TensorShape(codec.dims);
  266. }
  267. else if (codec.name == 'Key2') {
  268. this._dataType = 'key2';
  269. }
  270. else {
  271. throw new mlnet.Error("Unsupported data type '" + codec.name + "'.");
  272. }
  273. }
  274. get dataType() {
  275. return this._dataType;
  276. }
  277. get shape() {
  278. return this._shape;
  279. }
  280. toString() {
  281. return this.dataType + this._shape.toString();
  282. }
  283. };
  284. mlnet.TensorShape = class {
  285. constructor(dimensions) {
  286. this._dimensions = dimensions;
  287. }
  288. get dimensions() {
  289. return this._dimensions;
  290. }
  291. toString() {
  292. if (!this._dimensions || this._dimensions.length == 0) {
  293. return '';
  294. }
  295. return '[' + this._dimensions.join(',') + ']';
  296. }
  297. };
  298. mlnet.ModelReader = class {
  299. constructor(entries) {
  300. const catalog = new mlnet.ComponentCatalog();
  301. catalog.register('AffineNormExec', mlnet.AffineNormSerializationUtils);
  302. catalog.register('AnomalyPredXfer', mlnet.AnomalyPredictionTransformer);
  303. catalog.register('BinaryPredXfer', mlnet.BinaryPredictionTransformer);
  304. catalog.register('BinaryLoader', mlnet.BinaryLoader);
  305. catalog.register('CaliPredExec', mlnet.CalibratedPredictor);
  306. catalog.register('CdfNormalizeFunction', mlnet.CdfColumnFunction);
  307. catalog.register('CharToken', mlnet.TokenizingByCharactersTransformer);
  308. catalog.register('ChooseColumnsTransform', mlnet.ColumnSelectingTransformer);
  309. catalog.register('ClusteringPredXfer', mlnet.ClusteringPredictionTransformer);
  310. catalog.register('ConcatTransform', mlnet.ColumnConcatenatingTransformer);
  311. catalog.register('CopyTransform', mlnet.ColumnCopyingTransformer);
  312. catalog.register('ConvertTransform', mlnet.TypeConvertingTransformer);
  313. catalog.register('CSharpTransform', mlnet.CSharpTransform);
  314. catalog.register('DropColumnsTransform', mlnet.DropColumnsTransform);
  315. catalog.register('FAFMPredXfer', mlnet.FieldAwareFactorizationMachinePredictionTransformer);
  316. catalog.register('FastForestBinaryExec', mlnet.FastForestClassificationPredictor);
  317. catalog.register('FastTreeBinaryExec', mlnet.FastTreeBinaryModelParameters);
  318. catalog.register('FastTreeTweedieExec', mlnet.FastTreeTweedieModelParameters);
  319. catalog.register('FastTreeRankerExec', mlnet.FastTreeRankingModelParameters);
  320. catalog.register('FastTreeRegressionExec', mlnet.FastTreeRegressionModelParameters);
  321. catalog.register('FeatWCaliPredExec', mlnet.FeatureWeightsCalibratedModelParameters);
  322. catalog.register('FieldAwareFactMacPredict', mlnet.FieldAwareFactorizationMachineModelParameters);
  323. catalog.register('GcnTransform', mlnet.LpNormNormalizingTransformer);
  324. catalog.register('GenericScoreTransform', mlnet.GenericScoreTransform);
  325. catalog.register('IidChangePointDetector', mlnet.IidChangePointDetector);
  326. catalog.register('IidSpikeDetector', mlnet.IidSpikeDetector);
  327. catalog.register('ImageClassificationTrans', mlnet.ImageClassificationTransformer);
  328. catalog.register('ImageClassificationPred', mlnet.ImageClassificationModelParameters);
  329. catalog.register('ImageLoaderTransform', mlnet.ImageLoadingTransformer);
  330. catalog.register('ImageScalerTransform', mlnet.ImageResizingTransformer);
  331. catalog.register('ImagePixelExtractor', mlnet.ImagePixelExtractingTransformer);
  332. catalog.register('KeyToValueTransform', mlnet.KeyToValueMappingTransformer);
  333. catalog.register('KeyToVectorTransform', mlnet.KeyToVectorMappingTransformer);
  334. catalog.register('KMeansPredictor', mlnet.KMeansModelParameters);
  335. catalog.register('LinearRegressionExec', mlnet.LinearRegressionModelParameters);
  336. catalog.register('LightGBMRegressionExec', mlnet.LightGbmRegressionModelParameters);
  337. catalog.register('LightGBMBinaryExec', mlnet.LightGbmBinaryModelParameters);
  338. catalog.register('Linear2CExec', mlnet.LinearBinaryModelParameters);
  339. catalog.register('LinearModelStats', mlnet.LinearModelParameterStatistics);
  340. catalog.register('MaFactPredXf', mlnet.MatrixFactorizationPredictionTransformer);
  341. catalog.register('MFPredictor', mlnet.MatrixFactorizationModelParameters);
  342. catalog.register('MulticlassLinear', mlnet.LinearMulticlassModelParameters);
  343. catalog.register('MultiClassLRExec', mlnet.MaximumEntropyModelParameters);
  344. catalog.register('MultiClassNaiveBayesPred', mlnet.NaiveBayesMulticlassModelParameters);
  345. catalog.register('MultiClassNetPredictor', mlnet.MultiClassNetPredictor);
  346. catalog.register('MulticlassPredXfer', mlnet.MulticlassPredictionTransformer);
  347. catalog.register('NAReplaceTransform', mlnet.MissingValueReplacingTransformer);
  348. catalog.register('NgramTransform', mlnet.NgramExtractingTransformer);
  349. catalog.register('NgramHashTransform', mlnet.NgramHashingTransformer);
  350. catalog.register('NltTokenizeTransform', mlnet.NltTokenizeTransform);
  351. catalog.register('Normalizer', mlnet.NormalizingTransformer);
  352. catalog.register('NormalizeTransform', mlnet.NormalizeTransform);
  353. catalog.register('OnnxTransform', mlnet.OnnxTransformer);
  354. catalog.register('OptColTransform', mlnet.OptionalColumnTransform);
  355. catalog.register('OVAExec', mlnet.OneVersusAllModelParameters);
  356. catalog.register('pcaAnomExec', mlnet.PcaModelParameters);
  357. catalog.register('PcaTransform', mlnet.PrincipalComponentAnalysisTransformer);
  358. catalog.register('PipeDataLoader', mlnet.CompositeDataLoader);
  359. catalog.register('PlattCaliExec', mlnet.PlattCalibrator);
  360. catalog.register('PMixCaliPredExec', mlnet.ParameterMixingCalibratedModelParameters);
  361. catalog.register('PoissonRegressionExec', mlnet.PoissonRegressionModelParameters);
  362. catalog.register('ProtonNNMCPred', mlnet.ProtonNNMCPred);
  363. catalog.register('RegressionPredXfer', mlnet.RegressionPredictionTransformer);
  364. catalog.register('RowToRowMapper', mlnet.RowToRowMapperTransform);
  365. catalog.register('SsaForecasting', mlnet.SsaForecastingTransformer);
  366. catalog.register('SSAModel', mlnet.AdaptiveSingularSpectrumSequenceModelerInternal);
  367. catalog.register('SelectColumnsTransform', mlnet.ColumnSelectingTransformer);
  368. catalog.register('StopWordsTransform', mlnet.StopWordsTransform);
  369. catalog.register('TensorFlowTransform', mlnet.TensorFlowTransformer);
  370. catalog.register('TermLookupTransform', mlnet.ValueMappingTransformer);
  371. catalog.register('TermTransform', mlnet.ValueToKeyMappingTransformer);
  372. catalog.register('TermManager', mlnet.TermManager);
  373. catalog.register('Text', mlnet.TextFeaturizingEstimator);
  374. catalog.register('TextLoader', mlnet.TextLoader);
  375. catalog.register('TextNormalizerTransform', mlnet.TextNormalizingTransformer);
  376. catalog.register('TokenizeTextTransform', mlnet.WordTokenizingTransformer);
  377. catalog.register('TransformerChain', mlnet.TransformerChain);
  378. catalog.register('ValueMappingTransformer', mlnet.ValueMappingTransformer);
  379. catalog.register('XGBoostMulticlass', mlnet.XGBoostMulticlass);
  380. const root = new mlnet.ModelHeader(catalog, entries, '', null);
  381. const version = root.openText('TrainingInfo/Version.txt');
  382. if (version) {
  383. this.version = version.split(' ').shift().split('\r').shift();
  384. }
  385. const schemaReader = root.openBinary('Schema');
  386. if (schemaReader) {
  387. this.schema = new mlnet.BinaryLoader(null, schemaReader).schema;
  388. }
  389. const transformerChain = root.open('TransformerChain');
  390. if (transformerChain) {
  391. this.transformerChain = transformerChain;
  392. }
  393. const dataLoaderModel = root.open('DataLoaderModel');
  394. if (dataLoaderModel) {
  395. this.dataLoaderModel = dataLoaderModel;
  396. }
  397. const predictor = root.open('Predictor');
  398. if (predictor) {
  399. this.predictor = predictor;
  400. }
  401. }
  402. };
  403. mlnet.ComponentCatalog = class {
  404. constructor() {
  405. this._map = new Map();
  406. }
  407. register(signature, type) {
  408. this._map.set(signature, type);
  409. }
  410. create(signature, context) {
  411. if (!this._map.has(signature)) {
  412. throw new mlnet.Error("Unsupported loader signature '" + signature + "'.");
  413. }
  414. const type = this._map.get(signature);
  415. return Reflect.construct(type, [ context ]);
  416. }
  417. };
  418. mlnet.ModelHeader = class {
  419. constructor(catalog, entries, directory, data) {
  420. this._entries = entries;
  421. this._catalog = catalog;
  422. this._directory = directory;
  423. if (data) {
  424. const reader = new mlnet.BinaryReader(data);
  425. const decoder = new TextDecoder('ascii');
  426. reader.assert('ML\0MODEL');
  427. this.versionWritten = reader.uint32();
  428. this.versionReadable = reader.uint32();
  429. const modelBlockOffset = reader.uint64();
  430. /* let modelBlockSize = */ reader.uint64();
  431. const stringTableOffset = reader.uint64();
  432. const stringTableSize = reader.uint64();
  433. const stringCharsOffset = reader.uint64();
  434. /* v stringCharsSize = */ reader.uint64();
  435. this.modelSignature = decoder.decode(reader.read(8));
  436. this.modelVersionWritten = reader.uint32();
  437. this.modelVersionReadable = reader.uint32();
  438. this.loaderSignature = decoder.decode(reader.read(24).filter((c) => c != 0));
  439. this.loaderSignatureAlt = decoder.decode(reader.read(24).filter((c) => c != 0));
  440. const tailOffset = reader.uint64();
  441. /* let tailLimit = */ reader.uint64();
  442. const assemblyNameOffset = reader.uint64();
  443. const assemblyNameSize = reader.uint32();
  444. if (stringTableOffset != 0 && stringCharsOffset != 0) {
  445. reader.seek(stringTableOffset);
  446. const stringCount = stringTableSize >> 3;
  447. const stringSizes = [];
  448. let previousStringSize = 0;
  449. for (let i = 0; i < stringCount; i++) {
  450. const stringSize = reader.uint64();
  451. stringSizes.push(stringSize - previousStringSize);
  452. previousStringSize = stringSize;
  453. }
  454. reader.seek(stringCharsOffset);
  455. this.strings = [];
  456. for (let i = 0; i < stringCount; i++) {
  457. const cch = stringSizes[i] >> 1;
  458. let sb = '';
  459. for (let ich = 0; ich < cch; ich++) {
  460. sb += String.fromCharCode(reader.uint16());
  461. }
  462. this.strings.push(sb);
  463. }
  464. }
  465. if (assemblyNameOffset != 0) {
  466. reader.seek(assemblyNameOffset);
  467. this.assemblyName = decoder.decode(reader.read(assemblyNameSize));
  468. }
  469. reader.seek(tailOffset);
  470. reader.assert('LEDOM\0LM');
  471. this._reader = reader;
  472. this._reader.seek(modelBlockOffset);
  473. }
  474. }
  475. get reader() {
  476. return this._reader;
  477. }
  478. string(empty) {
  479. const id = this.reader.int32();
  480. if (empty === null && id < 0) {
  481. return null;
  482. }
  483. return this.strings[id];
  484. }
  485. open(name) {
  486. const dir = this._directory.length > 0 ? this._directory + '/' : this._directory;
  487. name = dir + name;
  488. const key = name + '/Model.key';
  489. const stream = this._entries.get(key) || this._entries.get(key.replace(/\//g, '\\'));
  490. if (stream) {
  491. const buffer = stream.peek();
  492. const context = new mlnet.ModelHeader(this._catalog, this._entries, name, buffer);
  493. const value = this._catalog.create(context.loaderSignature, context);
  494. value.__type__ = value.__type__ || context.loaderSignature;
  495. value.__name__ = name;
  496. return value;
  497. }
  498. return null;
  499. }
  500. openBinary(name) {
  501. const dir = this._directory.length > 0 ? this._directory + '/' : this._directory;
  502. name = dir + name;
  503. const stream = this._entries.get(name) || this._entries.get(name.replace(/\//g, '\\'));
  504. if (stream) {
  505. return new mlnet.BinaryReader(stream);
  506. }
  507. return null;
  508. }
  509. openText(name) {
  510. const dir = this._directory.length > 0 ? this._directory + '/' : this._directory;
  511. name = dir + name;
  512. const stream = this._entries.get(name) || this._entries.get(name.replace(/\//g, '\\'));
  513. if (stream) {
  514. const buffer = stream.peek();
  515. const decoder = new TextDecoder();
  516. return decoder.decode(buffer);
  517. }
  518. return null;
  519. }
  520. check(signature, verWrittenCur, verWeCanReadBack) {
  521. return signature === this.modelSignature && verWrittenCur >= this.modelVersionReadable && verWeCanReadBack <= this.modelVersionWritten;
  522. }
  523. };
  524. mlnet.BinaryReader = class extends base.BinaryReader {
  525. match(text) {
  526. const position = this.position;
  527. for (let i = 0; i < text.length; i++) {
  528. if (this.byte() != text.charCodeAt(i)) {
  529. this.seek(position);
  530. return false;
  531. }
  532. }
  533. return true;
  534. }
  535. assert(text) {
  536. if (!this.match(text)) {
  537. throw new mlnet.Error("Invalid '" + text.split('\0').join('') + "' signature.");
  538. }
  539. }
  540. booleans(count) {
  541. const values = [];
  542. for (let i = 0; i < count; i++) {
  543. values.push(this.boolean());
  544. }
  545. return values;
  546. }
  547. int32s(count) {
  548. const values = [];
  549. for (let i = 0; i < count; i++) {
  550. values.push(this.int32());
  551. }
  552. return values;
  553. }
  554. uint32s(count) {
  555. const values = [];
  556. for (let i = 0; i < count; i++) {
  557. values.push(this.uint32());
  558. }
  559. return values;
  560. }
  561. int64() {
  562. const low = this.uint32();
  563. const hi = this.uint32();
  564. if (low == 0xffffffff && hi == 0x7fffffff) {
  565. return Number.MAX_SAFE_INTEGER;
  566. }
  567. if (hi == -1) {
  568. return -low;
  569. }
  570. if (hi != 0) {
  571. throw new mlnet.Error('Value not in 48-bit range.');
  572. }
  573. return (hi << 32) | low;
  574. }
  575. float32s(count) {
  576. const values = [];
  577. for (let i = 0; i < count; i++) {
  578. values.push(this.float32());
  579. }
  580. return values;
  581. }
  582. float64s(count) {
  583. const values = [];
  584. for (let i = 0; i < count; i++) {
  585. values.push(this.float64());
  586. }
  587. return values;
  588. }
  589. string() {
  590. const size = this.leb128();
  591. const buffer = this.read(size);
  592. return new TextDecoder('utf-8').decode(buffer);
  593. }
  594. leb128() {
  595. let result = 0;
  596. let shift = 0;
  597. let value;
  598. do {
  599. value = this.byte();
  600. result |= (value & 0x7F) << shift;
  601. shift += 7;
  602. } while ((value & 0x80) != 0);
  603. return result;
  604. }
  605. };
  606. mlnet.BinaryLoader = class { // 'BINLOADR'
  607. constructor(context, reader) {
  608. if (context) {
  609. if (context.modelVersionWritten >= 0x00010002) {
  610. this.Threads = context.reader.int32();
  611. this.GeneratedRowIndexName = context.string(null);
  612. }
  613. this.ShuffleBlocks = context.modelVersionWritten >= 0x00010003 ? context.reader.float64() : 4;
  614. reader = context.openBinary('Schema.idv');
  615. }
  616. // https://github.com/dotnet/machinelearning/blob/master/docs/code/IdvFileFormat.md
  617. reader.assert('CML\0DVB\0');
  618. reader.skip(8); // version
  619. reader.skip(8); // compatibleVersion
  620. const tableOfContentsOffset = reader.uint64();
  621. const tailOffset = reader.int64();
  622. reader.int64(); // rowCount
  623. const columnCount = reader.int32();
  624. reader.seek(tailOffset);
  625. reader.assert('\0BVD\0LMC');
  626. reader.seek(tableOfContentsOffset);
  627. this.schema = {};
  628. this.schema.inputs = [];
  629. for (let c = 0; c < columnCount; c ++) {
  630. const input = {};
  631. input.name = reader.string();
  632. input.type = new mlnet.Codec(reader);
  633. input.compression = reader.byte(); // None = 0, Deflate = 1
  634. input.rowsPerBlock = reader.leb128();
  635. input.lookupOffset = reader.int64();
  636. input.metadataTocOffset = reader.int64();
  637. this.schema.inputs.push(input);
  638. }
  639. }
  640. };
  641. mlnet.TransformerChain = class {
  642. constructor(context) {
  643. const reader = context.reader;
  644. const length = reader.int32();
  645. this.scopes = [];
  646. this.chain = [];
  647. for (let i = 0; i < length; i++) {
  648. this.scopes.push(reader.int32()); // 0x01 = Training, 0x02 = Testing, 0x04 = Scoring
  649. const dirName = 'Transform_' + ('00' + i).slice(-3);
  650. const transformer = context.open(dirName);
  651. this.chain.push(transformer);
  652. }
  653. }
  654. };
  655. mlnet.TransformBase = class {
  656. constructor(/* context */) {
  657. }
  658. };
  659. mlnet.RowToRowTransformBase = class extends mlnet.TransformBase {
  660. constructor(context) {
  661. super(context);
  662. }
  663. };
  664. mlnet.RowToRowTransformerBase = class {
  665. constructor(/* context */) {
  666. }
  667. };
  668. mlnet.RowToRowMapperTransformBase = class extends mlnet.RowToRowTransformBase {
  669. constructor(context) {
  670. super(context);
  671. }
  672. };
  673. mlnet.OneToOneTransformerBase = class {
  674. constructor(context) {
  675. const reader = context.reader;
  676. const n = reader.int32();
  677. this.inputs = [];
  678. this.outputs = [];
  679. for (let i = 0; i < n; i++) {
  680. const output = context.string();
  681. const input = context.string();
  682. this.outputs.push({ name: output });
  683. this.inputs.push({ name: input });
  684. }
  685. }
  686. };
  687. mlnet.ColumnCopyingTransformer = class {
  688. constructor(context) {
  689. const reader = context.reader;
  690. const length = reader.uint32();
  691. this.inputs = [];
  692. this.outputs = [];
  693. for (let i = 0; i < length; i++) {
  694. this.outputs.push({ name: context.string() });
  695. this.inputs.push({ name: context.string() });
  696. }
  697. }
  698. };
  699. mlnet.ColumnConcatenatingTransformer = class {
  700. constructor(context) {
  701. const reader = context.reader;
  702. if (context.modelVersionReadable >= 0x00010003) {
  703. const count = reader.int32();
  704. for (let i = 0; i < count; i++) {
  705. this.outputs = [];
  706. this.outputs.push({ name: context.string() });
  707. const n = reader.int32();
  708. this.inputs = [];
  709. for (let j = 0; j < n; j++) {
  710. const input = {
  711. name: context.string()
  712. };
  713. const alias = context.string(null);
  714. if (alias) {
  715. input.alias = alias;
  716. }
  717. this.inputs.push(input);
  718. }
  719. }
  720. }
  721. else {
  722. this.precision = reader.int32();
  723. const n = reader.int32();
  724. const names = [];
  725. const inputs = [];
  726. for (let i = 0; i < n; i++) {
  727. names.push(context.string());
  728. const numSources = reader.int32();
  729. const input = [];
  730. for (let j = 0; j < numSources; j++) {
  731. input.push(context.string());
  732. }
  733. inputs.push(input);
  734. }
  735. const aliases = [];
  736. if (context.modelVersionReadable >= 0x00010002) {
  737. for (let i = 0; i < n; i++) {
  738. /* let length = */ inputs[i].length;
  739. const alias = {};
  740. aliases.push(alias);
  741. if (context.modelVersionReadable >= 0x00010002) {
  742. for (;;) {
  743. const j = reader.int32();
  744. if (j == -1) {
  745. break;
  746. }
  747. alias[j] = context.string();
  748. }
  749. }
  750. }
  751. }
  752. if (n > 1) {
  753. throw new mlnet.Error('');
  754. }
  755. this.outputs = [];
  756. for (let i = 0; i < n; i++) {
  757. this.outputs.push({
  758. name: names[i]
  759. });
  760. this.inputs = inputs[i];
  761. }
  762. }
  763. }
  764. };
  765. mlnet.PredictionTransformerBase = class {
  766. constructor(context) {
  767. this.Model = context.open('Model');
  768. const trainSchemaReader = context.openBinary('TrainSchema');
  769. if (trainSchemaReader) {
  770. new mlnet.BinaryLoader(null, trainSchemaReader).schema;
  771. }
  772. }
  773. };
  774. mlnet.MatrixFactorizationModelParameters = class {
  775. constructor(context) {
  776. const reader = context.reader;
  777. this.NumberOfRows = reader.int32();
  778. if (context.modelVersionWritten < 0x00010002) {
  779. reader.uint64(); // mMin
  780. }
  781. this.NumberOfColumns = reader.int32();
  782. if (context.modelVersionWritten < 0x00010002) {
  783. reader.uint64(); // nMin
  784. }
  785. this.ApproximationRank = reader.int32();
  786. this._leftFactorMatrix = reader.float32s(this.NumberOfRows * this.ApproximationRank);
  787. this._rightFactorMatrix = reader.float32s(this.NumberOfColumns * this.ApproximationRank);
  788. }
  789. };
  790. mlnet.MatrixFactorizationPredictionTransformer = class extends mlnet.PredictionTransformerBase {
  791. constructor(context) {
  792. super(context);
  793. this.MatrixColumnIndexColumnName = context.string();
  794. this.MatrixRowIndexColumnName = context.string();
  795. // TODO
  796. }
  797. };
  798. mlnet.FieldAwareFactorizationMachinePredictionTransformer = class extends mlnet.PredictionTransformerBase {
  799. constructor(context) {
  800. super(context);
  801. const reader = context.reader;
  802. this.inputs = [];
  803. for (let i = 0; i < this.FieldCount; i++) {
  804. this.inputs.push({ name: context.string() });
  805. }
  806. this.Threshold = reader.float32();
  807. this.ThresholdColumn = context.string();
  808. this.inputs.push({ name: this.ThresholdColumn });
  809. }
  810. };
  811. mlnet.SingleFeaturePredictionTransformerBase = class extends mlnet.PredictionTransformerBase {
  812. constructor(context) {
  813. super(context);
  814. const featureColumn = context.string(null);
  815. this.inputs = [];
  816. this.inputs.push({ name: featureColumn });
  817. this.outputs = [];
  818. this.outputs.push({ name: featureColumn });
  819. }
  820. };
  821. mlnet.ClusteringPredictionTransformer = class extends mlnet.SingleFeaturePredictionTransformerBase {
  822. constructor(context) {
  823. super(context);
  824. }
  825. };
  826. mlnet.AnomalyPredictionTransformer = class extends mlnet.SingleFeaturePredictionTransformerBase {
  827. constructor(context) {
  828. super(context);
  829. const reader = context.reader;
  830. this.Threshold = reader.float32();
  831. this.ThresholdColumn = context.string();
  832. }
  833. };
  834. mlnet.AffineNormSerializationUtils = class {
  835. constructor(context) {
  836. const reader = context.reader;
  837. /* cbFloat = */ reader.int32();
  838. this.NumFeatures = reader.int32();
  839. const morphCount = reader.int32();
  840. if (morphCount == -1) {
  841. this.ScalesSparse = reader.float32s(reader.int32());
  842. this.OffsetsSparse = reader.float32s(reader.int32());
  843. }
  844. else {
  845. // debugger;
  846. }
  847. }
  848. };
  849. mlnet.RegressionPredictionTransformer = class extends mlnet.SingleFeaturePredictionTransformerBase {
  850. constructor(context) {
  851. super(context);
  852. }
  853. };
  854. mlnet.BinaryPredictionTransformer = class extends mlnet.SingleFeaturePredictionTransformerBase {
  855. constructor(context) {
  856. super(context);
  857. const reader = context.reader;
  858. this.Threshold = reader.float32();
  859. this.ThresholdColumn = context.string();
  860. }
  861. };
  862. mlnet.MulticlassPredictionTransformer = class extends mlnet.SingleFeaturePredictionTransformerBase {
  863. constructor(context) {
  864. super(context);
  865. this.TrainLabelColumn = context.string(null);
  866. this.inputs.push({ name: this.TrainLabelColumn });
  867. }
  868. };
  869. mlnet.MissingValueReplacingTransformer = class extends mlnet.OneToOneTransformerBase {
  870. constructor(context) {
  871. super(context);
  872. const reader = context.reader;
  873. for (let i = 0; i < this.inputs.length; i++) {
  874. const codec = new mlnet.Codec(reader);
  875. const count = reader.int32();
  876. this.values = codec.read(reader, count);
  877. }
  878. }
  879. };
  880. mlnet.PredictorBase = class {
  881. constructor(context) {
  882. const reader = context.reader;
  883. if (reader.int32() != 4) {
  884. throw new mlnet.Error('Invalid float type size.');
  885. }
  886. }
  887. };
  888. mlnet.ModelParametersBase = class {
  889. constructor(context) {
  890. const reader = context.reader;
  891. const cbFloat = reader.int32();
  892. if (cbFloat !== 4) {
  893. throw new mlnet.Error('This file was saved by an incompatible version.');
  894. }
  895. }
  896. };
  897. mlnet.ImageClassificationModelParameters = class extends mlnet.ModelParametersBase {
  898. constructor(context) {
  899. super(context);
  900. const reader = context.reader;
  901. this.classCount = reader.int32();
  902. this.imagePreprocessorTensorInput = reader.string();
  903. this.imagePreprocessorTensorOutput = reader.string();
  904. this.graphInputTensor = reader.string();
  905. this.graphOutputTensor = reader.string();
  906. this.modelFile = 'TFModel';
  907. // const modelBytes = context.openBinary('TFModel');
  908. // first uint32 is size of TensorFlow model
  909. // inputType = new VectorDataViewType(uint8);
  910. // outputType = new VectorDataViewType(float32, classCount);
  911. }
  912. };
  913. mlnet.NaiveBayesMulticlassModelParameters = class extends mlnet.ModelParametersBase {
  914. constructor(context) {
  915. super(context);
  916. const reader = context.reader;
  917. this._labelHistogram = reader.int32s(reader.int32());
  918. this._featureCount = reader.int32();
  919. this._featureHistogram = [];
  920. for (let i = 0; i < this._labelHistogram.length; i++) {
  921. if (this._labelHistogram[i] > 0) {
  922. this._featureHistogram.push(reader.int32s(this._featureCount));
  923. }
  924. }
  925. this._absentFeaturesLogProb = reader.float64s(this._labelHistogram.length);
  926. }
  927. };
  928. mlnet.LinearModelParameters = class extends mlnet.ModelParametersBase {
  929. constructor(context) {
  930. super(context);
  931. const reader = context.reader;
  932. this.Bias = reader.float32();
  933. /* let len = */ reader.int32();
  934. this.Indices = reader.int32s(reader.int32());
  935. this.Weights = reader.float32s(reader.int32());
  936. }
  937. };
  938. mlnet.LinearBinaryModelParameters = class extends mlnet.LinearModelParameters {
  939. constructor(context) {
  940. super(context);
  941. if (context.modelVersionWritten > 0x00020001) {
  942. this.Statistics = context.open('ModelStats');
  943. }
  944. }
  945. };
  946. mlnet.ModelStatisticsBase = class {
  947. constructor(context) {
  948. const reader = context.reader;
  949. this.ParametersCount = reader.int32();
  950. this.TrainingExampleCount = reader.int64();
  951. this.Deviance = reader.float32();
  952. this.NullDeviance = reader.float32();
  953. }
  954. };
  955. mlnet.LinearModelParameterStatistics = class extends mlnet.ModelStatisticsBase {
  956. constructor(context) {
  957. super(context);
  958. const reader = context.reader;
  959. if (context.modelVersionWritten < 0x00010002) {
  960. if (!reader.boolean()) {
  961. return;
  962. }
  963. }
  964. const stdErrorValues = reader.float32s(this.ParametersCount);
  965. const length = reader.int32();
  966. if (length == this.ParametersCount) {
  967. this._coeffStdError = stdErrorValues;
  968. }
  969. else {
  970. this.stdErrorIndices = reader.int32s(this.ParametersCount);
  971. this._coeffStdError = stdErrorValues;
  972. }
  973. this._bias = reader.float32();
  974. const isWeightsDense = reader.byte();
  975. const weightsLength = reader.int32();
  976. const weightsValues = reader.float32s(weightsLength);
  977. if (isWeightsDense) {
  978. this._weights = weightsValues;
  979. }
  980. else {
  981. this.weightsIndices = reader.int32s(weightsLength);
  982. }
  983. }
  984. };
  985. mlnet.LinearMulticlassModelParametersBase = class extends mlnet.ModelParametersBase {
  986. constructor(context) {
  987. super(context);
  988. const reader = context.reader;
  989. const numberOfFeatures = reader.int32();
  990. const numberOfClasses = reader.int32();
  991. this.Biases = reader.float32s(numberOfClasses);
  992. const numStarts = reader.int32();
  993. if (numStarts == 0) {
  994. /* let numIndices = */ reader.int32();
  995. /* let numWeights = */ reader.int32();
  996. this.Weights = [];
  997. for (let i = 0; i < numberOfClasses; i++) {
  998. const w = reader.float32s(numberOfFeatures);
  999. this.Weights.push(w);
  1000. }
  1001. }
  1002. else {
  1003. const starts = reader.int32s(reader.int32());
  1004. /* let numIndices = */ reader.int32();
  1005. const indices = [];
  1006. for (let i = 0; i < numberOfClasses; i++) {
  1007. indices.push(reader.int32s(starts[i + 1] - starts[i]));
  1008. }
  1009. /* let numValues = */ reader.int32();
  1010. this.Weights = [];
  1011. for (let i = 0; i < numberOfClasses; i++) {
  1012. const values = reader.float32s(starts[i + 1] - starts[i]);
  1013. this.Weights.push(values);
  1014. }
  1015. }
  1016. const labelNamesReader = context.openBinary('LabelNames');
  1017. if (labelNamesReader) {
  1018. this.LabelNames = [];
  1019. for (let i = 0; i < numberOfClasses; i++) {
  1020. const id = labelNamesReader.int32();
  1021. this.LabelNames.push(context.strings[id]);
  1022. }
  1023. }
  1024. const statistics = context.open('ModelStats');
  1025. if (statistics) {
  1026. this.Statistics = statistics;
  1027. }
  1028. }
  1029. };
  1030. mlnet.LinearMulticlassModelParameters = class extends mlnet.LinearMulticlassModelParametersBase {
  1031. constructor(context) {
  1032. super(context);
  1033. }
  1034. };
  1035. mlnet.RegressionModelParameters = class extends mlnet.LinearModelParameters {
  1036. constructor(context) {
  1037. super(context);
  1038. }
  1039. };
  1040. mlnet.PoissonRegressionModelParameters = class extends mlnet.RegressionModelParameters {
  1041. constructor(context) {
  1042. super(context);
  1043. }
  1044. };
  1045. mlnet.LinearRegressionModelParameters = class extends mlnet.RegressionModelParameters {
  1046. constructor(context) {
  1047. super(context);
  1048. }
  1049. };
  1050. mlnet.MaximumEntropyModelParameters = class extends mlnet.LinearMulticlassModelParametersBase {
  1051. constructor(context) {
  1052. super(context);
  1053. }
  1054. };
  1055. mlnet.TokenizingByCharactersTransformer = class extends mlnet.OneToOneTransformerBase {
  1056. constructor(context) {
  1057. super(context);
  1058. const reader = context.reader;
  1059. this.UseMarkerChars = reader.boolean();
  1060. this.IsSeparatorStartEnd = context.modelVersionReadable < 0x00010002 ? true : reader.boolean();
  1061. }
  1062. };
  1063. mlnet.SequencePool = class {
  1064. constructor(reader) {
  1065. this.idLim = reader.int32();
  1066. this.start = reader.int32s(this.idLim + 1);
  1067. this.bytes = reader.read(this.start[this.idLim]);
  1068. }
  1069. };
  1070. mlnet.NgramExtractingTransformer = class extends mlnet.OneToOneTransformerBase {
  1071. constructor(context) {
  1072. super(context);
  1073. const reader = context.reader;
  1074. if (this.inputs.length == 1) {
  1075. this._option(context, reader, this);
  1076. }
  1077. else {
  1078. // debugger;
  1079. }
  1080. }
  1081. _option(context, reader, option) {
  1082. const readWeighting = context.modelVersionReadable >= 0x00010002;
  1083. option.NgramLength = reader.int32();
  1084. option.SkipLength = reader.int32();
  1085. if (readWeighting) {
  1086. option.Weighting = reader.int32();
  1087. }
  1088. option.NonEmptyLevels = reader.booleans(option.NgramLength);
  1089. option.NgramMap = new mlnet.SequencePool(reader);
  1090. if (readWeighting) {
  1091. option.InvDocFreqs = reader.float64s(reader.int32());
  1092. }
  1093. }
  1094. };
  1095. // mlnet.NgramExtractingTransformer.WeightingCriteria
  1096. mlnet.NgramHashingTransformer = class extends mlnet.RowToRowTransformerBase {
  1097. constructor(context) {
  1098. super(context);
  1099. const loadLegacy = context.modelVersionWritten < 0x00010003;
  1100. const reader = context.reader;
  1101. if (loadLegacy) {
  1102. reader.int32(); // cbFloat
  1103. }
  1104. this.inputs = [];
  1105. this.outputs = [];
  1106. const columnsLength = reader.int32();
  1107. if (loadLegacy) {
  1108. /* TODO
  1109. for (let i = 0; i < columnsLength; i++) {
  1110. this.Columns.push(new NgramHashingEstimator.ColumnOptions(context));
  1111. } */
  1112. }
  1113. else {
  1114. for (let i = 0; i < columnsLength; i++) {
  1115. this.outputs.push(context.string());
  1116. const csrc = reader.int32();
  1117. for (let j = 0; j < csrc; j++) {
  1118. const src = context.string();
  1119. this.inputs.push(src);
  1120. // TODO inputs[i][j] = src;
  1121. }
  1122. }
  1123. }
  1124. }
  1125. };
  1126. mlnet.WordTokenizingTransformer = class extends mlnet.OneToOneTransformerBase {
  1127. constructor(context) {
  1128. super(context);
  1129. const reader = context.reader;
  1130. if (this.inputs.length == 1) {
  1131. this.Separators = [];
  1132. const count = reader.int32();
  1133. for (let i = 0; i < count; i++) {
  1134. this.Separators.push(String.fromCharCode(reader.int16()));
  1135. }
  1136. }
  1137. else {
  1138. // debugger;
  1139. }
  1140. }
  1141. };
  1142. mlnet.TextNormalizingTransformer = class extends mlnet.OneToOneTransformerBase {
  1143. constructor(context) {
  1144. super(context);
  1145. const reader = context.reader;
  1146. this.CaseMode = reader.byte();
  1147. this.KeepDiacritics = reader.boolean();
  1148. this.KeepPunctuations = reader.boolean();
  1149. this.KeepNumbers = reader.boolean();
  1150. }
  1151. };
  1152. mlnet.TextNormalizingTransformer.CaseMode = {
  1153. Lower: 0,
  1154. Upper: 1,
  1155. None: 2
  1156. };
  1157. mlnet.PrincipalComponentAnalysisTransformer = class extends mlnet.OneToOneTransformerBase {
  1158. constructor(context) {
  1159. super(context);
  1160. const reader = context.reader;
  1161. if (context.modelVersionReadable === 0x00010001) {
  1162. if (reader.int32() !== 4) {
  1163. throw new mlnet.Error('This file was saved by an incompatible version.');
  1164. }
  1165. }
  1166. this.TransformInfos = [];
  1167. for (let i = 0; i < this.inputs.length; i++) {
  1168. const option = {};
  1169. option.Dimension = reader.int32();
  1170. option.Rank = reader.int32();
  1171. option.Eigenvectors = [];
  1172. for (let j = 0; j < option.Rank; j++) {
  1173. option.Eigenvectors.push(reader.float32s(option.Dimension));
  1174. }
  1175. option.MeanProjected = reader.float32s(reader.int32());
  1176. this.TransformInfos.push(option);
  1177. }
  1178. }
  1179. };
  1180. mlnet.LpNormNormalizingTransformer = class extends mlnet.OneToOneTransformerBase {
  1181. constructor(context) {
  1182. super(context);
  1183. const reader = context.reader;
  1184. if (context.modelVersionWritten <= 0x00010002) {
  1185. /* cbFloat */ reader.int32();
  1186. }
  1187. // let normKindSerialized = context.modelVersionWritten >= 0x00010002;
  1188. if (this.inputs.length == 1) {
  1189. this.EnsureZeroMean = reader.boolean();
  1190. this.Norm = reader.byte();
  1191. this.Scale = reader.float32();
  1192. }
  1193. else {
  1194. // debugger;
  1195. }
  1196. }
  1197. };
  1198. mlnet.KeyToVectorMappingTransformer = class extends mlnet.OneToOneTransformerBase {
  1199. constructor(context) {
  1200. super(context);
  1201. const reader = context.reader;
  1202. if (context.modelVersionWritten == 0x00010001) {
  1203. /* cbFloat = */ reader.int32();
  1204. }
  1205. const columnsLength = this.inputs.length;
  1206. this.Bags = reader.booleans(columnsLength);
  1207. }
  1208. };
  1209. mlnet.TypeConvertingTransformer = class extends mlnet.OneToOneTransformerBase {
  1210. constructor(context) {
  1211. super(context);
  1212. // debugger;
  1213. }
  1214. };
  1215. mlnet.ImageLoadingTransformer = class extends mlnet.OneToOneTransformerBase {
  1216. constructor(context) {
  1217. super(context);
  1218. this.ImageFolder = context.string(null);
  1219. }
  1220. };
  1221. mlnet.ImageResizingTransformer = class extends mlnet.OneToOneTransformerBase {
  1222. constructor(context) {
  1223. super(context);
  1224. const reader = context.reader;
  1225. if (this.inputs.length == 1) {
  1226. this._option(reader, this);
  1227. }
  1228. else {
  1229. this.Options = [];
  1230. for (let i = 0; i < this.inputs.length; i++) {
  1231. const option = {};
  1232. this._option(reader, option);
  1233. this.Options.push(option);
  1234. }
  1235. }
  1236. }
  1237. _option(reader, option) {
  1238. option.Width = reader.int32();
  1239. option.Height = reader.int32();
  1240. option.Resizing = reader.byte();
  1241. option.Anchor = reader.byte();
  1242. }
  1243. };
  1244. mlnet.ImageResizingTransformer.ResizingKind = {
  1245. IsoPad: 0,
  1246. IsoCrop: 1,
  1247. Fill: 2
  1248. };
  1249. mlnet.ImageResizingTransformer.Anchor = {
  1250. Right: 0,
  1251. Left: 1,
  1252. Top: 2,
  1253. Bottom: 3,
  1254. Center: 4
  1255. };
  1256. mlnet.ImagePixelExtractingTransformer = class extends mlnet.OneToOneTransformerBase {
  1257. constructor(context) {
  1258. super(context);
  1259. const reader = context.reader;
  1260. if (this.inputs.length == 1) {
  1261. this._option(context, reader, this);
  1262. }
  1263. else {
  1264. this.Options = [];
  1265. for (let i = 0; i < this.inputs.length; i++) {
  1266. const option = {};
  1267. this._option(context, reader, option);
  1268. this.Options.push(option);
  1269. }
  1270. }
  1271. }
  1272. _option(context, reader, option) {
  1273. option.ColorsToExtract = reader.byte();
  1274. option.OrderOfExtraction = context.modelVersionWritten <= 0x00010002 ? mlnet.ImagePixelExtractingTransformer.ColorsOrder.ARGB : reader.byte();
  1275. let planes = option.ColorsToExtract;
  1276. planes = (planes & 0x05) + ((planes >> 1) & 0x05);
  1277. planes = (planes & 0x03) + ((planes >> 2) & 0x03);
  1278. option.Planes = planes & 0xFF;
  1279. option.OutputAsFloatArray = reader.boolean();
  1280. option.OffsetImage = reader.float32();
  1281. option.ScaleImage = reader.float32();
  1282. option.InterleavePixelColors = reader.boolean();
  1283. }
  1284. };
  1285. mlnet.ImagePixelExtractingTransformer.ColorBits = {
  1286. Alpha: 0x01,
  1287. Red: 0x02,
  1288. Green: 0x04,
  1289. Blue: 0x08,
  1290. Rgb: 0x0E,
  1291. All: 0x0F
  1292. };
  1293. mlnet.ImagePixelExtractingTransformer.ColorsOrder = {
  1294. ARGB: 1,
  1295. ARBG: 2,
  1296. ABRG: 3,
  1297. ABGR: 4,
  1298. AGRB: 5,
  1299. AGBR: 6
  1300. };
  1301. mlnet.NormalizingTransformer = class extends mlnet.OneToOneTransformerBase {
  1302. constructor(context) {
  1303. super(context);
  1304. const reader = context.reader;
  1305. this.Options = [];
  1306. for (let i = 0; i < this.inputs.length; i++) {
  1307. let isVector = false;
  1308. let shape = 0;
  1309. let itemKind = '';
  1310. if (context.modelVersionWritten < 0x00010002) {
  1311. isVector = reader.boolean();
  1312. shape = [ reader.int32() ];
  1313. itemKind = reader.byte();
  1314. }
  1315. else {
  1316. isVector = reader.boolean();
  1317. itemKind = reader.byte();
  1318. shape = reader.int32s(reader.int32());
  1319. }
  1320. let itemType = '';
  1321. switch (itemKind) {
  1322. case 9: itemType = 'float32'; break;
  1323. case 10: itemType = 'float64'; break;
  1324. default: throw new mlnet.Error("Unsupported NormalizingTransformer item kind '" + itemKind + "'.");
  1325. }
  1326. const type = itemType + (!isVector ? '' : '[' + shape.map((dim) => dim.toString()).join(',') + ']');
  1327. const name = 'Normalizer_' + ('00' + i).slice(-3);
  1328. const func = context.open(name);
  1329. this.Options.push({ type: type, func: func });
  1330. }
  1331. }
  1332. };
  1333. mlnet.KeyToValueMappingTransformer = class extends mlnet.OneToOneTransformerBase {
  1334. constructor(context) {
  1335. super(context);
  1336. }
  1337. };
  1338. mlnet.ValueToKeyMappingTransformer = class extends mlnet.OneToOneTransformerBase {
  1339. constructor(context) {
  1340. super(context);
  1341. const reader = context.reader;
  1342. if (context.modelVersionWritten >= 0x00010003) {
  1343. this.textMetadata = reader.booleans(this.outputs.length + this.inputs.length);
  1344. }
  1345. else {
  1346. this.textMetadata = [];
  1347. for (let i = 0; i < this.columnPairs.length; i++) {
  1348. this.textMetadata.push(false);
  1349. }
  1350. }
  1351. const vocabulary = context.open('Vocabulary');
  1352. if (vocabulary) {
  1353. this.termMap = vocabulary.termMap;
  1354. }
  1355. }
  1356. };
  1357. mlnet.TermMap = class {
  1358. constructor(context) {
  1359. const reader = context.reader;
  1360. const mtype = reader.byte();
  1361. switch (mtype) {
  1362. case 0: { // Text
  1363. this.values = [];
  1364. const cstr = reader.int32();
  1365. for (let i = 0; i < cstr; i++) {
  1366. this.values.push(context.string());
  1367. }
  1368. break;
  1369. }
  1370. case 1: { // Codec
  1371. const codec = new mlnet.Codec(reader);
  1372. const count = reader.int32();
  1373. this.values = codec.read(reader, count);
  1374. break;
  1375. }
  1376. default:
  1377. throw new mlnet.Error("Unsupported term map type '" + mtype.toString() + "'.");
  1378. }
  1379. }
  1380. };
  1381. mlnet.TermManager = class {
  1382. constructor(context) {
  1383. const reader = context.reader;
  1384. const cmap = reader.int32();
  1385. this.termMap = [];
  1386. if (context.modelVersionWritten >= 0x00010002) {
  1387. for (let i = 0; i < cmap; ++i) {
  1388. this.termMap.push(new mlnet.TermMap(context));
  1389. // debugger;
  1390. // termMap[i] = TermMap.Load(c, host, CodecFactory);
  1391. }
  1392. }
  1393. else {
  1394. throw new mlnet.Error('Unsupported TermManager version.');
  1395. // for (let i = 0; i < cmap; ++i) {
  1396. // debugger;
  1397. // // termMap[i] = TermMap.TextImpl.Create(c, host)
  1398. // }
  1399. }
  1400. }
  1401. };
  1402. mlnet.ValueMappingTransformer = class extends mlnet.OneToOneTransformerBase {
  1403. constructor(context) {
  1404. super(context);
  1405. this.keyColumnName = 'Key';
  1406. if (context.check('TXTLOOKT', 0x00010002, 0x00010002)) {
  1407. this.keyColumnName = 'Term';
  1408. }
  1409. // TODO
  1410. }
  1411. };
  1412. mlnet.KeyToVectorTransform = class {
  1413. constructor(/* context */) {
  1414. }
  1415. };
  1416. mlnet.GenericScoreTransform = class {
  1417. constructor(/* context */) {
  1418. }
  1419. };
  1420. mlnet.CompositeDataLoader = class {
  1421. constructor(context) {
  1422. /* let loader = */ context.open('Loader');
  1423. const reader = context.reader;
  1424. // LoadTransforms
  1425. reader.int32(); // floatSize
  1426. const cxf = reader.int32();
  1427. const tagData = [];
  1428. for (let i = 0; i < cxf; i++) {
  1429. let tag = '';
  1430. let args = null;
  1431. if (context.modelVersionReadable >= 0x00010002) {
  1432. tag = context.string();
  1433. args = context.string(null);
  1434. }
  1435. tagData.push([ tag, args ]);
  1436. }
  1437. this.chain = [];
  1438. for (let j = 0; j < cxf; j++) {
  1439. const name = 'Transform_' + ('00' + j).slice(-3);
  1440. const transform = context.open(name);
  1441. this.chain.push(transform);
  1442. }
  1443. }
  1444. };
  1445. mlnet.RowToRowMapperTransform = class extends mlnet.RowToRowTransformBase {
  1446. constructor(context) {
  1447. super(context);
  1448. const mapper = context.open('Mapper');
  1449. this.__type__ = mapper.__type__;
  1450. for (const key of Object.keys(mapper)) {
  1451. this[key] = mapper[key];
  1452. }
  1453. }
  1454. };
  1455. mlnet.ImageClassificationTransformer = class extends mlnet.RowToRowTransformerBase {
  1456. constructor(context) {
  1457. super(context);
  1458. const reader = context.reader;
  1459. this.addBatchDimensionInput = reader.boolean();
  1460. const numInputs = reader.int32();
  1461. this.inputs = [];
  1462. for (let i = 0; i < numInputs; i++) {
  1463. this.inputs.push({ name: context.string() });
  1464. }
  1465. this.outputs = [];
  1466. const numOutputs = reader.int32();
  1467. for (let i = 0; i < numOutputs; i++) {
  1468. this.outputs.push({ name: context.string() });
  1469. }
  1470. this.labelColumn = reader.string();
  1471. this.checkpointName = reader.string();
  1472. this.arch = reader.int32(); // Architecture
  1473. this.scoreColumnName = reader.string();
  1474. this.predictedColumnName = reader.string();
  1475. this.learningRate = reader.float32();
  1476. this.classCount = reader.int32();
  1477. this.keyValueAnnotations = [];
  1478. for (let i = 0; i < this.classCount; i++) {
  1479. this.keyValueAnnotations.push(context.string());
  1480. }
  1481. this.predictionTensorName = reader.string();
  1482. this.softMaxTensorName = reader.string();
  1483. this.jpegDataTensorName = reader.string();
  1484. this.resizeTensorName = reader.string();
  1485. }
  1486. };
  1487. mlnet.OnnxTransformer = class extends mlnet.RowToRowTransformerBase {
  1488. constructor(context) {
  1489. super(context);
  1490. const reader = context.reader;
  1491. this.modelFile = 'OnnxModel';
  1492. // const modelBytes = context.openBinary('OnnxModel');
  1493. // first uint32 is size of .onnx model
  1494. const numInputs = context.modelVersionWritten > 0x00010001 ? reader.int32() : 1;
  1495. this.inputs = [];
  1496. for (let i = 0; i < numInputs; i++) {
  1497. this.inputs.push({ name: context.string() });
  1498. }
  1499. const numOutputs = context.modelVersionWritten > 0x00010001 ? reader.int32() : 1;
  1500. this.outputs = [];
  1501. for (let i = 0; i < numOutputs; i++) {
  1502. this.outputs.push({ name: context.string() });
  1503. }
  1504. if (context.modelVersionWritten > 0x0001000C) {
  1505. const customShapeInfosLength = reader.int32();
  1506. this.LoadedCustomShapeInfos = [];
  1507. for (let i = 0; i < customShapeInfosLength; i++) {
  1508. this.LoadedCustomShapeInfos.push({
  1509. name: context.string(),
  1510. shape: reader.int32s(reader.int32())
  1511. });
  1512. }
  1513. }
  1514. }
  1515. };
  1516. mlnet.OptionalColumnTransform = class extends mlnet.RowToRowMapperTransformBase {
  1517. constructor(context) {
  1518. super(context);
  1519. }
  1520. };
  1521. mlnet.TensorFlowTransformer = class extends mlnet.RowToRowTransformerBase {
  1522. constructor(context) {
  1523. super(context);
  1524. const reader = context.reader;
  1525. this.IsFrozen = context.modelVersionReadable >= 0x00010002 ? reader.boolean() : true;
  1526. this.AddBatchDimensionInput = context.modelVersionReadable >= 0x00010003 ? reader.boolean() : true;
  1527. const numInputs = reader.int32();
  1528. this.inputs = [];
  1529. for (let i = 0; i < numInputs; i++) {
  1530. this.inputs.push({ name: context.string() });
  1531. }
  1532. const numOutputs = context.modelVersionReadable >= 0x00010002 ? reader.int32() : 1;
  1533. this.outputs = [];
  1534. for (let i = 0; i < numOutputs; i++) {
  1535. this.outputs.push({ name: context.string() });
  1536. }
  1537. }
  1538. };
  1539. mlnet.OneVersusAllModelParameters = class extends mlnet.ModelParametersBase {
  1540. constructor(context) {
  1541. super(context);
  1542. const reader = context.reader;
  1543. this.UseDist = reader.boolean();
  1544. const len = reader.int32();
  1545. this.chain = [];
  1546. for (let i = 0; i < len; i++) {
  1547. const name = 'SubPredictor_' + ('00' + i).slice(-3);
  1548. const predictor = context.open(name);
  1549. this.chain.push(predictor);
  1550. }
  1551. }
  1552. };
  1553. mlnet.TextFeaturizingEstimator = class {
  1554. constructor(context) {
  1555. if (context.modelVersionReadable === 0x00010001) {
  1556. const reader = context.reader;
  1557. const n = reader.int32();
  1558. this.chain = [];
  1559. /* let loader = */ context.open('Loader');
  1560. for (let i = 0; i < n; i++) {
  1561. const name = 'Step_' + ('00' + i).slice(-3);
  1562. const transformer = context.open(name);
  1563. this.chain.push(transformer);
  1564. // debugger;
  1565. }
  1566. // throw new mlnet.Error('Unsupported TextFeaturizingEstimator format.');
  1567. }
  1568. else {
  1569. const chain = context.open('Chain');
  1570. this.chain = chain.chain;
  1571. }
  1572. }
  1573. };
  1574. mlnet.TextLoader = class {
  1575. constructor(context) {
  1576. const reader = context.reader;
  1577. reader.int32(); // floatSize
  1578. this.MaxRows = reader.int64();
  1579. this.Flags = reader.uint32();
  1580. this.InputSize = reader.int32();
  1581. const separatorCount = reader.int32();
  1582. this.Separators = [];
  1583. for (let i = 0; i < separatorCount; i++) {
  1584. this.Separators.push(String.fromCharCode(reader.uint16()));
  1585. }
  1586. this.Bindinds = new mlnet.TextLoader.Bindinds(context);
  1587. }
  1588. };
  1589. mlnet.TextLoader.Bindinds = class {
  1590. constructor(context) {
  1591. const reader = context.reader;
  1592. const cinfo = reader.int32();
  1593. for (let i = 0; i < cinfo; i++) {
  1594. // debugger;
  1595. }
  1596. }
  1597. };
  1598. mlnet.CalibratedPredictorBase = class {
  1599. constructor(predictor, calibrator) {
  1600. this.SubPredictor = predictor;
  1601. this.Calibrator = calibrator;
  1602. }
  1603. };
  1604. mlnet.ValueMapperCalibratedPredictorBase = class extends mlnet.CalibratedPredictorBase {
  1605. constructor(predictor, calibrator) {
  1606. super(predictor, calibrator);
  1607. }
  1608. };
  1609. mlnet.CalibratedModelParametersBase = class {
  1610. constructor(context) {
  1611. this.Predictor = context.open('Predictor');
  1612. this.Calibrator = context.open('Calibrator');
  1613. }
  1614. };
  1615. mlnet.ValueMapperCalibratedModelParametersBase = class extends mlnet.CalibratedModelParametersBase {
  1616. constructor(context) {
  1617. super(context);
  1618. // debugger;
  1619. }
  1620. };
  1621. mlnet.CalibratedPredictor = class extends mlnet.ValueMapperCalibratedPredictorBase {
  1622. constructor(context) {
  1623. const predictor = context.open('Predictor');
  1624. const calibrator = context.open('Calibrator');
  1625. super(predictor, calibrator);
  1626. }
  1627. };
  1628. mlnet.ParameterMixingCalibratedModelParameters = class extends mlnet.ValueMapperCalibratedModelParametersBase {
  1629. constructor(context) {
  1630. super(context);
  1631. }
  1632. };
  1633. mlnet.FieldAwareFactorizationMachineModelParameters = class {
  1634. constructor(context) {
  1635. const reader = context.reader;
  1636. this.Norm = reader.boolean();
  1637. this.FieldCount = reader.int32();
  1638. this.FeatureCount = reader.int32();
  1639. this.LatentDim = reader.int32();
  1640. this.LinearWeights = reader.float32s(reader.int32());
  1641. this.LatentWeights = reader.float32s(reader.int32());
  1642. }
  1643. };
  1644. mlnet.KMeansModelParameters = class extends mlnet.ModelParametersBase {
  1645. constructor(context) {
  1646. super(context);
  1647. const reader = context.reader;
  1648. this.k = reader.int32();
  1649. this.Dimensionality = reader.int32();
  1650. this.Centroids = [];
  1651. for (let i = 0; i < this.k; i++) {
  1652. const count = context.modelVersionWritten >= 0x00010002 ? reader.int32() : this.Dimensionality;
  1653. const indices = count < this.Dimensionality ? reader.int32s(count) : null;
  1654. const values = reader.float32s(count);
  1655. this.Centroids.push({ indices: indices, values: values });
  1656. }
  1657. // input type = float32[dimensionality]
  1658. // output type = float32[k]
  1659. }
  1660. };
  1661. mlnet.PcaModelParameters = class extends mlnet.ModelParametersBase {
  1662. constructor(context) {
  1663. super(context);
  1664. const reader = context.reader;
  1665. this.Dimension = reader.int32();
  1666. this.Rank = reader.int32();
  1667. const center = reader.boolean();
  1668. if (center) {
  1669. this.Mean = reader.float32s(this.Dimension);
  1670. }
  1671. else {
  1672. this.Mean = [];
  1673. }
  1674. this.EigenVectors = [];
  1675. for (let i = 0; i < this.Rank; ++i) {
  1676. this.EigenVectors.push(reader.float32s(this.Dimension));
  1677. }
  1678. // input type -> float32[Dimension]
  1679. }
  1680. };
  1681. mlnet.TreeEnsembleModelParameters = class extends mlnet.ModelParametersBase {
  1682. constructor(context) {
  1683. super(context);
  1684. const reader = context.reader;
  1685. const usingDefaultValues = context.modelVersionWritten >= this.VerDefaultValueSerialized;
  1686. const categoricalSplits = context.modelVersionWritten >= this.VerCategoricalSplitSerialized;
  1687. this.TrainedEnsemble = new mlnet.InternalTreeEnsemble(context, usingDefaultValues, categoricalSplits);
  1688. this.InnerOptions = context.string(null);
  1689. if (context.modelVersionWritten >= this.verNumFeaturesSerialized) {
  1690. this.NumFeatures = reader.int32();
  1691. }
  1692. // input type -> float32[NumFeatures]
  1693. // output type -> float32
  1694. }
  1695. };
  1696. mlnet.InternalTreeEnsemble = class {
  1697. constructor(context, usingDefaultValues, categoricalSplits) {
  1698. const reader = context.reader;
  1699. this.Trees = [];
  1700. const numTrees = reader.int32();
  1701. for (let i = 0; i < numTrees; i++) {
  1702. switch (reader.byte()) {
  1703. case mlnet.InternalTreeEnsemble.TreeType.Regression:
  1704. this.Trees.push(new mlnet.InternalRegressionTree(context, usingDefaultValues, categoricalSplits));
  1705. break;
  1706. case mlnet.InternalTreeEnsemble.TreeType.FastForest:
  1707. this.Trees.push(new mlnet.InternalQuantileRegressionTree(context, usingDefaultValues, categoricalSplits));
  1708. break;
  1709. case mlnet.InternalTreeEnsemble.TreeType.Affine:
  1710. // Affine regression trees do not actually work, nor is it clear how they ever
  1711. // could have worked within TLC, so the chance of this happening seems remote.
  1712. throw new mlnet.Error('Affine regression trees unsupported.');
  1713. default:
  1714. throw new mlnet.Error('Unsupported ensemble tree type.');
  1715. }
  1716. }
  1717. this.Bias = reader.float64();
  1718. this.FirstInputInitializationContent = context.string(null);
  1719. }
  1720. };
  1721. mlnet.InternalRegressionTree = class {
  1722. constructor(context, usingDefaultValue, categoricalSplits) {
  1723. const reader = context.reader;
  1724. this.NumLeaves = reader.int32();
  1725. this.MaxOuptut = reader.float64();
  1726. this.Weight = reader.float64();
  1727. this.LteChild = reader.int32s(reader.int32());
  1728. this.GtChild = reader.int32s(reader.int32());
  1729. this.SplitFeatures = reader.int32s(reader.int32());
  1730. if (categoricalSplits) {
  1731. const categoricalNodeIndices = reader.int32s(reader.int32());
  1732. if (categoricalNodeIndices.length > 0) {
  1733. this.CategoricalSplitFeatures = [];
  1734. this.CategoricalSplitFeatureRanges = [];
  1735. for (const index of categoricalNodeIndices) {
  1736. this.CategoricalSplitFeatures[index] = reader.int32s(reader.int32());
  1737. this.CategoricalSplitFeatureRanges[index] = reader.int32s(2);
  1738. }
  1739. }
  1740. }
  1741. this.Thresholds = reader.uint32s(reader.int32());
  1742. this.RawThresholds = reader.float32s(reader.int32());
  1743. this.DefaultValueForMissing = usingDefaultValue ? reader.float32s(reader.int32()) : null;
  1744. this.LeafValues = reader.float64s(reader.int32());
  1745. this.SplitGain = reader.float64s(reader.int32());
  1746. this.GainPValue = reader.float64s(reader.int32());
  1747. this.PreviousLeafValue = reader.float64s(reader.int32());
  1748. }
  1749. };
  1750. mlnet.InternalTreeEnsemble.TreeType = {
  1751. Regression: 0,
  1752. Affine: 1,
  1753. FastForest: 2
  1754. };
  1755. mlnet.TreeEnsembleModelParametersBasedOnRegressionTree = class extends mlnet.TreeEnsembleModelParameters {
  1756. constructor(context) {
  1757. super(context);
  1758. }
  1759. };
  1760. mlnet.FastTreeTweedieModelParameters = class extends mlnet.TreeEnsembleModelParametersBasedOnRegressionTree {
  1761. constructor(context) {
  1762. super(context);
  1763. }
  1764. get VerNumFeaturesSerialized() {
  1765. return 0x00010001;
  1766. }
  1767. get VerDefaultValueSerialized() {
  1768. return 0x00010002;
  1769. }
  1770. get VerCategoricalSplitSerialized() {
  1771. return 0x00010003;
  1772. }
  1773. };
  1774. mlnet.FastTreeRankingModelParameters = class extends mlnet.TreeEnsembleModelParametersBasedOnRegressionTree {
  1775. constructor(context) {
  1776. super(context);
  1777. }
  1778. get VerNumFeaturesSerialized() {
  1779. return 0x00010002;
  1780. }
  1781. get VerDefaultValueSerialized() {
  1782. return 0x00010004;
  1783. }
  1784. get VerCategoricalSplitSerialized() {
  1785. return 0x00010005;
  1786. }
  1787. };
  1788. mlnet.FastTreeBinaryModelParameters = class extends mlnet.TreeEnsembleModelParametersBasedOnRegressionTree {
  1789. constructor(context) {
  1790. super(context);
  1791. }
  1792. get VerNumFeaturesSerialized() {
  1793. return 0x00010002;
  1794. }
  1795. get VerDefaultValueSerialized() {
  1796. return 0x00010004;
  1797. }
  1798. get VerCategoricalSplitSerialized() {
  1799. return 0x00010005;
  1800. }
  1801. };
  1802. mlnet.FastTreeRegressionModelParameters = class extends mlnet.TreeEnsembleModelParametersBasedOnRegressionTree {
  1803. constructor(context) {
  1804. super(context);
  1805. }
  1806. get VerNumFeaturesSerialized() {
  1807. return 0x00010002;
  1808. }
  1809. get VerDefaultValueSerialized() {
  1810. return 0x00010004;
  1811. }
  1812. get VerCategoricalSplitSerialized() {
  1813. return 0x00010005;
  1814. }
  1815. };
  1816. mlnet.LightGbmRegressionModelParameters = class extends mlnet.TreeEnsembleModelParametersBasedOnRegressionTree {
  1817. constructor(context) {
  1818. super(context);
  1819. }
  1820. get VerNumFeaturesSerialized() {
  1821. return 0x00010002;
  1822. }
  1823. get VerDefaultValueSerialized() {
  1824. return 0x00010004;
  1825. }
  1826. get VerCategoricalSplitSerialized() {
  1827. return 0x00010005;
  1828. }
  1829. };
  1830. mlnet.LightGbmBinaryModelParameters = class extends mlnet.TreeEnsembleModelParametersBasedOnRegressionTree {
  1831. constructor(context) {
  1832. super(context);
  1833. }
  1834. get VerNumFeaturesSerialized() {
  1835. return 0x00010002;
  1836. }
  1837. get VerDefaultValueSerialized() {
  1838. return 0x00010004;
  1839. }
  1840. get VerCategoricalSplitSerialized() {
  1841. return 0x00010005;
  1842. }
  1843. };
  1844. mlnet.FeatureWeightsCalibratedModelParameters = class extends mlnet.ValueMapperCalibratedModelParametersBase {
  1845. constructor(context) {
  1846. super(context);
  1847. // debugger;
  1848. }
  1849. };
  1850. mlnet.FastTreePredictionWrapper = class {
  1851. constructor(/* context */) {
  1852. }
  1853. };
  1854. mlnet.FastForestClassificationPredictor = class extends mlnet.FastTreePredictionWrapper {
  1855. constructor(context) {
  1856. super(context);
  1857. }
  1858. };
  1859. mlnet.PlattCalibrator = class {
  1860. constructor(context) {
  1861. const reader = context.reader;
  1862. this.ParamA = reader.float64();
  1863. this.ParamB = reader.float64();
  1864. }
  1865. };
  1866. mlnet.Codec = class {
  1867. constructor(reader) {
  1868. this.name = reader.string();
  1869. const size = reader.leb128();
  1870. const data = reader.read(size);
  1871. reader = new mlnet.BinaryReader(data);
  1872. switch (this.name) {
  1873. case 'Boolean': break;
  1874. case 'Single': break;
  1875. case 'Double': break;
  1876. case 'Byte': break;
  1877. case 'Int32': break;
  1878. case 'UInt32': break;
  1879. case 'Int64': break;
  1880. case 'TextSpan': break;
  1881. case 'VBuffer':
  1882. this.itemType = new mlnet.Codec(reader);
  1883. this.dims = reader.int32s(reader.int32());
  1884. break;
  1885. case 'Key':
  1886. case 'Key2':
  1887. this.itemType = new mlnet.Codec(reader);
  1888. this.count = reader.uint64();
  1889. break;
  1890. default:
  1891. throw new mlnet.Error("Unsupported codec '" + this.name + "'.");
  1892. }
  1893. }
  1894. read(reader, count) {
  1895. const values = [];
  1896. switch (this.name) {
  1897. case 'Single':
  1898. for (let i = 0; i < count; i++) {
  1899. values.push(reader.float32());
  1900. }
  1901. break;
  1902. case 'Int32':
  1903. for (let i = 0; i < count; i++) {
  1904. values.push(reader.int32());
  1905. }
  1906. break;
  1907. case 'Int64':
  1908. for (let i = 0; i < count; i++) {
  1909. values.push(reader.int64());
  1910. }
  1911. break;
  1912. default:
  1913. throw new mlnet.Error("Unsupported codec read operation '" + this.name + "'.");
  1914. }
  1915. return values;
  1916. }
  1917. };
  1918. mlnet.SequentialTransformerBase = class {
  1919. constructor(context) {
  1920. const reader = context.reader;
  1921. this.WindowSize = reader.int32();
  1922. this.InitialWindowSize = reader.int32();
  1923. this.inputs = [];
  1924. this.inputs.push({ name: context.string() });
  1925. this.outputs = [];
  1926. this.outputs.push({ name: context.string() });
  1927. this.ConfidenceLowerBoundColumn = reader.string();
  1928. this.ConfidenceUpperBoundColumn = reader.string();
  1929. this.Type = new mlnet.Codec(reader);
  1930. }
  1931. };
  1932. mlnet.AnomalyDetectionStateBase = class {
  1933. constructor(context) {
  1934. const reader = context.reader;
  1935. this.LogMartingaleUpdateBuffer = mlnet.AnomalyDetectionStateBase._deserializeFixedSizeQueueDouble(reader);
  1936. this.RawScoreBuffer = mlnet.AnomalyDetectionStateBase._deserializeFixedSizeQueueDouble(reader);
  1937. this.LogMartingaleValue = reader.float64();
  1938. this.SumSquaredDist = reader.float64();
  1939. this.MartingaleAlertCounter = reader.int32();
  1940. }
  1941. static _deserializeFixedSizeQueueDouble(reader) {
  1942. /* let capacity = */ reader.int32();
  1943. const count = reader.int32();
  1944. const queue = [];
  1945. for (let i = 0; i < count; i++) {
  1946. queue.push(reader.float64());
  1947. }
  1948. return queue;
  1949. }
  1950. };
  1951. mlnet.SequentialAnomalyDetectionTransformBase = class extends mlnet.SequentialTransformerBase {
  1952. constructor(context) {
  1953. super(context);
  1954. const reader = context.reader;
  1955. this.Martingale = reader.byte();
  1956. this.ThresholdScore = reader.byte();
  1957. this.Side = reader.byte();
  1958. this.PowerMartingaleEpsilon = reader.float64();
  1959. this.AlertThreshold = reader.float64();
  1960. this.State = new mlnet.AnomalyDetectionStateBase(context);
  1961. }
  1962. };
  1963. mlnet.TimeSeriesUtils = class {
  1964. static deserializeFixedSizeQueueSingle(reader) {
  1965. /* const capacity = */ reader.int32();
  1966. const count = reader.int32();
  1967. const queue = [];
  1968. for (let i = 0; i < count; i++) {
  1969. queue.push(reader.float32());
  1970. }
  1971. return queue;
  1972. }
  1973. };
  1974. mlnet.IidAnomalyDetectionBase = class extends mlnet.SequentialAnomalyDetectionTransformBase {
  1975. constructor(context) {
  1976. super(context);
  1977. const reader = context.reader;
  1978. this.WindowedBuffer = mlnet.TimeSeriesUtils.deserializeFixedSizeQueueSingle(reader);
  1979. this.InitialWindowedBuffer = mlnet.TimeSeriesUtils.deserializeFixedSizeQueueSingle(reader);
  1980. }
  1981. };
  1982. mlnet.IidAnomalyDetectionBaseWrapper = class {
  1983. constructor(context) {
  1984. const internalTransform = new mlnet.IidAnomalyDetectionBase(context);
  1985. for (const key of Object.keys(internalTransform)) {
  1986. this[key] = internalTransform[key];
  1987. }
  1988. }
  1989. };
  1990. mlnet.IidChangePointDetector = class extends mlnet.IidAnomalyDetectionBaseWrapper {
  1991. constructor(context) {
  1992. super(context);
  1993. }
  1994. };
  1995. mlnet.IidSpikeDetector = class extends mlnet.IidAnomalyDetectionBaseWrapper {
  1996. constructor(context) {
  1997. super(context);
  1998. }
  1999. };
  2000. mlnet.SequenceModelerBase = class {
  2001. constructor(/* context */) {
  2002. }
  2003. };
  2004. mlnet.RankSelectionMethod = {
  2005. Fixed: 0,
  2006. Exact: 1,
  2007. Fact: 2
  2008. };
  2009. mlnet.AdaptiveSingularSpectrumSequenceModelerInternal = class extends mlnet.SequenceModelerBase {
  2010. constructor(context) {
  2011. super(context);
  2012. const reader = context.reader;
  2013. this._seriesLength = reader.int32();
  2014. this._windowSize = reader.int32();
  2015. this._trainSize = reader.int32();
  2016. this._rank = reader.int32();
  2017. this._discountFactor = reader.float32();
  2018. this._rankSelectionMethod = reader.byte(); // RankSelectionMethod
  2019. const isWeightSet = reader.byte();
  2020. this._alpha = reader.float32s(reader.int32());
  2021. if (context.modelVersionReadable >= 0x00010002) {
  2022. this._state = reader.float32s(reader.int32());
  2023. }
  2024. this.ShouldComputeForecastIntervals = reader.byte();
  2025. this._observationNoiseVariance = reader.float32();
  2026. this._autoregressionNoiseVariance = reader.float32();
  2027. this._observationNoiseMean = reader.float32();
  2028. this._autoregressionNoiseMean = reader.float32();
  2029. if (context.modelVersionReadable >= 0x00010002) {
  2030. this._nextPrediction = reader.float32();
  2031. }
  2032. this._maxRank = reader.int32();
  2033. this._shouldStablize = reader.byte();
  2034. this._shouldMaintainInfo = reader.byte();
  2035. this._maxTrendRatio = reader.float64();
  2036. if (isWeightSet) {
  2037. this._wTrans = reader.float32s(reader.int32());
  2038. this._y = reader.float32s(reader.int32());
  2039. }
  2040. this._buffer = mlnet.TimeSeriesUtils.deserializeFixedSizeQueueSingle(reader);
  2041. }
  2042. };
  2043. mlnet.SequentialForecastingTransformBase = class extends mlnet.SequentialTransformerBase {
  2044. constructor(context) {
  2045. super(context);
  2046. const reader = context.reader;
  2047. this._outputLength = reader.int32();
  2048. }
  2049. };
  2050. mlnet.SsaForecastingBaseWrapper = class extends mlnet.SequentialForecastingTransformBase {
  2051. constructor(context) {
  2052. super(context);
  2053. const reader = context.reader;
  2054. this.IsAdaptive = reader.boolean();
  2055. this.Horizon = reader.int32();
  2056. this.ConfidenceLevel = reader.float32();
  2057. this.WindowedBuffer = mlnet.TimeSeriesUtils.deserializeFixedSizeQueueSingle(reader);
  2058. this.InitialWindowedBuffer = mlnet.TimeSeriesUtils.deserializeFixedSizeQueueSingle(reader);
  2059. this.Model = context.open('SSA');
  2060. }
  2061. };
  2062. mlnet.SsaForecastingTransformer = class extends mlnet.SsaForecastingBaseWrapper {
  2063. constructor(context) {
  2064. super(context);
  2065. }
  2066. };
  2067. mlnet.ColumnSelectingTransformer = class {
  2068. constructor(context) {
  2069. const reader = context.reader;
  2070. if (context.check('DRPCOLST', 0x00010002, 0x00010002)) {
  2071. throw new mlnet.Error("'LoadDropColumnsTransform' not supported.");
  2072. }
  2073. else if (context.check('CHSCOLSF', 0x00010001, 0x00010001)) {
  2074. reader.int32(); // cbFloat
  2075. this.KeepHidden = this._getHiddenOption(reader.byte());
  2076. const count = reader.int32();
  2077. this.inputs = [];
  2078. for (let colIdx = 0; colIdx < count; colIdx++) {
  2079. const dst = context.string();
  2080. this.inputs.push(dst);
  2081. context.string(); // src
  2082. this._getHiddenOption(reader.byte()); // colKeepHidden
  2083. }
  2084. }
  2085. else {
  2086. const keepColumns = reader.boolean();
  2087. this.KeepHidden = reader.boolean();
  2088. this.IgnoreMissing = reader.boolean();
  2089. const length = reader.int32();
  2090. this.inputs = [];
  2091. for (let i = 0; i < length; i++) {
  2092. this.inputs.push({ name: context.string() });
  2093. }
  2094. if (keepColumns) {
  2095. this.ColumnsToKeep = this.inputs;
  2096. }
  2097. else {
  2098. this.ColumnsToDrop = this.inputs;
  2099. }
  2100. }
  2101. }
  2102. _getHiddenOption(value) {
  2103. switch (value) {
  2104. case 1: return true;
  2105. case 2: return false;
  2106. default: throw new mlnet.Error('Unsupported hide option specified');
  2107. }
  2108. }
  2109. };
  2110. mlnet.XGBoostMulticlass = class {};
  2111. mlnet.NltTokenizeTransform = class {};
  2112. mlnet.DropColumnsTransform = class {};
  2113. mlnet.StopWordsTransform = class {};
  2114. mlnet.CSharpTransform = class {};
  2115. mlnet.GenericScoreTransform = class {};
  2116. mlnet.NormalizeTransform = class {};
  2117. mlnet.CdfColumnFunction = class {
  2118. constructor(/* context, typeSrc */) {
  2119. // TODO
  2120. }
  2121. };
  2122. mlnet.MultiClassNetPredictor = class {};
  2123. mlnet.ProtonNNMCPred = class {};
  2124. mlnet.Error = class extends Error {
  2125. constructor(message) {
  2126. super(message);
  2127. this.name = 'Error loading ML.NET model.';
  2128. }
  2129. };
  2130. if (module && module.exports) {
  2131. module.exports.ModelFactory = mlnet.ModelFactory;
  2132. }