GlobOpt.cpp 641 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246924792489249925092519252925392549255925692579258925992609261926292639264926592669267926892699270927192729273927492759276927792789279928092819282928392849285928692879288928992909291929292939294929592969297929892999300930193029303930493059306930793089309931093119312931393149315931693179318931993209321932293239324932593269327932893299330933193329333933493359336933793389339934093419342934393449345934693479348934993509351935293539354935593569357935893599360936193629363936493659366936793689369937093719372937393749375937693779378937993809381938293839384938593869387938893899390939193929393939493959396939793989399940094019402940394049405940694079408940994109411941294139414941594169417941894199420942194229423942494259426942794289429943094319432943394349435943694379438943994409441944294439444944594469447944894499450945194529453945494559456945794589459946094619462946394649465946694679468946994709471947294739474947594769477947894799480948194829483948494859486948794889489949094919492949394949495949694979498949995009501950295039504950595069507950895099510951195129513951495159516951795189519952095219522952395249525952695279528952995309531953295339534953595369537953895399540954195429543954495459546954795489549955095519552955395549555955695579558955995609561956295639564956595669567956895699570957195729573957495759576957795789579958095819582958395849585958695879588958995909591959295939594959595969597959895999600960196029603960496059606960796089609961096119612961396149615961696179618961996209621962296239624962596269627962896299630963196329633963496359636963796389639964096419642964396449645964696479648964996509651965296539654965596569657965896599660966196629663966496659666966796689669967096719672967396749675967696779678967996809681968296839684968596869687968896899690969196929693969496959696969796989699970097019702970397049705970697079708970997109711971297139714971597169717971897199720972197229723972497259726972797289729973097319732973397349735973697379738973997409741974297439744974597469747974897499750975197529753975497559756975797589759976097619762976397649765976697679768976997709771977297739774977597769777977897799780978197829783978497859786978797889789979097919792979397949795979697979798979998009801980298039804980598069807980898099810981198129813981498159816981798189819982098219822982398249825982698279828982998309831983298339834983598369837983898399840984198429843984498459846984798489849985098519852985398549855985698579858985998609861986298639864986598669867986898699870987198729873987498759876987798789879988098819882988398849885988698879888988998909891989298939894989598969897989898999900990199029903990499059906990799089909991099119912991399149915991699179918991999209921992299239924992599269927992899299930993199329933993499359936993799389939994099419942994399449945994699479948994999509951995299539954995599569957995899599960996199629963996499659966996799689969997099719972997399749975997699779978997999809981998299839984998599869987998899899990999199929993999499959996999799989999100001000110002100031000410005100061000710008100091001010011100121001310014100151001610017100181001910020100211002210023100241002510026100271002810029100301003110032100331003410035100361003710038100391004010041100421004310044100451004610047100481004910050100511005210053100541005510056100571005810059100601006110062100631006410065100661006710068100691007010071100721007310074100751007610077100781007910080100811008210083100841008510086100871008810089100901009110092100931009410095100961009710098100991010010101101021010310104101051010610107101081010910110101111011210113101141011510116101171011810119101201012110122101231012410125101261012710128101291013010131101321013310134101351013610137101381013910140101411014210143101441014510146101471014810149101501015110152101531015410155101561015710158101591016010161101621016310164101651016610167101681016910170101711017210173101741017510176101771017810179101801018110182101831018410185101861018710188101891019010191101921019310194101951019610197101981019910200102011020210203102041020510206102071020810209102101021110212102131021410215102161021710218102191022010221102221022310224102251022610227102281022910230102311023210233102341023510236102371023810239102401024110242102431024410245102461024710248102491025010251102521025310254102551025610257102581025910260102611026210263102641026510266102671026810269102701027110272102731027410275102761027710278102791028010281102821028310284102851028610287102881028910290102911029210293102941029510296102971029810299103001030110302103031030410305103061030710308103091031010311103121031310314103151031610317103181031910320103211032210323103241032510326103271032810329103301033110332103331033410335103361033710338103391034010341103421034310344103451034610347103481034910350103511035210353103541035510356103571035810359103601036110362103631036410365103661036710368103691037010371103721037310374103751037610377103781037910380103811038210383103841038510386103871038810389103901039110392103931039410395103961039710398103991040010401104021040310404104051040610407104081040910410104111041210413104141041510416104171041810419104201042110422104231042410425104261042710428104291043010431104321043310434104351043610437104381043910440104411044210443104441044510446104471044810449104501045110452104531045410455104561045710458104591046010461104621046310464104651046610467104681046910470104711047210473104741047510476104771047810479104801048110482104831048410485104861048710488104891049010491104921049310494104951049610497104981049910500105011050210503105041050510506105071050810509105101051110512105131051410515105161051710518105191052010521105221052310524105251052610527105281052910530105311053210533105341053510536105371053810539105401054110542105431054410545105461054710548105491055010551105521055310554105551055610557105581055910560105611056210563105641056510566105671056810569105701057110572105731057410575105761057710578105791058010581105821058310584105851058610587105881058910590105911059210593105941059510596105971059810599106001060110602106031060410605106061060710608106091061010611106121061310614106151061610617106181061910620106211062210623106241062510626106271062810629106301063110632106331063410635106361063710638106391064010641106421064310644106451064610647106481064910650106511065210653106541065510656106571065810659106601066110662106631066410665106661066710668106691067010671106721067310674106751067610677106781067910680106811068210683106841068510686106871068810689106901069110692106931069410695106961069710698106991070010701107021070310704107051070610707107081070910710107111071210713107141071510716107171071810719107201072110722107231072410725107261072710728107291073010731107321073310734107351073610737107381073910740107411074210743107441074510746107471074810749107501075110752107531075410755107561075710758107591076010761107621076310764107651076610767107681076910770107711077210773107741077510776107771077810779107801078110782107831078410785107861078710788107891079010791107921079310794107951079610797107981079910800108011080210803108041080510806108071080810809108101081110812108131081410815108161081710818108191082010821108221082310824108251082610827108281082910830108311083210833108341083510836108371083810839108401084110842108431084410845108461084710848108491085010851108521085310854108551085610857108581085910860108611086210863108641086510866108671086810869108701087110872108731087410875108761087710878108791088010881108821088310884108851088610887108881088910890108911089210893108941089510896108971089810899109001090110902109031090410905109061090710908109091091010911109121091310914109151091610917109181091910920109211092210923109241092510926109271092810929109301093110932109331093410935109361093710938109391094010941109421094310944109451094610947109481094910950109511095210953109541095510956109571095810959109601096110962109631096410965109661096710968109691097010971109721097310974109751097610977109781097910980109811098210983109841098510986109871098810989109901099110992109931099410995109961099710998109991100011001110021100311004110051100611007110081100911010110111101211013110141101511016110171101811019110201102111022110231102411025110261102711028110291103011031110321103311034110351103611037110381103911040110411104211043110441104511046110471104811049110501105111052110531105411055110561105711058110591106011061110621106311064110651106611067110681106911070110711107211073110741107511076110771107811079110801108111082110831108411085110861108711088110891109011091110921109311094110951109611097110981109911100111011110211103111041110511106111071110811109111101111111112111131111411115111161111711118111191112011121111221112311124111251112611127111281112911130111311113211133111341113511136111371113811139111401114111142111431114411145111461114711148111491115011151111521115311154111551115611157111581115911160111611116211163111641116511166111671116811169111701117111172111731117411175111761117711178111791118011181111821118311184111851118611187111881118911190111911119211193111941119511196111971119811199112001120111202112031120411205112061120711208112091121011211112121121311214112151121611217112181121911220112211122211223112241122511226112271122811229112301123111232112331123411235112361123711238112391124011241112421124311244112451124611247112481124911250112511125211253112541125511256112571125811259112601126111262112631126411265112661126711268112691127011271112721127311274112751127611277112781127911280112811128211283112841128511286112871128811289112901129111292112931129411295112961129711298112991130011301113021130311304113051130611307113081130911310113111131211313113141131511316113171131811319113201132111322113231132411325113261132711328113291133011331113321133311334113351133611337113381133911340113411134211343113441134511346113471134811349113501135111352113531135411355113561135711358113591136011361113621136311364113651136611367113681136911370113711137211373113741137511376113771137811379113801138111382113831138411385113861138711388113891139011391113921139311394113951139611397113981139911400114011140211403114041140511406114071140811409114101141111412114131141411415114161141711418114191142011421114221142311424114251142611427114281142911430114311143211433114341143511436114371143811439114401144111442114431144411445114461144711448114491145011451114521145311454114551145611457114581145911460114611146211463114641146511466114671146811469114701147111472114731147411475114761147711478114791148011481114821148311484114851148611487114881148911490114911149211493114941149511496114971149811499115001150111502115031150411505115061150711508115091151011511115121151311514115151151611517115181151911520115211152211523115241152511526115271152811529115301153111532115331153411535115361153711538115391154011541115421154311544115451154611547115481154911550115511155211553115541155511556115571155811559115601156111562115631156411565115661156711568115691157011571115721157311574115751157611577115781157911580115811158211583115841158511586115871158811589115901159111592115931159411595115961159711598115991160011601116021160311604116051160611607116081160911610116111161211613116141161511616116171161811619116201162111622116231162411625116261162711628116291163011631116321163311634116351163611637116381163911640116411164211643116441164511646116471164811649116501165111652116531165411655116561165711658116591166011661116621166311664116651166611667116681166911670116711167211673116741167511676116771167811679116801168111682116831168411685116861168711688116891169011691116921169311694116951169611697116981169911700117011170211703117041170511706117071170811709117101171111712117131171411715117161171711718117191172011721117221172311724117251172611727117281172911730117311173211733117341173511736117371173811739117401174111742117431174411745117461174711748117491175011751117521175311754117551175611757117581175911760117611176211763117641176511766117671176811769117701177111772117731177411775117761177711778117791178011781117821178311784117851178611787117881178911790117911179211793117941179511796117971179811799118001180111802118031180411805118061180711808118091181011811118121181311814118151181611817118181181911820118211182211823118241182511826118271182811829118301183111832118331183411835118361183711838118391184011841118421184311844118451184611847118481184911850118511185211853118541185511856118571185811859118601186111862118631186411865118661186711868118691187011871118721187311874118751187611877118781187911880118811188211883118841188511886118871188811889118901189111892118931189411895118961189711898118991190011901119021190311904119051190611907119081190911910119111191211913119141191511916119171191811919119201192111922119231192411925119261192711928119291193011931119321193311934119351193611937119381193911940119411194211943119441194511946119471194811949119501195111952119531195411955119561195711958119591196011961119621196311964119651196611967119681196911970119711197211973119741197511976119771197811979119801198111982119831198411985119861198711988119891199011991119921199311994119951199611997119981199912000120011200212003120041200512006120071200812009120101201112012120131201412015120161201712018120191202012021120221202312024120251202612027120281202912030120311203212033120341203512036120371203812039120401204112042120431204412045120461204712048120491205012051120521205312054120551205612057120581205912060120611206212063120641206512066120671206812069120701207112072120731207412075120761207712078120791208012081120821208312084120851208612087120881208912090120911209212093120941209512096120971209812099121001210112102121031210412105121061210712108121091211012111121121211312114121151211612117121181211912120121211212212123121241212512126121271212812129121301213112132121331213412135121361213712138121391214012141121421214312144121451214612147121481214912150121511215212153121541215512156121571215812159121601216112162121631216412165121661216712168121691217012171121721217312174121751217612177121781217912180121811218212183121841218512186121871218812189121901219112192121931219412195121961219712198121991220012201122021220312204122051220612207122081220912210122111221212213122141221512216122171221812219122201222112222122231222412225122261222712228122291223012231122321223312234122351223612237122381223912240122411224212243122441224512246122471224812249122501225112252122531225412255122561225712258122591226012261122621226312264122651226612267122681226912270122711227212273122741227512276122771227812279122801228112282122831228412285122861228712288122891229012291122921229312294122951229612297122981229912300123011230212303123041230512306123071230812309123101231112312123131231412315123161231712318123191232012321123221232312324123251232612327123281232912330123311233212333123341233512336123371233812339123401234112342123431234412345123461234712348123491235012351123521235312354123551235612357123581235912360123611236212363123641236512366123671236812369123701237112372123731237412375123761237712378123791238012381123821238312384123851238612387123881238912390123911239212393123941239512396123971239812399124001240112402124031240412405124061240712408124091241012411124121241312414124151241612417124181241912420124211242212423124241242512426124271242812429124301243112432124331243412435124361243712438124391244012441124421244312444124451244612447124481244912450124511245212453124541245512456124571245812459124601246112462124631246412465124661246712468124691247012471124721247312474124751247612477124781247912480124811248212483124841248512486124871248812489124901249112492124931249412495124961249712498124991250012501125021250312504125051250612507125081250912510125111251212513125141251512516125171251812519125201252112522125231252412525125261252712528125291253012531125321253312534125351253612537125381253912540125411254212543125441254512546125471254812549125501255112552125531255412555125561255712558125591256012561125621256312564125651256612567125681256912570125711257212573125741257512576125771257812579125801258112582125831258412585125861258712588125891259012591125921259312594125951259612597125981259912600126011260212603126041260512606126071260812609126101261112612126131261412615126161261712618126191262012621126221262312624126251262612627126281262912630126311263212633126341263512636126371263812639126401264112642126431264412645126461264712648126491265012651126521265312654126551265612657126581265912660126611266212663126641266512666126671266812669126701267112672126731267412675126761267712678126791268012681126821268312684126851268612687126881268912690126911269212693126941269512696126971269812699127001270112702127031270412705127061270712708127091271012711127121271312714127151271612717127181271912720127211272212723127241272512726127271272812729127301273112732127331273412735127361273712738127391274012741127421274312744127451274612747127481274912750127511275212753127541275512756127571275812759127601276112762127631276412765127661276712768127691277012771127721277312774127751277612777127781277912780127811278212783127841278512786127871278812789127901279112792127931279412795127961279712798127991280012801128021280312804128051280612807128081280912810128111281212813128141281512816128171281812819128201282112822128231282412825128261282712828128291283012831128321283312834128351283612837128381283912840128411284212843128441284512846128471284812849128501285112852128531285412855128561285712858128591286012861128621286312864128651286612867128681286912870128711287212873128741287512876128771287812879128801288112882128831288412885128861288712888128891289012891128921289312894128951289612897128981289912900129011290212903129041290512906129071290812909129101291112912129131291412915129161291712918129191292012921129221292312924129251292612927129281292912930129311293212933129341293512936129371293812939129401294112942129431294412945129461294712948129491295012951129521295312954129551295612957129581295912960129611296212963129641296512966129671296812969129701297112972129731297412975129761297712978129791298012981129821298312984129851298612987129881298912990129911299212993129941299512996129971299812999130001300113002130031300413005130061300713008130091301013011130121301313014130151301613017130181301913020130211302213023130241302513026130271302813029130301303113032130331303413035130361303713038130391304013041130421304313044130451304613047130481304913050130511305213053130541305513056130571305813059130601306113062130631306413065130661306713068130691307013071130721307313074130751307613077130781307913080130811308213083130841308513086130871308813089130901309113092130931309413095130961309713098130991310013101131021310313104131051310613107131081310913110131111311213113131141311513116131171311813119131201312113122131231312413125131261312713128131291313013131131321313313134131351313613137131381313913140131411314213143131441314513146131471314813149131501315113152131531315413155131561315713158131591316013161131621316313164131651316613167131681316913170131711317213173131741317513176131771317813179131801318113182131831318413185131861318713188131891319013191131921319313194131951319613197131981319913200132011320213203132041320513206132071320813209132101321113212132131321413215132161321713218132191322013221132221322313224132251322613227132281322913230132311323213233132341323513236132371323813239132401324113242132431324413245132461324713248132491325013251132521325313254132551325613257132581325913260132611326213263132641326513266132671326813269132701327113272132731327413275132761327713278132791328013281132821328313284132851328613287132881328913290132911329213293132941329513296132971329813299133001330113302133031330413305133061330713308133091331013311133121331313314133151331613317133181331913320133211332213323133241332513326133271332813329133301333113332133331333413335133361333713338133391334013341133421334313344133451334613347133481334913350133511335213353133541335513356133571335813359133601336113362133631336413365133661336713368133691337013371133721337313374133751337613377133781337913380133811338213383133841338513386133871338813389133901339113392133931339413395133961339713398133991340013401134021340313404134051340613407134081340913410134111341213413134141341513416134171341813419134201342113422134231342413425134261342713428134291343013431134321343313434134351343613437134381343913440134411344213443134441344513446134471344813449134501345113452134531345413455134561345713458134591346013461134621346313464134651346613467134681346913470134711347213473134741347513476134771347813479134801348113482134831348413485134861348713488134891349013491134921349313494134951349613497134981349913500135011350213503135041350513506135071350813509135101351113512135131351413515135161351713518135191352013521135221352313524135251352613527135281352913530135311353213533135341353513536135371353813539135401354113542135431354413545135461354713548135491355013551135521355313554135551355613557135581355913560135611356213563135641356513566135671356813569135701357113572135731357413575135761357713578135791358013581135821358313584135851358613587135881358913590135911359213593135941359513596135971359813599136001360113602136031360413605136061360713608136091361013611136121361313614136151361613617136181361913620136211362213623136241362513626136271362813629136301363113632136331363413635136361363713638136391364013641136421364313644136451364613647136481364913650136511365213653136541365513656136571365813659136601366113662136631366413665136661366713668136691367013671136721367313674136751367613677136781367913680136811368213683136841368513686136871368813689136901369113692136931369413695136961369713698136991370013701137021370313704137051370613707137081370913710137111371213713137141371513716137171371813719137201372113722137231372413725137261372713728137291373013731137321373313734137351373613737137381373913740137411374213743137441374513746137471374813749137501375113752137531375413755137561375713758137591376013761137621376313764137651376613767137681376913770137711377213773137741377513776137771377813779137801378113782137831378413785137861378713788137891379013791137921379313794137951379613797137981379913800138011380213803138041380513806138071380813809138101381113812138131381413815138161381713818138191382013821138221382313824138251382613827138281382913830138311383213833138341383513836138371383813839138401384113842138431384413845138461384713848138491385013851138521385313854138551385613857138581385913860138611386213863138641386513866138671386813869138701387113872138731387413875138761387713878138791388013881138821388313884138851388613887138881388913890138911389213893138941389513896138971389813899139001390113902139031390413905139061390713908139091391013911139121391313914139151391613917139181391913920139211392213923139241392513926139271392813929139301393113932139331393413935139361393713938139391394013941139421394313944139451394613947139481394913950139511395213953139541395513956139571395813959139601396113962139631396413965139661396713968139691397013971139721397313974139751397613977139781397913980139811398213983139841398513986139871398813989139901399113992139931399413995139961399713998139991400014001140021400314004140051400614007140081400914010140111401214013140141401514016140171401814019140201402114022140231402414025140261402714028140291403014031140321403314034140351403614037140381403914040140411404214043140441404514046140471404814049140501405114052140531405414055140561405714058140591406014061140621406314064140651406614067140681406914070140711407214073140741407514076140771407814079140801408114082140831408414085140861408714088140891409014091140921409314094140951409614097140981409914100141011410214103141041410514106141071410814109141101411114112141131411414115141161411714118141191412014121141221412314124141251412614127141281412914130141311413214133141341413514136141371413814139141401414114142141431414414145141461414714148141491415014151141521415314154141551415614157141581415914160141611416214163141641416514166141671416814169141701417114172141731417414175141761417714178141791418014181141821418314184141851418614187141881418914190141911419214193141941419514196141971419814199142001420114202142031420414205142061420714208142091421014211142121421314214142151421614217142181421914220142211422214223142241422514226142271422814229142301423114232142331423414235142361423714238142391424014241142421424314244142451424614247142481424914250142511425214253142541425514256142571425814259142601426114262142631426414265142661426714268142691427014271142721427314274142751427614277142781427914280142811428214283142841428514286142871428814289142901429114292142931429414295142961429714298142991430014301143021430314304143051430614307143081430914310143111431214313143141431514316143171431814319143201432114322143231432414325143261432714328143291433014331143321433314334143351433614337143381433914340143411434214343143441434514346143471434814349143501435114352143531435414355143561435714358143591436014361143621436314364143651436614367143681436914370143711437214373143741437514376143771437814379143801438114382143831438414385143861438714388143891439014391143921439314394143951439614397143981439914400144011440214403144041440514406144071440814409144101441114412144131441414415144161441714418144191442014421144221442314424144251442614427144281442914430144311443214433144341443514436144371443814439144401444114442144431444414445144461444714448144491445014451144521445314454144551445614457144581445914460144611446214463144641446514466144671446814469144701447114472144731447414475144761447714478144791448014481144821448314484144851448614487144881448914490144911449214493144941449514496144971449814499145001450114502145031450414505145061450714508145091451014511145121451314514145151451614517145181451914520145211452214523145241452514526145271452814529145301453114532145331453414535145361453714538145391454014541145421454314544145451454614547145481454914550145511455214553145541455514556145571455814559145601456114562145631456414565145661456714568145691457014571145721457314574145751457614577145781457914580145811458214583145841458514586145871458814589145901459114592145931459414595145961459714598145991460014601146021460314604146051460614607146081460914610146111461214613146141461514616146171461814619146201462114622146231462414625146261462714628146291463014631146321463314634146351463614637146381463914640146411464214643146441464514646146471464814649146501465114652146531465414655146561465714658146591466014661146621466314664146651466614667146681466914670146711467214673146741467514676146771467814679146801468114682146831468414685146861468714688146891469014691146921469314694146951469614697146981469914700147011470214703147041470514706147071470814709147101471114712147131471414715147161471714718147191472014721147221472314724147251472614727147281472914730147311473214733147341473514736147371473814739147401474114742147431474414745147461474714748147491475014751147521475314754147551475614757147581475914760147611476214763147641476514766147671476814769147701477114772147731477414775147761477714778147791478014781147821478314784147851478614787147881478914790147911479214793147941479514796147971479814799148001480114802148031480414805148061480714808148091481014811148121481314814148151481614817148181481914820148211482214823148241482514826148271482814829148301483114832148331483414835148361483714838148391484014841148421484314844148451484614847148481484914850148511485214853148541485514856148571485814859148601486114862148631486414865148661486714868148691487014871148721487314874148751487614877148781487914880148811488214883148841488514886148871488814889148901489114892148931489414895148961489714898148991490014901149021490314904149051490614907149081490914910149111491214913149141491514916149171491814919149201492114922149231492414925149261492714928149291493014931149321493314934149351493614937149381493914940149411494214943149441494514946149471494814949149501495114952149531495414955149561495714958149591496014961149621496314964149651496614967149681496914970149711497214973149741497514976149771497814979149801498114982149831498414985149861498714988149891499014991149921499314994149951499614997149981499915000150011500215003150041500515006150071500815009150101501115012150131501415015150161501715018150191502015021150221502315024150251502615027150281502915030150311503215033150341503515036150371503815039150401504115042150431504415045150461504715048150491505015051150521505315054150551505615057150581505915060150611506215063150641506515066150671506815069150701507115072150731507415075150761507715078150791508015081150821508315084150851508615087150881508915090150911509215093150941509515096150971509815099151001510115102151031510415105151061510715108151091511015111151121511315114151151511615117151181511915120151211512215123151241512515126151271512815129151301513115132151331513415135151361513715138151391514015141151421514315144151451514615147151481514915150151511515215153151541515515156151571515815159151601516115162151631516415165151661516715168151691517015171151721517315174151751517615177151781517915180151811518215183151841518515186151871518815189151901519115192151931519415195151961519715198151991520015201152021520315204152051520615207152081520915210152111521215213152141521515216152171521815219152201522115222152231522415225152261522715228152291523015231152321523315234152351523615237152381523915240152411524215243152441524515246152471524815249152501525115252152531525415255152561525715258152591526015261152621526315264152651526615267152681526915270152711527215273152741527515276152771527815279152801528115282152831528415285152861528715288152891529015291152921529315294152951529615297152981529915300153011530215303153041530515306153071530815309153101531115312153131531415315153161531715318153191532015321153221532315324153251532615327153281532915330153311533215333153341533515336153371533815339153401534115342153431534415345153461534715348153491535015351153521535315354153551535615357153581535915360153611536215363153641536515366153671536815369153701537115372153731537415375153761537715378153791538015381153821538315384153851538615387153881538915390153911539215393153941539515396153971539815399154001540115402154031540415405154061540715408154091541015411154121541315414154151541615417154181541915420154211542215423154241542515426154271542815429154301543115432154331543415435154361543715438154391544015441154421544315444154451544615447154481544915450154511545215453154541545515456154571545815459154601546115462154631546415465154661546715468154691547015471154721547315474154751547615477154781547915480154811548215483154841548515486154871548815489154901549115492154931549415495154961549715498154991550015501155021550315504155051550615507155081550915510155111551215513155141551515516155171551815519155201552115522155231552415525155261552715528155291553015531155321553315534155351553615537155381553915540155411554215543155441554515546155471554815549155501555115552155531555415555155561555715558155591556015561155621556315564155651556615567155681556915570155711557215573155741557515576155771557815579155801558115582155831558415585155861558715588155891559015591155921559315594155951559615597155981559915600156011560215603156041560515606156071560815609156101561115612156131561415615156161561715618156191562015621156221562315624156251562615627156281562915630156311563215633156341563515636156371563815639156401564115642156431564415645156461564715648156491565015651156521565315654156551565615657156581565915660156611566215663156641566515666156671566815669156701567115672156731567415675156761567715678156791568015681156821568315684156851568615687156881568915690156911569215693156941569515696156971569815699157001570115702157031570415705157061570715708157091571015711157121571315714157151571615717157181571915720157211572215723157241572515726157271572815729157301573115732157331573415735157361573715738157391574015741157421574315744157451574615747157481574915750157511575215753157541575515756157571575815759157601576115762157631576415765157661576715768157691577015771157721577315774157751577615777157781577915780157811578215783157841578515786157871578815789157901579115792157931579415795157961579715798157991580015801158021580315804158051580615807158081580915810158111581215813158141581515816158171581815819158201582115822158231582415825158261582715828158291583015831158321583315834158351583615837158381583915840158411584215843158441584515846158471584815849158501585115852158531585415855158561585715858158591586015861158621586315864158651586615867158681586915870158711587215873158741587515876158771587815879158801588115882158831588415885158861588715888158891589015891158921589315894158951589615897158981589915900159011590215903159041590515906159071590815909159101591115912159131591415915159161591715918159191592015921159221592315924159251592615927159281592915930159311593215933159341593515936159371593815939159401594115942159431594415945159461594715948159491595015951159521595315954159551595615957159581595915960159611596215963159641596515966159671596815969159701597115972159731597415975159761597715978159791598015981159821598315984159851598615987159881598915990159911599215993159941599515996159971599815999160001600116002160031600416005160061600716008160091601016011160121601316014160151601616017160181601916020160211602216023160241602516026160271602816029160301603116032160331603416035160361603716038160391604016041160421604316044160451604616047160481604916050160511605216053160541605516056160571605816059160601606116062160631606416065160661606716068160691607016071160721607316074160751607616077160781607916080160811608216083160841608516086160871608816089160901609116092160931609416095160961609716098160991610016101161021610316104161051610616107161081610916110161111611216113161141611516116161171611816119161201612116122161231612416125161261612716128161291613016131161321613316134161351613616137161381613916140161411614216143161441614516146161471614816149161501615116152161531615416155161561615716158161591616016161161621616316164161651616616167161681616916170161711617216173161741617516176161771617816179161801618116182161831618416185161861618716188161891619016191161921619316194161951619616197161981619916200162011620216203162041620516206162071620816209162101621116212162131621416215162161621716218162191622016221162221622316224162251622616227162281622916230162311623216233162341623516236162371623816239162401624116242162431624416245162461624716248162491625016251162521625316254162551625616257162581625916260162611626216263162641626516266162671626816269162701627116272162731627416275162761627716278162791628016281162821628316284162851628616287162881628916290162911629216293162941629516296162971629816299163001630116302163031630416305163061630716308163091631016311163121631316314163151631616317163181631916320163211632216323163241632516326163271632816329163301633116332163331633416335163361633716338163391634016341163421634316344163451634616347163481634916350163511635216353163541635516356163571635816359163601636116362163631636416365163661636716368163691637016371163721637316374163751637616377163781637916380163811638216383163841638516386163871638816389163901639116392163931639416395163961639716398163991640016401164021640316404164051640616407164081640916410164111641216413164141641516416164171641816419164201642116422164231642416425164261642716428164291643016431164321643316434164351643616437164381643916440164411644216443164441644516446164471644816449164501645116452164531645416455164561645716458164591646016461164621646316464164651646616467164681646916470164711647216473164741647516476164771647816479164801648116482164831648416485164861648716488164891649016491164921649316494164951649616497164981649916500165011650216503165041650516506165071650816509165101651116512165131651416515165161651716518165191652016521165221652316524165251652616527165281652916530165311653216533165341653516536165371653816539165401654116542165431654416545165461654716548165491655016551165521655316554165551655616557165581655916560165611656216563165641656516566165671656816569165701657116572165731657416575165761657716578165791658016581165821658316584165851658616587165881658916590165911659216593165941659516596165971659816599166001660116602166031660416605166061660716608166091661016611166121661316614166151661616617166181661916620166211662216623166241662516626166271662816629166301663116632166331663416635166361663716638166391664016641166421664316644166451664616647166481664916650166511665216653166541665516656166571665816659166601666116662166631666416665166661666716668166691667016671166721667316674166751667616677166781667916680166811668216683166841668516686166871668816689166901669116692166931669416695166961669716698166991670016701167021670316704167051670616707167081670916710167111671216713167141671516716167171671816719167201672116722167231672416725167261672716728167291673016731167321673316734167351673616737167381673916740167411674216743167441674516746167471674816749167501675116752167531675416755167561675716758167591676016761167621676316764167651676616767167681676916770167711677216773167741677516776167771677816779167801678116782167831678416785167861678716788167891679016791167921679316794167951679616797167981679916800168011680216803168041680516806168071680816809168101681116812168131681416815168161681716818168191682016821168221682316824168251682616827168281682916830168311683216833168341683516836168371683816839168401684116842168431684416845168461684716848168491685016851168521685316854168551685616857168581685916860168611686216863168641686516866168671686816869168701687116872168731687416875168761687716878168791688016881168821688316884168851688616887168881688916890168911689216893168941689516896168971689816899169001690116902169031690416905169061690716908169091691016911169121691316914169151691616917169181691916920169211692216923169241692516926169271692816929169301693116932169331693416935169361693716938169391694016941169421694316944169451694616947169481694916950169511695216953169541695516956169571695816959169601696116962169631696416965169661696716968169691697016971169721697316974169751697616977169781697916980169811698216983169841698516986169871698816989169901699116992169931699416995169961699716998169991700017001170021700317004170051700617007170081700917010170111701217013170141701517016170171701817019170201702117022170231702417025170261702717028170291703017031170321703317034170351703617037170381703917040170411704217043170441704517046170471704817049170501705117052170531705417055170561705717058170591706017061170621706317064170651706617067170681706917070170711707217073170741707517076170771707817079170801708117082170831708417085170861708717088170891709017091170921709317094170951709617097170981709917100171011710217103171041710517106171071710817109171101711117112171131711417115171161711717118171191712017121171221712317124171251712617127171281712917130171311713217133171341713517136171371713817139171401714117142171431714417145171461714717148171491715017151171521715317154171551715617157171581715917160171611716217163171641716517166171671716817169171701717117172171731717417175171761717717178171791718017181171821718317184171851718617187171881718917190171911719217193171941719517196171971719817199172001720117202172031720417205172061720717208172091721017211172121721317214172151721617217172181721917220172211722217223172241722517226172271722817229172301723117232172331723417235172361723717238172391724017241172421724317244172451724617247172481724917250172511725217253172541725517256172571725817259172601726117262172631726417265172661726717268172691727017271172721727317274172751727617277172781727917280172811728217283172841728517286172871728817289172901729117292172931729417295172961729717298172991730017301173021730317304173051730617307173081730917310173111731217313173141731517316173171731817319173201732117322173231732417325173261732717328173291733017331173321733317334173351733617337173381733917340173411734217343173441734517346173471734817349173501735117352173531735417355173561735717358173591736017361173621736317364173651736617367173681736917370173711737217373173741737517376173771737817379173801738117382173831738417385173861738717388173891739017391173921739317394173951739617397173981739917400174011740217403174041740517406174071740817409174101741117412174131741417415174161741717418174191742017421174221742317424174251742617427174281742917430174311743217433174341743517436174371743817439174401744117442174431744417445174461744717448174491745017451174521745317454174551745617457174581745917460174611746217463174641746517466174671746817469174701747117472174731747417475174761747717478174791748017481174821748317484
  1. //-------------------------------------------------------------------------------------------------------
  2. // Copyright (C) Microsoft Corporation and contributors. All rights reserved.
  3. // Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
  4. //-------------------------------------------------------------------------------------------------------
  5. #include "Backend.h"
  6. #if DBG_DUMP
  7. #define DO_MEMOP_TRACE() (PHASE_TRACE(Js::MemOpPhase, this->func) ||\
  8. PHASE_TRACE(Js::MemSetPhase, this->func) ||\
  9. PHASE_TRACE(Js::MemCopyPhase, this->func))
  10. #define DO_MEMOP_TRACE_PHASE(phase) (PHASE_TRACE(Js::MemOpPhase, this->func) || PHASE_TRACE(Js::phase ## Phase, this->func))
  11. #define OUTPUT_MEMOP_TRACE(loop, instr, ...) {\
  12. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];\
  13. Output::Print(15, _u("Function: %s%s, Loop: %u: "), this->func->GetJITFunctionBody()->GetDisplayName(), this->func->GetDebugNumberSet(debugStringBuffer), loop->GetLoopNumber());\
  14. Output::Print(__VA_ARGS__);\
  15. IR::Instr* __instr__ = instr;\
  16. if(__instr__) __instr__->DumpByteCodeOffset();\
  17. if(__instr__) Output::Print(_u(" (%s)"), Js::OpCodeUtil::GetOpCodeName(__instr__->m_opcode));\
  18. Output::Print(_u("\n"));\
  19. Output::Flush(); \
  20. }
  21. #define TRACE_MEMOP(loop, instr, ...) \
  22. if (DO_MEMOP_TRACE()) {\
  23. Output::Print(_u("TRACE MemOp:"));\
  24. OUTPUT_MEMOP_TRACE(loop, instr, __VA_ARGS__)\
  25. }
  26. #define TRACE_MEMOP_VERBOSE(loop, instr, ...) if(CONFIG_FLAG(Verbose)) {TRACE_MEMOP(loop, instr, __VA_ARGS__)}
  27. #define TRACE_MEMOP_PHASE(phase, loop, instr, ...) \
  28. if (DO_MEMOP_TRACE_PHASE(phase))\
  29. {\
  30. Output::Print(_u("TRACE ") _u(#phase) _u(":"));\
  31. OUTPUT_MEMOP_TRACE(loop, instr, __VA_ARGS__)\
  32. }
  33. #define TRACE_MEMOP_PHASE_VERBOSE(phase, loop, instr, ...) if(CONFIG_FLAG(Verbose)) {TRACE_MEMOP_PHASE(phase, loop, instr, __VA_ARGS__)}
  34. #else
  35. #define DO_MEMOP_TRACE()
  36. #define DO_MEMOP_TRACE_PHASE(phase)
  37. #define OUTPUT_MEMOP_TRACE(loop, instr, ...)
  38. #define TRACE_MEMOP(loop, instr, ...)
  39. #define TRACE_MEMOP_VERBOSE(loop, instr, ...)
  40. #define TRACE_MEMOP_PHASE(phase, loop, instr, ...)
  41. #define TRACE_MEMOP_PHASE_VERBOSE(phase, loop, instr, ...)
  42. #endif
  43. class AutoRestoreVal
  44. {
  45. private:
  46. Value *const originalValue;
  47. Value *const tempValue;
  48. Value * *const valueRef;
  49. public:
  50. AutoRestoreVal(Value *const originalValue, Value * *const tempValueRef)
  51. : originalValue(originalValue), tempValue(*tempValueRef), valueRef(tempValueRef)
  52. {
  53. }
  54. ~AutoRestoreVal()
  55. {
  56. if(*valueRef == tempValue)
  57. {
  58. *valueRef = originalValue;
  59. }
  60. }
  61. PREVENT_COPY(AutoRestoreVal);
  62. };
  63. GlobOpt::GlobOpt(Func * func)
  64. : func(func),
  65. intConstantToStackSymMap(nullptr),
  66. intConstantToValueMap(nullptr),
  67. currentValue(FirstNewValueNumber),
  68. prePassLoop(nullptr),
  69. alloc(nullptr),
  70. isCallHelper(false),
  71. inInlinedBuiltIn(false),
  72. rootLoopPrePass(nullptr),
  73. noImplicitCallUsesToInsert(nullptr),
  74. valuesCreatedForClone(nullptr),
  75. valuesCreatedForMerge(nullptr),
  76. instrCountSinceLastCleanUp(0),
  77. isRecursiveCallOnLandingPad(false),
  78. updateInductionVariableValueNumber(false),
  79. isPerformingLoopBackEdgeCompensation(false),
  80. currentRegion(nullptr),
  81. changedSymsAfterIncBailoutCandidate(nullptr),
  82. doTypeSpec(
  83. !IsTypeSpecPhaseOff(func)),
  84. doAggressiveIntTypeSpec(
  85. doTypeSpec &&
  86. DoAggressiveIntTypeSpec(func)),
  87. doAggressiveMulIntTypeSpec(
  88. doTypeSpec &&
  89. !PHASE_OFF(Js::AggressiveMulIntTypeSpecPhase, func) &&
  90. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsAggressiveMulIntTypeSpecDisabled(func->IsLoopBody()))),
  91. doDivIntTypeSpec(
  92. doAggressiveIntTypeSpec &&
  93. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsDivIntTypeSpecDisabled(func->IsLoopBody()))),
  94. doLossyIntTypeSpec(
  95. doTypeSpec &&
  96. DoLossyIntTypeSpec(func)),
  97. doFloatTypeSpec(
  98. doTypeSpec &&
  99. DoFloatTypeSpec(func)),
  100. doArrayCheckHoist(
  101. DoArrayCheckHoist(func)),
  102. doArrayMissingValueCheckHoist(
  103. doArrayCheckHoist &&
  104. DoArrayMissingValueCheckHoist(func)),
  105. doArraySegmentHoist(
  106. doArrayCheckHoist &&
  107. DoArraySegmentHoist(ValueType::GetObject(ObjectType::Int32Array), func)),
  108. doJsArraySegmentHoist(
  109. doArraySegmentHoist &&
  110. DoArraySegmentHoist(ValueType::GetObject(ObjectType::Array), func)),
  111. doArrayLengthHoist(
  112. doArrayCheckHoist &&
  113. DoArrayLengthHoist(func)),
  114. doEliminateArrayAccessHelperCall(
  115. doArrayCheckHoist &&
  116. !PHASE_OFF(Js::EliminateArrayAccessHelperCallPhase, func)),
  117. doTrackRelativeIntBounds(
  118. doAggressiveIntTypeSpec &&
  119. DoPathDependentValues() &&
  120. !PHASE_OFF(Js::Phase::TrackRelativeIntBoundsPhase, func)),
  121. doBoundCheckElimination(
  122. doTrackRelativeIntBounds &&
  123. !PHASE_OFF(Js::Phase::BoundCheckEliminationPhase, func)),
  124. doBoundCheckHoist(
  125. doEliminateArrayAccessHelperCall &&
  126. doBoundCheckElimination &&
  127. DoConstFold() &&
  128. !PHASE_OFF(Js::Phase::BoundCheckHoistPhase, func) &&
  129. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsBoundCheckHoistDisabled(func->IsLoopBody()))),
  130. doLoopCountBasedBoundCheckHoist(
  131. doBoundCheckHoist &&
  132. !PHASE_OFF(Js::Phase::LoopCountBasedBoundCheckHoistPhase, func) &&
  133. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsLoopCountBasedBoundCheckHoistDisabled(func->IsLoopBody()))),
  134. doPowIntIntTypeSpec(
  135. doAggressiveIntTypeSpec &&
  136. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsPowIntIntTypeSpecDisabled())),
  137. doTagChecks(
  138. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsTagCheckDisabled())),
  139. isAsmJSFunc(func->GetJITFunctionBody()->IsAsmJsMode())
  140. {
  141. }
  142. void
  143. GlobOpt::BackwardPass(Js::Phase tag)
  144. {
  145. BEGIN_CODEGEN_PHASE(this->func, tag);
  146. ::BackwardPass backwardPass(this->func, this, tag);
  147. backwardPass.Optimize();
  148. END_CODEGEN_PHASE(this->func, tag);
  149. }
  150. void
  151. GlobOpt::Optimize()
  152. {
  153. this->objectTypeSyms = nullptr;
  154. this->func->argInsCount = this->func->GetInParamsCount() - 1; //Don't include "this" pointer in the count.
  155. if (!func->DoGlobOpt())
  156. {
  157. this->lengthEquivBv = nullptr;
  158. this->argumentsEquivBv = nullptr;
  159. this->callerEquivBv = nullptr;
  160. // Still need to run the dead store phase to calculate the live reg on back edge
  161. this->BackwardPass(Js::DeadStorePhase);
  162. CannotAllocateArgumentsObjectOnStack();
  163. return;
  164. }
  165. {
  166. this->lengthEquivBv = this->func->m_symTable->m_propertyEquivBvMap->Lookup(Js::PropertyIds::length, nullptr); // Used to kill live "length" properties
  167. this->argumentsEquivBv = func->m_symTable->m_propertyEquivBvMap->Lookup(Js::PropertyIds::arguments, nullptr); // Used to kill live "arguments" properties
  168. this->callerEquivBv = func->m_symTable->m_propertyEquivBvMap->Lookup(Js::PropertyIds::caller, nullptr); // Used to kill live "caller" properties
  169. // The backward phase needs the glob opt's allocator to allocate the propertyTypeValueMap
  170. // in GlobOpt::EnsurePropertyTypeValue and ranges of instructions where int overflow may be ignored.
  171. // (see BackwardPass::TrackIntUsage)
  172. PageAllocator * pageAllocator = this->func->m_alloc->GetPageAllocator();
  173. NoRecoverMemoryJitArenaAllocator localAlloc(_u("BE-GlobOpt"), pageAllocator, Js::Throw::OutOfMemory);
  174. this->alloc = &localAlloc;
  175. NoRecoverMemoryJitArenaAllocator localTempAlloc(_u("BE-GlobOpt temp"), pageAllocator, Js::Throw::OutOfMemory);
  176. this->tempAlloc = &localTempAlloc;
  177. // The forward passes use info (upwardExposedUses) from the backward pass. This info
  178. // isn't available for some of the symbols created during the backward pass, or the forward pass.
  179. // Keep track of the last symbol for which we're guaranteed to have data.
  180. this->maxInitialSymID = this->func->m_symTable->GetMaxSymID();
  181. #if DBG
  182. this->BackwardPass(Js::CaptureByteCodeRegUsePhase);
  183. #endif
  184. this->BackwardPass(Js::BackwardPhase);
  185. this->ForwardPass();
  186. this->BackwardPass(Js::DeadStorePhase);
  187. }
  188. this->TailDupPass();
  189. }
  190. bool GlobOpt::ShouldExpectConventionalArrayIndexValue(IR::IndirOpnd *const indirOpnd)
  191. {
  192. Assert(indirOpnd);
  193. if(!indirOpnd->GetIndexOpnd())
  194. {
  195. return indirOpnd->GetOffset() >= 0;
  196. }
  197. IR::RegOpnd *const indexOpnd = indirOpnd->GetIndexOpnd();
  198. if(indexOpnd->m_sym->m_isNotNumber)
  199. {
  200. // Typically, single-def or any sym-specific information for type-specialized syms should not be used because all of
  201. // their defs will not have been accounted for until after the forward pass. But m_isNotNumber is only ever changed from
  202. // false to true, so it's okay in this case.
  203. return false;
  204. }
  205. StackSym *indexVarSym = indexOpnd->m_sym;
  206. if(indexVarSym->IsTypeSpec())
  207. {
  208. indexVarSym = indexVarSym->GetVarEquivSym(nullptr);
  209. Assert(indexVarSym);
  210. }
  211. else if(!IsLoopPrePass())
  212. {
  213. // Don't use single-def info or const flags for type-specialized syms, as all of their defs will not have been accounted
  214. // for until after the forward pass. Also, don't use the const flags in a loop prepass because the const flags may not
  215. // be up-to-date.
  216. if (indexOpnd->IsNotInt())
  217. {
  218. return false;
  219. }
  220. StackSym *const indexSym = indexOpnd->m_sym;
  221. if(indexSym->IsIntConst())
  222. {
  223. return indexSym->GetIntConstValue() >= 0;
  224. }
  225. }
  226. Value *const indexValue = CurrentBlockData()->FindValue(indexVarSym);
  227. if(!indexValue)
  228. {
  229. // Treat it as Uninitialized, assume it's going to be valid
  230. return true;
  231. }
  232. ValueInfo *const indexValueInfo = indexValue->GetValueInfo();
  233. int32 indexConstantValue;
  234. if(indexValueInfo->TryGetIntConstantValue(&indexConstantValue))
  235. {
  236. return indexConstantValue >= 0;
  237. }
  238. if(indexValueInfo->IsUninitialized())
  239. {
  240. // Assume it's going to be valid
  241. return true;
  242. }
  243. return indexValueInfo->HasBeenNumber() && !indexValueInfo->HasBeenFloat();
  244. }
  245. //
  246. // Either result is float or 1/x or cst1/cst2 where cst1%cst2 != 0
  247. //
  248. ValueType GlobOpt::GetDivValueType(IR::Instr* instr, Value* src1Val, Value* src2Val, bool specialize)
  249. {
  250. ValueInfo *src1ValueInfo = (src1Val ? src1Val->GetValueInfo() : nullptr);
  251. ValueInfo *src2ValueInfo = (src2Val ? src2Val->GetValueInfo() : nullptr);
  252. if (instr->IsProfiledInstr() && instr->m_func->HasProfileInfo())
  253. {
  254. ValueType resultType = instr->m_func->GetReadOnlyProfileInfo()->GetDivProfileInfo(static_cast<Js::ProfileId>(instr->AsProfiledInstr()->u.profileId));
  255. if (resultType.IsLikelyInt())
  256. {
  257. if (specialize && src1ValueInfo && src2ValueInfo
  258. && ((src1ValueInfo->IsInt() && src2ValueInfo->IsInt()) ||
  259. (this->DoDivIntTypeSpec() && src1ValueInfo->IsLikelyInt() && src2ValueInfo->IsLikelyInt())))
  260. {
  261. return ValueType::GetInt(true);
  262. }
  263. return resultType;
  264. }
  265. // Consider: Checking that the sources are numbers.
  266. if (resultType.IsLikelyFloat())
  267. {
  268. return ValueType::Float;
  269. }
  270. return resultType;
  271. }
  272. int32 src1IntConstantValue;
  273. if(!src1ValueInfo || !src1ValueInfo->TryGetIntConstantValue(&src1IntConstantValue))
  274. {
  275. return ValueType::Number;
  276. }
  277. if (src1IntConstantValue == 1)
  278. {
  279. return ValueType::Float;
  280. }
  281. int32 src2IntConstantValue;
  282. if(!src2Val || !src2ValueInfo->TryGetIntConstantValue(&src2IntConstantValue))
  283. {
  284. return ValueType::Number;
  285. }
  286. if (src2IntConstantValue // Avoid divide by zero
  287. && !(src1IntConstantValue == 0x80000000 && src2IntConstantValue == -1) // Avoid integer overflow
  288. && (src1IntConstantValue % src2IntConstantValue) != 0)
  289. {
  290. return ValueType::Float;
  291. }
  292. return ValueType::Number;
  293. }
  294. void
  295. GlobOpt::ForwardPass()
  296. {
  297. BEGIN_CODEGEN_PHASE(this->func, Js::ForwardPhase);
  298. #if DBG_DUMP
  299. if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::GlobOptPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId()))
  300. {
  301. this->func->DumpHeader();
  302. }
  303. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::GlobOptPhase))
  304. {
  305. this->TraceSettings();
  306. }
  307. #endif
  308. // GetConstantCount() gives us the right size to pick for the SparseArray, but we may need more if we've inlined
  309. // functions with constants. There will be a gap in the symbol numbering between the main constants and
  310. // the inlined ones, so we'll most likely need a new array chunk. Make the min size of the array chunks be 64
  311. // in case we have a main function with very few constants and a bunch of constants from inlined functions.
  312. this->byteCodeConstantValueArray = SparseArray<Value>::New(this->alloc, max(this->func->GetJITFunctionBody()->GetConstCount(), 64U));
  313. this->byteCodeConstantValueNumbersBv = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  314. this->tempBv = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  315. this->prePassCopyPropSym = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  316. this->slotSyms = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  317. this->byteCodeUses = nullptr;
  318. this->propertySymUse = nullptr;
  319. // changedSymsAfterIncBailoutCandidate helps track building incremental bailout in ForwardPass
  320. this->changedSymsAfterIncBailoutCandidate = JitAnew(alloc, BVSparse<JitArenaAllocator>, alloc);
  321. #if DBG
  322. this->byteCodeUsesBeforeOpt = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  323. if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldCopyPropPhase) && this->DoFunctionFieldCopyProp())
  324. {
  325. Output::Print(_u("TRACE: CanDoFieldCopyProp Func: "));
  326. this->func->DumpFullFunctionName();
  327. Output::Print(_u("\n"));
  328. }
  329. #endif
  330. OpndList localNoImplicitCallUsesToInsert(alloc);
  331. this->noImplicitCallUsesToInsert = &localNoImplicitCallUsesToInsert;
  332. IntConstantToStackSymMap localIntConstantToStackSymMap(alloc);
  333. this->intConstantToStackSymMap = &localIntConstantToStackSymMap;
  334. IntConstantToValueMap localIntConstantToValueMap(alloc);
  335. this->intConstantToValueMap = &localIntConstantToValueMap;
  336. Int64ConstantToValueMap localInt64ConstantToValueMap(alloc);
  337. this->int64ConstantToValueMap = &localInt64ConstantToValueMap;
  338. AddrConstantToValueMap localAddrConstantToValueMap(alloc);
  339. this->addrConstantToValueMap = &localAddrConstantToValueMap;
  340. StringConstantToValueMap localStringConstantToValueMap(alloc);
  341. this->stringConstantToValueMap = &localStringConstantToValueMap;
  342. SymIdToInstrMap localPrePassInstrMap(alloc);
  343. this->prePassInstrMap = &localPrePassInstrMap;
  344. ValueSetByValueNumber localValuesCreatedForClone(alloc, 64);
  345. this->valuesCreatedForClone = &localValuesCreatedForClone;
  346. ValueNumberPairToValueMap localValuesCreatedForMerge(alloc, 64);
  347. this->valuesCreatedForMerge = &localValuesCreatedForMerge;
  348. #if DBG
  349. BVSparse<JitArenaAllocator> localFinishedStackLiteralInitFld(alloc);
  350. this->finishedStackLiteralInitFld = &localFinishedStackLiteralInitFld;
  351. #endif
  352. FOREACH_BLOCK_IN_FUNC_EDITING(block, this->func)
  353. {
  354. this->OptBlock(block);
  355. } NEXT_BLOCK_IN_FUNC_EDITING;
  356. if (!PHASE_OFF(Js::MemOpPhase, this->func))
  357. {
  358. ProcessMemOp();
  359. }
  360. this->noImplicitCallUsesToInsert = nullptr;
  361. this->intConstantToStackSymMap = nullptr;
  362. this->intConstantToValueMap = nullptr;
  363. this->int64ConstantToValueMap = nullptr;
  364. this->addrConstantToValueMap = nullptr;
  365. this->stringConstantToValueMap = nullptr;
  366. #if DBG
  367. this->finishedStackLiteralInitFld = nullptr;
  368. uint freedCount = 0;
  369. uint spilledCount = 0;
  370. #endif
  371. FOREACH_BLOCK_IN_FUNC(block, this->func)
  372. {
  373. #if DBG
  374. if (block->GetDataUseCount() == 0)
  375. {
  376. freedCount++;
  377. }
  378. else
  379. {
  380. spilledCount++;
  381. }
  382. #endif
  383. block->SetDataUseCount(0);
  384. if (block->cloneStrCandidates)
  385. {
  386. JitAdelete(this->alloc, block->cloneStrCandidates);
  387. block->cloneStrCandidates = nullptr;
  388. }
  389. } NEXT_BLOCK_IN_FUNC;
  390. // Make sure we free most of them.
  391. Assert(freedCount >= spilledCount);
  392. // this->alloc will be freed right after return, no need to free it here
  393. this->changedSymsAfterIncBailoutCandidate = nullptr;
  394. END_CODEGEN_PHASE(this->func, Js::ForwardPhase);
  395. }
  396. void
  397. GlobOpt::OptBlock(BasicBlock *block)
  398. {
  399. if (this->func->m_fg->RemoveUnreachableBlock(block, this))
  400. {
  401. GOPT_TRACE(_u("Removing unreachable block #%d\n"), block->GetBlockNum());
  402. return;
  403. }
  404. Loop * loop = block->loop;
  405. if (loop && block->isLoopHeader)
  406. {
  407. if (loop != this->prePassLoop)
  408. {
  409. OptLoops(loop);
  410. if (!IsLoopPrePass() && loop->parent)
  411. {
  412. loop->fieldPRESymStores->Or(loop->parent->fieldPRESymStores);
  413. }
  414. if (!this->IsLoopPrePass() && DoFieldPRE(loop))
  415. {
  416. // Note: !IsLoopPrePass means this was a root loop pre-pass. FieldPre() is called once per loop.
  417. this->FieldPRE(loop);
  418. // Re-optimize the landing pad
  419. BasicBlock *landingPad = loop->landingPad;
  420. this->isRecursiveCallOnLandingPad = true;
  421. this->OptBlock(landingPad);
  422. this->isRecursiveCallOnLandingPad = false;
  423. }
  424. }
  425. }
  426. this->currentBlock = block;
  427. PrepareLoopArrayCheckHoist();
  428. block->MergePredBlocksValueMaps(this);
  429. this->intOverflowCurrentlyMattersInRange = true;
  430. this->intOverflowDoesNotMatterRange = this->currentBlock->intOverflowDoesNotMatterRange;
  431. if (!DoFieldCopyProp() && !DoFieldRefOpts())
  432. {
  433. this->KillAllFields(CurrentBlockData()->liveFields);
  434. }
  435. this->tempAlloc->Reset();
  436. if(loop && block->isLoopHeader)
  437. {
  438. loop->firstValueNumberInLoop = this->currentValue;
  439. }
  440. GOPT_TRACE_BLOCK(block, true);
  441. FOREACH_INSTR_IN_BLOCK_EDITING(instr, instrNext, block)
  442. {
  443. GOPT_TRACE_INSTRTRACE(instr);
  444. BailOutInfo* oldBailOutInfo = nullptr;
  445. bool isCheckAuxBailoutNeeded = this->func->IsJitInDebugMode() && !this->IsLoopPrePass();
  446. if (isCheckAuxBailoutNeeded && instr->HasAuxBailOut() && !instr->HasBailOutInfo())
  447. {
  448. oldBailOutInfo = instr->GetBailOutInfo();
  449. Assert(oldBailOutInfo);
  450. }
  451. bool isInstrRemoved = false;
  452. instrNext = this->OptInstr(instr, &isInstrRemoved);
  453. // If we still have instrs with only aux bail out, convert aux bail out back to regular bail out and fill it.
  454. // During OptInstr some instr can be moved out to a different block, in this case bailout info is going to be replaced
  455. // with e.g. loop bailout info which is filled as part of processing that block, thus we don't need to fill it here.
  456. if (isCheckAuxBailoutNeeded && !isInstrRemoved && instr->HasAuxBailOut() && !instr->HasBailOutInfo())
  457. {
  458. if (instr->GetBailOutInfo() == oldBailOutInfo)
  459. {
  460. instr->PromoteAuxBailOut();
  461. FillBailOutInfo(block, instr);
  462. }
  463. else
  464. {
  465. AssertMsg(instr->GetBailOutInfo(), "With aux bailout, the bailout info should not be removed by OptInstr.");
  466. }
  467. }
  468. } NEXT_INSTR_IN_BLOCK_EDITING;
  469. GOPT_TRACE_BLOCK(block, false);
  470. if (block->loop)
  471. {
  472. if (IsLoopPrePass())
  473. {
  474. if (DoBoundCheckHoist())
  475. {
  476. DetectUnknownChangesToInductionVariables(&block->globOptData);
  477. }
  478. }
  479. else
  480. {
  481. isPerformingLoopBackEdgeCompensation = true;
  482. Assert(this->tempBv->IsEmpty());
  483. BVSparse<JitArenaAllocator> tempBv2(this->tempAlloc);
  484. // On loop back-edges, we need to restore the state of the type specialized
  485. // symbols to that of the loop header.
  486. FOREACH_SUCCESSOR_BLOCK(succ, block)
  487. {
  488. if (succ->isLoopHeader && succ->loop->IsDescendentOrSelf(block->loop))
  489. {
  490. BVSparse<JitArenaAllocator> *liveOnBackEdge = block->loop->regAlloc.liveOnBackEdgeSyms;
  491. liveOnBackEdge->Or(block->loop->fieldPRESymStores);
  492. this->tempBv->Minus(block->loop->varSymsOnEntry, block->globOptData.liveVarSyms);
  493. this->tempBv->And(liveOnBackEdge);
  494. this->ToVar(this->tempBv, block);
  495. // Lossy int in the loop header, and no int on the back-edge - need a lossy conversion to int
  496. this->tempBv->Minus(block->loop->lossyInt32SymsOnEntry, block->globOptData.liveInt32Syms);
  497. this->tempBv->And(liveOnBackEdge);
  498. this->ToInt32(this->tempBv, block, true /* lossy */);
  499. // Lossless int in the loop header, and no lossless int on the back-edge - need a lossless conversion to int
  500. this->tempBv->Minus(block->loop->int32SymsOnEntry, block->loop->lossyInt32SymsOnEntry);
  501. tempBv2.Minus(block->globOptData.liveInt32Syms, block->globOptData.liveLossyInt32Syms);
  502. this->tempBv->Minus(&tempBv2);
  503. this->tempBv->And(liveOnBackEdge);
  504. this->ToInt32(this->tempBv, block, false /* lossy */);
  505. this->tempBv->Minus(block->loop->float64SymsOnEntry, block->globOptData.liveFloat64Syms);
  506. this->tempBv->And(liveOnBackEdge);
  507. this->ToFloat64(this->tempBv, block);
  508. // For ints and floats, go aggressive and type specialize in the landing pad any symbol which was specialized on
  509. // entry to the loop body (in the loop header), and is still specialized on this tail, but wasn't specialized in
  510. // the landing pad.
  511. // Lossy int in the loop header and no int in the landing pad - need a lossy conversion to int
  512. // (entry.lossyInt32 - landingPad.int32)
  513. this->tempBv->Minus(block->loop->lossyInt32SymsOnEntry, block->loop->landingPad->globOptData.liveInt32Syms);
  514. this->tempBv->And(liveOnBackEdge);
  515. this->ToInt32(this->tempBv, block->loop->landingPad, true /* lossy */);
  516. // Lossless int in the loop header, and no lossless int in the landing pad - need a lossless conversion to int
  517. // ((entry.int32 - entry.lossyInt32) - (landingPad.int32 - landingPad.lossyInt32))
  518. this->tempBv->Minus(block->loop->int32SymsOnEntry, block->loop->lossyInt32SymsOnEntry);
  519. tempBv2.Minus(
  520. block->loop->landingPad->globOptData.liveInt32Syms,
  521. block->loop->landingPad->globOptData.liveLossyInt32Syms);
  522. this->tempBv->Minus(&tempBv2);
  523. this->tempBv->And(liveOnBackEdge);
  524. this->ToInt32(this->tempBv, block->loop->landingPad, false /* lossy */);
  525. // ((entry.float64 - landingPad.float64) & block.float64)
  526. this->tempBv->Minus(block->loop->float64SymsOnEntry, block->loop->landingPad->globOptData.liveFloat64Syms);
  527. this->tempBv->And(block->globOptData.liveFloat64Syms);
  528. this->tempBv->And(liveOnBackEdge);
  529. this->ToFloat64(this->tempBv, block->loop->landingPad);
  530. if (block->loop->symsRequiringCompensationToMergedValueInfoMap)
  531. {
  532. InsertValueCompensation(block, block->loop->symsRequiringCompensationToMergedValueInfoMap);
  533. }
  534. // Now that we're done with the liveFields within this loop, trim the set to those syms
  535. // that the backward pass told us were live out of the loop.
  536. // This assumes we have no further need of the liveFields within the loop.
  537. if (block->loop->liveOutFields)
  538. {
  539. block->globOptData.liveFields->And(block->loop->liveOutFields);
  540. }
  541. }
  542. } NEXT_SUCCESSOR_BLOCK;
  543. this->tempBv->ClearAll();
  544. isPerformingLoopBackEdgeCompensation = false;
  545. }
  546. }
  547. block->PathDepBranchFolding(this);
  548. #if DBG
  549. // The set of live lossy int32 syms should be a subset of all live int32 syms
  550. this->tempBv->And(block->globOptData.liveInt32Syms, block->globOptData.liveLossyInt32Syms);
  551. Assert(this->tempBv->Count() == block->globOptData.liveLossyInt32Syms->Count());
  552. // The set of live lossy int32 syms should be a subset of live var or float syms (var or float sym containing the lossless
  553. // value of the sym should be live)
  554. this->tempBv->Or(block->globOptData.liveVarSyms, block->globOptData.liveFloat64Syms);
  555. this->tempBv->And(block->globOptData.liveLossyInt32Syms);
  556. Assert(this->tempBv->Count() == block->globOptData.liveLossyInt32Syms->Count());
  557. this->tempBv->ClearAll();
  558. Assert(this->currentBlock == block);
  559. #endif
  560. }
  561. void
  562. GlobOpt::OptLoops(Loop *loop)
  563. {
  564. Assert(loop != nullptr);
  565. #if DBG
  566. if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldCopyPropPhase) &&
  567. !DoFunctionFieldCopyProp() && DoFieldCopyProp(loop))
  568. {
  569. Output::Print(_u("TRACE: CanDoFieldCopyProp Loop: "));
  570. this->func->DumpFullFunctionName();
  571. uint loopNumber = loop->GetLoopNumber();
  572. Assert(loopNumber != Js::LoopHeader::NoLoop);
  573. Output::Print(_u(" Loop: %d\n"), loopNumber);
  574. }
  575. #endif
  576. Loop *previousLoop = this->prePassLoop;
  577. this->prePassLoop = loop;
  578. if (previousLoop == nullptr)
  579. {
  580. Assert(this->rootLoopPrePass == nullptr);
  581. this->rootLoopPrePass = loop;
  582. this->prePassInstrMap->Clear();
  583. if (loop->parent == nullptr)
  584. {
  585. // Outer most loop...
  586. this->prePassCopyPropSym->ClearAll();
  587. }
  588. }
  589. Assert(loop->symsAssignedToInLoop != nullptr);
  590. if (loop->symsUsedBeforeDefined == nullptr)
  591. {
  592. loop->symsUsedBeforeDefined = JitAnew(alloc, BVSparse<JitArenaAllocator>, this->alloc);
  593. loop->likelyIntSymsUsedBeforeDefined = JitAnew(alloc, BVSparse<JitArenaAllocator>, this->alloc);
  594. loop->likelyNumberSymsUsedBeforeDefined = JitAnew(alloc, BVSparse<JitArenaAllocator>, this->alloc);
  595. loop->forceFloat64SymsOnEntry = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  596. loop->symsDefInLoop = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  597. loop->fieldKilled = JitAnew(alloc, BVSparse<JitArenaAllocator>, this->alloc);
  598. loop->fieldPRESymStores = JitAnew(alloc, BVSparse<JitArenaAllocator>, this->alloc);
  599. loop->allFieldsKilled = false;
  600. }
  601. else
  602. {
  603. loop->symsUsedBeforeDefined->ClearAll();
  604. loop->likelyIntSymsUsedBeforeDefined->ClearAll();
  605. loop->likelyNumberSymsUsedBeforeDefined->ClearAll();
  606. loop->forceFloat64SymsOnEntry->ClearAll();
  607. loop->symsDefInLoop->ClearAll();
  608. loop->fieldKilled->ClearAll();
  609. loop->allFieldsKilled = false;
  610. loop->initialValueFieldMap.Reset();
  611. }
  612. FOREACH_BLOCK_IN_LOOP(block, loop)
  613. {
  614. block->SetDataUseCount(block->GetSuccList()->Count());
  615. OptBlock(block);
  616. } NEXT_BLOCK_IN_LOOP;
  617. if (previousLoop == nullptr)
  618. {
  619. Assert(this->rootLoopPrePass == loop);
  620. this->rootLoopPrePass = nullptr;
  621. }
  622. this->prePassLoop = previousLoop;
  623. }
  624. void
  625. GlobOpt::TailDupPass()
  626. {
  627. FOREACH_LOOP_IN_FUNC_EDITING(loop, this->func)
  628. {
  629. BasicBlock* header = loop->GetHeadBlock();
  630. BasicBlock* loopTail = nullptr;
  631. FOREACH_PREDECESSOR_BLOCK(pred, header)
  632. {
  633. if (loop->IsDescendentOrSelf(pred->loop))
  634. {
  635. loopTail = pred;
  636. break;
  637. }
  638. } NEXT_PREDECESSOR_BLOCK;
  639. if (loopTail)
  640. {
  641. AssertMsg(loopTail->GetLastInstr()->IsBranchInstr(), "LastInstr of loop should always be a branch no?");
  642. if (!loopTail->GetPredList()->HasOne())
  643. {
  644. TryTailDup(loopTail->GetLastInstr()->AsBranchInstr());
  645. }
  646. }
  647. } NEXT_LOOP_IN_FUNC_EDITING;
  648. }
  649. bool
  650. GlobOpt::TryTailDup(IR::BranchInstr *tailBranch)
  651. {
  652. if (PHASE_OFF(Js::TailDupPhase, tailBranch->m_func->GetTopFunc()))
  653. {
  654. return false;
  655. }
  656. if (tailBranch->IsConditional())
  657. {
  658. return false;
  659. }
  660. IR::Instr *instr;
  661. uint instrCount = 0;
  662. for (instr = tailBranch->GetPrevRealInstrOrLabel(); !instr->IsLabelInstr(); instr = instr->GetPrevRealInstrOrLabel())
  663. {
  664. if (instr->HasBailOutInfo())
  665. {
  666. break;
  667. }
  668. if (!OpCodeAttr::CanCSE(instr->m_opcode))
  669. {
  670. // Consider: We could be more aggressive here
  671. break;
  672. }
  673. instrCount++;
  674. if (instrCount > 1)
  675. {
  676. // Consider: If copy handled single-def tmps renaming, we could do more instrs
  677. break;
  678. }
  679. }
  680. if (!instr->IsLabelInstr())
  681. {
  682. return false;
  683. }
  684. IR::LabelInstr *mergeLabel = instr->AsLabelInstr();
  685. IR::Instr *mergeLabelPrev = mergeLabel->m_prev;
  686. // Skip unreferenced labels
  687. while (mergeLabelPrev->IsLabelInstr() && mergeLabelPrev->AsLabelInstr()->labelRefs.Empty())
  688. {
  689. mergeLabelPrev = mergeLabelPrev->m_prev;
  690. }
  691. BasicBlock* labelBlock = mergeLabel->GetBasicBlock();
  692. uint origPredCount = labelBlock->GetPredList()->Count();
  693. uint dupCount = 0;
  694. // We are good to go. Let's do the tail duplication.
  695. FOREACH_SLISTCOUNTED_ENTRY_EDITING(IR::BranchInstr*, branchEntry, &mergeLabel->labelRefs, iter)
  696. {
  697. if (branchEntry->IsUnconditional() && !branchEntry->IsMultiBranch() && branchEntry != mergeLabelPrev && branchEntry != tailBranch)
  698. {
  699. for (instr = mergeLabel->m_next; instr != tailBranch; instr = instr->m_next)
  700. {
  701. branchEntry->InsertBefore(instr->Copy());
  702. }
  703. instr = branchEntry;
  704. branchEntry->ReplaceTarget(mergeLabel, tailBranch->GetTarget());
  705. while(!instr->IsLabelInstr())
  706. {
  707. instr = instr->m_prev;
  708. }
  709. BasicBlock* branchBlock = instr->AsLabelInstr()->GetBasicBlock();
  710. labelBlock->RemovePred(branchBlock, func->m_fg);
  711. func->m_fg->AddEdge(branchBlock, tailBranch->GetTarget()->GetBasicBlock());
  712. dupCount++;
  713. }
  714. } NEXT_SLISTCOUNTED_ENTRY_EDITING;
  715. // If we've duplicated everywhere, tail block is dead and should be removed.
  716. if (dupCount == origPredCount)
  717. {
  718. AssertMsg(mergeLabel->labelRefs.Empty(), "Should not remove block with referenced label.");
  719. func->m_fg->RemoveBlock(labelBlock, nullptr, true);
  720. }
  721. return true;
  722. }
  723. void
  724. GlobOpt::ToVar(BVSparse<JitArenaAllocator> *bv, BasicBlock *block)
  725. {
  726. FOREACH_BITSET_IN_SPARSEBV(id, bv)
  727. {
  728. StackSym *stackSym = this->func->m_symTable->FindStackSym(id);
  729. IR::RegOpnd *newOpnd = IR::RegOpnd::New(stackSym, TyVar, this->func);
  730. IR::Instr *lastInstr = block->GetLastInstr();
  731. if (lastInstr->IsBranchInstr() || lastInstr->m_opcode == Js::OpCode::BailTarget)
  732. {
  733. // If branch is using this symbol, hoist the operand as the ToVar load will get
  734. // inserted right before the branch.
  735. IR::Opnd *src1 = lastInstr->GetSrc1();
  736. if (src1)
  737. {
  738. if (src1->IsRegOpnd() && src1->AsRegOpnd()->m_sym == stackSym)
  739. {
  740. lastInstr->HoistSrc1(Js::OpCode::Ld_A);
  741. }
  742. IR::Opnd *src2 = lastInstr->GetSrc2();
  743. if (src2)
  744. {
  745. if (src2->IsRegOpnd() && src2->AsRegOpnd()->m_sym == stackSym)
  746. {
  747. lastInstr->HoistSrc2(Js::OpCode::Ld_A);
  748. }
  749. }
  750. }
  751. this->ToVar(lastInstr, newOpnd, block, nullptr, false);
  752. }
  753. else
  754. {
  755. IR::Instr *lastNextInstr = lastInstr->m_next;
  756. this->ToVar(lastNextInstr, newOpnd, block, nullptr, false);
  757. }
  758. } NEXT_BITSET_IN_SPARSEBV;
  759. }
  760. void
  761. GlobOpt::ToInt32(BVSparse<JitArenaAllocator> *bv, BasicBlock *block, bool lossy, IR::Instr *insertBeforeInstr)
  762. {
  763. return this->ToTypeSpec(bv, block, TyInt32, IR::BailOutIntOnly, lossy, insertBeforeInstr);
  764. }
  765. void
  766. GlobOpt::ToFloat64(BVSparse<JitArenaAllocator> *bv, BasicBlock *block)
  767. {
  768. return this->ToTypeSpec(bv, block, TyFloat64, IR::BailOutNumberOnly);
  769. }
  770. void
  771. GlobOpt::ToTypeSpec(BVSparse<JitArenaAllocator> *bv, BasicBlock *block, IRType toType, IR::BailOutKind bailOutKind, bool lossy, IR::Instr *insertBeforeInstr)
  772. {
  773. FOREACH_BITSET_IN_SPARSEBV(id, bv)
  774. {
  775. StackSym *stackSym = this->func->m_symTable->FindStackSym(id);
  776. IRType fromType = TyIllegal;
  777. // Win8 bug: 757126. If we are trying to type specialize the arguments object,
  778. // let's make sure stack args optimization is not enabled. This is a problem, particularly,
  779. // if the instruction comes from an unreachable block. In other cases, the pass on the
  780. // instruction itself should disable arguments object optimization.
  781. if(block->globOptData.argObjSyms && block->globOptData.IsArgumentsSymID(id))
  782. {
  783. CannotAllocateArgumentsObjectOnStack();
  784. }
  785. if (block->globOptData.liveVarSyms->Test(id))
  786. {
  787. fromType = TyVar;
  788. }
  789. else if (block->globOptData.liveInt32Syms->Test(id) && !block->globOptData.liveLossyInt32Syms->Test(id))
  790. {
  791. fromType = TyInt32;
  792. stackSym = stackSym->GetInt32EquivSym(this->func);
  793. }
  794. else if (block->globOptData.liveFloat64Syms->Test(id))
  795. {
  796. fromType = TyFloat64;
  797. stackSym = stackSym->GetFloat64EquivSym(this->func);
  798. }
  799. else
  800. {
  801. Assert(UNREACHED);
  802. }
  803. IR::RegOpnd *newOpnd = IR::RegOpnd::New(stackSym, fromType, this->func);
  804. this->ToTypeSpecUse(nullptr, newOpnd, block, nullptr, nullptr, toType, bailOutKind, lossy, insertBeforeInstr);
  805. } NEXT_BITSET_IN_SPARSEBV;
  806. }
  807. void GlobOpt::PRE::FindPossiblePRECandidates(Loop *loop, JitArenaAllocator *alloc)
  808. {
  809. // Find the set of PRE candidates
  810. BasicBlock *loopHeader = loop->GetHeadBlock();
  811. PRECandidates *candidates = nullptr;
  812. bool firstBackEdge = true;
  813. FOREACH_PREDECESSOR_BLOCK(blockPred, loopHeader)
  814. {
  815. if (!loop->IsDescendentOrSelf(blockPred->loop))
  816. {
  817. // Not a loop back-edge
  818. continue;
  819. }
  820. if (firstBackEdge)
  821. {
  822. candidates = this->globOpt->FindBackEdgePRECandidates(blockPred, alloc);
  823. }
  824. else
  825. {
  826. blockPred->globOptData.RemoveUnavailableCandidates(candidates);
  827. }
  828. } NEXT_PREDECESSOR_BLOCK;
  829. this->candidates = candidates;
  830. }
  831. BOOL GlobOpt::PRE::PreloadPRECandidate(Loop *loop, GlobHashBucket* candidate)
  832. {
  833. // Insert a load for each field PRE candidate.
  834. PropertySym *propertySym = candidate->value->AsPropertySym();
  835. if (!candidates->candidatesToProcess->TestAndClear(propertySym->m_id))
  836. {
  837. return false;
  838. }
  839. Value * propSymValueOnBackEdge = candidate->element;
  840. StackSym *objPtrSym = propertySym->m_stackSym;
  841. Sym * objPtrCopyPropSym = nullptr;
  842. if (!loop->landingPad->globOptData.IsLive(objPtrSym))
  843. {
  844. if (PHASE_OFF(Js::MakeObjSymLiveInLandingPadPhase, this->globOpt->func))
  845. {
  846. return false;
  847. }
  848. if (objPtrSym->IsSingleDef())
  849. {
  850. // We can still try to do PRE if the object sym is single def, even if its not live in the landing pad.
  851. // We'll have to add a def instruction for the object sym in the landing pad, and then we can continue
  852. // pre-loading the current PRE candidate.
  853. // Case in point:
  854. // $L1
  855. // value|symStore
  856. // t1 = o.x (v1|t3)
  857. // t2 = t1.y (v2|t4) <-- t1 is not live in the loop landing pad
  858. // jmp $L1
  859. if (!InsertSymDefinitionInLandingPad(objPtrSym, loop, &objPtrCopyPropSym))
  860. {
  861. #if DBG_DUMP
  862. TraceFailedPreloadInLandingPad(loop, propertySym, _u("Failed to insert load of object sym in landing pad"));
  863. #endif
  864. return false;
  865. }
  866. }
  867. else
  868. {
  869. #if DBG_DUMP
  870. TraceFailedPreloadInLandingPad(loop, propertySym, _u("Object sym not live in landing pad and not single-def"));
  871. #endif
  872. return false;
  873. }
  874. }
  875. Assert(loop->landingPad->globOptData.IsLive(objPtrSym));
  876. BasicBlock *landingPad = loop->landingPad;
  877. Sym *symStore = propSymValueOnBackEdge->GetValueInfo()->GetSymStore();
  878. // The symStore can't be live into the loop
  879. // The symStore needs to still have the same value
  880. Assert(symStore && symStore->IsStackSym());
  881. if (loop->landingPad->globOptData.IsLive(symStore))
  882. {
  883. // May have already been hoisted:
  884. // o.x = t1;
  885. // o.y = t1;
  886. return false;
  887. }
  888. Value *landingPadValue = landingPad->globOptData.FindValue(propertySym);
  889. // Value should be added as initial value or already be there.
  890. Assert(landingPadValue);
  891. IR::Instr * ldInstrInLoop = this->globOpt->prePassInstrMap->Lookup(propertySym->m_id, nullptr);
  892. Assert(ldInstrInLoop);
  893. Assert(ldInstrInLoop->GetDst() == nullptr);
  894. // Create instr to put in landing pad for compensation
  895. Assert(IsPREInstrCandidateLoad(ldInstrInLoop->m_opcode));
  896. IR::Instr * ldInstr = InsertPropertySymPreloadInLandingPad(ldInstrInLoop, loop, propertySym);
  897. if (!ldInstr)
  898. {
  899. return false;
  900. }
  901. Assert(ldInstr->GetDst() == nullptr);
  902. ldInstr->SetDst(IR::RegOpnd::New(symStore->AsStackSym(), TyVar, this->globOpt->func));
  903. loop->fieldPRESymStores->Set(symStore->m_id);
  904. landingPad->globOptData.liveVarSyms->Set(symStore->m_id);
  905. Value * objPtrValue = landingPad->globOptData.FindValue(objPtrSym);
  906. objPtrCopyPropSym = objPtrCopyPropSym ? objPtrCopyPropSym : objPtrValue ? landingPad->globOptData.GetCopyPropSym(objPtrSym, objPtrValue) : nullptr;
  907. if (objPtrCopyPropSym)
  908. {
  909. // If we inserted T4 = T1.y, and T3 is the copy prop sym for T1 in the landing pad, we need T3.y
  910. // to be live on back edges to have the merge produce a value for T3.y. Having a value for T1.y
  911. // produced from the merge is not enough as the T1.y in the loop will get obj-ptr-copy-propped to
  912. // T3.y
  913. // T3.y
  914. PropertySym *newPropSym = PropertySym::FindOrCreate(
  915. objPtrCopyPropSym->m_id, propertySym->m_propertyId, propertySym->GetPropertyIdIndex(), propertySym->GetInlineCacheIndex(), propertySym->m_fieldKind, this->globOpt->func);
  916. if (!landingPad->globOptData.FindValue(newPropSym))
  917. {
  918. landingPad->globOptData.SetValue(landingPadValue, newPropSym);
  919. landingPad->globOptData.liveFields->Set(newPropSym->m_id);
  920. MakePropertySymLiveOnBackEdges(newPropSym, loop, propSymValueOnBackEdge);
  921. }
  922. }
  923. ValueType valueType(ValueType::Uninitialized);
  924. Value *initialValue = nullptr;
  925. if (loop->initialValueFieldMap.TryGetValue(propertySym, &initialValue))
  926. {
  927. if (ldInstr->IsProfiledInstr())
  928. {
  929. if (initialValue->GetValueNumber() == propSymValueOnBackEdge->GetValueNumber())
  930. {
  931. if (propSymValueOnBackEdge->GetValueInfo()->IsUninitialized())
  932. {
  933. valueType = ldInstr->AsProfiledInstr()->u.FldInfo().valueType;
  934. }
  935. else
  936. {
  937. valueType = propSymValueOnBackEdge->GetValueInfo()->Type();
  938. }
  939. }
  940. else
  941. {
  942. valueType = ValueType::Uninitialized;
  943. }
  944. ldInstr->AsProfiledInstr()->u.FldInfo().valueType = valueType;
  945. }
  946. }
  947. else
  948. {
  949. valueType = landingPadValue->GetValueInfo()->Type();
  950. }
  951. loop->symsUsedBeforeDefined->Set(symStore->m_id);
  952. if (valueType.IsLikelyNumber())
  953. {
  954. loop->likelyNumberSymsUsedBeforeDefined->Set(symStore->m_id);
  955. if (globOpt->DoAggressiveIntTypeSpec() ? valueType.IsLikelyInt() : valueType.IsInt())
  956. {
  957. // Can only force int conversions in the landing pad based on likely-int values if aggressive int type
  958. // specialization is enabled
  959. loop->likelyIntSymsUsedBeforeDefined->Set(symStore->m_id);
  960. }
  961. }
  962. #if DBG_DUMP
  963. if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldPREPhase, this->globOpt->func->GetSourceContextId(), this->globOpt->func->GetLocalFunctionId()))
  964. {
  965. Output::Print(_u("** TRACE: Field PRE: field pre-loaded in landing pad of loop head #%-3d: "), loop->GetHeadBlock()->GetBlockNum());
  966. ldInstr->Dump();
  967. Output::Print(_u("\n"));
  968. Output::Flush();
  969. }
  970. #endif
  971. return true;
  972. }
  973. void GlobOpt::PRE::PreloadPRECandidates(Loop *loop)
  974. {
  975. // Insert loads in landing pad for field PRE candidates. Iterate while(changed)
  976. // for the o.x.y cases.
  977. BOOL changed = true;
  978. if (!candidates || !candidates->candidatesList)
  979. {
  980. return;
  981. }
  982. Assert(loop->landingPad->GetFirstInstr() == loop->landingPad->GetLastInstr());
  983. while (changed)
  984. {
  985. changed = false;
  986. FOREACH_SLIST_ENTRY_EDITING(GlobHashBucket*, candidate, (SList<GlobHashBucket*>*)candidates->candidatesList, iter)
  987. {
  988. if (this->PreloadPRECandidate(loop, candidate))
  989. {
  990. changed = true;
  991. iter.RemoveCurrent();
  992. }
  993. if (PHASE_TRACE(Js::FieldPREPhase, this->globOpt->func))
  994. {
  995. Output::Print(_u("============================\n"));
  996. Output::Flush();
  997. }
  998. } NEXT_SLIST_ENTRY_EDITING;
  999. }
  1000. }
  1001. void GlobOpt::FieldPRE(Loop *loop)
  1002. {
  1003. if (!DoFieldPRE(loop))
  1004. {
  1005. return;
  1006. }
  1007. GlobOpt::PRE pre(this);
  1008. pre.FieldPRE(loop);
  1009. }
  1010. void GlobOpt::InsertValueCompensation(
  1011. BasicBlock *const predecessor,
  1012. const SymToValueInfoMap *symsRequiringCompensationToMergedValueInfoMap)
  1013. {
  1014. Assert(predecessor);
  1015. Assert(symsRequiringCompensationToMergedValueInfoMap->Count() != 0);
  1016. IR::Instr *insertBeforeInstr = predecessor->GetLastInstr();
  1017. Func *const func = insertBeforeInstr->m_func;
  1018. bool setLastInstrInPredecessor;
  1019. if(insertBeforeInstr->IsBranchInstr() || insertBeforeInstr->m_opcode == Js::OpCode::BailTarget)
  1020. {
  1021. // Don't insert code between the branch and the corresponding ByteCodeUses instructions
  1022. while(insertBeforeInstr->m_prev->m_opcode == Js::OpCode::ByteCodeUses)
  1023. {
  1024. insertBeforeInstr = insertBeforeInstr->m_prev;
  1025. }
  1026. setLastInstrInPredecessor = false;
  1027. }
  1028. else
  1029. {
  1030. // Insert at the end of the block and set the last instruction
  1031. Assert(insertBeforeInstr->m_next);
  1032. insertBeforeInstr = insertBeforeInstr->m_next; // Instruction after the last instruction in the predecessor
  1033. setLastInstrInPredecessor = true;
  1034. }
  1035. GlobOptBlockData &predecessorBlockData = predecessor->globOptData;
  1036. GlobOptBlockData &successorBlockData = *CurrentBlockData();
  1037. struct DelayChangeValueInfo
  1038. {
  1039. Value* predecessorValue;
  1040. ArrayValueInfo* valueInfo;
  1041. void ChangeValueInfo(BasicBlock* predecessor, GlobOpt* g)
  1042. {
  1043. g->ChangeValueInfo(
  1044. predecessor,
  1045. predecessorValue,
  1046. valueInfo,
  1047. false /*allowIncompatibleType*/,
  1048. true /*compensated*/);
  1049. }
  1050. };
  1051. JsUtil::List<DelayChangeValueInfo, ArenaAllocator> delayChangeValueInfo(alloc);
  1052. for(auto it = symsRequiringCompensationToMergedValueInfoMap->GetIterator(); it.IsValid(); it.MoveNext())
  1053. {
  1054. const auto &entry = it.Current();
  1055. Sym *const sym = entry.Key();
  1056. Value *const predecessorValue = predecessorBlockData.FindValue(sym);
  1057. Assert(predecessorValue);
  1058. ValueInfo *const predecessorValueInfo = predecessorValue->GetValueInfo();
  1059. // Currently, array value infos are the only ones that require compensation based on values
  1060. Assert(predecessorValueInfo->IsAnyOptimizedArray());
  1061. const ArrayValueInfo *const predecessorArrayValueInfo = predecessorValueInfo->AsArrayValueInfo();
  1062. StackSym *const predecessorHeadSegmentSym = predecessorArrayValueInfo->HeadSegmentSym();
  1063. StackSym *const predecessorHeadSegmentLengthSym = predecessorArrayValueInfo->HeadSegmentLengthSym();
  1064. StackSym *const predecessorLengthSym = predecessorArrayValueInfo->LengthSym();
  1065. ValueInfo *const mergedValueInfo = entry.Value();
  1066. const ArrayValueInfo *const mergedArrayValueInfo = mergedValueInfo->AsArrayValueInfo();
  1067. StackSym *const mergedHeadSegmentSym = mergedArrayValueInfo->HeadSegmentSym();
  1068. StackSym *const mergedHeadSegmentLengthSym = mergedArrayValueInfo->HeadSegmentLengthSym();
  1069. StackSym *const mergedLengthSym = mergedArrayValueInfo->LengthSym();
  1070. Assert(!mergedHeadSegmentSym || predecessorHeadSegmentSym);
  1071. Assert(!mergedHeadSegmentLengthSym || predecessorHeadSegmentLengthSym);
  1072. Assert(!mergedLengthSym || predecessorLengthSym);
  1073. bool compensated = false;
  1074. if(mergedHeadSegmentSym && predecessorHeadSegmentSym != mergedHeadSegmentSym)
  1075. {
  1076. IR::Instr *const newInstr =
  1077. IR::Instr::New(
  1078. Js::OpCode::Ld_A,
  1079. IR::RegOpnd::New(mergedHeadSegmentSym, mergedHeadSegmentSym->GetType(), func),
  1080. IR::RegOpnd::New(predecessorHeadSegmentSym, predecessorHeadSegmentSym->GetType(), func),
  1081. func);
  1082. newInstr->GetDst()->SetIsJITOptimizedReg(true);
  1083. newInstr->GetSrc1()->SetIsJITOptimizedReg(true);
  1084. newInstr->SetByteCodeOffset(insertBeforeInstr);
  1085. insertBeforeInstr->InsertBefore(newInstr);
  1086. compensated = true;
  1087. }
  1088. if(mergedHeadSegmentLengthSym && predecessorHeadSegmentLengthSym != mergedHeadSegmentLengthSym)
  1089. {
  1090. IR::Instr *const newInstr =
  1091. IR::Instr::New(
  1092. Js::OpCode::Ld_I4,
  1093. IR::RegOpnd::New(mergedHeadSegmentLengthSym, mergedHeadSegmentLengthSym->GetType(), func),
  1094. IR::RegOpnd::New(predecessorHeadSegmentLengthSym, predecessorHeadSegmentLengthSym->GetType(), func),
  1095. func);
  1096. newInstr->GetDst()->SetIsJITOptimizedReg(true);
  1097. newInstr->GetSrc1()->SetIsJITOptimizedReg(true);
  1098. newInstr->SetByteCodeOffset(insertBeforeInstr);
  1099. insertBeforeInstr->InsertBefore(newInstr);
  1100. compensated = true;
  1101. // Merge the head segment length value
  1102. Assert(predecessorBlockData.liveVarSyms->Test(predecessorHeadSegmentLengthSym->m_id));
  1103. predecessorBlockData.liveVarSyms->Set(mergedHeadSegmentLengthSym->m_id);
  1104. successorBlockData.liveVarSyms->Set(mergedHeadSegmentLengthSym->m_id);
  1105. Value *const predecessorHeadSegmentLengthValue =
  1106. predecessorBlockData.FindValue(predecessorHeadSegmentLengthSym);
  1107. Assert(predecessorHeadSegmentLengthValue);
  1108. predecessorBlockData.SetValue(predecessorHeadSegmentLengthValue, mergedHeadSegmentLengthSym);
  1109. Value *const mergedHeadSegmentLengthValue = successorBlockData.FindValue(mergedHeadSegmentLengthSym);
  1110. if(mergedHeadSegmentLengthValue)
  1111. {
  1112. Assert(mergedHeadSegmentLengthValue->GetValueNumber() != predecessorHeadSegmentLengthValue->GetValueNumber());
  1113. if(predecessorHeadSegmentLengthValue->GetValueInfo() != mergedHeadSegmentLengthValue->GetValueInfo())
  1114. {
  1115. mergedHeadSegmentLengthValue->SetValueInfo(
  1116. ValueInfo::MergeLikelyIntValueInfo(
  1117. this->alloc,
  1118. mergedHeadSegmentLengthValue,
  1119. predecessorHeadSegmentLengthValue,
  1120. mergedHeadSegmentLengthValue->GetValueInfo()->Type()
  1121. .Merge(predecessorHeadSegmentLengthValue->GetValueInfo()->Type())));
  1122. }
  1123. }
  1124. else
  1125. {
  1126. successorBlockData.SetValue(CopyValue(predecessorHeadSegmentLengthValue), mergedHeadSegmentLengthSym);
  1127. }
  1128. }
  1129. if(mergedLengthSym && predecessorLengthSym != mergedLengthSym)
  1130. {
  1131. IR::Instr *const newInstr =
  1132. IR::Instr::New(
  1133. Js::OpCode::Ld_I4,
  1134. IR::RegOpnd::New(mergedLengthSym, mergedLengthSym->GetType(), func),
  1135. IR::RegOpnd::New(predecessorLengthSym, predecessorLengthSym->GetType(), func),
  1136. func);
  1137. newInstr->GetDst()->SetIsJITOptimizedReg(true);
  1138. newInstr->GetSrc1()->SetIsJITOptimizedReg(true);
  1139. newInstr->SetByteCodeOffset(insertBeforeInstr);
  1140. insertBeforeInstr->InsertBefore(newInstr);
  1141. compensated = true;
  1142. // Merge the length value
  1143. Assert(predecessorBlockData.liveVarSyms->Test(predecessorLengthSym->m_id));
  1144. predecessorBlockData.liveVarSyms->Set(mergedLengthSym->m_id);
  1145. successorBlockData.liveVarSyms->Set(mergedLengthSym->m_id);
  1146. Value *const predecessorLengthValue = predecessorBlockData.FindValue(predecessorLengthSym);
  1147. Assert(predecessorLengthValue);
  1148. predecessorBlockData.SetValue(predecessorLengthValue, mergedLengthSym);
  1149. Value *const mergedLengthValue = successorBlockData.FindValue(mergedLengthSym);
  1150. if(mergedLengthValue)
  1151. {
  1152. Assert(mergedLengthValue->GetValueNumber() != predecessorLengthValue->GetValueNumber());
  1153. if(predecessorLengthValue->GetValueInfo() != mergedLengthValue->GetValueInfo())
  1154. {
  1155. mergedLengthValue->SetValueInfo(
  1156. ValueInfo::MergeLikelyIntValueInfo(
  1157. this->alloc,
  1158. mergedLengthValue,
  1159. predecessorLengthValue,
  1160. mergedLengthValue->GetValueInfo()->Type().Merge(predecessorLengthValue->GetValueInfo()->Type())));
  1161. }
  1162. }
  1163. else
  1164. {
  1165. successorBlockData.SetValue(CopyValue(predecessorLengthValue), mergedLengthSym);
  1166. }
  1167. }
  1168. if(compensated)
  1169. {
  1170. // Save the new ValueInfo for later.
  1171. // We don't want other symbols needing compensation to see this new one
  1172. delayChangeValueInfo.Add({
  1173. predecessorValue,
  1174. ArrayValueInfo::New(
  1175. alloc,
  1176. predecessorValueInfo->Type(),
  1177. mergedHeadSegmentSym ? mergedHeadSegmentSym : predecessorHeadSegmentSym,
  1178. mergedHeadSegmentLengthSym ? mergedHeadSegmentLengthSym : predecessorHeadSegmentLengthSym,
  1179. mergedLengthSym ? mergedLengthSym : predecessorLengthSym,
  1180. predecessorValueInfo->GetSymStore())
  1181. });
  1182. }
  1183. }
  1184. // Once we've compensated all the symbols, update the new ValueInfo.
  1185. delayChangeValueInfo.Map([predecessor, this](int, DelayChangeValueInfo d) { d.ChangeValueInfo(predecessor, this); });
  1186. if(setLastInstrInPredecessor)
  1187. {
  1188. predecessor->SetLastInstr(insertBeforeInstr->m_prev);
  1189. }
  1190. }
  1191. bool
  1192. GlobOpt::AreFromSameBytecodeFunc(IR::RegOpnd const* src1, IR::RegOpnd const* dst) const
  1193. {
  1194. Assert(this->func->m_symTable->FindStackSym(src1->m_sym->m_id) == src1->m_sym);
  1195. Assert(this->func->m_symTable->FindStackSym(dst->m_sym->m_id) == dst->m_sym);
  1196. if (dst->m_sym->HasByteCodeRegSlot() && src1->m_sym->HasByteCodeRegSlot())
  1197. {
  1198. return src1->m_sym->GetByteCodeFunc() == dst->m_sym->GetByteCodeFunc();
  1199. }
  1200. return false;
  1201. }
  1202. /*
  1203. * This is for scope object removal along with Heap Arguments optimization.
  1204. * We track several instructions to facilitate the removal of scope object.
  1205. * - LdSlotArr - This instr is tracked to keep track of the formals array (the dest)
  1206. * - InlineeStart - To keep track of the stack syms for the formals of the inlinee.
  1207. */
  1208. void
  1209. GlobOpt::TrackInstrsForScopeObjectRemoval(IR::Instr * instr)
  1210. {
  1211. IR::Opnd* dst = instr->GetDst();
  1212. IR::Opnd* src1 = instr->GetSrc1();
  1213. if (instr->m_opcode == Js::OpCode::Ld_A && src1->IsRegOpnd())
  1214. {
  1215. AssertMsg(!instr->m_func->IsStackArgsEnabled() || !src1->IsScopeObjOpnd(instr->m_func), "There can be no aliasing for scope object.");
  1216. }
  1217. // The following is to track formals array for Stack Arguments optimization with Formals
  1218. if (instr->m_func->IsStackArgsEnabled() && !this->IsLoopPrePass())
  1219. {
  1220. if (instr->m_opcode == Js::OpCode::LdSlotArr)
  1221. {
  1222. if (instr->GetSrc1()->IsScopeObjOpnd(instr->m_func))
  1223. {
  1224. AssertMsg(!instr->m_func->GetJITFunctionBody()->HasImplicitArgIns(), "No mapping is required in this case. So it should already be generating ArgIns.");
  1225. instr->m_func->TrackFormalsArraySym(dst->GetStackSym()->m_id);
  1226. }
  1227. }
  1228. else if (instr->m_opcode == Js::OpCode::InlineeStart)
  1229. {
  1230. Assert(instr->m_func->IsInlined());
  1231. Js::ArgSlot actualsCount = instr->m_func->actualCount - 1;
  1232. Js::ArgSlot formalsCount = instr->m_func->GetJITFunctionBody()->GetInParamsCount() - 1;
  1233. Func * func = instr->m_func;
  1234. Func * inlinerFunc = func->GetParentFunc(); //Inliner's func
  1235. IR::Instr * argOutInstr = instr->GetSrc2()->GetStackSym()->GetInstrDef();
  1236. //The argout immediately before the InlineeStart will be the ArgOut for NewScObject
  1237. //So we don't want to track the stack sym for this argout.- Skipping it here.
  1238. if (instr->m_func->IsInlinedConstructor())
  1239. {
  1240. //PRE might introduce a second defintion for the Src1. So assert for the opcode only when it has single definition.
  1241. Assert(argOutInstr->GetSrc1()->GetStackSym()->GetInstrDef() == nullptr ||
  1242. argOutInstr->GetSrc1()->GetStackSym()->GetInstrDef()->m_opcode == Js::OpCode::NewScObjectNoCtor);
  1243. argOutInstr = argOutInstr->GetSrc2()->GetStackSym()->GetInstrDef();
  1244. }
  1245. if (formalsCount < actualsCount)
  1246. {
  1247. Js::ArgSlot extraActuals = actualsCount - formalsCount;
  1248. //Skipping extra actuals passed
  1249. for (Js::ArgSlot i = 0; i < extraActuals; i++)
  1250. {
  1251. argOutInstr = argOutInstr->GetSrc2()->GetStackSym()->GetInstrDef();
  1252. }
  1253. }
  1254. StackSym * undefinedSym = nullptr;
  1255. for (Js::ArgSlot param = formalsCount; param > 0; param--)
  1256. {
  1257. StackSym * argOutSym = nullptr;
  1258. if (argOutInstr->GetSrc1())
  1259. {
  1260. if (argOutInstr->GetSrc1()->IsRegOpnd())
  1261. {
  1262. argOutSym = argOutInstr->GetSrc1()->GetStackSym();
  1263. }
  1264. else
  1265. {
  1266. // We will always have ArgOut instr - so the source operand will not be removed.
  1267. argOutSym = StackSym::New(inlinerFunc);
  1268. IR::Opnd * srcOpnd = argOutInstr->GetSrc1();
  1269. IR::Opnd * dstOpnd = IR::RegOpnd::New(argOutSym, TyVar, inlinerFunc);
  1270. IR::Instr * assignInstr = IR::Instr::New(Js::OpCode::Ld_A, dstOpnd, srcOpnd, inlinerFunc);
  1271. instr->InsertBefore(assignInstr);
  1272. }
  1273. }
  1274. Assert(!func->HasStackSymForFormal(param - 1));
  1275. if (param <= actualsCount)
  1276. {
  1277. Assert(argOutSym);
  1278. func->TrackStackSymForFormalIndex(param - 1, argOutSym);
  1279. argOutInstr = argOutInstr->GetSrc2()->GetStackSym()->GetInstrDef();
  1280. }
  1281. else
  1282. {
  1283. /*When param is out of range of actuals count, load undefined*/
  1284. // TODO: saravind: This will insert undefined for each of the param not having an actual. - Clean up this by having a sym for undefined on func ?
  1285. Assert(formalsCount > actualsCount);
  1286. if (undefinedSym == nullptr)
  1287. {
  1288. undefinedSym = StackSym::New(inlinerFunc);
  1289. IR::Opnd * srcOpnd = IR::AddrOpnd::New(inlinerFunc->GetScriptContextInfo()->GetUndefinedAddr(), IR::AddrOpndKindDynamicMisc, inlinerFunc);
  1290. IR::Opnd * dstOpnd = IR::RegOpnd::New(undefinedSym, TyVar, inlinerFunc);
  1291. IR::Instr * assignUndefined = IR::Instr::New(Js::OpCode::Ld_A, dstOpnd, srcOpnd, inlinerFunc);
  1292. instr->InsertBefore(assignUndefined);
  1293. }
  1294. func->TrackStackSymForFormalIndex(param - 1, undefinedSym);
  1295. }
  1296. }
  1297. }
  1298. }
  1299. }
  1300. void
  1301. GlobOpt::OptArguments(IR::Instr *instr)
  1302. {
  1303. IR::Opnd* dst = instr->GetDst();
  1304. IR::Opnd* src1 = instr->GetSrc1();
  1305. IR::Opnd* src2 = instr->GetSrc2();
  1306. TrackInstrsForScopeObjectRemoval(instr);
  1307. if (!TrackArgumentsObject())
  1308. {
  1309. return;
  1310. }
  1311. if (instr->HasAnyLoadHeapArgsOpCode())
  1312. {
  1313. #ifdef ENABLE_DEBUG_CONFIG_OPTIONS
  1314. if (instr->m_func->IsStackArgsEnabled())
  1315. {
  1316. if (instr->GetSrc1()->IsRegOpnd() && instr->m_func->GetJITFunctionBody()->GetInParamsCount() > 1)
  1317. {
  1318. StackSym * scopeObjSym = instr->GetSrc1()->GetStackSym();
  1319. Assert(scopeObjSym);
  1320. Assert(scopeObjSym->GetInstrDef()->m_opcode == Js::OpCode::InitCachedScope || scopeObjSym->GetInstrDef()->m_opcode == Js::OpCode::NewScopeObject);
  1321. Assert(instr->m_func->GetScopeObjSym() == scopeObjSym);
  1322. if (PHASE_VERBOSE_TRACE1(Js::StackArgFormalsOptPhase))
  1323. {
  1324. Output::Print(_u("StackArgFormals : %s (%d) :Setting scopeObjSym in forward pass. \n"), instr->m_func->GetJITFunctionBody()->GetDisplayName(), instr->m_func->GetJITFunctionBody()->GetFunctionNumber());
  1325. Output::Flush();
  1326. }
  1327. }
  1328. }
  1329. #endif
  1330. if (instr->m_func->GetJITFunctionBody()->GetInParamsCount() != 1 && !instr->m_func->IsStackArgsEnabled())
  1331. {
  1332. CannotAllocateArgumentsObjectOnStack();
  1333. }
  1334. else
  1335. {
  1336. CurrentBlockData()->TrackArgumentsSym(dst->AsRegOpnd());
  1337. }
  1338. return;
  1339. }
  1340. // Keep track of arguments objects and its aliases
  1341. // LdHeapArguments loads the arguments object and Ld_A tracks the aliases.
  1342. if ((instr->m_opcode == Js::OpCode::Ld_A || instr->m_opcode == Js::OpCode::BytecodeArgOutCapture) && (src1->IsRegOpnd() && CurrentBlockData()->IsArgumentsOpnd(src1)))
  1343. {
  1344. // In the debug mode, we don't want to optimize away the aliases. Since we may have to show them on the inspection.
  1345. if (((!AreFromSameBytecodeFunc(src1->AsRegOpnd(), dst->AsRegOpnd()) || this->currentBlock->loop) && instr->m_opcode != Js::OpCode::BytecodeArgOutCapture) || this->func->IsJitInDebugMode())
  1346. {
  1347. CannotAllocateArgumentsObjectOnStack();
  1348. return;
  1349. }
  1350. if(!dst->AsRegOpnd()->GetStackSym()->m_nonEscapingArgObjAlias)
  1351. {
  1352. CurrentBlockData()->TrackArgumentsSym(dst->AsRegOpnd());
  1353. }
  1354. return;
  1355. }
  1356. if (!CurrentBlockData()->TestAnyArgumentsSym())
  1357. {
  1358. // There are no syms to track yet, don't start tracking arguments sym.
  1359. return;
  1360. }
  1361. // Avoid loop prepass
  1362. if (this->currentBlock->loop && this->IsLoopPrePass())
  1363. {
  1364. return;
  1365. }
  1366. SymID id = 0;
  1367. switch(instr->m_opcode)
  1368. {
  1369. case Js::OpCode::LdElemI_A:
  1370. case Js::OpCode::TypeofElem:
  1371. {
  1372. Assert(src1->IsIndirOpnd());
  1373. IR::RegOpnd *indexOpnd = src1->AsIndirOpnd()->GetIndexOpnd();
  1374. if (indexOpnd && CurrentBlockData()->IsArgumentsSymID(indexOpnd->m_sym->m_id))
  1375. {
  1376. // Pathological test cases such as a[arguments]
  1377. CannotAllocateArgumentsObjectOnStack();
  1378. return;
  1379. }
  1380. IR::RegOpnd *baseOpnd = src1->AsIndirOpnd()->GetBaseOpnd();
  1381. id = baseOpnd->m_sym->m_id;
  1382. if (CurrentBlockData()->IsArgumentsSymID(id))
  1383. {
  1384. instr->usesStackArgumentsObject = true;
  1385. }
  1386. break;
  1387. }
  1388. case Js::OpCode::LdLen_A:
  1389. {
  1390. Assert(src1->IsRegOpnd());
  1391. if(CurrentBlockData()->IsArgumentsOpnd(src1))
  1392. {
  1393. instr->usesStackArgumentsObject = true;
  1394. }
  1395. break;
  1396. }
  1397. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  1398. {
  1399. if (CurrentBlockData()->IsArgumentsOpnd(src1))
  1400. {
  1401. instr->usesStackArgumentsObject = true;
  1402. }
  1403. if (CurrentBlockData()->IsArgumentsOpnd(src1) &&
  1404. src1->AsRegOpnd()->m_sym->GetInstrDef()->m_opcode == Js::OpCode::BytecodeArgOutCapture)
  1405. {
  1406. // Apply inlining results in such usage - this is to ignore this sym that is def'd by ByteCodeArgOutCapture
  1407. // It's needed because we do not have block level merging of arguments object and this def due to inlining can turn off stack args opt.
  1408. IR::Instr* builtinStart = instr->GetNextRealInstr();
  1409. if (builtinStart->m_opcode == Js::OpCode::InlineBuiltInStart)
  1410. {
  1411. IR::Opnd* builtinOpnd = builtinStart->GetSrc1();
  1412. if (builtinStart->GetSrc1()->IsAddrOpnd())
  1413. {
  1414. Assert(builtinOpnd->AsAddrOpnd()->m_isFunction);
  1415. Js::BuiltinFunction builtinFunction = Js::JavascriptLibrary::GetBuiltInForFuncInfo(((FixedFieldInfo*)builtinOpnd->AsAddrOpnd()->m_metadata)->GetLocalFuncId());
  1416. if (builtinFunction == Js::BuiltinFunction::JavascriptFunction_Apply)
  1417. {
  1418. CurrentBlockData()->ClearArgumentsSym(src1->AsRegOpnd());
  1419. }
  1420. }
  1421. else if (builtinOpnd->IsRegOpnd())
  1422. {
  1423. if (builtinOpnd->AsRegOpnd()->m_sym->m_builtInIndex == Js::BuiltinFunction::JavascriptFunction_Apply)
  1424. {
  1425. CurrentBlockData()->ClearArgumentsSym(src1->AsRegOpnd());
  1426. }
  1427. }
  1428. }
  1429. }
  1430. break;
  1431. }
  1432. case Js::OpCode::BailOnNotStackArgs:
  1433. case Js::OpCode::ArgOut_A_FromStackArgs:
  1434. case Js::OpCode::BytecodeArgOutUse:
  1435. {
  1436. if (src1 && CurrentBlockData()->IsArgumentsOpnd(src1))
  1437. {
  1438. instr->usesStackArgumentsObject = true;
  1439. }
  1440. break;
  1441. }
  1442. default:
  1443. {
  1444. // Super conservative here, if we see the arguments or any of its alias being used in any
  1445. // other opcode just don't do this optimization. Revisit this to optimize further if we see any common
  1446. // case is missed.
  1447. if (src1)
  1448. {
  1449. if (src1->IsRegOpnd() || src1->IsSymOpnd() || src1->IsIndirOpnd())
  1450. {
  1451. if (CurrentBlockData()->IsArgumentsOpnd(src1))
  1452. {
  1453. #ifdef PERF_HINT
  1454. if (PHASE_TRACE1(Js::PerfHintPhase))
  1455. {
  1456. WritePerfHint(PerfHints::HeapArgumentsCreated, instr->m_func, instr->GetByteCodeOffset());
  1457. }
  1458. #endif
  1459. CannotAllocateArgumentsObjectOnStack();
  1460. return;
  1461. }
  1462. }
  1463. }
  1464. if (src2)
  1465. {
  1466. if (src2->IsRegOpnd() || src2->IsSymOpnd() || src2->IsIndirOpnd())
  1467. {
  1468. if (CurrentBlockData()->IsArgumentsOpnd(src2))
  1469. {
  1470. #ifdef PERF_HINT
  1471. if (PHASE_TRACE1(Js::PerfHintPhase))
  1472. {
  1473. WritePerfHint(PerfHints::HeapArgumentsCreated, instr->m_func, instr->GetByteCodeOffset());
  1474. }
  1475. #endif
  1476. CannotAllocateArgumentsObjectOnStack();
  1477. return;
  1478. }
  1479. }
  1480. }
  1481. // We should look at dst last to correctly handle cases where it's the same as one of the src operands.
  1482. if (dst)
  1483. {
  1484. if (dst->IsIndirOpnd() || dst->IsSymOpnd())
  1485. {
  1486. if (CurrentBlockData()->IsArgumentsOpnd(dst))
  1487. {
  1488. #ifdef PERF_HINT
  1489. if (PHASE_TRACE1(Js::PerfHintPhase))
  1490. {
  1491. WritePerfHint(PerfHints::HeapArgumentsModification, instr->m_func, instr->GetByteCodeOffset());
  1492. }
  1493. #endif
  1494. CannotAllocateArgumentsObjectOnStack();
  1495. return;
  1496. }
  1497. }
  1498. else if (dst->IsRegOpnd())
  1499. {
  1500. if (this->currentBlock->loop && CurrentBlockData()->IsArgumentsOpnd(dst))
  1501. {
  1502. #ifdef PERF_HINT
  1503. if (PHASE_TRACE1(Js::PerfHintPhase))
  1504. {
  1505. WritePerfHint(PerfHints::HeapArgumentsModification, instr->m_func, instr->GetByteCodeOffset());
  1506. }
  1507. #endif
  1508. CannotAllocateArgumentsObjectOnStack();
  1509. return;
  1510. }
  1511. CurrentBlockData()->ClearArgumentsSym(dst->AsRegOpnd());
  1512. }
  1513. }
  1514. }
  1515. break;
  1516. }
  1517. return;
  1518. }
  1519. void
  1520. GlobOpt::MarkArgumentsUsedForBranch(IR::Instr * instr)
  1521. {
  1522. // If it's a conditional branch instruction and the operand used for branching is one of the arguments
  1523. // to the function, tag the m_argUsedForBranch of the functionBody so that it can be used later for inlining decisions.
  1524. if (instr->IsBranchInstr() && !instr->AsBranchInstr()->IsUnconditional())
  1525. {
  1526. IR::BranchInstr * bInstr = instr->AsBranchInstr();
  1527. IR::Opnd *src1 = bInstr->GetSrc1();
  1528. IR::Opnd *src2 = bInstr->GetSrc2();
  1529. // These are used because we don't want to rely on src1 or src2 to always be the register/constant
  1530. IR::RegOpnd *regOpnd = nullptr;
  1531. if (!src2 && (instr->m_opcode == Js::OpCode::BrFalse_A || instr->m_opcode == Js::OpCode::BrTrue_A) && src1->IsRegOpnd())
  1532. {
  1533. regOpnd = src1->AsRegOpnd();
  1534. }
  1535. // We need to check for (0===arg) and (arg===0); this is especially important since some minifiers
  1536. // change all instances of one to the other.
  1537. else if (src2 && src2->IsConstOpnd() && src1->IsRegOpnd())
  1538. {
  1539. regOpnd = src1->AsRegOpnd();
  1540. }
  1541. else if (src2 && src2->IsRegOpnd() && src1->IsConstOpnd())
  1542. {
  1543. regOpnd = src2->AsRegOpnd();
  1544. }
  1545. if (regOpnd != nullptr)
  1546. {
  1547. if (regOpnd->m_sym->IsSingleDef())
  1548. {
  1549. IR::Instr * defInst = regOpnd->m_sym->GetInstrDef();
  1550. IR::Opnd *defSym = defInst->GetSrc1();
  1551. if (defSym && defSym->IsSymOpnd() && defSym->AsSymOpnd()->m_sym->IsStackSym()
  1552. && defSym->AsSymOpnd()->m_sym->AsStackSym()->IsParamSlotSym())
  1553. {
  1554. uint16 param = defSym->AsSymOpnd()->m_sym->AsStackSym()->GetParamSlotNum();
  1555. // We only support functions with 13 arguments to ensure optimal size of callSiteInfo
  1556. if (param < Js::Constants::MaximumArgumentCountForConstantArgumentInlining)
  1557. {
  1558. this->func->GetJITOutput()->SetArgUsedForBranch((uint8)param);
  1559. }
  1560. }
  1561. }
  1562. }
  1563. }
  1564. }
  1565. const InductionVariable*
  1566. GlobOpt::GetInductionVariable(SymID sym, Loop *loop)
  1567. {
  1568. if (loop->inductionVariables)
  1569. {
  1570. for (auto it = loop->inductionVariables->GetIterator(); it.IsValid(); it.MoveNext())
  1571. {
  1572. InductionVariable* iv = &it.CurrentValueReference();
  1573. if (!iv->IsChangeDeterminate() || !iv->IsChangeUnidirectional())
  1574. {
  1575. continue;
  1576. }
  1577. if (iv->Sym()->m_id == sym)
  1578. {
  1579. return iv;
  1580. }
  1581. }
  1582. }
  1583. return nullptr;
  1584. }
  1585. bool
  1586. GlobOpt::IsSymIDInductionVariable(SymID sym, Loop *loop)
  1587. {
  1588. return GetInductionVariable(sym, loop) != nullptr;
  1589. }
  1590. SymID
  1591. GlobOpt::GetVarSymID(StackSym *sym)
  1592. {
  1593. if (sym && sym->m_type != TyVar)
  1594. {
  1595. sym = sym->GetVarEquivSym(nullptr);
  1596. }
  1597. if (!sym)
  1598. {
  1599. return Js::Constants::InvalidSymID;
  1600. }
  1601. return sym->m_id;
  1602. }
  1603. bool
  1604. GlobOpt::IsAllowedForMemOpt(IR::Instr* instr, bool isMemset, IR::RegOpnd *baseOpnd, IR::Opnd *indexOpnd)
  1605. {
  1606. Assert(instr);
  1607. if (!baseOpnd || !indexOpnd)
  1608. {
  1609. return false;
  1610. }
  1611. Loop* loop = this->currentBlock->loop;
  1612. const ValueType baseValueType(baseOpnd->GetValueType());
  1613. const ValueType indexValueType(indexOpnd->GetValueType());
  1614. // Validate the array and index types
  1615. if (
  1616. !indexValueType.IsInt() ||
  1617. !(
  1618. baseValueType.IsTypedIntOrFloatArray() ||
  1619. baseValueType.IsArray()
  1620. )
  1621. )
  1622. {
  1623. #if DBG_DUMP
  1624. wchar indexValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  1625. indexValueType.ToString(indexValueTypeStr);
  1626. wchar baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  1627. baseValueType.ToString(baseValueTypeStr);
  1628. TRACE_MEMOP_VERBOSE(loop, instr, _u("Index[%s] or Array[%s] value type is invalid"), indexValueTypeStr, baseValueTypeStr);
  1629. #endif
  1630. return false;
  1631. }
  1632. // The following is conservative and works around a bug in induction variable analysis.
  1633. if (baseOpnd->IsArrayRegOpnd())
  1634. {
  1635. IR::ArrayRegOpnd *baseArrayOp = baseOpnd->AsArrayRegOpnd();
  1636. bool hasBoundChecksRemoved = (
  1637. baseArrayOp->EliminatedLowerBoundCheck() &&
  1638. baseArrayOp->EliminatedUpperBoundCheck() &&
  1639. !instr->extractedUpperBoundCheckWithoutHoisting &&
  1640. !instr->loadedArrayHeadSegment &&
  1641. !instr->loadedArrayHeadSegmentLength
  1642. );
  1643. if (!hasBoundChecksRemoved)
  1644. {
  1645. TRACE_MEMOP_VERBOSE(loop, instr, _u("Missing bounds check optimization"));
  1646. return false;
  1647. }
  1648. }
  1649. else
  1650. {
  1651. return false;
  1652. }
  1653. if (!baseValueType.IsTypedArray())
  1654. {
  1655. // Check if the instr can kill the value type of the array
  1656. JsArrayKills arrayKills = CheckJsArrayKills(instr);
  1657. if (arrayKills.KillsValueType(baseValueType))
  1658. {
  1659. TRACE_MEMOP_VERBOSE(loop, instr, _u("The array (s%d) can lose its value type"), GetVarSymID(baseOpnd->GetStackSym()));
  1660. return false;
  1661. }
  1662. }
  1663. // Process the Index Operand
  1664. if (!this->OptIsInvariant(baseOpnd, this->currentBlock, loop, CurrentBlockData()->FindValue(baseOpnd->m_sym), false, true))
  1665. {
  1666. TRACE_MEMOP_VERBOSE(loop, instr, _u("Base (s%d) is not invariant"), GetVarSymID(baseOpnd->GetStackSym()));
  1667. return false;
  1668. }
  1669. // Validate the index
  1670. Assert(indexOpnd->GetStackSym());
  1671. SymID indexSymID = GetVarSymID(indexOpnd->GetStackSym());
  1672. const InductionVariable* iv = GetInductionVariable(indexSymID, loop);
  1673. if (!iv)
  1674. {
  1675. // If the index is not an induction variable return
  1676. TRACE_MEMOP_VERBOSE(loop, instr, _u("Index (s%d) is not an induction variable"), indexSymID);
  1677. return false;
  1678. }
  1679. Assert(iv->IsChangeDeterminate() && iv->IsChangeUnidirectional());
  1680. const IntConstantBounds & bounds = iv->ChangeBounds();
  1681. if (loop->memOpInfo)
  1682. {
  1683. // Only accept induction variables that increments by 1
  1684. Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 0, 0 };
  1685. inductionVariableChangeInfo = loop->memOpInfo->inductionVariableChangeInfoMap->Lookup(indexSymID, inductionVariableChangeInfo);
  1686. if (
  1687. (bounds.LowerBound() != 1 && bounds.LowerBound() != -1) ||
  1688. (bounds.UpperBound() != bounds.LowerBound()) ||
  1689. inductionVariableChangeInfo.unroll > 1 // Must be 0 (not seen yet) or 1 (already seen)
  1690. )
  1691. {
  1692. TRACE_MEMOP_VERBOSE(loop, instr, _u("The index does not change by 1: %d><%d, unroll=%d"), bounds.LowerBound(), bounds.UpperBound(), inductionVariableChangeInfo.unroll);
  1693. return false;
  1694. }
  1695. // Check if the index is the same in all MemOp optimization in this loop
  1696. if (!loop->memOpInfo->candidates->Empty())
  1697. {
  1698. Loop::MemOpCandidate* previousCandidate = loop->memOpInfo->candidates->Head();
  1699. // All MemOp operations within the same loop must use the same index
  1700. if (previousCandidate->index != indexSymID)
  1701. {
  1702. TRACE_MEMOP_VERBOSE(loop, instr, _u("The index is not the same as other MemOp in the loop"));
  1703. return false;
  1704. }
  1705. }
  1706. }
  1707. return true;
  1708. }
  1709. bool
  1710. GlobOpt::CollectMemcopyLdElementI(IR::Instr *instr, Loop *loop)
  1711. {
  1712. Assert(instr->GetSrc1()->IsIndirOpnd());
  1713. IR::IndirOpnd *src1 = instr->GetSrc1()->AsIndirOpnd();
  1714. IR::Opnd *indexOpnd = src1->GetIndexOpnd();
  1715. IR::RegOpnd *baseOpnd = src1->GetBaseOpnd()->AsRegOpnd();
  1716. SymID baseSymID = GetVarSymID(baseOpnd->GetStackSym());
  1717. if (!IsAllowedForMemOpt(instr, false, baseOpnd, indexOpnd))
  1718. {
  1719. return false;
  1720. }
  1721. SymID inductionSymID = GetVarSymID(indexOpnd->GetStackSym());
  1722. Assert(IsSymIDInductionVariable(inductionSymID, loop));
  1723. loop->EnsureMemOpVariablesInitialized();
  1724. bool isIndexPreIncr = loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID);
  1725. IR::Opnd * dst = instr->GetDst();
  1726. if (!dst->IsRegOpnd() || !dst->AsRegOpnd()->GetStackSym()->IsSingleDef())
  1727. {
  1728. return false;
  1729. }
  1730. Loop::MemCopyCandidate* memcopyInfo = memcopyInfo = JitAnewStruct(this->func->GetTopFunc()->m_fg->alloc, Loop::MemCopyCandidate);
  1731. memcopyInfo->ldBase = baseSymID;
  1732. memcopyInfo->ldCount = 1;
  1733. memcopyInfo->count = 0;
  1734. memcopyInfo->bIndexAlreadyChanged = isIndexPreIncr;
  1735. memcopyInfo->base = Js::Constants::InvalidSymID; //need to find the stElem first
  1736. memcopyInfo->index = inductionSymID;
  1737. memcopyInfo->transferSym = dst->AsRegOpnd()->GetStackSym();
  1738. loop->memOpInfo->candidates->Prepend(memcopyInfo);
  1739. return true;
  1740. }
  1741. bool
  1742. GlobOpt::CollectMemsetStElementI(IR::Instr *instr, Loop *loop)
  1743. {
  1744. Assert(instr->GetDst()->IsIndirOpnd());
  1745. IR::IndirOpnd *dst = instr->GetDst()->AsIndirOpnd();
  1746. IR::Opnd *indexOp = dst->GetIndexOpnd();
  1747. IR::RegOpnd *baseOp = dst->GetBaseOpnd()->AsRegOpnd();
  1748. if (!IsAllowedForMemOpt(instr, true, baseOp, indexOp))
  1749. {
  1750. return false;
  1751. }
  1752. SymID baseSymID = GetVarSymID(baseOp->GetStackSym());
  1753. IR::Opnd *srcDef = instr->GetSrc1();
  1754. StackSym *srcSym = nullptr;
  1755. if (srcDef->IsRegOpnd())
  1756. {
  1757. IR::RegOpnd* opnd = srcDef->AsRegOpnd();
  1758. if (this->OptIsInvariant(opnd, this->currentBlock, loop, CurrentBlockData()->FindValue(opnd->m_sym), true, true))
  1759. {
  1760. srcSym = opnd->GetStackSym();
  1761. }
  1762. }
  1763. BailoutConstantValue constant = {TyIllegal, 0};
  1764. if (srcDef->IsFloatConstOpnd())
  1765. {
  1766. constant.InitFloatConstValue(srcDef->AsFloatConstOpnd()->m_value);
  1767. }
  1768. else if (srcDef->IsIntConstOpnd())
  1769. {
  1770. constant.InitIntConstValue(srcDef->AsIntConstOpnd()->GetValue(), srcDef->AsIntConstOpnd()->GetType());
  1771. }
  1772. else if (srcDef->IsAddrOpnd())
  1773. {
  1774. constant.InitVarConstValue(srcDef->AsAddrOpnd()->m_address);
  1775. }
  1776. else if(!srcSym)
  1777. {
  1778. TRACE_MEMOP_PHASE_VERBOSE(MemSet, loop, instr, _u("Source is not an invariant"));
  1779. return false;
  1780. }
  1781. // Process the Index Operand
  1782. Assert(indexOp->GetStackSym());
  1783. SymID inductionSymID = GetVarSymID(indexOp->GetStackSym());
  1784. Assert(IsSymIDInductionVariable(inductionSymID, loop));
  1785. loop->EnsureMemOpVariablesInitialized();
  1786. bool isIndexPreIncr = loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID);
  1787. Loop::MemSetCandidate* memsetInfo = JitAnewStruct(this->func->GetTopFunc()->m_fg->alloc, Loop::MemSetCandidate);
  1788. memsetInfo->base = baseSymID;
  1789. memsetInfo->index = inductionSymID;
  1790. memsetInfo->constant = constant;
  1791. memsetInfo->srcSym = srcSym;
  1792. memsetInfo->count = 1;
  1793. memsetInfo->bIndexAlreadyChanged = isIndexPreIncr;
  1794. loop->memOpInfo->candidates->Prepend(memsetInfo);
  1795. return true;
  1796. }
  1797. bool GlobOpt::CollectMemcopyStElementI(IR::Instr *instr, Loop *loop)
  1798. {
  1799. if (!loop->memOpInfo || loop->memOpInfo->candidates->Empty())
  1800. {
  1801. // There is no ldElem matching this stElem
  1802. return false;
  1803. }
  1804. Assert(instr->GetDst()->IsIndirOpnd());
  1805. IR::IndirOpnd *dst = instr->GetDst()->AsIndirOpnd();
  1806. IR::Opnd *indexOp = dst->GetIndexOpnd();
  1807. IR::RegOpnd *baseOp = dst->GetBaseOpnd()->AsRegOpnd();
  1808. SymID baseSymID = GetVarSymID(baseOp->GetStackSym());
  1809. if (!instr->GetSrc1()->IsRegOpnd())
  1810. {
  1811. return false;
  1812. }
  1813. IR::RegOpnd* src1 = instr->GetSrc1()->AsRegOpnd();
  1814. if (!src1->GetIsDead())
  1815. {
  1816. // This must be the last use of the register.
  1817. // It will invalidate `var m = a[i]; b[i] = m;` but this is not a very interesting case.
  1818. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Source (s%d) is still alive after StElemI"), baseSymID);
  1819. return false;
  1820. }
  1821. if (!IsAllowedForMemOpt(instr, false, baseOp, indexOp))
  1822. {
  1823. return false;
  1824. }
  1825. SymID srcSymID = GetVarSymID(src1->GetStackSym());
  1826. // Prepare the memcopyCandidate entry
  1827. Loop::MemOpCandidate* previousCandidate = loop->memOpInfo->candidates->Head();
  1828. if (!previousCandidate->IsMemCopy())
  1829. {
  1830. return false;
  1831. }
  1832. Loop::MemCopyCandidate* memcopyInfo = previousCandidate->AsMemCopy();
  1833. // The previous candidate has to have been created by the matching ldElem
  1834. if (
  1835. memcopyInfo->base != Js::Constants::InvalidSymID ||
  1836. GetVarSymID(memcopyInfo->transferSym) != srcSymID
  1837. )
  1838. {
  1839. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("No matching LdElem found (s%d)"), baseSymID);
  1840. return false;
  1841. }
  1842. Assert(indexOp->GetStackSym());
  1843. SymID inductionSymID = GetVarSymID(indexOp->GetStackSym());
  1844. Assert(IsSymIDInductionVariable(inductionSymID, loop));
  1845. bool isIndexPreIncr = loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID);
  1846. if (isIndexPreIncr != memcopyInfo->bIndexAlreadyChanged)
  1847. {
  1848. // The index changed between the load and the store
  1849. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Index value changed between ldElem and stElem"));
  1850. return false;
  1851. }
  1852. // Consider: Can we remove the count field?
  1853. memcopyInfo->count++;
  1854. memcopyInfo->base = baseSymID;
  1855. return true;
  1856. }
  1857. bool
  1858. GlobOpt::CollectMemOpLdElementI(IR::Instr *instr, Loop *loop)
  1859. {
  1860. Assert(instr->m_opcode == Js::OpCode::LdElemI_A);
  1861. return (!PHASE_OFF(Js::MemCopyPhase, this->func) && CollectMemcopyLdElementI(instr, loop));
  1862. }
  1863. bool
  1864. GlobOpt::CollectMemOpStElementI(IR::Instr *instr, Loop *loop)
  1865. {
  1866. Assert(instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict);
  1867. Assert(instr->GetSrc1());
  1868. return (!PHASE_OFF(Js::MemSetPhase, this->func) && CollectMemsetStElementI(instr, loop)) ||
  1869. (!PHASE_OFF(Js::MemCopyPhase, this->func) && CollectMemcopyStElementI(instr, loop));
  1870. }
  1871. bool
  1872. GlobOpt::CollectMemOpInfo(IR::Instr *instrBegin, IR::Instr *instr, Value *src1Val, Value *src2Val)
  1873. {
  1874. Assert(this->currentBlock->loop);
  1875. Loop *loop = this->currentBlock->loop;
  1876. if (!loop->blockList.HasTwo())
  1877. {
  1878. // We support memcopy and memset for loops which have only two blocks.
  1879. return false;
  1880. }
  1881. if (loop->GetLoopFlags().isInterpreted && !loop->GetLoopFlags().memopMinCountReached)
  1882. {
  1883. TRACE_MEMOP_VERBOSE(loop, instr, _u("minimum loop count not reached"))
  1884. loop->doMemOp = false;
  1885. return false;
  1886. }
  1887. Assert(loop->doMemOp);
  1888. bool isIncr = true, isChangedByOne = false;
  1889. switch (instr->m_opcode)
  1890. {
  1891. case Js::OpCode::StElemI_A:
  1892. case Js::OpCode::StElemI_A_Strict:
  1893. if (!CollectMemOpStElementI(instr, loop))
  1894. {
  1895. loop->doMemOp = false;
  1896. return false;
  1897. }
  1898. break;
  1899. case Js::OpCode::LdElemI_A:
  1900. if (!CollectMemOpLdElementI(instr, loop))
  1901. {
  1902. loop->doMemOp = false;
  1903. return false;
  1904. }
  1905. break;
  1906. case Js::OpCode::Decr_A:
  1907. isIncr = false;
  1908. case Js::OpCode::Incr_A:
  1909. isChangedByOne = true;
  1910. goto MemOpCheckInductionVariable;
  1911. case Js::OpCode::Sub_I4:
  1912. case Js::OpCode::Sub_A:
  1913. isIncr = false;
  1914. case Js::OpCode::Add_A:
  1915. case Js::OpCode::Add_I4:
  1916. {
  1917. MemOpCheckInductionVariable:
  1918. StackSym *sym = instr->GetSrc1()->GetStackSym();
  1919. if (!sym)
  1920. {
  1921. sym = instr->GetSrc2()->GetStackSym();
  1922. }
  1923. SymID inductionSymID = GetVarSymID(sym);
  1924. if (IsSymIDInductionVariable(inductionSymID, this->currentBlock->loop))
  1925. {
  1926. if (!isChangedByOne)
  1927. {
  1928. IR::Opnd *src1, *src2;
  1929. src1 = instr->GetSrc1();
  1930. src2 = instr->GetSrc2();
  1931. if (src2->IsRegOpnd())
  1932. {
  1933. Value *val = CurrentBlockData()->FindValue(src2->AsRegOpnd()->m_sym);
  1934. if (val)
  1935. {
  1936. ValueInfo *vi = val->GetValueInfo();
  1937. int constValue;
  1938. if (vi && vi->TryGetIntConstantValue(&constValue))
  1939. {
  1940. if (constValue == 1)
  1941. {
  1942. isChangedByOne = true;
  1943. }
  1944. }
  1945. }
  1946. }
  1947. else if (src2->IsIntConstOpnd())
  1948. {
  1949. if (src2->AsIntConstOpnd()->GetValue() == 1)
  1950. {
  1951. isChangedByOne = true;
  1952. }
  1953. }
  1954. }
  1955. loop->EnsureMemOpVariablesInitialized();
  1956. if (!isChangedByOne)
  1957. {
  1958. Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { Js::Constants::InvalidLoopUnrollFactor, 0 };
  1959. if (!loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID))
  1960. {
  1961. loop->memOpInfo->inductionVariableChangeInfoMap->Add(inductionSymID, inductionVariableChangeInfo);
  1962. }
  1963. else
  1964. {
  1965. loop->memOpInfo->inductionVariableChangeInfoMap->Item(inductionSymID, inductionVariableChangeInfo);
  1966. }
  1967. }
  1968. else
  1969. {
  1970. if (!loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID))
  1971. {
  1972. Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 1, isIncr };
  1973. loop->memOpInfo->inductionVariableChangeInfoMap->Add(inductionSymID, inductionVariableChangeInfo);
  1974. }
  1975. else
  1976. {
  1977. Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 0, 0 };
  1978. inductionVariableChangeInfo = loop->memOpInfo->inductionVariableChangeInfoMap->Lookup(inductionSymID, inductionVariableChangeInfo);
  1979. inductionVariableChangeInfo.unroll++;
  1980. inductionVariableChangeInfo.isIncremental = isIncr;
  1981. loop->memOpInfo->inductionVariableChangeInfoMap->Item(inductionSymID, inductionVariableChangeInfo);
  1982. }
  1983. }
  1984. break;
  1985. }
  1986. // Fallthrough if not an induction variable
  1987. }
  1988. default:
  1989. FOREACH_INSTR_IN_RANGE(chkInstr, instrBegin->m_next, instr)
  1990. {
  1991. if (IsInstrInvalidForMemOp(chkInstr, loop, src1Val, src2Val))
  1992. {
  1993. loop->doMemOp = false;
  1994. return false;
  1995. }
  1996. // Make sure this instruction doesn't use the memcopy transfer sym before it is checked by StElemI
  1997. if (loop->memOpInfo && !loop->memOpInfo->candidates->Empty())
  1998. {
  1999. Loop::MemOpCandidate* prevCandidate = loop->memOpInfo->candidates->Head();
  2000. if (prevCandidate->IsMemCopy())
  2001. {
  2002. Loop::MemCopyCandidate* memcopyCandidate = prevCandidate->AsMemCopy();
  2003. if (memcopyCandidate->base == Js::Constants::InvalidSymID)
  2004. {
  2005. if (chkInstr->HasSymUse(memcopyCandidate->transferSym))
  2006. {
  2007. loop->doMemOp = false;
  2008. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, chkInstr, _u("Found illegal use of LdElemI value(s%d)"), GetVarSymID(memcopyCandidate->transferSym));
  2009. return false;
  2010. }
  2011. }
  2012. }
  2013. }
  2014. }
  2015. NEXT_INSTR_IN_RANGE;
  2016. }
  2017. return true;
  2018. }
  2019. bool
  2020. GlobOpt::IsInstrInvalidForMemOp(IR::Instr *instr, Loop *loop, Value *src1Val, Value *src2Val)
  2021. {
  2022. // List of instruction that are valid with memop (ie: instr that gets removed if memop is emitted)
  2023. if (
  2024. this->currentBlock != loop->GetHeadBlock() &&
  2025. !instr->IsLabelInstr() &&
  2026. instr->IsRealInstr() &&
  2027. instr->m_opcode != Js::OpCode::IncrLoopBodyCount &&
  2028. instr->m_opcode != Js::OpCode::StLoopBodyCount &&
  2029. instr->m_opcode != Js::OpCode::Ld_A &&
  2030. instr->m_opcode != Js::OpCode::Ld_I4 &&
  2031. !(instr->IsBranchInstr() && instr->AsBranchInstr()->IsUnconditional())
  2032. )
  2033. {
  2034. TRACE_MEMOP_VERBOSE(loop, instr, _u("Instruction not accepted for memop"));
  2035. return true;
  2036. }
  2037. // Check prev instr because it could have been added by an optimization and we won't see it here.
  2038. if (OpCodeAttr::FastFldInstr(instr->m_opcode) || (instr->m_prev && OpCodeAttr::FastFldInstr(instr->m_prev->m_opcode)))
  2039. {
  2040. // Refuse any operations interacting with Fields
  2041. TRACE_MEMOP_VERBOSE(loop, instr, _u("Field interaction detected"));
  2042. return true;
  2043. }
  2044. if (Js::OpCodeUtil::GetOpCodeLayout(instr->m_opcode) == Js::OpLayoutType::ElementSlot)
  2045. {
  2046. // Refuse any operations interacting with slots
  2047. TRACE_MEMOP_VERBOSE(loop, instr, _u("Slot interaction detected"));
  2048. return true;
  2049. }
  2050. if (this->MayNeedBailOnImplicitCall(instr, src1Val, src2Val))
  2051. {
  2052. TRACE_MEMOP_VERBOSE(loop, instr, _u("Implicit call bailout detected"));
  2053. return true;
  2054. }
  2055. return false;
  2056. }
  2057. void
  2058. GlobOpt::TryReplaceLdLen(IR::Instr *& instr)
  2059. {
  2060. // Change LdLen on objects other than arrays, strings, and 'arguments' to LdFld. Otherwise, convert the SymOpnd to a RegOpnd here.
  2061. if (instr->m_opcode == Js::OpCode::LdLen_A && instr->GetSrc1() && instr->GetSrc1()->IsSymOpnd())
  2062. {
  2063. IR::SymOpnd * opnd = instr->GetSrc1()->AsSymOpnd();
  2064. Sym *sym = opnd->m_sym;
  2065. Assert(sym->IsPropertySym());
  2066. PropertySym *originalPropertySym = sym->AsPropertySym();
  2067. IR::RegOpnd* newopnd = IR::RegOpnd::New(originalPropertySym->m_stackSym, IRType::TyVar, instr->m_func);
  2068. ValueInfo *const objectValueInfo = CurrentBlockData()->FindValue(originalPropertySym->m_stackSym)->GetValueInfo();
  2069. // things we'd emit a fast path for
  2070. if (
  2071. objectValueInfo->IsLikelyAnyArray() ||
  2072. objectValueInfo->HasHadStringTag() ||
  2073. objectValueInfo->IsLikelyString() ||
  2074. newopnd->IsArgumentsObject() ||
  2075. (CurrentBlockData()->argObjSyms && CurrentBlockData()->IsArgumentsOpnd(newopnd))
  2076. )
  2077. {
  2078. // We need to properly transfer over the information from the old operand, which is
  2079. // a SymOpnd, to the new one, which is a RegOpnd. Unfortunately, the types mean the
  2080. // normal copy methods won't work here, so we're going to directly copy data.
  2081. newopnd->SetIsJITOptimizedReg(opnd->GetIsJITOptimizedReg());
  2082. newopnd->SetValueType(objectValueInfo->Type());
  2083. newopnd->SetIsDead(opnd->GetIsDead());
  2084. instr->ReplaceSrc1(newopnd);
  2085. }
  2086. else
  2087. {
  2088. // otherwise, change the instruction to an LdFld here.
  2089. instr->m_opcode = Js::OpCode::LdFld;
  2090. }
  2091. }
  2092. }
  2093. IR::Instr *
  2094. GlobOpt::OptInstr(IR::Instr *&instr, bool* isInstrRemoved)
  2095. {
  2096. Assert(instr->m_func->IsTopFunc() || instr->m_func->isGetterSetter || instr->m_func->callSiteIdInParentFunc != UINT16_MAX);
  2097. IR::Opnd *src1, *src2;
  2098. Value *src1Val = nullptr, *src2Val = nullptr, *dstVal = nullptr;
  2099. Value *src1IndirIndexVal = nullptr, *dstIndirIndexVal = nullptr;
  2100. IR::Instr *instrPrev = instr->m_prev;
  2101. IR::Instr *instrNext = instr->m_next;
  2102. if (instr->IsLabelInstr() && this->func->HasTry() && this->func->DoOptimizeTry())
  2103. {
  2104. this->currentRegion = instr->AsLabelInstr()->GetRegion();
  2105. Assert(this->currentRegion);
  2106. }
  2107. if(PrepareForIgnoringIntOverflow(instr))
  2108. {
  2109. if(!IsLoopPrePass())
  2110. {
  2111. *isInstrRemoved = true;
  2112. currentBlock->RemoveInstr(instr);
  2113. }
  2114. return instrNext;
  2115. }
  2116. if (!instr->IsRealInstr() || instr->IsByteCodeUsesInstr() || instr->m_opcode == Js::OpCode::Conv_Bool)
  2117. {
  2118. return instrNext;
  2119. }
  2120. if (instr->m_opcode == Js::OpCode::Yield)
  2121. {
  2122. // TODO[generators][ianhall]: Can this and the FillBailOutInfo call below be moved to after Src1 and Src2 so that Yield can be optimized right up to the actual yield?
  2123. CurrentBlockData()->KillStateForGeneratorYield();
  2124. }
  2125. if (!IsLoopPrePass())
  2126. {
  2127. // Change LdLen on objects other than arrays, strings, and 'arguments' to LdFld.
  2128. this->TryReplaceLdLen(instr);
  2129. }
  2130. // Consider: Do we ever get post-op bailout here, and if so is the FillBailOutInfo call in the right place?
  2131. if (instr->HasBailOutInfo() && !this->IsLoopPrePass())
  2132. {
  2133. this->FillBailOutInfo(this->currentBlock, instr);
  2134. }
  2135. this->instrCountSinceLastCleanUp++;
  2136. instr = this->PreOptPeep(instr);
  2137. this->OptArguments(instr);
  2138. //StackArguments Optimization - We bail out if the index is out of range of actuals.
  2139. if ((instr->m_opcode == Js::OpCode::LdElemI_A || instr->m_opcode == Js::OpCode::TypeofElem) &&
  2140. instr->DoStackArgsOpt(this->func) && !this->IsLoopPrePass())
  2141. {
  2142. GenerateBailAtOperation(&instr, IR::BailOnStackArgsOutOfActualsRange);
  2143. }
  2144. #if DBG
  2145. PropertySym *propertySymUseBefore = nullptr;
  2146. Assert(this->byteCodeUses == nullptr);
  2147. this->byteCodeUsesBeforeOpt->ClearAll();
  2148. GlobOpt::TrackByteCodeSymUsed(instr, this->byteCodeUsesBeforeOpt, &propertySymUseBefore);
  2149. Assert(noImplicitCallUsesToInsert->Count() == 0);
  2150. #endif
  2151. this->ignoredIntOverflowForCurrentInstr = false;
  2152. this->ignoredNegativeZeroForCurrentInstr = false;
  2153. src1 = instr->GetSrc1();
  2154. src2 = instr->GetSrc2();
  2155. if (src1)
  2156. {
  2157. src1Val = this->OptSrc(src1, &instr, &src1IndirIndexVal);
  2158. GOPT_TRACE_VALUENUMBER(_u("[src1] "), instr->GetSrc1(), _u("%d"), src1Val ? src1Val->GetValueNumber() : -1);
  2159. instr = this->SetTypeCheckBailOut(instr->GetSrc1(), instr, nullptr);
  2160. if (src2)
  2161. {
  2162. src2Val = this->OptSrc(src2, &instr);
  2163. GOPT_TRACE_VALUENUMBER(_u("[src2] "), instr->GetSrc2(), _u("%d"), src2Val ? src2Val->GetValueNumber() : -1);
  2164. }
  2165. }
  2166. if(instr->GetDst() && instr->GetDst()->IsIndirOpnd())
  2167. {
  2168. this->OptSrc(instr->GetDst(), &instr, &dstIndirIndexVal);
  2169. }
  2170. MarkArgumentsUsedForBranch(instr);
  2171. CSEOptimize(this->currentBlock, &instr, &src1Val, &src2Val, &src1IndirIndexVal);
  2172. OptimizeChecks(instr);
  2173. OptArraySrc(&instr, &src1Val, &src2Val);
  2174. OptNewScObject(&instr, src1Val);
  2175. instr = this->OptPeep(instr, src1Val, src2Val);
  2176. if (instr->m_opcode == Js::OpCode::Nop ||
  2177. (instr->m_opcode == Js::OpCode::CheckThis &&
  2178. instr->GetSrc1()->IsRegOpnd() &&
  2179. instr->GetSrc1()->AsRegOpnd()->m_sym->m_isSafeThis))
  2180. {
  2181. instrNext = instr->m_next;
  2182. InsertNoImplicitCallUses(instr);
  2183. if (this->byteCodeUses)
  2184. {
  2185. this->InsertByteCodeUses(instr);
  2186. }
  2187. *isInstrRemoved = true;
  2188. this->currentBlock->RemoveInstr(instr);
  2189. return instrNext;
  2190. }
  2191. else if (instr->m_opcode == Js::OpCode::GetNewScObject && !this->IsLoopPrePass() && src1Val->GetValueInfo()->IsPrimitive())
  2192. {
  2193. // Constructor returned (src1) a primitive value, so fold this into "dst = Ld_A src2", where src2 is the new object that
  2194. // was passed into the constructor as its 'this' parameter
  2195. instr->FreeSrc1();
  2196. instr->SetSrc1(instr->UnlinkSrc2());
  2197. instr->m_opcode = Js::OpCode::Ld_A;
  2198. src1Val = src2Val;
  2199. src2Val = nullptr;
  2200. }
  2201. else if ((instr->m_opcode == Js::OpCode::TryCatch && this->func->DoOptimizeTry()) || (instr->m_opcode == Js::OpCode::TryFinally && this->func->DoOptimizeTry()))
  2202. {
  2203. ProcessTryHandler(instr);
  2204. }
  2205. else if (instr->m_opcode == Js::OpCode::BrOnException || instr->m_opcode == Js::OpCode::BrOnNoException)
  2206. {
  2207. if (this->ProcessExceptionHandlingEdges(instr))
  2208. {
  2209. *isInstrRemoved = true;
  2210. return instrNext;
  2211. }
  2212. }
  2213. bool isAlreadyTypeSpecialized = false;
  2214. if (!IsLoopPrePass() && instr->HasBailOutInfo())
  2215. {
  2216. if (instr->GetBailOutKind() == IR::BailOutExpectingInteger)
  2217. {
  2218. isAlreadyTypeSpecialized = TypeSpecializeBailoutExpectedInteger(instr, src1Val, &dstVal);
  2219. }
  2220. else if (instr->GetBailOutKind() == IR::BailOutExpectingString)
  2221. {
  2222. if (instr->GetSrc1()->IsRegOpnd())
  2223. {
  2224. if (!src1Val || !src1Val->GetValueInfo()->IsLikelyString())
  2225. {
  2226. // Disable SwitchOpt if the source is definitely not a string - This may be realized only in Globopt
  2227. Assert(IsSwitchOptEnabled());
  2228. throw Js::RejitException(RejitReason::DisableSwitchOptExpectingString);
  2229. }
  2230. }
  2231. }
  2232. }
  2233. bool forceInvariantHoisting = false;
  2234. const bool ignoreIntOverflowInRangeForInstr = instr->ignoreIntOverflowInRange; // Save it since the instr can change
  2235. if (!isAlreadyTypeSpecialized)
  2236. {
  2237. bool redoTypeSpec;
  2238. instr = this->TypeSpecialization(instr, &src1Val, &src2Val, &dstVal, &redoTypeSpec, &forceInvariantHoisting);
  2239. if(redoTypeSpec && instr->m_opcode != Js::OpCode::Nop)
  2240. {
  2241. forceInvariantHoisting = false;
  2242. instr = this->TypeSpecialization(instr, &src1Val, &src2Val, &dstVal, &redoTypeSpec, &forceInvariantHoisting);
  2243. Assert(!redoTypeSpec);
  2244. }
  2245. if (instr->m_opcode == Js::OpCode::Nop)
  2246. {
  2247. InsertNoImplicitCallUses(instr);
  2248. if (this->byteCodeUses)
  2249. {
  2250. this->InsertByteCodeUses(instr);
  2251. }
  2252. instrNext = instr->m_next;
  2253. *isInstrRemoved = true;
  2254. this->currentBlock->RemoveInstr(instr);
  2255. return instrNext;
  2256. }
  2257. }
  2258. if (ignoreIntOverflowInRangeForInstr)
  2259. {
  2260. VerifyIntSpecForIgnoringIntOverflow(instr);
  2261. }
  2262. // Track calls after any pre-op bailouts have been inserted before the call, because they will need to restore out params.
  2263. this->TrackCalls(instr);
  2264. if (instr->GetSrc1())
  2265. {
  2266. this->UpdateObjPtrValueType(instr->GetSrc1(), instr);
  2267. }
  2268. IR::Opnd *dst = instr->GetDst();
  2269. if (dst)
  2270. {
  2271. // Copy prop dst uses and mark live/available type syms before tracking kills.
  2272. CopyPropDstUses(dst, instr, src1Val);
  2273. }
  2274. // Track mark temp object before we process the dst so we can generate pre-op bailout
  2275. instr = this->TrackMarkTempObject(instrPrev->m_next, instr);
  2276. bool removed = OptTagChecks(instr);
  2277. if (removed)
  2278. {
  2279. *isInstrRemoved = true;
  2280. return instrNext;
  2281. }
  2282. dstVal = this->OptDst(&instr, dstVal, src1Val, src2Val, dstIndirIndexVal, src1IndirIndexVal);
  2283. if (dst)
  2284. {
  2285. GOPT_TRACE_VALUENUMBER(_u("[dst] "), instr->GetDst(), _u("%d\n"), dstVal ? dstVal->GetValueNumber() : -1);
  2286. }
  2287. dst = instr->GetDst();
  2288. instrNext = instr->m_next;
  2289. if (dst)
  2290. {
  2291. if (this->func->HasTry() && this->func->DoOptimizeTry())
  2292. {
  2293. this->InsertToVarAtDefInTryRegion(instr, dst);
  2294. }
  2295. instr = this->SetTypeCheckBailOut(dst, instr, nullptr);
  2296. this->UpdateObjPtrValueType(dst, instr);
  2297. }
  2298. BVSparse<JitArenaAllocator> instrByteCodeStackSymUsedAfter(this->alloc);
  2299. PropertySym *propertySymUseAfter = nullptr;
  2300. if (this->byteCodeUses != nullptr)
  2301. {
  2302. GlobOpt::TrackByteCodeSymUsed(instr, &instrByteCodeStackSymUsedAfter, &propertySymUseAfter);
  2303. }
  2304. #if DBG
  2305. else
  2306. {
  2307. GlobOpt::TrackByteCodeSymUsed(instr, &instrByteCodeStackSymUsedAfter, &propertySymUseAfter);
  2308. instrByteCodeStackSymUsedAfter.Equal(this->byteCodeUsesBeforeOpt);
  2309. Assert(propertySymUseAfter == propertySymUseBefore);
  2310. }
  2311. #endif
  2312. bool isHoisted = false;
  2313. if (this->currentBlock->loop && !this->IsLoopPrePass())
  2314. {
  2315. isHoisted = this->TryHoistInvariant(instr, this->currentBlock, dstVal, src1Val, src2Val, true, false, forceInvariantHoisting);
  2316. }
  2317. src1 = instr->GetSrc1();
  2318. if (!this->IsLoopPrePass() && src1)
  2319. {
  2320. // instr const, nonConst => canonicalize by swapping operands
  2321. // This simplifies lowering. (somewhat machine dependent)
  2322. // Note that because of Var overflows, src1 may not have been constant prop'd to an IntConst
  2323. this->PreLowerCanonicalize(instr, &src1Val, &src2Val);
  2324. }
  2325. if (!PHASE_OFF(Js::MemOpPhase, this->func) &&
  2326. !isHoisted &&
  2327. !(instr->IsJitProfilingInstr()) &&
  2328. this->currentBlock->loop && !IsLoopPrePass() &&
  2329. !func->IsJitInDebugMode() &&
  2330. (func->HasProfileInfo() && !func->GetReadOnlyProfileInfo()->IsMemOpDisabled()) &&
  2331. this->currentBlock->loop->doMemOp)
  2332. {
  2333. CollectMemOpInfo(instrPrev, instr, src1Val, src2Val);
  2334. }
  2335. InsertNoImplicitCallUses(instr);
  2336. if (this->byteCodeUses != nullptr)
  2337. {
  2338. // Optimization removed some uses from the instruction.
  2339. // Need to insert fake uses so we can get the correct live register to restore in bailout.
  2340. this->byteCodeUses->Minus(&instrByteCodeStackSymUsedAfter);
  2341. if (this->propertySymUse == propertySymUseAfter)
  2342. {
  2343. this->propertySymUse = nullptr;
  2344. }
  2345. this->InsertByteCodeUses(instr);
  2346. }
  2347. if (!this->IsLoopPrePass() && !isHoisted && this->IsImplicitCallBailOutCurrentlyNeeded(instr, src1Val, src2Val))
  2348. {
  2349. IR::BailOutKind kind = IR::BailOutOnImplicitCalls;
  2350. if(instr->HasBailOutInfo())
  2351. {
  2352. Assert(instr->GetBailOutInfo()->bailOutOffset == instr->GetByteCodeOffset());
  2353. const IR::BailOutKind bailOutKind = instr->GetBailOutKind();
  2354. if((bailOutKind & ~IR::BailOutKindBits) != IR::BailOutOnImplicitCallsPreOp)
  2355. {
  2356. Assert(!(bailOutKind & ~IR::BailOutKindBits));
  2357. instr->SetBailOutKind(bailOutKind + IR::BailOutOnImplicitCallsPreOp);
  2358. }
  2359. }
  2360. else if (instr->forcePreOpBailOutIfNeeded || this->isRecursiveCallOnLandingPad)
  2361. {
  2362. // We can't have a byte code reg slot as dst to generate a
  2363. // pre-op implicit call after we have processed the dst.
  2364. // Consider: This might miss an opportunity to use a copy prop sym to restore
  2365. // some other byte code reg if the dst is that copy prop that we already killed.
  2366. Assert(!instr->GetDst()
  2367. || !instr->GetDst()->IsRegOpnd()
  2368. || instr->GetDst()->AsRegOpnd()->GetIsJITOptimizedReg()
  2369. || !instr->GetDst()->AsRegOpnd()->m_sym->HasByteCodeRegSlot());
  2370. this->GenerateBailAtOperation(&instr, IR::BailOutOnImplicitCallsPreOp);
  2371. }
  2372. else
  2373. {
  2374. // Capture value of the bailout after the operation is done.
  2375. this->GenerateBailAfterOperation(&instr, kind);
  2376. }
  2377. }
  2378. if (CurrentBlockData()->capturedValuesCandidate && !this->IsLoopPrePass())
  2379. {
  2380. this->CommitCapturedValuesCandidate();
  2381. }
  2382. #if DBG
  2383. if (CONFIG_FLAG(ValidateIntRanges) && !IsLoopPrePass())
  2384. {
  2385. if (instr->ShouldEmitIntRangeCheck())
  2386. {
  2387. this->EmitIntRangeChecks(instr);
  2388. }
  2389. }
  2390. #endif
  2391. return instrNext;
  2392. }
  2393. bool
  2394. GlobOpt::OptTagChecks(IR::Instr *instr)
  2395. {
  2396. if (PHASE_OFF(Js::OptTagChecksPhase, this->func) || !this->DoTagChecks())
  2397. {
  2398. return false;
  2399. }
  2400. StackSym *stackSym = nullptr;
  2401. IR::SymOpnd *symOpnd = nullptr;
  2402. IR::RegOpnd *regOpnd = nullptr;
  2403. switch(instr->m_opcode)
  2404. {
  2405. case Js::OpCode::LdFld:
  2406. case Js::OpCode::LdMethodFld:
  2407. case Js::OpCode::CheckFixedFld:
  2408. case Js::OpCode::CheckPropertyGuardAndLoadType:
  2409. symOpnd = instr->GetSrc1()->AsSymOpnd();
  2410. stackSym = symOpnd->m_sym->AsPropertySym()->m_stackSym;
  2411. break;
  2412. case Js::OpCode::BailOnNotObject:
  2413. case Js::OpCode::BailOnNotArray:
  2414. if (instr->GetSrc1()->IsRegOpnd())
  2415. {
  2416. regOpnd = instr->GetSrc1()->AsRegOpnd();
  2417. stackSym = regOpnd->m_sym;
  2418. }
  2419. break;
  2420. case Js::OpCode::StFld:
  2421. symOpnd = instr->GetDst()->AsSymOpnd();
  2422. stackSym = symOpnd->m_sym->AsPropertySym()->m_stackSym;
  2423. break;
  2424. }
  2425. if (stackSym)
  2426. {
  2427. Value *value = CurrentBlockData()->FindValue(stackSym);
  2428. if (value)
  2429. {
  2430. ValueInfo *valInfo = value->GetValueInfo();
  2431. if (valInfo->GetSymStore() && valInfo->GetSymStore()->IsStackSym() && valInfo->GetSymStore()->AsStackSym()->IsFromByteCodeConstantTable())
  2432. {
  2433. return false;
  2434. }
  2435. ValueType valueType = value->GetValueInfo()->Type();
  2436. if (instr->m_opcode == Js::OpCode::BailOnNotObject)
  2437. {
  2438. if (valueType.CanBeTaggedValue())
  2439. {
  2440. // We're not adding new information to the value other than changing the value type. Preserve any existing
  2441. // information and just change the value type.
  2442. ChangeValueType(nullptr, value, valueType.SetCanBeTaggedValue(false), true /*preserveSubClassInfo*/);
  2443. return false;
  2444. }
  2445. if (!this->IsLoopPrePass())
  2446. {
  2447. if (this->byteCodeUses)
  2448. {
  2449. this->InsertByteCodeUses(instr);
  2450. }
  2451. this->currentBlock->RemoveInstr(instr);
  2452. }
  2453. return true;
  2454. }
  2455. if (valueType.CanBeTaggedValue() &&
  2456. !valueType.HasBeenNumber() &&
  2457. !this->IsLoopPrePass())
  2458. {
  2459. ValueType newValueType = valueType.SetCanBeTaggedValue(false);
  2460. // Split out the tag check as a separate instruction.
  2461. IR::Instr *bailOutInstr;
  2462. bailOutInstr = IR::BailOutInstr::New(Js::OpCode::BailOnNotObject, IR::BailOutOnTaggedValue, instr, instr->m_func);
  2463. if (!this->IsLoopPrePass())
  2464. {
  2465. FillBailOutInfo(this->currentBlock, bailOutInstr);
  2466. }
  2467. IR::RegOpnd *srcOpnd = regOpnd;
  2468. if (!srcOpnd)
  2469. {
  2470. srcOpnd = IR::RegOpnd::New(stackSym, stackSym->GetType(), instr->m_func);
  2471. AnalysisAssert(symOpnd);
  2472. if (symOpnd->GetIsJITOptimizedReg())
  2473. {
  2474. srcOpnd->SetIsJITOptimizedReg(true);
  2475. }
  2476. }
  2477. bailOutInstr->SetSrc1(srcOpnd);
  2478. bailOutInstr->GetSrc1()->SetValueType(valueType);
  2479. bailOutInstr->SetByteCodeOffset(instr);
  2480. instr->InsertBefore(bailOutInstr);
  2481. if (this->currentBlock->loop)
  2482. {
  2483. // Try hoisting the BailOnNotObject instr.
  2484. // But since this isn't the current instr being optimized, we need to play tricks with
  2485. // the byteCodeUse fields...
  2486. TrackByteCodeUsesForInstrAddedInOptInstr(bailOutInstr, [&]()
  2487. {
  2488. TryHoistInvariant(bailOutInstr, this->currentBlock, nullptr, value, nullptr, true, false, false, IR::BailOutOnTaggedValue);
  2489. });
  2490. }
  2491. if (symOpnd)
  2492. {
  2493. symOpnd->SetPropertyOwnerValueType(newValueType);
  2494. }
  2495. else
  2496. {
  2497. regOpnd->SetValueType(newValueType);
  2498. }
  2499. ChangeValueType(nullptr, value, newValueType, false);
  2500. }
  2501. }
  2502. }
  2503. return false;
  2504. }
  2505. bool
  2506. GlobOpt::TypeSpecializeBailoutExpectedInteger(IR::Instr* instr, Value* src1Val, Value** dstVal)
  2507. {
  2508. bool isAlreadyTypeSpecialized = false;
  2509. if(instr->GetSrc1()->IsRegOpnd())
  2510. {
  2511. if (!src1Val || !src1Val->GetValueInfo()->IsLikelyInt() || instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber)
  2512. {
  2513. Assert(IsSwitchOptEnabledForIntTypeSpec());
  2514. throw Js::RejitException(RejitReason::DisableSwitchOptExpectingInteger);
  2515. }
  2516. // Attach the BailOutExpectingInteger to FromVar and Remove the bail out info on the Ld_A (Begin Switch) instr.
  2517. this->ToTypeSpecUse(instr, instr->GetSrc1(), this->currentBlock, src1Val, nullptr, TyInt32, IR::BailOutExpectingInteger, false, instr);
  2518. //TypeSpecialize the dst of Ld_A
  2519. TypeSpecializeIntDst(instr, instr->m_opcode, src1Val, src1Val, nullptr, IR::BailOutInvalid, INT32_MIN, INT32_MAX, dstVal);
  2520. isAlreadyTypeSpecialized = true;
  2521. }
  2522. instr->ClearBailOutInfo();
  2523. return isAlreadyTypeSpecialized;
  2524. }
  2525. Value*
  2526. GlobOpt::OptDst(
  2527. IR::Instr ** pInstr,
  2528. Value *dstVal,
  2529. Value *src1Val,
  2530. Value *src2Val,
  2531. Value *dstIndirIndexVal,
  2532. Value *src1IndirIndexVal)
  2533. {
  2534. IR::Instr *&instr = *pInstr;
  2535. IR::Opnd *opnd = instr->GetDst();
  2536. if (opnd)
  2537. {
  2538. if (opnd->IsSymOpnd() && opnd->AsSymOpnd()->IsPropertySymOpnd())
  2539. {
  2540. this->FinishOptPropOp(instr, opnd->AsPropertySymOpnd());
  2541. }
  2542. if (opnd->IsIndirOpnd() && !this->IsLoopPrePass())
  2543. {
  2544. IR::RegOpnd *baseOpnd = opnd->AsIndirOpnd()->GetBaseOpnd();
  2545. const ValueType baseValueType(baseOpnd->GetValueType());
  2546. if ((
  2547. baseValueType.IsLikelyNativeArray() ||
  2548. #ifdef _M_IX86
  2549. (
  2550. !AutoSystemInfo::Data.SSE2Available() &&
  2551. baseValueType.IsLikelyObject() &&
  2552. (
  2553. baseValueType.GetObjectType() == ObjectType::Float32Array ||
  2554. baseValueType.GetObjectType() == ObjectType::Float64Array
  2555. )
  2556. )
  2557. #else
  2558. false
  2559. #endif
  2560. ) &&
  2561. instr->GetSrc1()->IsVar())
  2562. {
  2563. if(instr->m_opcode == Js::OpCode::StElemC)
  2564. {
  2565. // StElemC has different code that handles native array conversion or missing value stores. Add a bailout
  2566. // for those cases.
  2567. Assert(baseValueType.IsLikelyNativeArray());
  2568. Assert(!instr->HasBailOutInfo());
  2569. GenerateBailAtOperation(&instr, IR::BailOutConventionalNativeArrayAccessOnly);
  2570. }
  2571. else if(instr->HasBailOutInfo())
  2572. {
  2573. // The lowerer is not going to generate a fast path for this case. Remove any bailouts that require the fast
  2574. // path. Note that the removed bailouts should not be necessary for correctness. Bailout on native array
  2575. // conversion will be handled automatically as normal.
  2576. IR::BailOutKind bailOutKind = instr->GetBailOutKind();
  2577. if(bailOutKind & IR::BailOutOnArrayAccessHelperCall)
  2578. {
  2579. bailOutKind -= IR::BailOutOnArrayAccessHelperCall;
  2580. }
  2581. if(bailOutKind == IR::BailOutOnImplicitCallsPreOp)
  2582. {
  2583. bailOutKind -= IR::BailOutOnImplicitCallsPreOp;
  2584. }
  2585. if(bailOutKind)
  2586. {
  2587. instr->SetBailOutKind(bailOutKind);
  2588. }
  2589. else
  2590. {
  2591. instr->ClearBailOutInfo();
  2592. }
  2593. }
  2594. }
  2595. }
  2596. }
  2597. this->ProcessKills(instr);
  2598. if (opnd)
  2599. {
  2600. if (dstVal == nullptr)
  2601. {
  2602. dstVal = ValueNumberDst(pInstr, src1Val, src2Val);
  2603. }
  2604. if (this->IsLoopPrePass())
  2605. {
  2606. // Keep track of symbols defined in the loop.
  2607. if (opnd->IsRegOpnd())
  2608. {
  2609. StackSym *symDst = opnd->AsRegOpnd()->m_sym;
  2610. rootLoopPrePass->symsDefInLoop->Set(symDst->m_id);
  2611. }
  2612. }
  2613. else if (dstVal)
  2614. {
  2615. opnd->SetValueType(dstVal->GetValueInfo()->Type());
  2616. if(currentBlock->loop &&
  2617. !IsLoopPrePass() &&
  2618. (instr->m_opcode == Js::OpCode::Ld_A || instr->m_opcode == Js::OpCode::Ld_I4) &&
  2619. instr->GetSrc1()->IsRegOpnd() &&
  2620. !func->IsJitInDebugMode() &&
  2621. func->DoGlobOptsForGeneratorFunc())
  2622. {
  2623. // Look for the following patterns:
  2624. //
  2625. // Pattern 1:
  2626. // s1[liveOnBackEdge] = s3[dead]
  2627. //
  2628. // Pattern 2:
  2629. // s3 = operation(s1[liveOnBackEdge], s2)
  2630. // s1[liveOnBackEdge] = s3
  2631. //
  2632. // In both patterns, s1 and s3 have the same value by the end. Prefer to use s1 as the sym store instead of s3
  2633. // since s1 is live on back-edge, as otherwise, their lifetimes overlap, requiring two registers to hold the
  2634. // value instead of one.
  2635. do
  2636. {
  2637. IR::RegOpnd *const src = instr->GetSrc1()->AsRegOpnd();
  2638. StackSym *srcVarSym = src->m_sym;
  2639. if(srcVarSym->IsTypeSpec())
  2640. {
  2641. srcVarSym = srcVarSym->GetVarEquivSym(nullptr);
  2642. Assert(srcVarSym);
  2643. }
  2644. if(dstVal->GetValueInfo()->GetSymStore() != srcVarSym)
  2645. {
  2646. break;
  2647. }
  2648. IR::RegOpnd *const dst = opnd->AsRegOpnd();
  2649. StackSym *dstVarSym = dst->m_sym;
  2650. if(dstVarSym->IsTypeSpec())
  2651. {
  2652. dstVarSym = dstVarSym->GetVarEquivSym(nullptr);
  2653. Assert(dstVarSym);
  2654. }
  2655. if(!currentBlock->loop->regAlloc.liveOnBackEdgeSyms->Test(dstVarSym->m_id))
  2656. {
  2657. break;
  2658. }
  2659. Value *const srcValue = CurrentBlockData()->FindValue(srcVarSym);
  2660. if(srcValue->GetValueNumber() != dstVal->GetValueNumber())
  2661. {
  2662. break;
  2663. }
  2664. if(!src->GetIsDead())
  2665. {
  2666. IR::Instr *const prevInstr = instr->GetPrevRealInstrOrLabel();
  2667. IR::Opnd *const prevDst = prevInstr->GetDst();
  2668. if(!prevDst ||
  2669. !src->IsEqualInternal(prevDst) ||
  2670. !(
  2671. (prevInstr->GetSrc1() && dst->IsEqual(prevInstr->GetSrc1())) ||
  2672. (prevInstr->GetSrc2() && dst->IsEqual(prevInstr->GetSrc2()))
  2673. ))
  2674. {
  2675. break;
  2676. }
  2677. }
  2678. this->SetSymStoreDirect(dstVal->GetValueInfo(), dstVarSym);
  2679. } while(false);
  2680. }
  2681. }
  2682. this->ValueNumberObjectType(opnd, instr);
  2683. }
  2684. this->CSEAddInstr(this->currentBlock, *pInstr, dstVal, src1Val, src2Val, dstIndirIndexVal, src1IndirIndexVal);
  2685. return dstVal;
  2686. }
  2687. void
  2688. GlobOpt::CopyPropDstUses(IR::Opnd *opnd, IR::Instr *instr, Value *src1Val)
  2689. {
  2690. if (opnd->IsSymOpnd())
  2691. {
  2692. IR::SymOpnd *symOpnd = opnd->AsSymOpnd();
  2693. if (symOpnd->m_sym->IsPropertySym())
  2694. {
  2695. PropertySym * originalPropertySym = symOpnd->m_sym->AsPropertySym();
  2696. Value *const objectValue = CurrentBlockData()->FindValue(originalPropertySym->m_stackSym);
  2697. symOpnd->SetPropertyOwnerValueType(objectValue ? objectValue->GetValueInfo()->Type() : ValueType::Uninitialized);
  2698. this->CopyPropPropertySymObj(symOpnd, instr);
  2699. }
  2700. }
  2701. }
  2702. void
  2703. GlobOpt::SetLoopFieldInitialValue(Loop *loop, IR::Instr *instr, PropertySym *propertySym, PropertySym *originalPropertySym)
  2704. {
  2705. Value *initialValue = nullptr;
  2706. StackSym *symStore;
  2707. if (loop->allFieldsKilled || loop->fieldKilled->Test(originalPropertySym->m_id) || loop->fieldKilled->Test(propertySym->m_id))
  2708. {
  2709. return;
  2710. }
  2711. // Value already exists
  2712. if (CurrentBlockData()->FindValue(propertySym))
  2713. {
  2714. return;
  2715. }
  2716. // If this initial value was already added, we would find in the current value table.
  2717. Assert(!loop->initialValueFieldMap.TryGetValue(propertySym, &initialValue));
  2718. // If propertySym is live in landingPad, we don't need an initial value.
  2719. if (loop->landingPad->globOptData.liveFields->Test(propertySym->m_id))
  2720. {
  2721. return;
  2722. }
  2723. StackSym * objectSym = propertySym->m_stackSym;
  2724. Value *landingPadObjPtrVal, *currentObjPtrVal;
  2725. landingPadObjPtrVal = loop->landingPad->globOptData.FindValue(objectSym);
  2726. currentObjPtrVal = CurrentBlockData()->FindValue(objectSym);
  2727. auto CanSetInitialValue = [&]() -> bool {
  2728. if (!currentObjPtrVal)
  2729. {
  2730. return false;
  2731. }
  2732. if (landingPadObjPtrVal)
  2733. {
  2734. return currentObjPtrVal->GetValueNumber() == landingPadObjPtrVal->GetValueNumber();
  2735. }
  2736. else
  2737. {
  2738. if (!objectSym->IsSingleDef())
  2739. {
  2740. return false;
  2741. }
  2742. IR::Instr * defInstr = objectSym->GetInstrDef();
  2743. IR::Opnd * src1 = defInstr->GetSrc1();
  2744. while (!(src1 && src1->IsSymOpnd() && src1->AsSymOpnd()->m_sym->IsPropertySym()))
  2745. {
  2746. if (src1 && src1->IsRegOpnd() && src1->AsRegOpnd()->GetStackSym()->IsSingleDef())
  2747. {
  2748. defInstr = src1->AsRegOpnd()->GetStackSym()->GetInstrDef();
  2749. src1 = defInstr->GetSrc1();
  2750. }
  2751. else
  2752. {
  2753. return false;
  2754. }
  2755. }
  2756. return true;
  2757. // Todo: allow other kinds of operands as src1 of instr def of the object sym of the current propertySym
  2758. // SymOpnd, but not PropertySymOpnd - LdSlotArr, some LdSlots (?)
  2759. // nullptr - NewScObject
  2760. }
  2761. };
  2762. if (!CanSetInitialValue())
  2763. {
  2764. // objPtr has a different value in the landing pad.
  2765. return;
  2766. }
  2767. // The opnd's value type has not yet been initialized. Since the property sym doesn't have a value, it effectively has an
  2768. // Uninitialized value type. Use the profiled value type from the instruction.
  2769. const ValueType profiledValueType =
  2770. instr->IsProfiledInstr() ? instr->AsProfiledInstr()->u.FldInfo().valueType : ValueType::Uninitialized;
  2771. Assert(!profiledValueType.IsDefinite()); // Hence the values created here don't need to be tracked for kills
  2772. initialValue = this->NewGenericValue(profiledValueType, propertySym);
  2773. symStore = StackSym::New(this->func);
  2774. initialValue->GetValueInfo()->SetSymStore(symStore);
  2775. loop->initialValueFieldMap.Add(propertySym, initialValue->Copy(this->alloc, initialValue->GetValueNumber()));
  2776. // Copy the initial value into the landing pad, but without a symStore
  2777. Value *landingPadInitialValue = Value::New(this->alloc, initialValue->GetValueNumber(),
  2778. ValueInfo::New(this->alloc, initialValue->GetValueInfo()->Type()));
  2779. loop->landingPad->globOptData.SetValue(landingPadInitialValue, propertySym);
  2780. loop->landingPad->globOptData.liveFields->Set(propertySym->m_id);
  2781. #if DBG_DUMP
  2782. if (PHASE_TRACE(Js::FieldPREPhase, this->func))
  2783. {
  2784. Output::Print(_u("** TRACE: Field PRE initial value for loop head #%d. Val:%d symStore:"),
  2785. loop->GetHeadBlock()->GetBlockNum(), initialValue->GetValueNumber());
  2786. symStore->Dump();
  2787. Output::Print(_u("\n Instr: "));
  2788. instr->Dump();
  2789. Output::Flush();
  2790. }
  2791. #endif
  2792. // Add initial value to all the previous blocks in the loop.
  2793. FOREACH_BLOCK_BACKWARD_IN_RANGE(block, this->currentBlock->GetPrev(), loop->GetHeadBlock())
  2794. {
  2795. if (block->GetDataUseCount() == 0)
  2796. {
  2797. // All successor blocks have been processed, no point in adding the value.
  2798. continue;
  2799. }
  2800. Value *newValue = initialValue->Copy(this->alloc, initialValue->GetValueNumber());
  2801. block->globOptData.SetValue(newValue, propertySym);
  2802. block->globOptData.liveFields->Set(propertySym->m_id);
  2803. block->globOptData.SetValue(newValue, symStore);
  2804. block->globOptData.liveVarSyms->Set(symStore->m_id);
  2805. } NEXT_BLOCK_BACKWARD_IN_RANGE;
  2806. CurrentBlockData()->SetValue(initialValue, symStore);
  2807. CurrentBlockData()->liveVarSyms->Set(symStore->m_id);
  2808. CurrentBlockData()->liveFields->Set(propertySym->m_id);
  2809. }
  2810. // Examine src, apply copy prop and value number it
  2811. Value*
  2812. GlobOpt::OptSrc(IR::Opnd *opnd, IR::Instr * *pInstr, Value **indirIndexValRef, IR::IndirOpnd *parentIndirOpnd)
  2813. {
  2814. IR::Instr * &instr = *pInstr;
  2815. Assert(!indirIndexValRef || !*indirIndexValRef);
  2816. Assert(
  2817. parentIndirOpnd
  2818. ? opnd == parentIndirOpnd->GetBaseOpnd() || opnd == parentIndirOpnd->GetIndexOpnd()
  2819. : opnd == instr->GetSrc1() || opnd == instr->GetSrc2() || opnd == instr->GetDst() && opnd->IsIndirOpnd());
  2820. Sym *sym;
  2821. Value *val;
  2822. PropertySym *originalPropertySym = nullptr;
  2823. switch(opnd->GetKind())
  2824. {
  2825. case IR::OpndKindIntConst:
  2826. val = this->GetIntConstantValue(opnd->AsIntConstOpnd()->AsInt32(), instr);
  2827. opnd->SetValueType(val->GetValueInfo()->Type());
  2828. return val;
  2829. case IR::OpndKindInt64Const:
  2830. val = this->GetIntConstantValue(opnd->AsInt64ConstOpnd()->GetValue(), instr);
  2831. opnd->SetValueType(val->GetValueInfo()->Type());
  2832. return val;
  2833. case IR::OpndKindFloatConst:
  2834. {
  2835. const FloatConstType floatValue = opnd->AsFloatConstOpnd()->m_value;
  2836. int32 int32Value;
  2837. if(Js::JavascriptNumber::TryGetInt32Value(floatValue, &int32Value))
  2838. {
  2839. val = GetIntConstantValue(int32Value, instr);
  2840. }
  2841. else
  2842. {
  2843. val = NewFloatConstantValue(floatValue);
  2844. }
  2845. opnd->SetValueType(val->GetValueInfo()->Type());
  2846. return val;
  2847. }
  2848. case IR::OpndKindAddr:
  2849. {
  2850. IR::AddrOpnd *addrOpnd = opnd->AsAddrOpnd();
  2851. if (addrOpnd->m_isFunction)
  2852. {
  2853. AssertMsg(!PHASE_OFF(Js::FixedMethodsPhase, instr->m_func), "Fixed function address operand with fixed method calls phase disabled?");
  2854. val = NewFixedFunctionValue((Js::JavascriptFunction *)addrOpnd->m_address, addrOpnd);
  2855. opnd->SetValueType(val->GetValueInfo()->Type());
  2856. return val;
  2857. }
  2858. else if (addrOpnd->IsVar() && Js::TaggedInt::Is(addrOpnd->m_address))
  2859. {
  2860. val = this->GetIntConstantValue(Js::TaggedInt::ToInt32(addrOpnd->m_address), instr);
  2861. opnd->SetValueType(val->GetValueInfo()->Type());
  2862. return val;
  2863. }
  2864. val = this->GetVarConstantValue(addrOpnd);
  2865. return val;
  2866. }
  2867. case IR::OpndKindSym:
  2868. {
  2869. // Clear the opnd's value type up-front, so that this code cannot accidentally use the value type set from a previous
  2870. // OptSrc on the same instruction (for instance, from an earlier loop prepass). The value type will be set from the
  2871. // value if available, before returning from this function.
  2872. opnd->SetValueType(ValueType::Uninitialized);
  2873. sym = opnd->AsSymOpnd()->m_sym;
  2874. // Don't create a new value for ArgSlots and don't copy prop them away.
  2875. if (sym->IsStackSym() && sym->AsStackSym()->IsArgSlotSym())
  2876. {
  2877. return nullptr;
  2878. }
  2879. // Unless we have profile info, don't create a new value for ArgSlots and don't copy prop them away.
  2880. if (sym->IsStackSym() && sym->AsStackSym()->IsParamSlotSym())
  2881. {
  2882. if (!instr->m_func->IsLoopBody() && instr->m_func->HasProfileInfo())
  2883. {
  2884. // Skip "this" pointer.
  2885. int paramSlotNum = sym->AsStackSym()->GetParamSlotNum() - 2;
  2886. if (paramSlotNum >= 0)
  2887. {
  2888. const auto parameterType = instr->m_func->GetReadOnlyProfileInfo()->GetParameterInfo(static_cast<Js::ArgSlot>(paramSlotNum));
  2889. val = NewGenericValue(parameterType);
  2890. opnd->SetValueType(val->GetValueInfo()->Type());
  2891. return val;
  2892. }
  2893. }
  2894. return nullptr;
  2895. }
  2896. if (!sym->IsPropertySym())
  2897. {
  2898. break;
  2899. }
  2900. originalPropertySym = sym->AsPropertySym();
  2901. // Dont give a vale to 'arguments' property sym to prevent field copy prop of 'arguments'
  2902. if (originalPropertySym->AsPropertySym()->m_propertyId == Js::PropertyIds::arguments &&
  2903. originalPropertySym->AsPropertySym()->m_fieldKind == PropertyKindData)
  2904. {
  2905. return nullptr;
  2906. }
  2907. Value *const objectValue = CurrentBlockData()->FindValue(originalPropertySym->m_stackSym);
  2908. opnd->AsSymOpnd()->SetPropertyOwnerValueType(
  2909. objectValue ? objectValue->GetValueInfo()->Type() : ValueType::Uninitialized);
  2910. sym = this->CopyPropPropertySymObj(opnd->AsSymOpnd(), instr);
  2911. if (!DoFieldCopyProp())
  2912. {
  2913. if (opnd->AsSymOpnd()->IsPropertySymOpnd())
  2914. {
  2915. this->FinishOptPropOp(instr, opnd->AsPropertySymOpnd());
  2916. }
  2917. return nullptr;
  2918. }
  2919. switch (instr->m_opcode)
  2920. {
  2921. // These need the symbolic reference to the field, don't copy prop the value of the field
  2922. case Js::OpCode::DeleteFld:
  2923. case Js::OpCode::DeleteRootFld:
  2924. case Js::OpCode::DeleteFldStrict:
  2925. case Js::OpCode::DeleteRootFldStrict:
  2926. case Js::OpCode::ScopedDeleteFld:
  2927. case Js::OpCode::ScopedDeleteFldStrict:
  2928. case Js::OpCode::LdMethodFromFlags:
  2929. case Js::OpCode::BrOnNoProperty:
  2930. case Js::OpCode::BrOnHasProperty:
  2931. case Js::OpCode::LdMethodFldPolyInlineMiss:
  2932. case Js::OpCode::StSlotChkUndecl:
  2933. case Js::OpCode::ScopedLdInst:
  2934. return nullptr;
  2935. };
  2936. if (instr->CallsGetter())
  2937. {
  2938. return nullptr;
  2939. }
  2940. if (this->IsLoopPrePass() && this->DoFieldPRE(this->rootLoopPrePass))
  2941. {
  2942. if (!this->prePassLoop->allFieldsKilled && !this->prePassLoop->fieldKilled->Test(sym->m_id))
  2943. {
  2944. this->SetLoopFieldInitialValue(this->rootLoopPrePass, instr, sym->AsPropertySym(), originalPropertySym);
  2945. }
  2946. if (this->IsPREInstrCandidateLoad(instr->m_opcode))
  2947. {
  2948. // Foreach property sym, remember the first instruction that loads it.
  2949. // Can this be done in one call?
  2950. if (!this->prePassInstrMap->ContainsKey(sym->m_id))
  2951. {
  2952. this->prePassInstrMap->AddNew(sym->m_id, instr->CopyWithoutDst());
  2953. }
  2954. }
  2955. }
  2956. break;
  2957. }
  2958. case IR::OpndKindReg:
  2959. // Clear the opnd's value type up-front, so that this code cannot accidentally use the value type set from a previous
  2960. // OptSrc on the same instruction (for instance, from an earlier loop prepass). The value type will be set from the
  2961. // value if available, before returning from this function.
  2962. opnd->SetValueType(ValueType::Uninitialized);
  2963. sym = opnd->AsRegOpnd()->m_sym;
  2964. CurrentBlockData()->MarkTempLastUse(instr, opnd->AsRegOpnd());
  2965. if (sym->AsStackSym()->IsTypeSpec())
  2966. {
  2967. sym = sym->AsStackSym()->GetVarEquivSym(this->func);
  2968. }
  2969. break;
  2970. case IR::OpndKindIndir:
  2971. this->OptimizeIndirUses(opnd->AsIndirOpnd(), &instr, indirIndexValRef);
  2972. return nullptr;
  2973. default:
  2974. return nullptr;
  2975. }
  2976. val = CurrentBlockData()->FindValue(sym);
  2977. if (val)
  2978. {
  2979. Assert(CurrentBlockData()->IsLive(sym) || (sym->IsPropertySym()));
  2980. if (instr)
  2981. {
  2982. opnd = this->CopyProp(opnd, instr, val, parentIndirOpnd);
  2983. }
  2984. // Check if we freed the operand.
  2985. if (opnd == nullptr)
  2986. {
  2987. return nullptr;
  2988. }
  2989. // In a loop prepass, determine stack syms that are used before they are defined in the root loop for which the prepass
  2990. // is being done. This information is used to do type specialization conversions in the landing pad where appropriate.
  2991. if(IsLoopPrePass() &&
  2992. sym->IsStackSym() &&
  2993. !rootLoopPrePass->symsUsedBeforeDefined->Test(sym->m_id) &&
  2994. rootLoopPrePass->landingPad->globOptData.IsLive(sym) && !isAsmJSFunc) // no typespec in asmjs and hence skipping this
  2995. {
  2996. Value *const landingPadValue = rootLoopPrePass->landingPad->globOptData.FindValue(sym);
  2997. if(landingPadValue && val->GetValueNumber() == landingPadValue->GetValueNumber())
  2998. {
  2999. rootLoopPrePass->symsUsedBeforeDefined->Set(sym->m_id);
  3000. ValueInfo *landingPadValueInfo = landingPadValue->GetValueInfo();
  3001. if(landingPadValueInfo->IsLikelyNumber())
  3002. {
  3003. rootLoopPrePass->likelyNumberSymsUsedBeforeDefined->Set(sym->m_id);
  3004. if(DoAggressiveIntTypeSpec() ? landingPadValueInfo->IsLikelyInt() : landingPadValueInfo->IsInt())
  3005. {
  3006. // Can only force int conversions in the landing pad based on likely-int values if aggressive int type
  3007. // specialization is enabled.
  3008. rootLoopPrePass->likelyIntSymsUsedBeforeDefined->Set(sym->m_id);
  3009. }
  3010. }
  3011. }
  3012. }
  3013. }
  3014. else if ((instr->TransfersSrcValue() || OpCodeAttr::CanCSE(instr->m_opcode)) && (opnd == instr->GetSrc1() || opnd == instr->GetSrc2()))
  3015. {
  3016. if (sym->IsPropertySym())
  3017. {
  3018. val = this->CreateFieldSrcValue(sym->AsPropertySym(), originalPropertySym, &opnd, instr);
  3019. }
  3020. else
  3021. {
  3022. val = this->NewGenericValue(ValueType::Uninitialized, opnd);
  3023. }
  3024. }
  3025. if (opnd->IsSymOpnd() && opnd->AsSymOpnd()->IsPropertySymOpnd())
  3026. {
  3027. TryOptimizeInstrWithFixedDataProperty(&instr);
  3028. this->FinishOptPropOp(instr, opnd->AsPropertySymOpnd());
  3029. }
  3030. if (val)
  3031. {
  3032. ValueType valueType(val->GetValueInfo()->Type());
  3033. // This block uses per-instruction profile information on array types to optimize using the best available profile
  3034. // information and to prevent infinite bailouts by ensuring array type information is updated on bailouts.
  3035. if (valueType.IsLikelyArray() && !valueType.IsDefinite() && !valueType.IsObject() && instr->IsProfiledInstr())
  3036. {
  3037. // See if we have profile data for the array type
  3038. IR::ProfiledInstr *const profiledInstr = instr->AsProfiledInstr();
  3039. ValueType profiledArrayType;
  3040. bool useAggressiveSpecialization = true;
  3041. switch(instr->m_opcode)
  3042. {
  3043. case Js::OpCode::LdElemI_A:
  3044. if(instr->GetSrc1()->IsIndirOpnd() && opnd == instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd())
  3045. {
  3046. profiledArrayType = profiledInstr->u.ldElemInfo->GetArrayType();
  3047. useAggressiveSpecialization = !profiledInstr->u.ldElemInfo->IsAggressiveSpecializationDisabled();
  3048. }
  3049. break;
  3050. case Js::OpCode::StElemI_A:
  3051. case Js::OpCode::StElemI_A_Strict:
  3052. case Js::OpCode::StElemC:
  3053. if(instr->GetDst()->IsIndirOpnd() && opnd == instr->GetDst()->AsIndirOpnd()->GetBaseOpnd())
  3054. {
  3055. profiledArrayType = profiledInstr->u.stElemInfo->GetArrayType();
  3056. useAggressiveSpecialization = !profiledInstr->u.stElemInfo->IsAggressiveSpecializationDisabled();
  3057. }
  3058. break;
  3059. case Js::OpCode::LdLen_A:
  3060. if(instr->GetSrc1()->IsRegOpnd() && opnd == instr->GetSrc1())
  3061. {
  3062. profiledArrayType = profiledInstr->u.LdLenInfo().GetArrayType();
  3063. useAggressiveSpecialization = !profiledInstr->u.LdLenInfo().IsAggressiveSpecializationDisabled();
  3064. }
  3065. break;
  3066. case Js::OpCode::IsIn:
  3067. if (instr->GetSrc2()->IsRegOpnd() && opnd == instr->GetSrc2())
  3068. {
  3069. profiledArrayType = profiledInstr->u.ldElemInfo->GetArrayType();
  3070. useAggressiveSpecialization = !profiledInstr->u.ldElemInfo->IsAggressiveSpecializationDisabled();
  3071. }
  3072. break;
  3073. }
  3074. if (profiledArrayType.IsLikelyObject())
  3075. {
  3076. // Ideally we want to use the most specialized type seen by this path, but when that causes bailouts use the least specialized type instead.
  3077. if (useAggressiveSpecialization &&
  3078. profiledArrayType.GetObjectType() == valueType.GetObjectType() &&
  3079. !valueType.IsLikelyNativeIntArray() &&
  3080. (
  3081. profiledArrayType.HasIntElements() || (valueType.HasVarElements() && profiledArrayType.HasFloatElements())
  3082. ))
  3083. {
  3084. // use the more specialized type profiled by the instruction.
  3085. valueType = profiledArrayType.SetHasNoMissingValues(valueType.HasNoMissingValues());
  3086. ChangeValueType(this->currentBlock, CurrentBlockData()->FindValue(opnd->AsRegOpnd()->m_sym), valueType, false);
  3087. }
  3088. else if (!useAggressiveSpecialization &&
  3089. (profiledArrayType.GetObjectType() != valueType.GetObjectType() ||
  3090. (
  3091. valueType.IsLikelyNativeArray() &&
  3092. (
  3093. profiledArrayType.HasVarElements() || (valueType.HasIntElements() && profiledArrayType.HasFloatElements())
  3094. )
  3095. )
  3096. ))
  3097. {
  3098. // Merge array type we pulled from profile with type propagated by dataflow.
  3099. if (profiledArrayType.IsLikelyArray())
  3100. {
  3101. valueType = valueType.Merge(profiledArrayType).SetHasNoMissingValues(valueType.HasNoMissingValues());
  3102. }
  3103. else
  3104. {
  3105. valueType = valueType.Merge(profiledArrayType);
  3106. }
  3107. ChangeValueType(this->currentBlock, CurrentBlockData()->FindValue(opnd->AsRegOpnd()->m_sym), valueType, false, true);
  3108. }
  3109. }
  3110. }
  3111. opnd->SetValueType(valueType);
  3112. if(!IsLoopPrePass() && opnd->IsSymOpnd() && valueType.IsDefinite())
  3113. {
  3114. if (opnd->AsSymOpnd()->m_sym->IsPropertySym())
  3115. {
  3116. // A property sym can only be guaranteed to have a definite value type when implicit calls are disabled from the
  3117. // point where the sym was defined with the definite value type. Insert an instruction to indicate to the
  3118. // dead-store pass that implicit calls need to be kept disabled until after this instruction.
  3119. Assert(DoFieldCopyProp());
  3120. CaptureNoImplicitCallUses(opnd, false, instr);
  3121. }
  3122. }
  3123. }
  3124. else
  3125. {
  3126. opnd->SetValueType(ValueType::Uninitialized);
  3127. }
  3128. return val;
  3129. }
  3130. /*
  3131. * GlobOpt::TryOptimizeInstrWithFixedDataProperty
  3132. * Converts Ld[Root]Fld instr to
  3133. * * CheckFixedFld
  3134. * * Dst = Ld_A <int Constant value>
  3135. * This API assumes that the source operand is a Sym/PropertySym kind.
  3136. */
  3137. void
  3138. GlobOpt::TryOptimizeInstrWithFixedDataProperty(IR::Instr ** const pInstr)
  3139. {
  3140. Assert(pInstr);
  3141. IR::Instr * &instr = *pInstr;
  3142. IR::Opnd * src1 = instr->GetSrc1();
  3143. Assert(src1 && src1->IsSymOpnd() && src1->AsSymOpnd()->IsPropertySymOpnd());
  3144. if(PHASE_OFF(Js::UseFixedDataPropsPhase, instr->m_func))
  3145. {
  3146. return;
  3147. }
  3148. if (!this->IsLoopPrePass() && !this->isRecursiveCallOnLandingPad &&
  3149. OpCodeAttr::CanLoadFixedFields(instr->m_opcode))
  3150. {
  3151. instr->TryOptimizeInstrWithFixedDataProperty(&instr, this);
  3152. }
  3153. }
  3154. // Constant prop if possible, otherwise if this value already resides in another
  3155. // symbol, reuse this previous symbol. This should help register allocation.
  3156. IR::Opnd *
  3157. GlobOpt::CopyProp(IR::Opnd *opnd, IR::Instr *instr, Value *val, IR::IndirOpnd *parentIndirOpnd)
  3158. {
  3159. Assert(
  3160. parentIndirOpnd
  3161. ? opnd == parentIndirOpnd->GetBaseOpnd() || opnd == parentIndirOpnd->GetIndexOpnd()
  3162. : opnd == instr->GetSrc1() || opnd == instr->GetSrc2() || opnd == instr->GetDst() && opnd->IsIndirOpnd());
  3163. if (this->IsLoopPrePass())
  3164. {
  3165. // Transformations are not legal in prepass...
  3166. return opnd;
  3167. }
  3168. if (!this->func->DoGlobOptsForGeneratorFunc())
  3169. {
  3170. // Don't copy prop in generator functions because non-bytecode temps that span a yield
  3171. // cannot be saved and restored by the current bail-out mechanics utilized by generator
  3172. // yield/resume.
  3173. // TODO[generators][ianhall]: Enable copy-prop at least for in between yields.
  3174. return opnd;
  3175. }
  3176. if (instr->m_opcode == Js::OpCode::CheckFixedFld || instr->m_opcode == Js::OpCode::CheckPropertyGuardAndLoadType)
  3177. {
  3178. // Don't copy prop into CheckFixedFld or CheckPropertyGuardAndLoadType
  3179. return opnd;
  3180. }
  3181. // Don't copy-prop link operands of ExtendedArgs
  3182. if (instr->m_opcode == Js::OpCode::ExtendArg_A && opnd == instr->GetSrc2())
  3183. {
  3184. return opnd;
  3185. }
  3186. // Don't copy-prop operand of SIMD instr with ExtendedArg operands. Each instr should have its exclusive EA sequence.
  3187. if (
  3188. Js::IsSimd128Opcode(instr->m_opcode) &&
  3189. instr->GetSrc1() != nullptr &&
  3190. instr->GetSrc1()->IsRegOpnd() &&
  3191. instr->GetSrc2() == nullptr
  3192. )
  3193. {
  3194. StackSym *sym = instr->GetSrc1()->GetStackSym();
  3195. if (sym && sym->IsSingleDef() && sym->GetInstrDef()->m_opcode == Js::OpCode::ExtendArg_A)
  3196. {
  3197. return opnd;
  3198. }
  3199. }
  3200. ValueInfo *valueInfo = val->GetValueInfo();
  3201. if (this->func->HasFinally())
  3202. {
  3203. // s0 = undefined was added on functions with early exit in try-finally functions, that can get copy-proped and case incorrect results
  3204. if (instr->m_opcode == Js::OpCode::ArgOut_A_Inline && valueInfo->GetSymStore() &&
  3205. valueInfo->GetSymStore()->m_id == 0)
  3206. {
  3207. // We don't want to copy-prop s0 (return symbol) into inlinee code
  3208. return opnd;
  3209. }
  3210. }
  3211. // Constant prop?
  3212. int32 intConstantValue;
  3213. int64 int64ConstantValue;
  3214. if (valueInfo->TryGetIntConstantValue(&intConstantValue))
  3215. {
  3216. if (PHASE_OFF(Js::ConstPropPhase, this->func))
  3217. {
  3218. return opnd;
  3219. }
  3220. if ((
  3221. instr->m_opcode == Js::OpCode::StElemI_A ||
  3222. instr->m_opcode == Js::OpCode::StElemI_A_Strict ||
  3223. instr->m_opcode == Js::OpCode::StElemC
  3224. ) && instr->GetSrc1() == opnd)
  3225. {
  3226. // Disabling prop to src of native array store, because we were losing the chance to type specialize.
  3227. // Is it possible to type specialize this src if we allow constants, etc., to be prop'd here?
  3228. if (instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsLikelyNativeArray())
  3229. {
  3230. return opnd;
  3231. }
  3232. }
  3233. if(opnd != instr->GetSrc1() && opnd != instr->GetSrc2())
  3234. {
  3235. if(PHASE_OFF(Js::IndirCopyPropPhase, instr->m_func))
  3236. {
  3237. return opnd;
  3238. }
  3239. // Const-prop an indir opnd's constant index into its offset
  3240. IR::Opnd *srcs[] = { instr->GetSrc1(), instr->GetSrc2(), instr->GetDst() };
  3241. for(int i = 0; i < sizeof(srcs) / sizeof(srcs[0]); ++i)
  3242. {
  3243. const auto src = srcs[i];
  3244. if(!src || !src->IsIndirOpnd())
  3245. {
  3246. continue;
  3247. }
  3248. const auto indir = src->AsIndirOpnd();
  3249. if ((int64)indir->GetOffset() + intConstantValue > INT32_MAX)
  3250. {
  3251. continue;
  3252. }
  3253. if(opnd == indir->GetIndexOpnd())
  3254. {
  3255. Assert(indir->GetScale() == 0);
  3256. GOPT_TRACE_OPND(opnd, _u("Constant prop indir index into offset (value: %d)\n"), intConstantValue);
  3257. this->CaptureByteCodeSymUses(instr);
  3258. indir->SetOffset(indir->GetOffset() + intConstantValue);
  3259. indir->SetIndexOpnd(nullptr);
  3260. }
  3261. }
  3262. return opnd;
  3263. }
  3264. if (Js::TaggedInt::IsOverflow(intConstantValue))
  3265. {
  3266. return opnd;
  3267. }
  3268. IR::Opnd *constOpnd;
  3269. if (opnd->IsVar())
  3270. {
  3271. IR::AddrOpnd *addrOpnd = IR::AddrOpnd::New(Js::TaggedInt::ToVarUnchecked((int)intConstantValue), IR::AddrOpndKindConstantVar, instr->m_func);
  3272. GOPT_TRACE_OPND(opnd, _u("Constant prop %d (value:%d)\n"), addrOpnd->m_address, intConstantValue);
  3273. constOpnd = addrOpnd;
  3274. }
  3275. else
  3276. {
  3277. // Note: Jit loop body generates some i32 operands...
  3278. Assert(opnd->IsInt32() || opnd->IsInt64() || opnd->IsUInt32());
  3279. IRType opndType;
  3280. IntConstType constVal;
  3281. if (opnd->IsUInt32())
  3282. {
  3283. // avoid sign extension
  3284. constVal = (uint32)intConstantValue;
  3285. opndType = TyUint32;
  3286. }
  3287. else
  3288. {
  3289. constVal = intConstantValue;
  3290. opndType = TyInt32;
  3291. }
  3292. IR::IntConstOpnd *intOpnd = IR::IntConstOpnd::New(constVal, opndType, instr->m_func);
  3293. GOPT_TRACE_OPND(opnd, _u("Constant prop %d (value:%d)\n"), intOpnd->GetImmediateValue(instr->m_func), intConstantValue);
  3294. constOpnd = intOpnd;
  3295. }
  3296. #if ENABLE_DEBUG_CONFIG_OPTIONS
  3297. //Need to update DumpFieldCopyPropTestTrace for every new opcode that is added for fieldcopyprop
  3298. if(Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FieldCopyPropPhase))
  3299. {
  3300. instr->DumpFieldCopyPropTestTrace(this->isRecursiveCallOnLandingPad);
  3301. }
  3302. #endif
  3303. this->CaptureByteCodeSymUses(instr);
  3304. opnd = instr->ReplaceSrc(opnd, constOpnd);
  3305. switch (instr->m_opcode)
  3306. {
  3307. case Js::OpCode::LdSlot:
  3308. case Js::OpCode::LdSlotArr:
  3309. case Js::OpCode::LdFld:
  3310. case Js::OpCode::LdFldForTypeOf:
  3311. case Js::OpCode::LdRootFldForTypeOf:
  3312. case Js::OpCode::LdFldForCallApplyTarget:
  3313. case Js::OpCode::LdRootFld:
  3314. case Js::OpCode::LdMethodFld:
  3315. case Js::OpCode::LdRootMethodFld:
  3316. case Js::OpCode::LdMethodFromFlags:
  3317. case Js::OpCode::ScopedLdMethodFld:
  3318. case Js::OpCode::ScopedLdFld:
  3319. case Js::OpCode::ScopedLdFldForTypeOf:
  3320. instr->m_opcode = Js::OpCode::Ld_A;
  3321. case Js::OpCode::Ld_A:
  3322. {
  3323. IR::Opnd * dst = instr->GetDst();
  3324. if (dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->IsSingleDef())
  3325. {
  3326. dst->AsRegOpnd()->m_sym->SetIsIntConst((int)intConstantValue);
  3327. }
  3328. break;
  3329. }
  3330. case Js::OpCode::ArgOut_A:
  3331. case Js::OpCode::ArgOut_A_Inline:
  3332. case Js::OpCode::ArgOut_A_FixupForStackArgs:
  3333. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  3334. if (instr->GetDst()->IsRegOpnd())
  3335. {
  3336. Assert(instr->GetDst()->AsRegOpnd()->m_sym->m_isSingleDef);
  3337. instr->GetDst()->AsRegOpnd()->m_sym->AsStackSym()->SetIsIntConst((int)intConstantValue);
  3338. }
  3339. else
  3340. {
  3341. instr->GetDst()->AsSymOpnd()->m_sym->AsStackSym()->SetIsIntConst((int)intConstantValue);
  3342. }
  3343. break;
  3344. case Js::OpCode::TypeofElem:
  3345. instr->m_opcode = Js::OpCode::Typeof;
  3346. break;
  3347. case Js::OpCode::StSlotChkUndecl:
  3348. if (instr->GetSrc2() == opnd)
  3349. {
  3350. // Src2 here should refer to the same location as the Dst operand, which we need to keep live
  3351. // due to the implicit read for ChkUndecl.
  3352. instr->m_opcode = Js::OpCode::StSlot;
  3353. instr->FreeSrc2();
  3354. opnd = nullptr;
  3355. }
  3356. break;
  3357. }
  3358. return opnd;
  3359. }
  3360. else if (valueInfo->TryGetIntConstantValue(&int64ConstantValue, false))
  3361. {
  3362. if (PHASE_OFF(Js::ConstPropPhase, this->func) || !PHASE_ON(Js::Int64ConstPropPhase, this->func))
  3363. {
  3364. return opnd;
  3365. }
  3366. Assert(this->func->GetJITFunctionBody()->IsWasmFunction());
  3367. if (this->func->GetJITFunctionBody()->IsWasmFunction() && opnd->IsInt64())
  3368. {
  3369. IR::Int64ConstOpnd *intOpnd = IR::Int64ConstOpnd::New(int64ConstantValue, opnd->GetType(), instr->m_func);
  3370. GOPT_TRACE_OPND(opnd, _u("Constant prop %lld (value:%lld)\n"), intOpnd->GetImmediateValue(instr->m_func), int64ConstantValue);
  3371. this->CaptureByteCodeSymUses(instr);
  3372. opnd = instr->ReplaceSrc(opnd, intOpnd);
  3373. }
  3374. return opnd;
  3375. }
  3376. Sym *opndSym = nullptr;
  3377. if (opnd->IsRegOpnd())
  3378. {
  3379. IR::RegOpnd *regOpnd = opnd->AsRegOpnd();
  3380. opndSym = regOpnd->m_sym;
  3381. }
  3382. else if (opnd->IsSymOpnd())
  3383. {
  3384. IR::SymOpnd *symOpnd = opnd->AsSymOpnd();
  3385. opndSym = symOpnd->m_sym;
  3386. }
  3387. if (!opndSym)
  3388. {
  3389. return opnd;
  3390. }
  3391. if (PHASE_OFF(Js::CopyPropPhase, this->func))
  3392. {
  3393. this->SetSymStoreDirect(valueInfo, opndSym);
  3394. return opnd;
  3395. }
  3396. StackSym *copySym = CurrentBlockData()->GetCopyPropSym(opndSym, val);
  3397. if (copySym != nullptr)
  3398. {
  3399. Assert(!opndSym->IsStackSym() || copySym->GetSymSize() == opndSym->AsStackSym()->GetSymSize());
  3400. // Copy prop.
  3401. return CopyPropReplaceOpnd(instr, opnd, copySym, parentIndirOpnd);
  3402. }
  3403. else
  3404. {
  3405. if (valueInfo->GetSymStore() && instr->m_opcode == Js::OpCode::Ld_A && instr->GetDst()->IsRegOpnd()
  3406. && valueInfo->GetSymStore() == instr->GetDst()->AsRegOpnd()->m_sym)
  3407. {
  3408. // Avoid resetting symStore after fieldHoisting:
  3409. // t1 = LdFld field <- set symStore to fieldHoistSym
  3410. // fieldHoistSym = Ld_A t1 <- we're looking at t1 now, but want to copy-prop fieldHoistSym forward
  3411. return opnd;
  3412. }
  3413. this->SetSymStoreDirect(valueInfo, opndSym);
  3414. }
  3415. return opnd;
  3416. }
  3417. IR::Opnd *
  3418. GlobOpt::CopyPropReplaceOpnd(IR::Instr * instr, IR::Opnd * opnd, StackSym * copySym, IR::IndirOpnd *parentIndirOpnd)
  3419. {
  3420. Assert(
  3421. parentIndirOpnd
  3422. ? opnd == parentIndirOpnd->GetBaseOpnd() || opnd == parentIndirOpnd->GetIndexOpnd()
  3423. : opnd == instr->GetSrc1() || opnd == instr->GetSrc2() || opnd == instr->GetDst() && opnd->IsIndirOpnd());
  3424. Assert(CurrentBlockData()->IsLive(copySym));
  3425. IR::RegOpnd *regOpnd;
  3426. StackSym *newSym = copySym;
  3427. GOPT_TRACE_OPND(opnd, _u("Copy prop s%d\n"), newSym->m_id);
  3428. #if ENABLE_DEBUG_CONFIG_OPTIONS
  3429. //Need to update DumpFieldCopyPropTestTrace for every new opcode that is added for fieldcopyprop
  3430. if(Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FieldCopyPropPhase))
  3431. {
  3432. instr->DumpFieldCopyPropTestTrace(this->isRecursiveCallOnLandingPad);
  3433. }
  3434. #endif
  3435. this->CaptureByteCodeSymUses(instr);
  3436. if (opnd->IsRegOpnd())
  3437. {
  3438. regOpnd = opnd->AsRegOpnd();
  3439. regOpnd->m_sym = newSym;
  3440. regOpnd->SetIsJITOptimizedReg(true);
  3441. // The dead bit on the opnd is specific to the sym it is referencing. Since we replaced the sym, the bit is reset.
  3442. regOpnd->SetIsDead(false);
  3443. if(parentIndirOpnd)
  3444. {
  3445. return regOpnd;
  3446. }
  3447. }
  3448. else
  3449. {
  3450. // If this is an object type specialized field load inside a loop, and it produces a type value which wasn't live
  3451. // before, make sure the type check is left in the loop, because it may be the last type check in the loop protecting
  3452. // other fields which are not hoistable and are lexically upstream in the loop. If the check is not ultimately
  3453. // needed, the dead store pass will remove it.
  3454. if (this->currentBlock->loop != nullptr && opnd->IsSymOpnd() && opnd->AsSymOpnd()->IsPropertySymOpnd())
  3455. {
  3456. IR::PropertySymOpnd* propertySymOpnd = opnd->AsPropertySymOpnd();
  3457. if (CheckIfPropOpEmitsTypeCheck(instr, propertySymOpnd))
  3458. {
  3459. // We only set guarded properties in the dead store pass, so they shouldn't be set here yet. If they were
  3460. // we would need to move them from this operand to the operand which is being copy propagated.
  3461. Assert(propertySymOpnd->GetGuardedPropOps() == nullptr);
  3462. // We're creating a copy of this operand to be reused in the same spot in the flow, so we can copy all
  3463. // flow sensitive fields. However, we will do only a type check here (no property access) and only for
  3464. // the sake of downstream instructions, so the flags pertaining to this property access are irrelevant.
  3465. IR::PropertySymOpnd* checkObjTypeOpnd = CreateOpndForTypeCheckOnly(propertySymOpnd, instr->m_func);
  3466. IR::Instr* checkObjTypeInstr = IR::Instr::New(Js::OpCode::CheckObjType, instr->m_func);
  3467. checkObjTypeInstr->SetSrc1(checkObjTypeOpnd);
  3468. checkObjTypeInstr->SetByteCodeOffset(instr);
  3469. instr->InsertBefore(checkObjTypeInstr);
  3470. // Since we inserted this instruction before the one that is being processed in natural flow, we must process
  3471. // it for object type spec explicitly here.
  3472. FinishOptPropOp(checkObjTypeInstr, checkObjTypeOpnd);
  3473. Assert(!propertySymOpnd->IsTypeChecked());
  3474. checkObjTypeInstr = this->SetTypeCheckBailOut(checkObjTypeOpnd, checkObjTypeInstr, nullptr);
  3475. Assert(checkObjTypeInstr->HasBailOutInfo());
  3476. if (this->currentBlock->loop && !this->IsLoopPrePass())
  3477. {
  3478. // Try hoisting this checkObjType.
  3479. // But since this isn't the current instr being optimized, we need to play tricks with
  3480. // the byteCodeUse fields...
  3481. TrackByteCodeUsesForInstrAddedInOptInstr(checkObjTypeInstr, [&]()
  3482. {
  3483. TryHoistInvariant(checkObjTypeInstr, this->currentBlock, NULL, CurrentBlockData()->FindValue(copySym), NULL, true);
  3484. });
  3485. }
  3486. }
  3487. }
  3488. if (opnd->IsSymOpnd() && opnd->GetIsDead())
  3489. {
  3490. // Take the property sym out of the live fields set
  3491. this->EndFieldLifetime(opnd->AsSymOpnd());
  3492. }
  3493. regOpnd = IR::RegOpnd::New(newSym, opnd->GetType(), instr->m_func);
  3494. regOpnd->SetIsJITOptimizedReg(true);
  3495. instr->ReplaceSrc(opnd, regOpnd);
  3496. }
  3497. switch (instr->m_opcode)
  3498. {
  3499. case Js::OpCode::Ld_A:
  3500. if (instr->GetDst()->IsRegOpnd() && instr->GetSrc1()->IsRegOpnd() &&
  3501. instr->GetDst()->AsRegOpnd()->GetStackSym() == instr->GetSrc1()->AsRegOpnd()->GetStackSym())
  3502. {
  3503. this->InsertByteCodeUses(instr, true);
  3504. instr->m_opcode = Js::OpCode::Nop;
  3505. }
  3506. break;
  3507. case Js::OpCode::LdSlot:
  3508. case Js::OpCode::LdSlotArr:
  3509. if (instr->GetDst()->IsRegOpnd() && instr->GetSrc1()->IsRegOpnd() &&
  3510. instr->GetDst()->AsRegOpnd()->GetStackSym() == instr->GetSrc1()->AsRegOpnd()->GetStackSym())
  3511. {
  3512. this->InsertByteCodeUses(instr, true);
  3513. instr->m_opcode = Js::OpCode::Nop;
  3514. }
  3515. else
  3516. {
  3517. instr->m_opcode = Js::OpCode::Ld_A;
  3518. }
  3519. break;
  3520. case Js::OpCode::StSlotChkUndecl:
  3521. if (instr->GetSrc2()->IsRegOpnd())
  3522. {
  3523. // Src2 here should refer to the same location as the Dst operand, which we need to keep live
  3524. // due to the implicit read for ChkUndecl.
  3525. instr->m_opcode = Js::OpCode::StSlot;
  3526. instr->FreeSrc2();
  3527. return nullptr;
  3528. }
  3529. break;
  3530. case Js::OpCode::LdFld:
  3531. case Js::OpCode::LdFldForTypeOf:
  3532. case Js::OpCode::LdRootFldForTypeOf:
  3533. case Js::OpCode::LdFldForCallApplyTarget:
  3534. case Js::OpCode::LdRootFld:
  3535. case Js::OpCode::LdMethodFld:
  3536. case Js::OpCode::LdRootMethodFld:
  3537. case Js::OpCode::ScopedLdMethodFld:
  3538. case Js::OpCode::ScopedLdFld:
  3539. case Js::OpCode::ScopedLdFldForTypeOf:
  3540. instr->m_opcode = Js::OpCode::Ld_A;
  3541. break;
  3542. case Js::OpCode::LdMethodFromFlags:
  3543. // The bailout is checked on the loop top and we don't need to check bailout again in loop.
  3544. instr->m_opcode = Js::OpCode::Ld_A;
  3545. instr->ClearBailOutInfo();
  3546. break;
  3547. case Js::OpCode::TypeofElem:
  3548. instr->m_opcode = Js::OpCode::Typeof;
  3549. break;
  3550. }
  3551. CurrentBlockData()->MarkTempLastUse(instr, regOpnd);
  3552. return regOpnd;
  3553. }
  3554. ValueNumber
  3555. GlobOpt::NewValueNumber()
  3556. {
  3557. ValueNumber valueNumber = this->currentValue++;
  3558. if (valueNumber == 0)
  3559. {
  3560. Js::Throw::OutOfMemory();
  3561. }
  3562. return valueNumber;
  3563. }
  3564. Value *GlobOpt::NewValue(ValueInfo *const valueInfo)
  3565. {
  3566. return NewValue(NewValueNumber(), valueInfo);
  3567. }
  3568. Value *GlobOpt::NewValue(const ValueNumber valueNumber, ValueInfo *const valueInfo)
  3569. {
  3570. Assert(valueInfo);
  3571. return Value::New(alloc, valueNumber, valueInfo);
  3572. }
  3573. Value *GlobOpt::CopyValue(Value const *const value)
  3574. {
  3575. return CopyValue(value, NewValueNumber());
  3576. }
  3577. Value *GlobOpt::CopyValue(Value const *const value, const ValueNumber valueNumber)
  3578. {
  3579. Assert(value);
  3580. return value->Copy(alloc, valueNumber);
  3581. }
  3582. Value *
  3583. GlobOpt::NewGenericValue(const ValueType valueType)
  3584. {
  3585. return NewGenericValue(valueType, static_cast<IR::Opnd *>(nullptr));
  3586. }
  3587. Value *
  3588. GlobOpt::NewGenericValue(const ValueType valueType, IR::Opnd *const opnd)
  3589. {
  3590. // Shouldn't assign a likely-int value to something that is definitely not an int
  3591. Assert(!(valueType.IsLikelyInt() && opnd && opnd->IsNotInt()));
  3592. ValueInfo *valueInfo = ValueInfo::New(this->alloc, valueType);
  3593. Value *val = NewValue(valueInfo);
  3594. TrackNewValueForKills(val);
  3595. CurrentBlockData()->InsertNewValue(val, opnd);
  3596. return val;
  3597. }
  3598. Value *
  3599. GlobOpt::NewGenericValue(const ValueType valueType, Sym *const sym)
  3600. {
  3601. ValueInfo *valueInfo = ValueInfo::New(this->alloc, valueType);
  3602. Value *val = NewValue(valueInfo);
  3603. TrackNewValueForKills(val);
  3604. CurrentBlockData()->SetValue(val, sym);
  3605. return val;
  3606. }
  3607. Value *
  3608. GlobOpt::GetIntConstantValue(const int32 intConst, IR::Instr * instr, IR::Opnd *const opnd)
  3609. {
  3610. Value *value = nullptr;
  3611. Value *const cachedValue = this->intConstantToValueMap->Lookup(intConst, nullptr);
  3612. if(cachedValue)
  3613. {
  3614. // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since
  3615. // values are cloned for each block, we can't use the same value object. We also can't have two values with the same
  3616. // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast
  3617. // way to determine if a value with the same value number exists for this block. So the best we can do with a global
  3618. // cache is to check the sym-store's value in the current block to see if it has a value with the same number.
  3619. // Otherwise, we have to create a new value with a new value number.
  3620. Sym *const symStore = cachedValue->GetValueInfo()->GetSymStore();
  3621. if (symStore && CurrentBlockData()->IsLive(symStore))
  3622. {
  3623. Value *const symStoreValue = CurrentBlockData()->FindValue(symStore);
  3624. int32 symStoreIntConstantValue;
  3625. if (symStoreValue &&
  3626. symStoreValue->GetValueNumber() == cachedValue->GetValueNumber() &&
  3627. symStoreValue->GetValueInfo()->TryGetIntConstantValue(&symStoreIntConstantValue) &&
  3628. symStoreIntConstantValue == intConst)
  3629. {
  3630. value = symStoreValue;
  3631. }
  3632. }
  3633. }
  3634. if (!value)
  3635. {
  3636. value = NewIntConstantValue(intConst, instr, !Js::TaggedInt::IsOverflow(intConst));
  3637. }
  3638. return CurrentBlockData()->InsertNewValue(value, opnd);
  3639. }
  3640. Value *
  3641. GlobOpt::GetIntConstantValue(const int64 intConst, IR::Instr * instr, IR::Opnd *const opnd)
  3642. {
  3643. Assert(instr->m_func->GetJITFunctionBody()->IsWasmFunction());
  3644. Value *value = nullptr;
  3645. Value *const cachedValue = this->int64ConstantToValueMap->Lookup(intConst, nullptr);
  3646. if (cachedValue)
  3647. {
  3648. // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since
  3649. // values are cloned for each block, we can't use the same value object. We also can't have two values with the same
  3650. // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast
  3651. // way to determine if a value with the same value number exists for this block. So the best we can do with a global
  3652. // cache is to check the sym-store's value in the current block to see if it has a value with the same number.
  3653. // Otherwise, we have to create a new value with a new value number.
  3654. Sym *const symStore = cachedValue->GetValueInfo()->GetSymStore();
  3655. if (symStore && this->currentBlock->globOptData.IsLive(symStore))
  3656. {
  3657. Value *const symStoreValue = this->currentBlock->globOptData.FindValue(symStore);
  3658. int64 symStoreIntConstantValue;
  3659. if (symStoreValue &&
  3660. symStoreValue->GetValueNumber() == cachedValue->GetValueNumber() &&
  3661. symStoreValue->GetValueInfo()->TryGetInt64ConstantValue(&symStoreIntConstantValue, false) &&
  3662. symStoreIntConstantValue == intConst)
  3663. {
  3664. value = symStoreValue;
  3665. }
  3666. }
  3667. }
  3668. if (!value)
  3669. {
  3670. value = NewInt64ConstantValue(intConst, instr);
  3671. }
  3672. return this->currentBlock->globOptData.InsertNewValue(value, opnd);
  3673. }
  3674. Value *
  3675. GlobOpt::NewInt64ConstantValue(const int64 intConst, IR::Instr* instr)
  3676. {
  3677. Value * value = NewValue(Int64ConstantValueInfo::New(this->alloc, intConst));
  3678. this->int64ConstantToValueMap->Item(intConst, value);
  3679. if (!value->GetValueInfo()->GetSymStore() &&
  3680. (instr->m_opcode == Js::OpCode::LdC_A_I4 || instr->m_opcode == Js::OpCode::Ld_I4))
  3681. {
  3682. StackSym * sym = instr->GetDst()->GetStackSym();
  3683. Assert(sym && !sym->IsTypeSpec());
  3684. this->currentBlock->globOptData.SetValue(value, sym);
  3685. this->currentBlock->globOptData.liveVarSyms->Set(sym->m_id);
  3686. }
  3687. return value;
  3688. }
  3689. Value *
  3690. GlobOpt::NewIntConstantValue(const int32 intConst, IR::Instr * instr, bool isTaggable)
  3691. {
  3692. Value * value = NewValue(IntConstantValueInfo::New(this->alloc, intConst));
  3693. this->intConstantToValueMap->Item(intConst, value);
  3694. if (isTaggable &&
  3695. !PHASE_OFF(Js::HoistConstIntPhase, this->func))
  3696. {
  3697. // When creating a new int constant value, make sure it gets a symstore. If the int const doesn't have a symstore,
  3698. // any downstream instruction using the same int will have to create a new value (object) for the int.
  3699. // This gets in the way of CSE.
  3700. value = HoistConstantLoadAndPropagateValueBackward(Js::TaggedInt::ToVarUnchecked(intConst), instr, value);
  3701. if (!value->GetValueInfo()->GetSymStore() &&
  3702. (instr->m_opcode == Js::OpCode::LdC_A_I4 || instr->m_opcode == Js::OpCode::Ld_I4))
  3703. {
  3704. StackSym * sym = instr->GetDst()->GetStackSym();
  3705. Assert(sym);
  3706. if (sym->IsTypeSpec())
  3707. {
  3708. Assert(sym->IsInt32());
  3709. StackSym * varSym = sym->GetVarEquivSym(instr->m_func);
  3710. CurrentBlockData()->SetValue(value, varSym);
  3711. CurrentBlockData()->liveInt32Syms->Set(varSym->m_id);
  3712. }
  3713. else
  3714. {
  3715. CurrentBlockData()->SetValue(value, sym);
  3716. CurrentBlockData()->liveVarSyms->Set(sym->m_id);
  3717. }
  3718. }
  3719. }
  3720. return value;
  3721. }
  3722. ValueInfo *
  3723. GlobOpt::NewIntRangeValueInfo(const int32 min, const int32 max, const bool wasNegativeZeroPreventedByBailout)
  3724. {
  3725. return ValueInfo::NewIntRangeValueInfo(this->alloc, min, max, wasNegativeZeroPreventedByBailout);
  3726. }
  3727. ValueInfo *GlobOpt::NewIntRangeValueInfo(
  3728. const ValueInfo *const originalValueInfo,
  3729. const int32 min,
  3730. const int32 max) const
  3731. {
  3732. Assert(originalValueInfo);
  3733. ValueInfo *valueInfo;
  3734. if(min == max)
  3735. {
  3736. // Since int constant values are const-propped, negative zero tracking does not track them, and so it's okay to ignore
  3737. // 'wasNegativeZeroPreventedByBailout'
  3738. valueInfo = IntConstantValueInfo::New(alloc, min);
  3739. }
  3740. else
  3741. {
  3742. valueInfo =
  3743. IntRangeValueInfo::New(
  3744. alloc,
  3745. min,
  3746. max,
  3747. min <= 0 && max >= 0 && originalValueInfo->WasNegativeZeroPreventedByBailout());
  3748. }
  3749. valueInfo->SetSymStore(originalValueInfo->GetSymStore());
  3750. return valueInfo;
  3751. }
  3752. Value *
  3753. GlobOpt::NewIntRangeValue(
  3754. const int32 min,
  3755. const int32 max,
  3756. const bool wasNegativeZeroPreventedByBailout,
  3757. IR::Opnd *const opnd)
  3758. {
  3759. ValueInfo *valueInfo = this->NewIntRangeValueInfo(min, max, wasNegativeZeroPreventedByBailout);
  3760. Value *val = NewValue(valueInfo);
  3761. if (opnd)
  3762. {
  3763. GOPT_TRACE_OPND(opnd, _u("Range %d (0x%X) to %d (0x%X)\n"), min, min, max, max);
  3764. }
  3765. CurrentBlockData()->InsertNewValue(val, opnd);
  3766. return val;
  3767. }
  3768. IntBoundedValueInfo *GlobOpt::NewIntBoundedValueInfo(
  3769. const ValueInfo *const originalValueInfo,
  3770. const IntBounds *const bounds) const
  3771. {
  3772. Assert(originalValueInfo);
  3773. bounds->Verify();
  3774. IntBoundedValueInfo *const valueInfo =
  3775. IntBoundedValueInfo::New(
  3776. originalValueInfo->Type(),
  3777. bounds,
  3778. (
  3779. bounds->ConstantLowerBound() <= 0 &&
  3780. bounds->ConstantUpperBound() >= 0 &&
  3781. originalValueInfo->WasNegativeZeroPreventedByBailout()
  3782. ),
  3783. alloc);
  3784. valueInfo->SetSymStore(originalValueInfo->GetSymStore());
  3785. return valueInfo;
  3786. }
  3787. Value *GlobOpt::NewIntBoundedValue(
  3788. const ValueType valueType,
  3789. const IntBounds *const bounds,
  3790. const bool wasNegativeZeroPreventedByBailout,
  3791. IR::Opnd *const opnd)
  3792. {
  3793. Value *const value = NewValue(IntBoundedValueInfo::New(valueType, bounds, wasNegativeZeroPreventedByBailout, alloc));
  3794. CurrentBlockData()->InsertNewValue(value, opnd);
  3795. return value;
  3796. }
  3797. Value *
  3798. GlobOpt::NewFloatConstantValue(const FloatConstType floatValue, IR::Opnd *const opnd)
  3799. {
  3800. FloatConstantValueInfo *valueInfo = FloatConstantValueInfo::New(this->alloc, floatValue);
  3801. Value *val = NewValue(valueInfo);
  3802. CurrentBlockData()->InsertNewValue(val, opnd);
  3803. return val;
  3804. }
  3805. Value *
  3806. GlobOpt::GetVarConstantValue(IR::AddrOpnd *addrOpnd)
  3807. {
  3808. bool isVar = addrOpnd->IsVar();
  3809. bool isString = isVar && addrOpnd->m_localAddress && JITJavascriptString::Is(addrOpnd->m_localAddress);
  3810. Value *val = nullptr;
  3811. Value *cachedValue = nullptr;
  3812. if(this->addrConstantToValueMap->TryGetValue(addrOpnd->m_address, &cachedValue))
  3813. {
  3814. // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since
  3815. // values are cloned for each block, we can't use the same value object. We also can't have two values with the same
  3816. // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast
  3817. // way to determine if a value with the same value number exists for this block. So the best we can do with a global
  3818. // cache is to check the sym-store's value in the current block to see if it has a value with the same number.
  3819. // Otherwise, we have to create a new value with a new value number.
  3820. Sym *symStore = cachedValue->GetValueInfo()->GetSymStore();
  3821. if(symStore && CurrentBlockData()->IsLive(symStore))
  3822. {
  3823. Value *const symStoreValue = CurrentBlockData()->FindValue(symStore);
  3824. if(symStoreValue && symStoreValue->GetValueNumber() == cachedValue->GetValueNumber())
  3825. {
  3826. ValueInfo *const symStoreValueInfo = symStoreValue->GetValueInfo();
  3827. if(symStoreValueInfo->IsVarConstant() && symStoreValueInfo->AsVarConstant()->VarValue() == addrOpnd->m_address)
  3828. {
  3829. val = symStoreValue;
  3830. }
  3831. }
  3832. }
  3833. }
  3834. else if (isString)
  3835. {
  3836. JITJavascriptString* jsString = JITJavascriptString::FromVar(addrOpnd->m_localAddress);
  3837. Js::InternalString internalString(jsString->GetString(), jsString->GetLength());
  3838. if (this->stringConstantToValueMap->TryGetValue(internalString, &cachedValue))
  3839. {
  3840. Sym *symStore = cachedValue->GetValueInfo()->GetSymStore();
  3841. if (symStore && CurrentBlockData()->IsLive(symStore))
  3842. {
  3843. Value *const symStoreValue = CurrentBlockData()->FindValue(symStore);
  3844. if (symStoreValue && symStoreValue->GetValueNumber() == cachedValue->GetValueNumber())
  3845. {
  3846. ValueInfo *const symStoreValueInfo = symStoreValue->GetValueInfo();
  3847. if (symStoreValueInfo->IsVarConstant())
  3848. {
  3849. JITJavascriptString * cachedString = JITJavascriptString::FromVar(symStoreValue->GetValueInfo()->AsVarConstant()->VarValue(true));
  3850. Js::InternalString cachedInternalString(cachedString->GetString(), cachedString->GetLength());
  3851. if (Js::InternalStringComparer::Equals(internalString, cachedInternalString))
  3852. {
  3853. val = symStoreValue;
  3854. }
  3855. }
  3856. }
  3857. }
  3858. }
  3859. }
  3860. if(!val)
  3861. {
  3862. val = NewVarConstantValue(addrOpnd, isString);
  3863. }
  3864. addrOpnd->SetValueType(val->GetValueInfo()->Type());
  3865. return val;
  3866. }
  3867. Value *
  3868. GlobOpt::NewVarConstantValue(IR::AddrOpnd *addrOpnd, bool isString)
  3869. {
  3870. VarConstantValueInfo *valueInfo = VarConstantValueInfo::New(this->alloc, addrOpnd->m_address, addrOpnd->GetValueType(), false, addrOpnd->m_localAddress);
  3871. Value * value = NewValue(valueInfo);
  3872. this->addrConstantToValueMap->Item(addrOpnd->m_address, value);
  3873. if (isString)
  3874. {
  3875. JITJavascriptString* jsString = JITJavascriptString::FromVar(addrOpnd->m_localAddress);
  3876. Js::InternalString internalString(jsString->GetString(), jsString->GetLength());
  3877. this->stringConstantToValueMap->Item(internalString, value);
  3878. }
  3879. return value;
  3880. }
  3881. Value *
  3882. GlobOpt::HoistConstantLoadAndPropagateValueBackward(Js::Var varConst, IR::Instr * origInstr, Value * value)
  3883. {
  3884. if (this->IsLoopPrePass() ||
  3885. ((this->currentBlock == this->func->m_fg->blockList) &&
  3886. origInstr->TransfersSrcValue()))
  3887. {
  3888. return value;
  3889. }
  3890. // Only hoisting taggable int const loads for now. Could be extended to other constants (floats, strings, addr opnds) if we see some benefit.
  3891. Assert(Js::TaggedInt::Is(varConst));
  3892. // Insert a load of the constant at the top of the function
  3893. StackSym * dstSym = StackSym::New(this->func);
  3894. IR::RegOpnd * constRegOpnd = IR::RegOpnd::New(dstSym, TyVar, this->func);
  3895. IR::Instr * loadInstr = IR::Instr::NewConstantLoad(constRegOpnd, (intptr_t)varConst, ValueType::GetInt(true), this->func);
  3896. this->func->m_fg->blockList->GetFirstInstr()->InsertAfter(loadInstr);
  3897. // Type-spec the load (Support for floats needs to be added when we start hoisting float constants).
  3898. bool typeSpecedToInt = false;
  3899. if (Js::TaggedInt::Is(varConst) && !IsTypeSpecPhaseOff(this->func))
  3900. {
  3901. typeSpecedToInt = true;
  3902. loadInstr->m_opcode = Js::OpCode::Ld_I4;
  3903. ToInt32Dst(loadInstr, loadInstr->GetDst()->AsRegOpnd(), this->currentBlock);
  3904. loadInstr->GetDst()->GetStackSym()->SetIsConst();
  3905. }
  3906. else
  3907. {
  3908. CurrentBlockData()->liveVarSyms->Set(dstSym->m_id);
  3909. }
  3910. // Add the value (object) to the current block's symToValueMap and propagate the value backward to all relevant blocks so it is available on merges.
  3911. value = CurrentBlockData()->InsertNewValue(value, constRegOpnd);
  3912. BVSparse<JitArenaAllocator>* GlobOptBlockData::*bv;
  3913. bv = typeSpecedToInt ? &GlobOptBlockData::liveInt32Syms : &GlobOptBlockData::liveVarSyms; // Will need to be expanded when we start hoisting float constants.
  3914. if (this->currentBlock != this->func->m_fg->blockList)
  3915. {
  3916. for (InvariantBlockBackwardIterator it(this, this->currentBlock, this->func->m_fg->blockList, nullptr);
  3917. it.IsValid();
  3918. it.MoveNext())
  3919. {
  3920. BasicBlock * block = it.Block();
  3921. (block->globOptData.*bv)->Set(dstSym->m_id);
  3922. if (!block->globOptData.FindValue(dstSym))
  3923. {
  3924. Value *const valueCopy = CopyValue(value, value->GetValueNumber());
  3925. block->globOptData.SetValue(valueCopy, dstSym);
  3926. }
  3927. }
  3928. }
  3929. return value;
  3930. }
  3931. Value *
  3932. GlobOpt::NewFixedFunctionValue(Js::JavascriptFunction *function, IR::AddrOpnd *addrOpnd)
  3933. {
  3934. Assert(function != nullptr);
  3935. Value *val = nullptr;
  3936. Value *cachedValue = nullptr;
  3937. if(this->addrConstantToValueMap->TryGetValue(addrOpnd->m_address, &cachedValue))
  3938. {
  3939. // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since
  3940. // values are cloned for each block, we can't use the same value object. We also can't have two values with the same
  3941. // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast
  3942. // way to determine if a value with the same value number exists for this block. So the best we can do with a global
  3943. // cache is to check the sym-store's value in the current block to see if it has a value with the same number.
  3944. // Otherwise, we have to create a new value with a new value number.
  3945. Sym *symStore = cachedValue->GetValueInfo()->GetSymStore();
  3946. if(symStore && CurrentBlockData()->IsLive(symStore))
  3947. {
  3948. Value *const symStoreValue = CurrentBlockData()->FindValue(symStore);
  3949. if(symStoreValue && symStoreValue->GetValueNumber() == cachedValue->GetValueNumber())
  3950. {
  3951. ValueInfo *const symStoreValueInfo = symStoreValue->GetValueInfo();
  3952. if(symStoreValueInfo->IsVarConstant())
  3953. {
  3954. VarConstantValueInfo *const symStoreVarConstantValueInfo = symStoreValueInfo->AsVarConstant();
  3955. if(symStoreVarConstantValueInfo->VarValue() == addrOpnd->m_address &&
  3956. symStoreVarConstantValueInfo->IsFunction())
  3957. {
  3958. val = symStoreValue;
  3959. }
  3960. }
  3961. }
  3962. }
  3963. }
  3964. if(!val)
  3965. {
  3966. VarConstantValueInfo *valueInfo = VarConstantValueInfo::New(this->alloc, function, addrOpnd->GetValueType(), true, addrOpnd->m_localAddress);
  3967. val = NewValue(valueInfo);
  3968. this->addrConstantToValueMap->AddNew(addrOpnd->m_address, val);
  3969. }
  3970. CurrentBlockData()->InsertNewValue(val, addrOpnd);
  3971. return val;
  3972. }
  3973. StackSym *GlobOpt::GetTaggedIntConstantStackSym(const int32 intConstantValue) const
  3974. {
  3975. Assert(!Js::TaggedInt::IsOverflow(intConstantValue));
  3976. return intConstantToStackSymMap->Lookup(intConstantValue, nullptr);
  3977. }
  3978. StackSym *GlobOpt::GetOrCreateTaggedIntConstantStackSym(const int32 intConstantValue) const
  3979. {
  3980. StackSym *stackSym = GetTaggedIntConstantStackSym(intConstantValue);
  3981. if(stackSym)
  3982. {
  3983. return stackSym;
  3984. }
  3985. stackSym = StackSym::New(TyVar,func);
  3986. intConstantToStackSymMap->Add(intConstantValue, stackSym);
  3987. return stackSym;
  3988. }
  3989. Sym *
  3990. GlobOpt::SetSymStore(ValueInfo *valueInfo, Sym *sym)
  3991. {
  3992. if (sym->IsStackSym())
  3993. {
  3994. StackSym *stackSym = sym->AsStackSym();
  3995. if (stackSym->IsTypeSpec())
  3996. {
  3997. stackSym = stackSym->GetVarEquivSym(this->func);
  3998. sym = stackSym;
  3999. }
  4000. }
  4001. if (valueInfo->GetSymStore() == nullptr || valueInfo->GetSymStore()->IsPropertySym())
  4002. {
  4003. SetSymStoreDirect(valueInfo, sym);
  4004. }
  4005. return sym;
  4006. }
  4007. void
  4008. GlobOpt::SetSymStoreDirect(ValueInfo * valueInfo, Sym * sym)
  4009. {
  4010. Sym * prevSymStore = valueInfo->GetSymStore();
  4011. CurrentBlockData()->SetChangedSym(prevSymStore);
  4012. valueInfo->SetSymStore(sym);
  4013. }
  4014. // Figure out the Value of this dst.
  4015. Value *
  4016. GlobOpt::ValueNumberDst(IR::Instr **pInstr, Value *src1Val, Value *src2Val)
  4017. {
  4018. IR::Instr *&instr = *pInstr;
  4019. IR::Opnd *dst = instr->GetDst();
  4020. Value *dstVal = nullptr;
  4021. Sym *sym;
  4022. if (instr->CallsSetter())
  4023. {
  4024. return nullptr;
  4025. }
  4026. if (dst == nullptr)
  4027. {
  4028. return nullptr;
  4029. }
  4030. switch (dst->GetKind())
  4031. {
  4032. case IR::OpndKindSym:
  4033. sym = dst->AsSymOpnd()->m_sym;
  4034. break;
  4035. case IR::OpndKindReg:
  4036. sym = dst->AsRegOpnd()->m_sym;
  4037. if (OpCodeAttr::TempNumberProducing(instr->m_opcode))
  4038. {
  4039. CurrentBlockData()->isTempSrc->Set(sym->m_id);
  4040. }
  4041. else if (OpCodeAttr::TempNumberTransfer(instr->m_opcode))
  4042. {
  4043. IR::Opnd *src1 = instr->GetSrc1();
  4044. if (src1->IsRegOpnd() && CurrentBlockData()->isTempSrc->Test(src1->AsRegOpnd()->m_sym->m_id))
  4045. {
  4046. StackSym *src1Sym = src1->AsRegOpnd()->m_sym;
  4047. // isTempSrc is used for marking isTempLastUse, which is used to generate AddLeftDead()
  4048. // calls instead of the normal Add helpers. It tells the runtime that concats can use string
  4049. // builders.
  4050. // We need to be careful in the case where src1 points to a string builder and is getting aliased.
  4051. // Clear the bit on src and dst of the transfer instr in this case, unless we can prove src1
  4052. // isn't pointing at a string builder, like if it is single def and the def instr is not an Add,
  4053. // but TempProducing.
  4054. if (src1Sym->IsSingleDef() && src1Sym->m_instrDef->m_opcode != Js::OpCode::Add_A
  4055. && OpCodeAttr::TempNumberProducing(src1Sym->m_instrDef->m_opcode))
  4056. {
  4057. CurrentBlockData()->isTempSrc->Set(sym->m_id);
  4058. }
  4059. else
  4060. {
  4061. CurrentBlockData()->isTempSrc->Clear(src1->AsRegOpnd()->m_sym->m_id);
  4062. CurrentBlockData()->isTempSrc->Clear(sym->m_id);
  4063. }
  4064. }
  4065. else
  4066. {
  4067. CurrentBlockData()->isTempSrc->Clear(sym->m_id);
  4068. }
  4069. }
  4070. else
  4071. {
  4072. CurrentBlockData()->isTempSrc->Clear(sym->m_id);
  4073. }
  4074. break;
  4075. case IR::OpndKindIndir:
  4076. return nullptr;
  4077. default:
  4078. return nullptr;
  4079. }
  4080. int32 min1, max1, min2, max2, newMin, newMax;
  4081. ValueInfo *src1ValueInfo = (src1Val ? src1Val->GetValueInfo() : nullptr);
  4082. ValueInfo *src2ValueInfo = (src2Val ? src2Val->GetValueInfo() : nullptr);
  4083. switch (instr->m_opcode)
  4084. {
  4085. case Js::OpCode::Conv_PrimStr:
  4086. AssertMsg(instr->GetDst()->GetValueType().IsString(),
  4087. "Creator of this instruction should have set the type");
  4088. if (this->IsLoopPrePass() || src1ValueInfo == nullptr || !src1ValueInfo->IsPrimitive())
  4089. {
  4090. break;
  4091. }
  4092. instr->m_opcode = Js::OpCode::Conv_Str;
  4093. // fall-through
  4094. case Js::OpCode::Conv_Str:
  4095. // This opcode is commented out since we don't track regex information in GlobOpt now.
  4096. //case Js::OpCode::Coerce_Regex:
  4097. case Js::OpCode::Coerce_Str:
  4098. AssertMsg(instr->GetDst()->GetValueType().IsString(),
  4099. "Creator of this instruction should have set the type");
  4100. // fall-through
  4101. case Js::OpCode::Coerce_StrOrRegex:
  4102. // We don't set the ValueType of src1 for Coerce_StrOrRegex, hence skip the ASSERT
  4103. if (this->IsLoopPrePass() || src1ValueInfo == nullptr || !src1ValueInfo->IsString())
  4104. {
  4105. break;
  4106. }
  4107. instr->m_opcode = Js::OpCode::Ld_A;
  4108. // fall-through
  4109. case Js::OpCode::BytecodeArgOutCapture:
  4110. case Js::OpCode::InitConst:
  4111. case Js::OpCode::LdAsmJsFunc:
  4112. case Js::OpCode::Ld_A:
  4113. case Js::OpCode::Ld_I4:
  4114. // Propagate sym attributes across the reg copy.
  4115. if (!this->IsLoopPrePass() && instr->GetSrc1()->IsRegOpnd())
  4116. {
  4117. if (dst->AsRegOpnd()->m_sym->IsSingleDef())
  4118. {
  4119. dst->AsRegOpnd()->m_sym->CopySymAttrs(instr->GetSrc1()->AsRegOpnd()->m_sym);
  4120. }
  4121. }
  4122. if (instr->IsProfiledInstr())
  4123. {
  4124. const ValueType profiledValueType(instr->AsProfiledInstr()->u.FldInfo().valueType);
  4125. if(!(
  4126. profiledValueType.IsLikelyInt() &&
  4127. (
  4128. (dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->m_isNotNumber) ||
  4129. (instr->GetSrc1()->IsRegOpnd() && instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber)
  4130. )
  4131. ))
  4132. {
  4133. if(!src1ValueInfo)
  4134. {
  4135. dstVal = this->NewGenericValue(profiledValueType, dst);
  4136. }
  4137. else if(src1ValueInfo->IsUninitialized())
  4138. {
  4139. if(IsLoopPrePass())
  4140. {
  4141. dstVal = this->NewGenericValue(profiledValueType, dst);
  4142. }
  4143. else
  4144. {
  4145. // Assuming the profile data gives more precise value types based on the path it took at runtime, we
  4146. // can improve the original value type.
  4147. src1ValueInfo->Type() = profiledValueType;
  4148. instr->GetSrc1()->SetValueType(profiledValueType);
  4149. }
  4150. }
  4151. }
  4152. }
  4153. if (dstVal == nullptr)
  4154. {
  4155. // Ld_A is just transferring the value
  4156. dstVal = this->ValueNumberTransferDst(instr, src1Val);
  4157. }
  4158. break;
  4159. case Js::OpCode::ExtendArg_A:
  4160. {
  4161. // SIMD_JS
  4162. // We avoid transforming EAs to Lds to keep the IR shape consistent and avoid CSEing of EAs.
  4163. // CSEOptimize only assigns a Value to the EA dst, and doesn't turn it to a Ld. If this happened, we shouldn't assign a new Value here.
  4164. if (DoCSE())
  4165. {
  4166. IR::Opnd * currDst = instr->GetDst();
  4167. Value * currDstVal = CurrentBlockData()->FindValue(currDst->GetStackSym());
  4168. if (currDstVal != nullptr)
  4169. {
  4170. return currDstVal;
  4171. }
  4172. }
  4173. break;
  4174. }
  4175. case Js::OpCode::CheckFixedFld:
  4176. AssertMsg(false, "CheckFixedFld doesn't have a dst, so we should never get here");
  4177. break;
  4178. case Js::OpCode::LdSlot:
  4179. case Js::OpCode::LdSlotArr:
  4180. case Js::OpCode::LdFld:
  4181. case Js::OpCode::LdFldForTypeOf:
  4182. case Js::OpCode::LdFldForCallApplyTarget:
  4183. // Do not transfer value type on LdRootFldForTypeOf to prevent copy-prop to LdRootFld in case the field doesn't exist since LdRootFldForTypeOf does not throw.
  4184. // Same goes for ScopedLdFldForTypeOf as we'll end up loading the property from the root object if the property is not in the scope chain.
  4185. //case Js::OpCode::LdRootFldForTypeOf:
  4186. //case Js::OpCode::ScopedLdFldForTypeOf:
  4187. case Js::OpCode::LdRootFld:
  4188. case Js::OpCode::LdMethodFld:
  4189. case Js::OpCode::LdRootMethodFld:
  4190. case Js::OpCode::ScopedLdMethodFld:
  4191. case Js::OpCode::LdMethodFromFlags:
  4192. case Js::OpCode::ScopedLdFld:
  4193. if (instr->IsProfiledInstr())
  4194. {
  4195. ValueType profiledValueType(instr->AsProfiledInstr()->u.FldInfo().valueType);
  4196. if(!(profiledValueType.IsLikelyInt() && dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->m_isNotNumber))
  4197. {
  4198. if(!src1ValueInfo)
  4199. {
  4200. dstVal = this->NewGenericValue(profiledValueType, dst);
  4201. }
  4202. else if(src1ValueInfo->IsUninitialized())
  4203. {
  4204. if(IsLoopPrePass() && (!dst->IsRegOpnd() || !dst->AsRegOpnd()->m_sym->IsSingleDef()))
  4205. {
  4206. dstVal = this->NewGenericValue(profiledValueType, dst);
  4207. }
  4208. else
  4209. {
  4210. // Assuming the profile data gives more precise value types based on the path it took at runtime, we
  4211. // can improve the original value type.
  4212. src1ValueInfo->Type() = profiledValueType;
  4213. instr->GetSrc1()->SetValueType(profiledValueType);
  4214. }
  4215. }
  4216. }
  4217. }
  4218. if (dstVal == nullptr)
  4219. {
  4220. dstVal = this->ValueNumberTransferDst(instr, src1Val);
  4221. }
  4222. if(!this->IsLoopPrePass())
  4223. {
  4224. // We cannot transfer value if the field hasn't been copy prop'd because we don't generate
  4225. // an implicit call bailout between those values if we don't have "live fields" unless, we are hoisting the field.
  4226. ValueInfo *dstValueInfo = (dstVal ? dstVal->GetValueInfo() : nullptr);
  4227. // Update symStore if it isn't a stackSym
  4228. if (dstVal && (!dstValueInfo->GetSymStore() || !dstValueInfo->GetSymStore()->IsStackSym()))
  4229. {
  4230. Assert(dst->IsRegOpnd());
  4231. this->SetSymStoreDirect(dstValueInfo, dst->AsRegOpnd()->m_sym);
  4232. }
  4233. if (src1Val != dstVal)
  4234. {
  4235. CurrentBlockData()->SetValue(dstVal, instr->GetSrc1());
  4236. }
  4237. }
  4238. break;
  4239. case Js::OpCode::LdC_A_R8:
  4240. case Js::OpCode::LdC_A_I4:
  4241. case Js::OpCode::ArgIn_A:
  4242. dstVal = src1Val;
  4243. break;
  4244. case Js::OpCode::LdStr:
  4245. if (src1Val == nullptr)
  4246. {
  4247. src1Val = NewGenericValue(ValueType::String, dst);
  4248. }
  4249. dstVal = src1Val;
  4250. break;
  4251. // LdElemUndef only assign undef if the field doesn't exist.
  4252. // So we don't actually know what the value is, so we can't really copy prop it.
  4253. //case Js::OpCode::LdElemUndef:
  4254. case Js::OpCode::StSlot:
  4255. case Js::OpCode::StSlotChkUndecl:
  4256. case Js::OpCode::StFld:
  4257. case Js::OpCode::StRootFld:
  4258. case Js::OpCode::StFldStrict:
  4259. case Js::OpCode::StRootFldStrict:
  4260. case Js::OpCode::InitFld:
  4261. case Js::OpCode::InitComputedProperty:
  4262. if (DoFieldCopyProp())
  4263. {
  4264. if (src1Val == nullptr)
  4265. {
  4266. // src1 may have no value if it's not a valid var, e.g., NULL for let/const initialization.
  4267. // Consider creating generic values for such things.
  4268. return nullptr;
  4269. }
  4270. AssertMsg(!src2Val, "Bad src Values...");
  4271. Assert(sym->IsPropertySym());
  4272. SymID symId = sym->m_id;
  4273. Assert(instr->m_opcode == Js::OpCode::StSlot || instr->m_opcode == Js::OpCode::StSlotChkUndecl || !CurrentBlockData()->liveFields->Test(symId));
  4274. CurrentBlockData()->liveFields->Set(symId);
  4275. if (!this->IsLoopPrePass() && dst->GetIsDead())
  4276. {
  4277. // Take the property sym out of the live fields set (with special handling for loops).
  4278. this->EndFieldLifetime(dst->AsSymOpnd());
  4279. }
  4280. dstVal = this->ValueNumberTransferDst(instr, src1Val);
  4281. }
  4282. else
  4283. {
  4284. return nullptr;
  4285. }
  4286. break;
  4287. case Js::OpCode::Conv_Num:
  4288. if(src1ValueInfo->IsNumber())
  4289. {
  4290. dstVal = ValueNumberTransferDst(instr, src1Val);
  4291. }
  4292. else
  4293. {
  4294. return NewGenericValue(src1ValueInfo->Type().ToDefiniteAnyNumber(), dst);
  4295. }
  4296. break;
  4297. case Js::OpCode::Not_A:
  4298. {
  4299. if (!src1Val || !src1ValueInfo->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec()))
  4300. {
  4301. min1 = INT32_MIN;
  4302. max1 = INT32_MAX;
  4303. }
  4304. this->PropagateIntRangeForNot(min1, max1, &newMin, &newMax);
  4305. return CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, src2Val);
  4306. }
  4307. case Js::OpCode::Xor_A:
  4308. case Js::OpCode::Or_A:
  4309. case Js::OpCode::And_A:
  4310. case Js::OpCode::Shl_A:
  4311. case Js::OpCode::Shr_A:
  4312. case Js::OpCode::ShrU_A:
  4313. {
  4314. if (!src1Val || !src1ValueInfo->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec()))
  4315. {
  4316. min1 = INT32_MIN;
  4317. max1 = INT32_MAX;
  4318. }
  4319. if (!src2Val || !src2ValueInfo->GetIntValMinMax(&min2, &max2, this->DoAggressiveIntTypeSpec()))
  4320. {
  4321. min2 = INT32_MIN;
  4322. max2 = INT32_MAX;
  4323. }
  4324. if (instr->m_opcode == Js::OpCode::ShrU_A &&
  4325. min1 < 0 &&
  4326. IntConstantBounds(min2, max2).And_0x1f().Contains(0))
  4327. {
  4328. // Src1 may be too large to represent as a signed int32, and src2 may be zero.
  4329. // Since the result can therefore be too large to represent as a signed int32,
  4330. // include Number in the value type.
  4331. return CreateDstUntransferredValue(
  4332. ValueType::AnyNumber.SetCanBeTaggedValue(true), instr, src1Val, src2Val);
  4333. }
  4334. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  4335. return CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, src2Val);
  4336. }
  4337. case Js::OpCode::Incr_A:
  4338. case Js::OpCode::Decr_A:
  4339. {
  4340. ValueType valueType;
  4341. if(src1Val)
  4342. {
  4343. valueType = src1Val->GetValueInfo()->Type().ToDefiniteAnyNumber();
  4344. }
  4345. else
  4346. {
  4347. valueType = ValueType::Number;
  4348. }
  4349. return CreateDstUntransferredValue(valueType, instr, src1Val, src2Val);
  4350. }
  4351. case Js::OpCode::Add_A:
  4352. {
  4353. ValueType valueType;
  4354. if (src1Val && src1ValueInfo->IsLikelyNumber() && src2Val && src2ValueInfo->IsLikelyNumber())
  4355. {
  4356. if(src1ValueInfo->IsLikelyInt() && src2ValueInfo->IsLikelyInt())
  4357. {
  4358. // When doing aggressiveIntType, just assume the result is likely going to be int
  4359. // if both input is int.
  4360. const bool isLikelyTagged = src1ValueInfo->IsLikelyTaggedInt() && src2ValueInfo->IsLikelyTaggedInt();
  4361. if(src1ValueInfo->IsNumber() && src2ValueInfo->IsNumber())
  4362. {
  4363. // If both of them are numbers then we can definitely say that the result is a number.
  4364. valueType = ValueType::GetNumberAndLikelyInt(isLikelyTagged);
  4365. }
  4366. else
  4367. {
  4368. // This is only likely going to be int but can be a string as well.
  4369. valueType = ValueType::GetInt(isLikelyTagged).ToLikely();
  4370. }
  4371. }
  4372. else
  4373. {
  4374. // We can only be certain of any thing if both of them are numbers.
  4375. // Otherwise, the result could be string.
  4376. if (src1ValueInfo->IsNumber() && src2ValueInfo->IsNumber())
  4377. {
  4378. if (src1ValueInfo->IsFloat() || src2ValueInfo->IsFloat())
  4379. {
  4380. // If one of them is a float, the result probably is a float instead of just int
  4381. // but should always be a number.
  4382. valueType = ValueType::Float;
  4383. }
  4384. else
  4385. {
  4386. // Could be int, could be number
  4387. valueType = ValueType::Number;
  4388. }
  4389. }
  4390. else if (src1ValueInfo->IsLikelyFloat() || src2ValueInfo->IsLikelyFloat())
  4391. {
  4392. // Result is likely a float (but can be anything)
  4393. valueType = ValueType::Float.ToLikely();
  4394. }
  4395. else
  4396. {
  4397. // Otherwise it is a likely int or float (but can be anything)
  4398. valueType = ValueType::Number.ToLikely();
  4399. }
  4400. }
  4401. }
  4402. else if((src1Val && src1ValueInfo->IsString()) || (src2Val && src2ValueInfo->IsString()))
  4403. {
  4404. // String + anything should always result in a string
  4405. valueType = ValueType::String;
  4406. }
  4407. else if((src1Val && src1ValueInfo->IsNotString() && src1ValueInfo->IsPrimitive())
  4408. && (src2Val && src2ValueInfo->IsNotString() && src2ValueInfo->IsPrimitive()))
  4409. {
  4410. // If src1 and src2 are not strings and primitive, add should yield a number.
  4411. valueType = ValueType::Number;
  4412. }
  4413. else if((src1Val && src1ValueInfo->IsLikelyString()) || (src2Val && src2ValueInfo->IsLikelyString()))
  4414. {
  4415. // likelystring + anything should always result in a likelystring
  4416. valueType = ValueType::String.ToLikely();
  4417. }
  4418. else
  4419. {
  4420. // Number or string. Could make the value a merge of Number and String, but Uninitialized is more useful at the moment.
  4421. Assert(valueType.IsUninitialized());
  4422. }
  4423. return CreateDstUntransferredValue(valueType, instr, src1Val, src2Val);
  4424. }
  4425. case Js::OpCode::Div_A:
  4426. {
  4427. ValueType divValueType = GetDivValueType(instr, src1Val, src2Val, false);
  4428. if (divValueType.IsLikelyInt() || divValueType.IsFloat())
  4429. {
  4430. return CreateDstUntransferredValue(divValueType, instr, src1Val, src2Val);
  4431. }
  4432. }
  4433. // fall-through
  4434. case Js::OpCode::Sub_A:
  4435. case Js::OpCode::Mul_A:
  4436. case Js::OpCode::Rem_A:
  4437. {
  4438. ValueType valueType;
  4439. if( src1Val &&
  4440. src1ValueInfo->IsLikelyInt() &&
  4441. src2Val &&
  4442. src2ValueInfo->IsLikelyInt() &&
  4443. instr->m_opcode != Js::OpCode::Div_A)
  4444. {
  4445. const bool isLikelyTagged =
  4446. src1ValueInfo->IsLikelyTaggedInt() && (src2ValueInfo->IsLikelyTaggedInt() || instr->m_opcode == Js::OpCode::Rem_A);
  4447. if(src1ValueInfo->IsNumber() && src2ValueInfo->IsNumber())
  4448. {
  4449. valueType = ValueType::GetNumberAndLikelyInt(isLikelyTagged);
  4450. }
  4451. else
  4452. {
  4453. valueType = ValueType::GetInt(isLikelyTagged).ToLikely();
  4454. }
  4455. }
  4456. else if ((src1Val && src1ValueInfo->IsLikelyFloat()) || (src2Val && src2ValueInfo->IsLikelyFloat()))
  4457. {
  4458. // This should ideally be NewNumberAndLikelyFloatValue since we know the result is a number but not sure if it will
  4459. // be a float value. However, that Number/LikelyFloat value type doesn't exist currently and all the necessary
  4460. // checks are done for float values (tagged int checks, etc.) so it's sufficient to just create a float value here.
  4461. valueType = ValueType::Float;
  4462. }
  4463. else
  4464. {
  4465. valueType = ValueType::Number;
  4466. }
  4467. return CreateDstUntransferredValue(valueType, instr, src1Val, src2Val);
  4468. }
  4469. case Js::OpCode::CallI:
  4470. Assert(dst->IsRegOpnd());
  4471. return NewGenericValue(dst->AsRegOpnd()->GetValueType(), dst);
  4472. case Js::OpCode::LdElemI_A:
  4473. {
  4474. dstVal = ValueNumberLdElemDst(pInstr, src1Val);
  4475. const ValueType baseValueType(instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType());
  4476. if( (
  4477. baseValueType.IsLikelyNativeArray() ||
  4478. #ifdef _M_IX86
  4479. (
  4480. !AutoSystemInfo::Data.SSE2Available() &&
  4481. baseValueType.IsLikelyObject() &&
  4482. (
  4483. baseValueType.GetObjectType() == ObjectType::Float32Array ||
  4484. baseValueType.GetObjectType() == ObjectType::Float64Array
  4485. )
  4486. )
  4487. #else
  4488. false
  4489. #endif
  4490. ) &&
  4491. instr->GetDst()->IsVar() &&
  4492. instr->HasBailOutInfo())
  4493. {
  4494. // The lowerer is not going to generate a fast path for this case. Remove any bailouts that require the fast
  4495. // path. Note that the removed bailouts should not be necessary for correctness.
  4496. IR::BailOutKind bailOutKind = instr->GetBailOutKind();
  4497. if(bailOutKind & IR::BailOutOnArrayAccessHelperCall)
  4498. {
  4499. bailOutKind -= IR::BailOutOnArrayAccessHelperCall;
  4500. }
  4501. if(bailOutKind == IR::BailOutOnImplicitCallsPreOp)
  4502. {
  4503. bailOutKind -= IR::BailOutOnImplicitCallsPreOp;
  4504. }
  4505. if(bailOutKind)
  4506. {
  4507. instr->SetBailOutKind(bailOutKind);
  4508. }
  4509. else
  4510. {
  4511. instr->ClearBailOutInfo();
  4512. }
  4513. }
  4514. return dstVal;
  4515. }
  4516. case Js::OpCode::LdMethodElem:
  4517. // Not worth profiling this, just assume it's likely object (should be likely function but ValueType does not track
  4518. // functions currently, so using ObjectType::Object instead)
  4519. dstVal = NewGenericValue(ValueType::GetObject(ObjectType::Object).ToLikely(), dst);
  4520. if(instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsLikelyNativeArray() && instr->HasBailOutInfo())
  4521. {
  4522. // The lowerer is not going to generate a fast path for this case. Remove any bailouts that require the fast
  4523. // path. Note that the removed bailouts should not be necessary for correctness.
  4524. IR::BailOutKind bailOutKind = instr->GetBailOutKind();
  4525. if(bailOutKind & IR::BailOutOnArrayAccessHelperCall)
  4526. {
  4527. bailOutKind -= IR::BailOutOnArrayAccessHelperCall;
  4528. }
  4529. if(bailOutKind == IR::BailOutOnImplicitCallsPreOp)
  4530. {
  4531. bailOutKind -= IR::BailOutOnImplicitCallsPreOp;
  4532. }
  4533. if(bailOutKind)
  4534. {
  4535. instr->SetBailOutKind(bailOutKind);
  4536. }
  4537. else
  4538. {
  4539. instr->ClearBailOutInfo();
  4540. }
  4541. }
  4542. return dstVal;
  4543. case Js::OpCode::StElemI_A:
  4544. case Js::OpCode::StElemI_A_Strict:
  4545. dstVal = this->ValueNumberTransferDst(instr, src1Val);
  4546. break;
  4547. case Js::OpCode::LdLen_A:
  4548. if (instr->IsProfiledInstr())
  4549. {
  4550. const ValueType profiledValueType(instr->AsProfiledInstr()->u.FldInfo().valueType);
  4551. if(!(profiledValueType.IsLikelyInt() && dst->AsRegOpnd()->m_sym->m_isNotNumber))
  4552. {
  4553. return this->NewGenericValue(profiledValueType, dst);
  4554. }
  4555. }
  4556. break;
  4557. case Js::OpCode::BrOnEmpty:
  4558. case Js::OpCode::BrOnNotEmpty:
  4559. Assert(dst->IsRegOpnd());
  4560. Assert(dst->GetValueType().IsString());
  4561. return this->NewGenericValue(ValueType::String, dst);
  4562. case Js::OpCode::IsInst:
  4563. case Js::OpCode::LdTrue:
  4564. case Js::OpCode::LdFalse:
  4565. return this->NewGenericValue(ValueType::Boolean, dst);
  4566. case Js::OpCode::LdUndef:
  4567. return this->NewGenericValue(ValueType::Undefined, dst);
  4568. case Js::OpCode::LdC_A_Null:
  4569. return this->NewGenericValue(ValueType::Null, dst);
  4570. case Js::OpCode::LdThis:
  4571. if (!PHASE_OFF(Js::OptTagChecksPhase, this->func) &&
  4572. (src1ValueInfo == nullptr || src1ValueInfo->IsUninitialized()))
  4573. {
  4574. return this->NewGenericValue(ValueType::GetObject(ObjectType::Object).ToLikely().SetCanBeTaggedValue(false), dst);
  4575. }
  4576. break;
  4577. case Js::OpCode::Typeof:
  4578. case Js::OpCode::TypeofElem:
  4579. return this->NewGenericValue(ValueType::String, dst);
  4580. case Js::OpCode::InitLocalClosure:
  4581. Assert(instr->GetDst());
  4582. Assert(instr->GetDst()->IsRegOpnd());
  4583. IR::RegOpnd *regOpnd = instr->GetDst()->AsRegOpnd();
  4584. StackSym *opndStackSym = regOpnd->m_sym;
  4585. Assert(opndStackSym != nullptr);
  4586. ObjectSymInfo *objectSymInfo = opndStackSym->m_objectInfo;
  4587. Assert(objectSymInfo != nullptr);
  4588. for (PropertySym *localVarSlotList = objectSymInfo->m_propertySymList; localVarSlotList; localVarSlotList = localVarSlotList->m_nextInStackSymList)
  4589. {
  4590. this->slotSyms->Set(localVarSlotList->m_id);
  4591. }
  4592. break;
  4593. }
  4594. if (dstVal == nullptr)
  4595. {
  4596. return this->NewGenericValue(dst->GetValueType(), dst);
  4597. }
  4598. return CurrentBlockData()->SetValue(dstVal, dst);
  4599. }
  4600. Value *
  4601. GlobOpt::ValueNumberLdElemDst(IR::Instr **pInstr, Value *srcVal)
  4602. {
  4603. IR::Instr *&instr = *pInstr;
  4604. IR::Opnd *dst = instr->GetDst();
  4605. Value *dstVal = nullptr;
  4606. int32 newMin, newMax;
  4607. ValueInfo *srcValueInfo = (srcVal ? srcVal->GetValueInfo() : nullptr);
  4608. ValueType profiledElementType;
  4609. if (instr->IsProfiledInstr())
  4610. {
  4611. profiledElementType = instr->AsProfiledInstr()->u.ldElemInfo->GetElementType();
  4612. if(!(profiledElementType.IsLikelyInt() && dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->m_isNotNumber) &&
  4613. srcVal &&
  4614. srcValueInfo->IsUninitialized())
  4615. {
  4616. if(IsLoopPrePass())
  4617. {
  4618. dstVal = NewGenericValue(profiledElementType, dst);
  4619. }
  4620. else
  4621. {
  4622. // Assuming the profile data gives more precise value types based on the path it took at runtime, we
  4623. // can improve the original value type.
  4624. srcValueInfo->Type() = profiledElementType;
  4625. instr->GetSrc1()->SetValueType(profiledElementType);
  4626. }
  4627. }
  4628. }
  4629. IR::IndirOpnd *src = instr->GetSrc1()->AsIndirOpnd();
  4630. const ValueType baseValueType(src->GetBaseOpnd()->GetValueType());
  4631. if (instr->DoStackArgsOpt(this->func) ||
  4632. !(
  4633. baseValueType.IsLikelyOptimizedTypedArray() ||
  4634. (baseValueType.IsLikelyNativeArray() && instr->IsProfiledInstr()) // Specialized native array lowering for LdElem requires that it is profiled.
  4635. ) ||
  4636. (!this->DoTypedArrayTypeSpec() && baseValueType.IsLikelyOptimizedTypedArray()) ||
  4637. // Don't do type spec on native array with a history of accessing gaps, as this is a bailout
  4638. (!this->DoNativeArrayTypeSpec() && baseValueType.IsLikelyNativeArray()) ||
  4639. !ShouldExpectConventionalArrayIndexValue(src))
  4640. {
  4641. if(DoTypedArrayTypeSpec() && !IsLoopPrePass())
  4642. {
  4643. GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access.\n"));
  4644. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  4645. {
  4646. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  4647. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  4648. baseValueType.ToString(baseValueTypeStr);
  4649. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not type specialize, because %s.\n"),
  4650. this->func->GetJITFunctionBody()->GetDisplayName(),
  4651. this->func->GetDebugNumberSet(debugStringBuffer),
  4652. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  4653. baseValueTypeStr,
  4654. instr->DoStackArgsOpt(this->func) ? _u("instruction uses the arguments object") :
  4655. baseValueType.IsLikelyOptimizedTypedArray() ? _u("index is negative or likely not int") : _u("of array type"));
  4656. Output::Flush();
  4657. }
  4658. }
  4659. if(!dstVal)
  4660. {
  4661. if(srcVal)
  4662. {
  4663. dstVal = this->ValueNumberTransferDst(instr, srcVal);
  4664. }
  4665. else
  4666. {
  4667. dstVal = NewGenericValue(profiledElementType, dst);
  4668. }
  4669. }
  4670. return dstVal;
  4671. }
  4672. Assert(instr->GetSrc1()->IsIndirOpnd());
  4673. IRType toType = TyVar;
  4674. IR::BailOutKind bailOutKind = IR::BailOutConventionalTypedArrayAccessOnly;
  4675. switch(baseValueType.GetObjectType())
  4676. {
  4677. case ObjectType::Int8Array:
  4678. case ObjectType::Int8VirtualArray:
  4679. case ObjectType::Int8MixedArray:
  4680. newMin = Int8ConstMin;
  4681. newMax = Int8ConstMax;
  4682. goto IntArrayCommon;
  4683. case ObjectType::Uint8Array:
  4684. case ObjectType::Uint8VirtualArray:
  4685. case ObjectType::Uint8MixedArray:
  4686. case ObjectType::Uint8ClampedArray:
  4687. case ObjectType::Uint8ClampedVirtualArray:
  4688. case ObjectType::Uint8ClampedMixedArray:
  4689. newMin = Uint8ConstMin;
  4690. newMax = Uint8ConstMax;
  4691. goto IntArrayCommon;
  4692. case ObjectType::Int16Array:
  4693. case ObjectType::Int16VirtualArray:
  4694. case ObjectType::Int16MixedArray:
  4695. newMin = Int16ConstMin;
  4696. newMax = Int16ConstMax;
  4697. goto IntArrayCommon;
  4698. case ObjectType::Uint16Array:
  4699. case ObjectType::Uint16VirtualArray:
  4700. case ObjectType::Uint16MixedArray:
  4701. newMin = Uint16ConstMin;
  4702. newMax = Uint16ConstMax;
  4703. goto IntArrayCommon;
  4704. case ObjectType::Int32Array:
  4705. case ObjectType::Int32VirtualArray:
  4706. case ObjectType::Int32MixedArray:
  4707. case ObjectType::Uint32Array: // int-specialized loads from uint32 arrays will bail out on values that don't fit in an int32
  4708. case ObjectType::Uint32VirtualArray:
  4709. case ObjectType::Uint32MixedArray:
  4710. Int32Array:
  4711. newMin = Int32ConstMin;
  4712. newMax = Int32ConstMax;
  4713. goto IntArrayCommon;
  4714. IntArrayCommon:
  4715. Assert(dst->IsRegOpnd());
  4716. // If int type spec is disabled, it is ok to load int values as they can help float type spec, and merging int32 with float64 => float64.
  4717. // But if float type spec is also disabled, we'll have problems because float64 merged with var => float64...
  4718. if (!this->DoAggressiveIntTypeSpec() && !this->DoFloatTypeSpec())
  4719. {
  4720. if (!dstVal)
  4721. {
  4722. if (srcVal)
  4723. {
  4724. dstVal = this->ValueNumberTransferDst(instr, srcVal);
  4725. }
  4726. else
  4727. {
  4728. dstVal = NewGenericValue(profiledElementType, dst);
  4729. }
  4730. }
  4731. return dstVal;
  4732. }
  4733. if (!this->IsLoopPrePass())
  4734. {
  4735. if (instr->HasBailOutInfo())
  4736. {
  4737. const IR::BailOutKind oldBailOutKind = instr->GetBailOutKind();
  4738. Assert(
  4739. (
  4740. !(oldBailOutKind & ~IR::BailOutKindBits) ||
  4741. (oldBailOutKind & ~IR::BailOutKindBits) == IR::BailOutOnImplicitCallsPreOp
  4742. ) &&
  4743. !(oldBailOutKind & IR::BailOutKindBits & ~(IR::BailOutOnArrayAccessHelperCall | IR::BailOutMarkTempObject)));
  4744. if (bailOutKind == IR::BailOutConventionalTypedArrayAccessOnly)
  4745. {
  4746. // BailOutConventionalTypedArrayAccessOnly also bails out if the array access is outside the head
  4747. // segment bounds, and guarantees no implicit calls. Override the bailout kind so that the instruction
  4748. // bails out for the right reason.
  4749. instr->SetBailOutKind(
  4750. bailOutKind | (oldBailOutKind & (IR::BailOutKindBits - IR::BailOutOnArrayAccessHelperCall)));
  4751. }
  4752. else
  4753. {
  4754. // BailOutConventionalNativeArrayAccessOnly by itself may generate a helper call, and may cause implicit
  4755. // calls to occur, so it must be merged in to eliminate generating the helper call
  4756. Assert(bailOutKind == IR::BailOutConventionalNativeArrayAccessOnly);
  4757. instr->SetBailOutKind(oldBailOutKind | bailOutKind);
  4758. }
  4759. }
  4760. else
  4761. {
  4762. GenerateBailAtOperation(&instr, bailOutKind);
  4763. }
  4764. }
  4765. TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, nullptr, nullptr, bailOutKind, newMin, newMax, &dstVal);
  4766. toType = TyInt32;
  4767. break;
  4768. case ObjectType::Float32Array:
  4769. case ObjectType::Float32VirtualArray:
  4770. case ObjectType::Float32MixedArray:
  4771. case ObjectType::Float64Array:
  4772. case ObjectType::Float64VirtualArray:
  4773. case ObjectType::Float64MixedArray:
  4774. Float64Array:
  4775. Assert(dst->IsRegOpnd());
  4776. // If float type spec is disabled, don't load float64 values
  4777. if (!this->DoFloatTypeSpec())
  4778. {
  4779. if (!dstVal)
  4780. {
  4781. if (srcVal)
  4782. {
  4783. dstVal = this->ValueNumberTransferDst(instr, srcVal);
  4784. }
  4785. else
  4786. {
  4787. dstVal = NewGenericValue(profiledElementType, dst);
  4788. }
  4789. }
  4790. return dstVal;
  4791. }
  4792. if (!this->IsLoopPrePass())
  4793. {
  4794. if (instr->HasBailOutInfo())
  4795. {
  4796. const IR::BailOutKind oldBailOutKind = instr->GetBailOutKind();
  4797. Assert(
  4798. (
  4799. !(oldBailOutKind & ~IR::BailOutKindBits) ||
  4800. (oldBailOutKind & ~IR::BailOutKindBits) == IR::BailOutOnImplicitCallsPreOp
  4801. ) &&
  4802. !(oldBailOutKind & IR::BailOutKindBits & ~(IR::BailOutOnArrayAccessHelperCall | IR::BailOutMarkTempObject)));
  4803. if (bailOutKind == IR::BailOutConventionalTypedArrayAccessOnly)
  4804. {
  4805. // BailOutConventionalTypedArrayAccessOnly also bails out if the array access is outside the head
  4806. // segment bounds, and guarantees no implicit calls. Override the bailout kind so that the instruction
  4807. // bails out for the right reason.
  4808. instr->SetBailOutKind(
  4809. bailOutKind | (oldBailOutKind & (IR::BailOutKindBits - IR::BailOutOnArrayAccessHelperCall)));
  4810. }
  4811. else
  4812. {
  4813. // BailOutConventionalNativeArrayAccessOnly by itself may generate a helper call, and may cause implicit
  4814. // calls to occur, so it must be merged in to eliminate generating the helper call
  4815. Assert(bailOutKind == IR::BailOutConventionalNativeArrayAccessOnly);
  4816. instr->SetBailOutKind(oldBailOutKind | bailOutKind);
  4817. }
  4818. }
  4819. else
  4820. {
  4821. GenerateBailAtOperation(&instr, bailOutKind);
  4822. }
  4823. }
  4824. TypeSpecializeFloatDst(instr, nullptr, nullptr, nullptr, &dstVal);
  4825. toType = TyFloat64;
  4826. break;
  4827. default:
  4828. Assert(baseValueType.IsLikelyNativeArray());
  4829. bailOutKind = IR::BailOutConventionalNativeArrayAccessOnly;
  4830. if(baseValueType.HasIntElements())
  4831. {
  4832. goto Int32Array;
  4833. }
  4834. Assert(baseValueType.HasFloatElements());
  4835. goto Float64Array;
  4836. }
  4837. if(!dstVal)
  4838. {
  4839. dstVal = NewGenericValue(profiledElementType, dst);
  4840. }
  4841. Assert(toType != TyVar);
  4842. GOPT_TRACE_INSTR(instr, _u("Type specialized array access.\n"));
  4843. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  4844. {
  4845. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  4846. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  4847. baseValueType.ToString(baseValueTypeStr);
  4848. char dstValTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  4849. dstVal->GetValueInfo()->Type().ToString(dstValTypeStr);
  4850. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, type specialized to %s producing %S"),
  4851. this->func->GetJITFunctionBody()->GetDisplayName(),
  4852. this->func->GetDebugNumberSet(debugStringBuffer),
  4853. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  4854. baseValueTypeStr,
  4855. toType == TyInt32 ? _u("int32") : _u("float64"),
  4856. dstValTypeStr);
  4857. #if DBG_DUMP
  4858. Output::Print(_u(" ("));
  4859. dstVal->Dump();
  4860. Output::Print(_u(").\n"));
  4861. #else
  4862. Output::Print(_u(".\n"));
  4863. #endif
  4864. Output::Flush();
  4865. }
  4866. return dstVal;
  4867. }
  4868. ValueType
  4869. GlobOpt::GetPrepassValueTypeForDst(
  4870. const ValueType desiredValueType,
  4871. IR::Instr *const instr,
  4872. Value *const src1Value,
  4873. Value *const src2Value,
  4874. bool const isValueInfoPrecise,
  4875. bool const isSafeToTransferInPrepass) const
  4876. {
  4877. // Values with definite types can be created in the loop prepass only when it is guaranteed that the value type will be the
  4878. // same on any iteration of the loop. The heuristics currently used are:
  4879. // - If the source sym is not live on the back-edge, then it acquires a new value for each iteration of the loop, so
  4880. // that value type can be definite
  4881. // - Consider: A better solution for this is to track values that originate in this loop, which can have definite value
  4882. // types. That catches more cases, should look into that in the future.
  4883. // - If the source sym has a constant value that doesn't change for the duration of the function
  4884. // - The operation always results in a definite value type. For instance, signed bitwise operations always result in an
  4885. // int32, conv_num and ++ always result in a number, etc.
  4886. // - For operations that always result in an int32, the resulting int range is precise only if the source syms pass
  4887. // the above heuristics. Otherwise, the range must be expanded to the full int32 range.
  4888. Assert(IsLoopPrePass());
  4889. Assert(instr);
  4890. if(!isValueInfoPrecise)
  4891. {
  4892. if(!desiredValueType.IsDefinite())
  4893. {
  4894. return isSafeToTransferInPrepass ? desiredValueType : desiredValueType.SetCanBeTaggedValue(true);
  4895. }
  4896. // If the desired value type is not precise, the value type of the destination is derived from the value types of the
  4897. // sources. Since the value type of a source sym is not definite, the destination value type also cannot be definite.
  4898. if(desiredValueType.IsInt() && OpCodeAttr::IsInt32(instr->m_opcode))
  4899. {
  4900. // The op always produces an int32, but not always a tagged int
  4901. return ValueType::GetInt(desiredValueType.IsLikelyTaggedInt());
  4902. }
  4903. if(desiredValueType.IsNumber() && OpCodeAttr::ProducesNumber(instr->m_opcode))
  4904. {
  4905. // The op always produces a number, but not always an int
  4906. return desiredValueType.ToDefiniteAnyNumber();
  4907. }
  4908. // Note: ToLikely() also sets CanBeTaggedValue
  4909. return desiredValueType.ToLikely();
  4910. }
  4911. return desiredValueType;
  4912. }
  4913. bool
  4914. GlobOpt::IsPrepassSrcValueInfoPrecise(IR::Instr *const instr, Value *const src1Value, Value *const src2Value, bool * isSafeToTransferInPrepass) const
  4915. {
  4916. return
  4917. (!instr->GetSrc1() || IsPrepassSrcValueInfoPrecise(instr->GetSrc1(), src1Value, isSafeToTransferInPrepass)) &&
  4918. (!instr->GetSrc2() || IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Value, isSafeToTransferInPrepass));
  4919. }
  4920. bool
  4921. GlobOpt::IsPrepassSrcValueInfoPrecise(IR::Opnd *const src, Value *const srcValue, bool * isSafeToTransferInPrepass) const
  4922. {
  4923. Assert(IsLoopPrePass());
  4924. Assert(src);
  4925. if (isSafeToTransferInPrepass)
  4926. {
  4927. *isSafeToTransferInPrepass = false;
  4928. }
  4929. if (src->IsAddrOpnd() &&
  4930. srcValue->GetValueInfo()->GetSymStore() &&
  4931. srcValue->GetValueInfo()->GetSymStore()->IsStackSym() &&
  4932. srcValue->GetValueInfo()->GetSymStore()->AsStackSym()->IsFromByteCodeConstantTable())
  4933. {
  4934. if (isSafeToTransferInPrepass)
  4935. {
  4936. *isSafeToTransferInPrepass = false;
  4937. }
  4938. return true;
  4939. }
  4940. if (!src->IsRegOpnd() || !srcValue)
  4941. {
  4942. return false;
  4943. }
  4944. ValueInfo *const srcValueInfo = srcValue->GetValueInfo();
  4945. bool isValueInfoDefinite = srcValueInfo->IsDefinite();
  4946. StackSym * srcSym = src->AsRegOpnd()->m_sym;
  4947. bool isSafeToTransfer = IsSafeToTransferInPrepass(srcSym, srcValueInfo);
  4948. if (isSafeToTransferInPrepass)
  4949. {
  4950. *isSafeToTransferInPrepass = isSafeToTransfer;
  4951. }
  4952. return isValueInfoDefinite && isSafeToTransfer;
  4953. }
  4954. bool
  4955. GlobOpt::IsSafeToTransferInPrepass(StackSym * const srcSym, ValueInfo *const srcValueInfo) const
  4956. {
  4957. int32 intConstantValue;
  4958. return
  4959. srcSym->IsFromByteCodeConstantTable() ||
  4960. (
  4961. srcValueInfo->TryGetIntConstantValue(&intConstantValue) &&
  4962. !Js::TaggedInt::IsOverflow(intConstantValue) &&
  4963. GetTaggedIntConstantStackSym(intConstantValue) == srcSym
  4964. ) ||
  4965. !currentBlock->loop->regAlloc.liveOnBackEdgeSyms->Test(srcSym->m_id) ||
  4966. !currentBlock->loop->IsSymAssignedToInSelfOrParents(srcSym);
  4967. }
  4968. bool
  4969. GlobOpt::SafeToCopyPropInPrepass(StackSym * const originalSym, StackSym * const copySym, Value *const value) const
  4970. {
  4971. Assert(this->currentBlock->globOptData.GetCopyPropSym(originalSym, value) == copySym);
  4972. // In the following example, to copy-prop s2 into s1, it is not enough to check if s1 and s2 are safe to transfer.
  4973. // In fact, both s1 and s2 are safe to transfer, but it is not legal to copy prop s2 into s1.
  4974. //
  4975. // s1 = s2
  4976. // $Loop:
  4977. // s3 = s1
  4978. // s2 = s4
  4979. // Br $Loop
  4980. //
  4981. // In general, requirements for copy-propping in prepass are more restricted than those for transferring values.
  4982. // For copy prop in prepass, if the original sym is live on back-edge, then the copy-prop sym should not be written to
  4983. // in the loop (or its parents)
  4984. ValueInfo* const valueInfo = value->GetValueInfo();
  4985. return IsSafeToTransferInPrepass(originalSym, valueInfo) &&
  4986. IsSafeToTransferInPrepass(copySym, valueInfo) &&
  4987. (!currentBlock->loop->regAlloc.liveOnBackEdgeSyms->Test(originalSym->m_id) || !currentBlock->loop->IsSymAssignedToInSelfOrParents(copySym));
  4988. }
  4989. Value *GlobOpt::CreateDstUntransferredIntValue(
  4990. const int32 min,
  4991. const int32 max,
  4992. IR::Instr *const instr,
  4993. Value *const src1Value,
  4994. Value *const src2Value)
  4995. {
  4996. Assert(instr);
  4997. Assert(instr->GetDst());
  4998. Assert(OpCodeAttr::ProducesNumber(instr->m_opcode)
  4999. || (instr->m_opcode == Js::OpCode::Add_A && src1Value->GetValueInfo()->IsNumber()
  5000. && src2Value->GetValueInfo()->IsNumber()));
  5001. ValueType valueType(ValueType::GetInt(IntConstantBounds(min, max).IsLikelyTaggable()));
  5002. Assert(valueType.IsInt());
  5003. bool isValueInfoPrecise;
  5004. if(IsLoopPrePass())
  5005. {
  5006. isValueInfoPrecise = IsPrepassSrcValueInfoPrecise(instr, src1Value, src2Value);
  5007. valueType = GetPrepassValueTypeForDst(valueType, instr, src1Value, src2Value, isValueInfoPrecise);
  5008. }
  5009. else
  5010. {
  5011. isValueInfoPrecise = true;
  5012. }
  5013. IR::Opnd *const dst = instr->GetDst();
  5014. if(isValueInfoPrecise)
  5015. {
  5016. Assert(valueType == ValueType::GetInt(IntConstantBounds(min, max).IsLikelyTaggable()));
  5017. Assert(!(dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->IsTypeSpec()));
  5018. return NewIntRangeValue(min, max, false, dst);
  5019. }
  5020. return NewGenericValue(valueType, dst);
  5021. }
  5022. Value *
  5023. GlobOpt::CreateDstUntransferredValue(
  5024. const ValueType desiredValueType,
  5025. IR::Instr *const instr,
  5026. Value *const src1Value,
  5027. Value *const src2Value)
  5028. {
  5029. Assert(instr);
  5030. Assert(instr->GetDst());
  5031. Assert(!desiredValueType.IsInt()); // use CreateDstUntransferredIntValue instead
  5032. ValueType valueType(desiredValueType);
  5033. if(IsLoopPrePass())
  5034. {
  5035. valueType = GetPrepassValueTypeForDst(valueType, instr, src1Value, src2Value, IsPrepassSrcValueInfoPrecise(instr, src1Value, src2Value));
  5036. }
  5037. return NewGenericValue(valueType, instr->GetDst());
  5038. }
  5039. Value *
  5040. GlobOpt::ValueNumberTransferDst(IR::Instr *const instr, Value * src1Val)
  5041. {
  5042. Value *dstVal = this->IsLoopPrePass() ? this->ValueNumberTransferDstInPrepass(instr, src1Val) : src1Val;
  5043. // Don't copy-prop a temp over a user symbol. This is likely to extend the temp's lifetime, as the user symbol
  5044. // is more likely to already have later references.
  5045. // REVIEW: Enabling this does cause perf issues...
  5046. #if 0
  5047. if (dstVal != src1Val)
  5048. {
  5049. return dstVal;
  5050. }
  5051. Sym *dstSym = dst->GetStackSym();
  5052. if (dstVal && dstSym && dstSym->IsStackSym() && !dstSym->AsStackSym()->m_isBytecodeTmp)
  5053. {
  5054. Sym *dstValSym = dstVal->GetValueInfo()->GetSymStore();
  5055. if (dstValSym && dstValSym->AsStackSym()->m_isBytecodeTmp /* src->GetIsDead()*/)
  5056. {
  5057. dstVal->GetValueInfo()->SetSymStore(dstSym);
  5058. }
  5059. }
  5060. #endif
  5061. return dstVal;
  5062. }
  5063. bool
  5064. GlobOpt::IsSafeToTransferInPrePass(IR::Opnd *src, Value *srcValue)
  5065. {
  5066. if (src->IsRegOpnd())
  5067. {
  5068. StackSym *srcSym = src->AsRegOpnd()->m_sym;
  5069. if (srcSym->IsFromByteCodeConstantTable())
  5070. {
  5071. return true;
  5072. }
  5073. ValueInfo *srcValueInfo = srcValue->GetValueInfo();
  5074. int32 srcIntConstantValue;
  5075. if (srcValueInfo->TryGetIntConstantValue(&srcIntConstantValue) && !Js::TaggedInt::IsOverflow(srcIntConstantValue)
  5076. && GetTaggedIntConstantStackSym(srcIntConstantValue) == srcSym)
  5077. {
  5078. return true;
  5079. }
  5080. }
  5081. return false;
  5082. }
  5083. Value *
  5084. GlobOpt::ValueNumberTransferDstInPrepass(IR::Instr *const instr, Value *const src1Val)
  5085. {
  5086. Value *dstVal = nullptr;
  5087. if (!src1Val)
  5088. {
  5089. return nullptr;
  5090. }
  5091. bool isValueInfoPrecise;
  5092. ValueInfo *const src1ValueInfo = src1Val->GetValueInfo();
  5093. // TODO: This conflicts with new values created by the type specialization code
  5094. // We should re-enable if we change that code to avoid the new values.
  5095. #if 0
  5096. if (this->IsSafeToTransferInPrePass(instr->GetSrc1(), src1Val))
  5097. {
  5098. return src1Val;
  5099. }
  5100. if (this->IsPREInstrCandidateLoad(instr->m_opcode) && instr->GetDst())
  5101. {
  5102. StackSym *dstSym = instr->GetDst()->AsRegOpnd()->m_sym;
  5103. for (Loop *curLoop = this->currentBlock->loop; curLoop; curLoop = curLoop->parent)
  5104. {
  5105. if (curLoop->fieldPRESymStore->Test(dstSym->m_id))
  5106. {
  5107. return src1Val;
  5108. }
  5109. }
  5110. }
  5111. if (instr->GetDst()->IsRegOpnd())
  5112. {
  5113. StackSym *stackSym = instr->GetDst()->AsRegOpnd()->m_sym;
  5114. if (stackSym->IsSingleDef() || this->IsLive(stackSym, this->prePassLoop->landingPad))
  5115. {
  5116. IntConstantBounds src1IntConstantBounds;
  5117. if (src1ValueInfo->TryGetIntConstantBounds(&src1IntConstantBounds) &&
  5118. !(
  5119. src1IntConstantBounds.LowerBound() == INT32_MIN &&
  5120. src1IntConstantBounds.UpperBound() == INT32_MAX
  5121. ))
  5122. {
  5123. const ValueType valueType(
  5124. GetPrepassValueTypeForDst(src1ValueInfo->Type(), instr, src1Val, nullptr, &isValueInfoPrecise));
  5125. if (isValueInfoPrecise)
  5126. {
  5127. return src1Val;
  5128. }
  5129. }
  5130. else
  5131. {
  5132. return src1Val;
  5133. }
  5134. }
  5135. }
  5136. #endif
  5137. // Src1's value could change later in the loop, so the value wouldn't be the same for each
  5138. // iteration. Since we don't iterate over loops "while (!changed)", go conservative on the
  5139. // first pass when transferring a value that is live on the back-edge.
  5140. // In prepass we are going to copy the value but with a different value number
  5141. // for aggressive int type spec.
  5142. bool isSafeToTransferInPrepass = false;
  5143. isValueInfoPrecise = IsPrepassSrcValueInfoPrecise(instr, src1Val, nullptr, &isSafeToTransferInPrepass);
  5144. const ValueType valueType(GetPrepassValueTypeForDst(src1ValueInfo->Type(), instr, src1Val, nullptr, isValueInfoPrecise, isSafeToTransferInPrepass));
  5145. if(isValueInfoPrecise || isSafeToTransferInPrepass)
  5146. {
  5147. Assert(valueType == src1ValueInfo->Type());
  5148. if (!PHASE_OFF1(Js::AVTInPrePassPhase))
  5149. {
  5150. dstVal = src1Val;
  5151. }
  5152. else
  5153. {
  5154. dstVal = CopyValue(src1Val);
  5155. TrackCopiedValueForKills(dstVal);
  5156. }
  5157. }
  5158. else if (valueType == src1ValueInfo->Type() && src1ValueInfo->IsGeneric()) // this else branch is probably not needed
  5159. {
  5160. Assert(valueType == src1ValueInfo->Type());
  5161. dstVal = CopyValue(src1Val);
  5162. TrackCopiedValueForKills(dstVal);
  5163. }
  5164. else
  5165. {
  5166. dstVal = NewGenericValue(valueType);
  5167. dstVal->GetValueInfo()->SetSymStore(src1ValueInfo->GetSymStore());
  5168. }
  5169. return dstVal;
  5170. }
  5171. void
  5172. GlobOpt::PropagateIntRangeForNot(int32 minimum, int32 maximum, int32 *pNewMin, int32* pNewMax)
  5173. {
  5174. int32 tmp;
  5175. Int32Math::Not(minimum, pNewMin);
  5176. *pNewMax = *pNewMin;
  5177. Int32Math::Not(maximum, &tmp);
  5178. *pNewMin = min(*pNewMin, tmp);
  5179. *pNewMax = max(*pNewMax, tmp);
  5180. }
  5181. void
  5182. GlobOpt::PropagateIntRangeBinary(IR::Instr *instr, int32 min1, int32 max1,
  5183. int32 min2, int32 max2, int32 *pNewMin, int32* pNewMax)
  5184. {
  5185. int32 min, max, tmp, tmp2;
  5186. min = INT32_MIN;
  5187. max = INT32_MAX;
  5188. switch (instr->m_opcode)
  5189. {
  5190. case Js::OpCode::Xor_A:
  5191. case Js::OpCode::Or_A:
  5192. // Find range with highest high order bit
  5193. tmp = ::max((uint32)min1, (uint32)max1);
  5194. tmp2 = ::max((uint32)min2, (uint32)max2);
  5195. if ((uint32)tmp > (uint32)tmp2)
  5196. {
  5197. max = tmp;
  5198. }
  5199. else
  5200. {
  5201. max = tmp2;
  5202. }
  5203. if (max < 0)
  5204. {
  5205. min = INT32_MIN; // REVIEW: conservative...
  5206. max = INT32_MAX;
  5207. }
  5208. else
  5209. {
  5210. // Turn values like 0x1010 into 0x1111
  5211. max = 1 << Math::Log2(max);
  5212. max = (uint32)(max << 1) - 1;
  5213. min = 0;
  5214. }
  5215. break;
  5216. case Js::OpCode::And_A:
  5217. if (min1 == INT32_MIN && min2 == INT32_MIN)
  5218. {
  5219. // Shortcut
  5220. break;
  5221. }
  5222. // Find range with lowest higher bit
  5223. tmp = ::max((uint32)min1, (uint32)max1);
  5224. tmp2 = ::max((uint32)min2, (uint32)max2);
  5225. if ((uint32)tmp < (uint32)tmp2)
  5226. {
  5227. min = min1;
  5228. max = max1;
  5229. }
  5230. else
  5231. {
  5232. min = min2;
  5233. max = max2;
  5234. }
  5235. // To compute max, look if min has higher high bit
  5236. if ((uint32)min > (uint32)max)
  5237. {
  5238. max = min;
  5239. }
  5240. // If max is negative, max let's assume it could be -1, so result in MAX_INT
  5241. if (max < 0)
  5242. {
  5243. max = INT32_MAX;
  5244. }
  5245. // If min is positive, the resulting min is zero
  5246. if (min >= 0)
  5247. {
  5248. min = 0;
  5249. }
  5250. else
  5251. {
  5252. min = INT32_MIN;
  5253. }
  5254. break;
  5255. case Js::OpCode::Shl_A:
  5256. {
  5257. // Shift count
  5258. if (min2 != max2 && ((uint32)min2 > 0x1F || (uint32)max2 > 0x1F))
  5259. {
  5260. min2 = 0;
  5261. max2 = 0x1F;
  5262. }
  5263. else
  5264. {
  5265. min2 &= 0x1F;
  5266. max2 &= 0x1F;
  5267. }
  5268. int32 min1FreeTopBitCount = min1 ? (sizeof(int32) * 8) - (Math::Log2(min1) + 1) : (sizeof(int32) * 8);
  5269. int32 max1FreeTopBitCount = max1 ? (sizeof(int32) * 8) - (Math::Log2(max1) + 1) : (sizeof(int32) * 8);
  5270. if (min1FreeTopBitCount <= max2 || max1FreeTopBitCount <= max2)
  5271. {
  5272. // If the shift is going to touch the sign bit return the max range
  5273. min = INT32_MIN;
  5274. max = INT32_MAX;
  5275. }
  5276. else
  5277. {
  5278. // Compute max
  5279. // Turn values like 0x1010 into 0x1111
  5280. if (min1)
  5281. {
  5282. min1 = 1 << Math::Log2(min1);
  5283. min1 = (min1 << 1) - 1;
  5284. }
  5285. if (max1)
  5286. {
  5287. max1 = 1 << Math::Log2(max1);
  5288. max1 = (uint32)(max1 << 1) - 1;
  5289. }
  5290. if (max1 > 0)
  5291. {
  5292. int32 nrTopBits = (sizeof(int32) * 8) - Math::Log2(max1);
  5293. if (nrTopBits < ::min(max2, 30))
  5294. max = INT32_MAX;
  5295. else
  5296. max = ::max((max1 << ::min(max2, 30)) & ~0x80000000, (min1 << min2) & ~0x80000000);
  5297. }
  5298. else
  5299. {
  5300. max = (max1 << min2) & ~0x80000000;
  5301. }
  5302. // Compute min
  5303. if (min1 < 0)
  5304. {
  5305. min = ::min(min1 << max2, max1 << max2);
  5306. }
  5307. else
  5308. {
  5309. min = ::min(min1 << min2, max1 << max2);
  5310. }
  5311. // Turn values like 0x1110 into 0x1000
  5312. if (min)
  5313. {
  5314. min = 1 << Math::Log2(min);
  5315. }
  5316. }
  5317. }
  5318. break;
  5319. case Js::OpCode::Shr_A:
  5320. // Shift count
  5321. if (min2 != max2 && ((uint32)min2 > 0x1F || (uint32)max2 > 0x1F))
  5322. {
  5323. min2 = 0;
  5324. max2 = 0x1F;
  5325. }
  5326. else
  5327. {
  5328. min2 &= 0x1F;
  5329. max2 &= 0x1F;
  5330. }
  5331. // Compute max
  5332. if (max1 < 0)
  5333. {
  5334. max = max1 >> max2;
  5335. }
  5336. else
  5337. {
  5338. max = max1 >> min2;
  5339. }
  5340. // Compute min
  5341. if (min1 < 0)
  5342. {
  5343. min = min1 >> min2;
  5344. }
  5345. else
  5346. {
  5347. min = min1 >> max2;
  5348. }
  5349. break;
  5350. case Js::OpCode::ShrU_A:
  5351. // shift count is constant zero
  5352. if ((min2 == max2) && (max2 & 0x1f) == 0)
  5353. {
  5354. // We can't encode uint32 result, so it has to be used as int32 only or the original value is positive.
  5355. Assert(instr->ignoreIntOverflow || min1 >= 0);
  5356. // We can transfer the signed int32 range.
  5357. min = min1;
  5358. max = max1;
  5359. break;
  5360. }
  5361. const IntConstantBounds src2NewBounds = IntConstantBounds(min2, max2).And_0x1f();
  5362. // Zero is only allowed if result is always a signed int32 or always used as a signed int32
  5363. Assert(min1 >= 0 || instr->ignoreIntOverflow || !src2NewBounds.Contains(0));
  5364. min2 = src2NewBounds.LowerBound();
  5365. max2 = src2NewBounds.UpperBound();
  5366. Assert(min2 <= max2);
  5367. // zero shift count is only allowed if result is used as int32 and/or value is positive
  5368. Assert(min2 > 0 || instr->ignoreIntOverflow || min1 >= 0);
  5369. uint32 umin1 = (uint32)min1;
  5370. uint32 umax1 = (uint32)max1;
  5371. if (umin1 > umax1)
  5372. {
  5373. uint32 temp = umax1;
  5374. umax1 = umin1;
  5375. umin1 = temp;
  5376. }
  5377. Assert(min2 >= 0 && max2 < 32);
  5378. // Compute max
  5379. if (min1 < 0)
  5380. {
  5381. umax1 = UINT32_MAX;
  5382. }
  5383. max = umax1 >> min2;
  5384. // Compute min
  5385. if (min1 <= 0 && max1 >=0)
  5386. {
  5387. min = 0;
  5388. }
  5389. else
  5390. {
  5391. min = umin1 >> max2;
  5392. }
  5393. // We should be able to fit uint32 range as int32
  5394. Assert(instr->ignoreIntOverflow || (min >= 0 && max >= 0) );
  5395. if (min > max)
  5396. {
  5397. // can only happen if shift count can be zero
  5398. Assert(min2 == 0 && (instr->ignoreIntOverflow || min1 >= 0));
  5399. min = Int32ConstMin;
  5400. max = Int32ConstMax;
  5401. }
  5402. break;
  5403. }
  5404. *pNewMin = min;
  5405. *pNewMax = max;
  5406. }
  5407. IR::Instr *
  5408. GlobOpt::TypeSpecialization(
  5409. IR::Instr *instr,
  5410. Value **pSrc1Val,
  5411. Value **pSrc2Val,
  5412. Value **pDstVal,
  5413. bool *redoTypeSpecRef,
  5414. bool *const forceInvariantHoistingRef)
  5415. {
  5416. Value *&src1Val = *pSrc1Val;
  5417. Value *&src2Val = *pSrc2Val;
  5418. *redoTypeSpecRef = false;
  5419. Assert(!*forceInvariantHoistingRef);
  5420. this->ignoredIntOverflowForCurrentInstr = false;
  5421. this->ignoredNegativeZeroForCurrentInstr = false;
  5422. // - Int32 values that can't be tagged are created as float constant values instead because a JavascriptNumber var is needed
  5423. // for that value at runtime. For the purposes of type specialization, recover the int32 values so that they will be
  5424. // treated as ints.
  5425. // - If int overflow does not matter for the instruction, we can additionally treat uint32 values as int32 values because
  5426. // the value resulting from the operation will eventually be converted to int32 anyway
  5427. Value *const src1OriginalVal = src1Val;
  5428. Value *const src2OriginalVal = src2Val;
  5429. if(!instr->ShouldCheckForIntOverflow())
  5430. {
  5431. if(src1Val && src1Val->GetValueInfo()->IsFloatConstant())
  5432. {
  5433. int32 int32Value;
  5434. bool isInt32;
  5435. if(Js::JavascriptNumber::TryGetInt32OrUInt32Value(
  5436. src1Val->GetValueInfo()->AsFloatConstant()->FloatValue(),
  5437. &int32Value,
  5438. &isInt32))
  5439. {
  5440. src1Val = GetIntConstantValue(int32Value, instr);
  5441. if(!isInt32)
  5442. {
  5443. this->ignoredIntOverflowForCurrentInstr = true;
  5444. }
  5445. }
  5446. }
  5447. if(src2Val && src2Val->GetValueInfo()->IsFloatConstant())
  5448. {
  5449. int32 int32Value;
  5450. bool isInt32;
  5451. if(Js::JavascriptNumber::TryGetInt32OrUInt32Value(
  5452. src2Val->GetValueInfo()->AsFloatConstant()->FloatValue(),
  5453. &int32Value,
  5454. &isInt32))
  5455. {
  5456. src2Val = GetIntConstantValue(int32Value, instr);
  5457. if(!isInt32)
  5458. {
  5459. this->ignoredIntOverflowForCurrentInstr = true;
  5460. }
  5461. }
  5462. }
  5463. }
  5464. const AutoRestoreVal autoRestoreSrc1Val(src1OriginalVal, &src1Val);
  5465. const AutoRestoreVal autoRestoreSrc2Val(src2OriginalVal, &src2Val);
  5466. if (src1Val && instr->GetSrc2() == nullptr)
  5467. {
  5468. // Unary
  5469. // Note make sure that native array StElemI gets to TypeSpecializeStElem. Do this for typed arrays, too?
  5470. int32 intConstantValue;
  5471. if (!this->IsLoopPrePass() &&
  5472. !instr->IsBranchInstr() &&
  5473. src1Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue) &&
  5474. !(
  5475. // Nothing to fold for element stores. Go into type specialization to see if they can at least be specialized.
  5476. instr->m_opcode == Js::OpCode::StElemI_A ||
  5477. instr->m_opcode == Js::OpCode::StElemI_A_Strict ||
  5478. instr->m_opcode == Js::OpCode::StElemC ||
  5479. instr->m_opcode == Js::OpCode::MultiBr ||
  5480. instr->m_opcode == Js::OpCode::InlineArrayPop
  5481. ))
  5482. {
  5483. if (OptConstFoldUnary(&instr, intConstantValue, src1Val == src1OriginalVal, pDstVal))
  5484. {
  5485. return instr;
  5486. }
  5487. }
  5488. else if (this->TypeSpecializeUnary(
  5489. &instr,
  5490. &src1Val,
  5491. pDstVal,
  5492. src1OriginalVal,
  5493. redoTypeSpecRef,
  5494. forceInvariantHoistingRef))
  5495. {
  5496. return instr;
  5497. }
  5498. else if(*redoTypeSpecRef)
  5499. {
  5500. return instr;
  5501. }
  5502. }
  5503. else if (instr->GetSrc2() && !instr->IsBranchInstr())
  5504. {
  5505. // Binary
  5506. if (!this->IsLoopPrePass())
  5507. {
  5508. if (GetIsAsmJSFunc())
  5509. {
  5510. if (CONFIG_FLAG(WasmFold))
  5511. {
  5512. bool success = instr->GetSrc1()->IsInt64() ?
  5513. this->OptConstFoldBinaryWasm<int64>(&instr, src1Val, src2Val, pDstVal) :
  5514. this->OptConstFoldBinaryWasm<int>(&instr, src1Val, src2Val, pDstVal);
  5515. if (success)
  5516. {
  5517. return instr;
  5518. }
  5519. }
  5520. }
  5521. else
  5522. {
  5523. // OptConstFoldBinary doesn't do type spec, so only deal with things we are sure are int (IntConstant and IntRange)
  5524. // and not just likely ints TypeSpecializeBinary will deal with type specializing them and fold them again
  5525. IntConstantBounds src1IntConstantBounds, src2IntConstantBounds;
  5526. if (src1Val && src1Val->GetValueInfo()->TryGetIntConstantBounds(&src1IntConstantBounds))
  5527. {
  5528. if (src2Val && src2Val->GetValueInfo()->TryGetIntConstantBounds(&src2IntConstantBounds))
  5529. {
  5530. if (this->OptConstFoldBinary(&instr, src1IntConstantBounds, src2IntConstantBounds, pDstVal))
  5531. {
  5532. return instr;
  5533. }
  5534. }
  5535. }
  5536. }
  5537. }
  5538. }
  5539. if (instr->GetSrc2() && this->TypeSpecializeBinary(&instr, pSrc1Val, pSrc2Val, pDstVal, src1OriginalVal, src2OriginalVal, redoTypeSpecRef))
  5540. {
  5541. if (!this->IsLoopPrePass() &&
  5542. instr->m_opcode != Js::OpCode::Nop &&
  5543. instr->m_opcode != Js::OpCode::Br && // We may have const fold a branch
  5544. // Cannot const-peep if the result of the operation is required for a bailout check
  5545. !(instr->HasBailOutInfo() && instr->GetBailOutKind() & IR::BailOutOnResultConditions))
  5546. {
  5547. if (src1Val && src1Val->GetValueInfo()->HasIntConstantValue())
  5548. {
  5549. if (this->OptConstPeep(instr, instr->GetSrc1(), pDstVal, src1Val->GetValueInfo()))
  5550. {
  5551. return instr;
  5552. }
  5553. }
  5554. else if (src2Val && src2Val->GetValueInfo()->HasIntConstantValue())
  5555. {
  5556. if (this->OptConstPeep(instr, instr->GetSrc2(), pDstVal, src2Val->GetValueInfo()))
  5557. {
  5558. return instr;
  5559. }
  5560. }
  5561. }
  5562. return instr;
  5563. }
  5564. else if(*redoTypeSpecRef)
  5565. {
  5566. return instr;
  5567. }
  5568. if (instr->IsBranchInstr() && !this->IsLoopPrePass())
  5569. {
  5570. if (this->OptConstFoldBranch(instr, src1Val, src2Val, pDstVal))
  5571. {
  5572. return instr;
  5573. }
  5574. }
  5575. // We didn't type specialize, make sure the srcs are unspecialized
  5576. IR::Opnd *src1 = instr->GetSrc1();
  5577. if (src1)
  5578. {
  5579. instr = this->ToVarUses(instr, src1, false, src1Val);
  5580. IR::Opnd *src2 = instr->GetSrc2();
  5581. if (src2)
  5582. {
  5583. instr = this->ToVarUses(instr, src2, false, src2Val);
  5584. }
  5585. }
  5586. IR::Opnd *dst = instr->GetDst();
  5587. if (dst)
  5588. {
  5589. instr = this->ToVarUses(instr, dst, true, nullptr);
  5590. // Handling for instructions other than built-ins that may require only dst type specialization
  5591. // should be added here.
  5592. if(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode) && !GetIsAsmJSFunc()) // don't need to do typespec for asmjs
  5593. {
  5594. this->TypeSpecializeInlineBuiltInDst(&instr, pDstVal);
  5595. return instr;
  5596. }
  5597. // Clear the int specialized bit on the dst.
  5598. if (dst->IsRegOpnd())
  5599. {
  5600. IR::RegOpnd *dstRegOpnd = dst->AsRegOpnd();
  5601. if (!dstRegOpnd->m_sym->IsTypeSpec())
  5602. {
  5603. this->ToVarRegOpnd(dstRegOpnd, this->currentBlock);
  5604. }
  5605. else if (dstRegOpnd->m_sym->IsInt32())
  5606. {
  5607. this->ToInt32Dst(instr, dstRegOpnd, this->currentBlock);
  5608. }
  5609. else if (dstRegOpnd->m_sym->IsUInt32() && GetIsAsmJSFunc())
  5610. {
  5611. this->ToUInt32Dst(instr, dstRegOpnd, this->currentBlock);
  5612. }
  5613. else if (dstRegOpnd->m_sym->IsFloat64())
  5614. {
  5615. this->ToFloat64Dst(instr, dstRegOpnd, this->currentBlock);
  5616. }
  5617. }
  5618. else if (dst->IsSymOpnd() && dst->AsSymOpnd()->m_sym->IsStackSym())
  5619. {
  5620. this->ToVarStackSym(dst->AsSymOpnd()->m_sym->AsStackSym(), this->currentBlock);
  5621. }
  5622. }
  5623. return instr;
  5624. }
  5625. bool
  5626. GlobOpt::OptConstPeep(IR::Instr *instr, IR::Opnd *constSrc, Value **pDstVal, ValueInfo *valuInfo)
  5627. {
  5628. int32 value;
  5629. IR::Opnd *src;
  5630. IR::Opnd *nonConstSrc = (constSrc == instr->GetSrc1() ? instr->GetSrc2() : instr->GetSrc1());
  5631. // Try to find the value from value info first
  5632. if (valuInfo->TryGetIntConstantValue(&value))
  5633. {
  5634. }
  5635. else if (constSrc->IsAddrOpnd())
  5636. {
  5637. IR::AddrOpnd *addrOpnd = constSrc->AsAddrOpnd();
  5638. #ifdef _M_X64
  5639. Assert(addrOpnd->IsVar() || Math::FitsInDWord((size_t)addrOpnd->m_address));
  5640. #else
  5641. Assert(sizeof(value) == sizeof(addrOpnd->m_address));
  5642. #endif
  5643. if (addrOpnd->IsVar())
  5644. {
  5645. value = Js::TaggedInt::ToInt32(addrOpnd->m_address);
  5646. }
  5647. else
  5648. {
  5649. // We asserted that the address will fit in a DWORD above
  5650. value = ::Math::PointerCastToIntegral<int32>(constSrc->AsAddrOpnd()->m_address);
  5651. }
  5652. }
  5653. else if (constSrc->IsIntConstOpnd())
  5654. {
  5655. value = constSrc->AsIntConstOpnd()->AsInt32();
  5656. }
  5657. else
  5658. {
  5659. return false;
  5660. }
  5661. switch(instr->m_opcode)
  5662. {
  5663. // Can't do all Add_A because of string concats.
  5664. // Sub_A cannot be transformed to a NEG_A because 0 - 0 != -0
  5665. case Js::OpCode::Add_A:
  5666. src = nonConstSrc;
  5667. if (!src->GetValueType().IsInt())
  5668. {
  5669. // 0 + -0 != -0
  5670. // "Foo" + 0 != "Foo
  5671. return false;
  5672. }
  5673. // fall-through
  5674. case Js::OpCode::Add_I4:
  5675. if (value != 0)
  5676. {
  5677. return false;
  5678. }
  5679. if (constSrc == instr->GetSrc1())
  5680. {
  5681. src = instr->GetSrc2();
  5682. }
  5683. else
  5684. {
  5685. src = instr->GetSrc1();
  5686. }
  5687. break;
  5688. case Js::OpCode::Mul_A:
  5689. case Js::OpCode::Mul_I4:
  5690. if (value == 0)
  5691. {
  5692. // -0 * 0 != 0
  5693. return false;
  5694. }
  5695. else if (value == 1)
  5696. {
  5697. src = nonConstSrc;
  5698. }
  5699. else
  5700. {
  5701. return false;
  5702. }
  5703. break;
  5704. case Js::OpCode::Div_A:
  5705. if (value == 1 && constSrc == instr->GetSrc2())
  5706. {
  5707. src = instr->GetSrc1();
  5708. }
  5709. else
  5710. {
  5711. return false;
  5712. }
  5713. break;
  5714. case Js::OpCode::Or_I4:
  5715. if (value == -1)
  5716. {
  5717. src = constSrc;
  5718. }
  5719. else if (value == 0)
  5720. {
  5721. src = nonConstSrc;
  5722. }
  5723. else
  5724. {
  5725. return false;
  5726. }
  5727. break;
  5728. case Js::OpCode::And_I4:
  5729. if (value == -1)
  5730. {
  5731. src = nonConstSrc;
  5732. }
  5733. else if (value == 0)
  5734. {
  5735. src = constSrc;
  5736. }
  5737. else
  5738. {
  5739. return false;
  5740. }
  5741. break;
  5742. case Js::OpCode::Shl_I4:
  5743. case Js::OpCode::ShrU_I4:
  5744. case Js::OpCode::Shr_I4:
  5745. if (value != 0 || constSrc != instr->GetSrc2())
  5746. {
  5747. return false;
  5748. }
  5749. src = instr->GetSrc1();
  5750. break;
  5751. default:
  5752. return false;
  5753. }
  5754. this->CaptureByteCodeSymUses(instr);
  5755. if (src == instr->GetSrc1())
  5756. {
  5757. instr->FreeSrc2();
  5758. }
  5759. else
  5760. {
  5761. Assert(src == instr->GetSrc2());
  5762. instr->ReplaceSrc1(instr->UnlinkSrc2());
  5763. }
  5764. instr->m_opcode = Js::OpCode::Ld_A;
  5765. InvalidateInductionVariables(instr);
  5766. return true;
  5767. }
  5768. Js::Var // TODO: michhol OOP JIT, shouldn't play with Vars
  5769. GlobOpt::GetConstantVar(IR::Opnd *opnd, Value *val)
  5770. {
  5771. ValueInfo *valueInfo = val->GetValueInfo();
  5772. if (valueInfo->IsVarConstant() && valueInfo->IsPrimitive())
  5773. {
  5774. return valueInfo->AsVarConstant()->VarValue();
  5775. }
  5776. if (opnd->IsAddrOpnd())
  5777. {
  5778. IR::AddrOpnd *addrOpnd = opnd->AsAddrOpnd();
  5779. if (addrOpnd->IsVar())
  5780. {
  5781. return addrOpnd->m_address;
  5782. }
  5783. }
  5784. else if (opnd->IsIntConstOpnd())
  5785. {
  5786. if (!Js::TaggedInt::IsOverflow(opnd->AsIntConstOpnd()->AsInt32()))
  5787. {
  5788. return Js::TaggedInt::ToVarUnchecked(opnd->AsIntConstOpnd()->AsInt32());
  5789. }
  5790. }
  5791. else if (opnd->IsRegOpnd() && opnd->AsRegOpnd()->m_sym->IsSingleDef())
  5792. {
  5793. if (valueInfo->IsBoolean())
  5794. {
  5795. IR::Instr * defInstr = opnd->AsRegOpnd()->m_sym->GetInstrDef();
  5796. if (defInstr->m_opcode != Js::OpCode::Ld_A || !defInstr->GetSrc1()->IsAddrOpnd())
  5797. {
  5798. return nullptr;
  5799. }
  5800. Assert(defInstr->GetSrc1()->AsAddrOpnd()->IsVar());
  5801. return defInstr->GetSrc1()->AsAddrOpnd()->m_address;
  5802. }
  5803. else if (valueInfo->IsUndefined())
  5804. {
  5805. return (Js::Var)this->func->GetScriptContextInfo()->GetUndefinedAddr();
  5806. }
  5807. else if (valueInfo->IsNull())
  5808. {
  5809. return (Js::Var)this->func->GetScriptContextInfo()->GetNullAddr();
  5810. }
  5811. }
  5812. return nullptr;
  5813. }
  5814. bool BoolAndIntStaticAndTypeMismatch(Value* src1Val, Value* src2Val, Js::Var src1Var, Js::Var src2Var)
  5815. {
  5816. ValueInfo *src1ValInfo = src1Val->GetValueInfo();
  5817. ValueInfo *src2ValInfo = src2Val->GetValueInfo();
  5818. return (src1ValInfo->IsNumber() && src1Var && src2ValInfo->IsBoolean() && src1Var != Js::TaggedInt::ToVarUnchecked(0) && src1Var != Js::TaggedInt::ToVarUnchecked(1)) ||
  5819. (src2ValInfo->IsNumber() && src2Var && src1ValInfo->IsBoolean() && src2Var != Js::TaggedInt::ToVarUnchecked(0) && src2Var != Js::TaggedInt::ToVarUnchecked(1));
  5820. }
  5821. bool
  5822. GlobOpt::CanProveConditionalBranch(IR::Instr *instr, Value *src1Val, Value *src2Val, Js::Var src1Var, Js::Var src2Var, bool *result)
  5823. {
  5824. auto AreSourcesEqual = [&](Value * val1, Value * val2, bool undefinedCmp) -> bool
  5825. {
  5826. // NaN !== NaN, and objects can have valueOf/toString
  5827. if (val1->IsEqualTo(val2))
  5828. {
  5829. if (val1->GetValueInfo()->IsUndefined())
  5830. {
  5831. return undefinedCmp;
  5832. }
  5833. return val1->GetValueInfo()->IsPrimitive() && val1->GetValueInfo()->IsNotFloat();
  5834. }
  5835. return false;
  5836. };
  5837. // Make sure GetConstantVar only returns primitives.
  5838. // TODO: OOP JIT, enabled these asserts
  5839. //Assert(!src1Var || !Js::JavascriptOperators::IsObject(src1Var));
  5840. //Assert(!src2Var || !Js::JavascriptOperators::IsObject(src2Var));
  5841. int64 left64, right64;
  5842. int32 left, right;
  5843. int32 constVal;
  5844. switch (instr->m_opcode)
  5845. {
  5846. #define BRANCHSIGNED(OPCODE,CMP,TYPE,UNSIGNEDNESS,UNDEFINEDCMP) \
  5847. case Js::OpCode::##OPCODE: \
  5848. if (src1Val && src2Val) \
  5849. { \
  5850. if (src1Val->GetValueInfo()->TryGetIntConstantValue(&left, UNSIGNEDNESS) && \
  5851. src2Val->GetValueInfo()->TryGetIntConstantValue(&right, UNSIGNEDNESS)) \
  5852. { \
  5853. *result = (TYPE)left CMP(TYPE)right; \
  5854. } \
  5855. if (src1Val->GetValueInfo()->TryGetInt64ConstantValue(&left64, UNSIGNEDNESS) && \
  5856. src2Val->GetValueInfo()->TryGetInt64ConstantValue(&right64, UNSIGNEDNESS)) \
  5857. { \
  5858. *result = (TYPE)left64 CMP(TYPE)right64; \
  5859. } \
  5860. else if (AreSourcesEqual(src1Val, src2Val, UNDEFINEDCMP)) \
  5861. { \
  5862. *result = 0 CMP 0; \
  5863. } \
  5864. else \
  5865. { \
  5866. return false; \
  5867. } \
  5868. } \
  5869. else \
  5870. { \
  5871. return false; \
  5872. } \
  5873. break;
  5874. BRANCHSIGNED(BrEq_I4, == , int64, false, true)
  5875. BRANCHSIGNED(BrGe_I4, >= , int64, false, false)
  5876. BRANCHSIGNED(BrGt_I4, > , int64, false, false)
  5877. BRANCHSIGNED(BrLt_I4, < , int64, false, false)
  5878. BRANCHSIGNED(BrLe_I4, <= , int64, false, false)
  5879. BRANCHSIGNED(BrNeq_I4, != , int64, false, false)
  5880. BRANCHSIGNED(BrUnGe_I4, >= , uint64, true, false)
  5881. BRANCHSIGNED(BrUnGt_I4, > , uint64, true, false)
  5882. BRANCHSIGNED(BrUnLt_I4, < , uint64, true, false)
  5883. BRANCHSIGNED(BrUnLe_I4, <= , uint64, true, false)
  5884. #undef BRANCHSIGNED
  5885. #define BRANCH(OPCODE,CMP,VARCMPFUNC,UNDEFINEDCMP) \
  5886. case Js::OpCode::##OPCODE: \
  5887. if (src1Val && src2Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&left) && \
  5888. src2Val->GetValueInfo()->TryGetIntConstantValue(&right)) \
  5889. { \
  5890. *result = left CMP right; \
  5891. } \
  5892. else if (src1Val && src2Val && AreSourcesEqual(src1Val, src2Val, UNDEFINEDCMP)) \
  5893. { \
  5894. *result = 0 CMP 0; \
  5895. } \
  5896. else if (src1Var && src2Var) \
  5897. { \
  5898. if (func->IsOOPJIT() || !CONFIG_FLAG(OOPJITMissingOpts)) \
  5899. { \
  5900. return false; \
  5901. } \
  5902. *result = VARCMPFUNC(src1Var, src2Var, this->func->GetScriptContext()); \
  5903. } \
  5904. else \
  5905. { \
  5906. return false; \
  5907. } \
  5908. break;
  5909. BRANCH(BrGe_A, >= , Js::JavascriptOperators::GreaterEqual, /*undefinedEquality*/ false)
  5910. BRANCH(BrNotGe_A, <, !Js::JavascriptOperators::GreaterEqual, false)
  5911. BRANCH(BrLt_A, <, Js::JavascriptOperators::Less, false)
  5912. BRANCH(BrNotLt_A, >= , !Js::JavascriptOperators::Less, false)
  5913. BRANCH(BrGt_A, >, Js::JavascriptOperators::Greater, false)
  5914. BRANCH(BrNotGt_A, <= , !Js::JavascriptOperators::Greater, false)
  5915. BRANCH(BrLe_A, <= , Js::JavascriptOperators::LessEqual, false)
  5916. BRANCH(BrNotLe_A, >, !Js::JavascriptOperators::LessEqual, false)
  5917. #undef BRANCH
  5918. case Js::OpCode::BrEq_A:
  5919. case Js::OpCode::BrNotNeq_A:
  5920. if (src1Val && src2Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&left) &&
  5921. src2Val->GetValueInfo()->TryGetIntConstantValue(&right))
  5922. {
  5923. *result = left == right;
  5924. }
  5925. else if (src1Val && src2Val && AreSourcesEqual(src1Val, src2Val, true))
  5926. {
  5927. *result = true;
  5928. }
  5929. else if (!src1Var || !src2Var)
  5930. {
  5931. if (BoolAndIntStaticAndTypeMismatch(src1Val, src2Val, src1Var, src2Var))
  5932. {
  5933. *result = false;
  5934. }
  5935. else
  5936. {
  5937. return false;
  5938. }
  5939. }
  5940. else
  5941. {
  5942. if (func->IsOOPJIT() || !CONFIG_FLAG(OOPJITMissingOpts))
  5943. {
  5944. // TODO: OOP JIT, const folding
  5945. return false;
  5946. }
  5947. *result = Js::JavascriptOperators::Equal(src1Var, src2Var, this->func->GetScriptContext());
  5948. }
  5949. break;
  5950. case Js::OpCode::BrNeq_A:
  5951. case Js::OpCode::BrNotEq_A:
  5952. if (src1Val && src2Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&left) &&
  5953. src2Val->GetValueInfo()->TryGetIntConstantValue(&right))
  5954. {
  5955. *result = left != right;
  5956. }
  5957. else if (src1Val && src2Val && AreSourcesEqual(src1Val, src2Val, true))
  5958. {
  5959. *result = false;
  5960. }
  5961. else if (!src1Var || !src2Var)
  5962. {
  5963. if (BoolAndIntStaticAndTypeMismatch(src1Val, src2Val, src1Var, src2Var))
  5964. {
  5965. *result = true;
  5966. }
  5967. else
  5968. {
  5969. return false;
  5970. }
  5971. }
  5972. else
  5973. {
  5974. if (func->IsOOPJIT() || !CONFIG_FLAG(OOPJITMissingOpts))
  5975. {
  5976. // TODO: OOP JIT, const folding
  5977. return false;
  5978. }
  5979. *result = Js::JavascriptOperators::NotEqual(src1Var, src2Var, this->func->GetScriptContext());
  5980. }
  5981. break;
  5982. case Js::OpCode::BrSrEq_A:
  5983. case Js::OpCode::BrSrNotNeq_A:
  5984. if (!src1Var || !src2Var)
  5985. {
  5986. ValueInfo *src1ValInfo = src1Val->GetValueInfo();
  5987. ValueInfo *src2ValInfo = src2Val->GetValueInfo();
  5988. if (
  5989. (src1ValInfo->IsUndefined() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenUndefined()) ||
  5990. (src1ValInfo->IsNull() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNull()) ||
  5991. (src1ValInfo->IsBoolean() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenBoolean()) ||
  5992. (src1ValInfo->IsNumber() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNumber()) ||
  5993. (src1ValInfo->IsString() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenString()) ||
  5994. (src2ValInfo->IsUndefined() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenUndefined()) ||
  5995. (src2ValInfo->IsNull() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNull()) ||
  5996. (src2ValInfo->IsBoolean() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenBoolean()) ||
  5997. (src2ValInfo->IsNumber() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNumber()) ||
  5998. (src2ValInfo->IsString() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenString())
  5999. )
  6000. {
  6001. *result = false;
  6002. }
  6003. else if (AreSourcesEqual(src1Val, src2Val, true))
  6004. {
  6005. *result = true;
  6006. }
  6007. else
  6008. {
  6009. return false;
  6010. }
  6011. }
  6012. else
  6013. {
  6014. if (func->IsOOPJIT() || !CONFIG_FLAG(OOPJITMissingOpts))
  6015. {
  6016. // TODO: OOP JIT, const folding
  6017. return false;
  6018. }
  6019. *result = Js::JavascriptOperators::StrictEqual(src1Var, src2Var, this->func->GetScriptContext());
  6020. }
  6021. break;
  6022. case Js::OpCode::BrSrNeq_A:
  6023. case Js::OpCode::BrSrNotEq_A:
  6024. if (!src1Var || !src2Var)
  6025. {
  6026. ValueInfo *src1ValInfo = src1Val->GetValueInfo();
  6027. ValueInfo *src2ValInfo = src2Val->GetValueInfo();
  6028. if (
  6029. (src1ValInfo->IsUndefined() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenUndefined()) ||
  6030. (src1ValInfo->IsNull() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNull()) ||
  6031. (src1ValInfo->IsBoolean() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenBoolean()) ||
  6032. (src1ValInfo->IsNumber() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNumber()) ||
  6033. (src1ValInfo->IsString() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenString()) ||
  6034. (src2ValInfo->IsUndefined() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenUndefined()) ||
  6035. (src2ValInfo->IsNull() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNull()) ||
  6036. (src2ValInfo->IsBoolean() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenBoolean()) ||
  6037. (src2ValInfo->IsNumber() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNumber()) ||
  6038. (src2ValInfo->IsString() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenString())
  6039. )
  6040. {
  6041. *result = true;
  6042. }
  6043. else if (AreSourcesEqual(src1Val, src2Val, true))
  6044. {
  6045. *result = false;
  6046. }
  6047. else
  6048. {
  6049. return false;
  6050. }
  6051. }
  6052. else
  6053. {
  6054. if (func->IsOOPJIT() || !CONFIG_FLAG(OOPJITMissingOpts))
  6055. {
  6056. // TODO: OOP JIT, const folding
  6057. return false;
  6058. }
  6059. *result = Js::JavascriptOperators::NotStrictEqual(src1Var, src2Var, this->func->GetScriptContext());
  6060. }
  6061. break;
  6062. case Js::OpCode::BrFalse_A:
  6063. case Js::OpCode::BrTrue_A:
  6064. {
  6065. ValueInfo *const src1ValueInfo = src1Val->GetValueInfo();
  6066. if (src1ValueInfo->IsNull() || src1ValueInfo->IsUndefined())
  6067. {
  6068. *result = instr->m_opcode == Js::OpCode::BrFalse_A;
  6069. break;
  6070. }
  6071. if (src1ValueInfo->IsObject() && src1ValueInfo->GetObjectType() > ObjectType::Object)
  6072. {
  6073. // Specific object types that are tracked are equivalent to 'true'
  6074. *result = instr->m_opcode == Js::OpCode::BrTrue_A;
  6075. break;
  6076. }
  6077. if (func->IsOOPJIT() || !CONFIG_FLAG(OOPJITMissingOpts))
  6078. {
  6079. // TODO: OOP JIT, const folding
  6080. return false;
  6081. }
  6082. if (!src1Var)
  6083. {
  6084. return false;
  6085. }
  6086. *result = Js::JavascriptConversion::ToBoolean(src1Var, this->func->GetScriptContext());
  6087. if (instr->m_opcode == Js::OpCode::BrFalse_A)
  6088. {
  6089. *result = !(*result);
  6090. }
  6091. break;
  6092. }
  6093. case Js::OpCode::BrFalse_I4:
  6094. {
  6095. constVal = 0;
  6096. if (!src1Val->GetValueInfo()->TryGetIntConstantValue(&constVal))
  6097. {
  6098. return false;
  6099. }
  6100. *result = constVal == 0;
  6101. break;
  6102. }
  6103. case Js::OpCode::BrOnObject_A:
  6104. {
  6105. ValueInfo *const src1ValueInfo = src1Val->GetValueInfo();
  6106. if (!src1ValueInfo->IsDefinite())
  6107. {
  6108. return false;
  6109. }
  6110. *result = !src1ValueInfo->IsPrimitive();
  6111. break;
  6112. }
  6113. default:
  6114. return false;
  6115. }
  6116. return true;
  6117. }
  6118. bool
  6119. GlobOpt::OptConstFoldBranch(IR::Instr *instr, Value *src1Val, Value*src2Val, Value **pDstVal)
  6120. {
  6121. if (!src1Val)
  6122. {
  6123. return false;
  6124. }
  6125. Js::Var src1Var = this->GetConstantVar(instr->GetSrc1(), src1Val);
  6126. Js::Var src2Var = nullptr;
  6127. if (instr->GetSrc2())
  6128. {
  6129. if (!src2Val)
  6130. {
  6131. return false;
  6132. }
  6133. src2Var = this->GetConstantVar(instr->GetSrc2(), src2Val);
  6134. }
  6135. bool result;
  6136. if (!CanProveConditionalBranch(instr, src1Val, src2Val, src1Var, src2Var, &result))
  6137. {
  6138. return false;
  6139. }
  6140. this->OptConstFoldBr(!!result, instr);
  6141. return true;
  6142. }
  6143. bool
  6144. GlobOpt::OptConstFoldUnary(
  6145. IR::Instr * *pInstr,
  6146. const int32 intConstantValue,
  6147. const bool isUsingOriginalSrc1Value,
  6148. Value **pDstVal)
  6149. {
  6150. IR::Instr * &instr = *pInstr;
  6151. int32 value = 0;
  6152. IR::Opnd *constOpnd;
  6153. bool isInt = true;
  6154. bool doSetDstVal = true;
  6155. FloatConstType fValue = 0.0;
  6156. if (!DoConstFold())
  6157. {
  6158. return false;
  6159. }
  6160. if (instr->GetDst() && !instr->GetDst()->IsRegOpnd())
  6161. {
  6162. return false;
  6163. }
  6164. switch(instr->m_opcode)
  6165. {
  6166. case Js::OpCode::Neg_A:
  6167. if (intConstantValue == 0)
  6168. {
  6169. // Could fold to -0.0
  6170. return false;
  6171. }
  6172. if (Int32Math::Neg(intConstantValue, &value))
  6173. {
  6174. return false;
  6175. }
  6176. break;
  6177. case Js::OpCode::Not_A:
  6178. Int32Math::Not(intConstantValue, &value);
  6179. break;
  6180. case Js::OpCode::Ld_A:
  6181. if (instr->HasBailOutInfo())
  6182. {
  6183. //The profile data for switch expr can be string and in GlobOpt we realize it is an int.
  6184. if(instr->GetBailOutKind() == IR::BailOutExpectingString)
  6185. {
  6186. throw Js::RejitException(RejitReason::DisableSwitchOptExpectingString);
  6187. }
  6188. Assert(instr->GetBailOutKind() == IR::BailOutExpectingInteger);
  6189. instr->ClearBailOutInfo();
  6190. }
  6191. value = intConstantValue;
  6192. if(isUsingOriginalSrc1Value)
  6193. {
  6194. doSetDstVal = false; // Let OptDst do it by copying src1Val
  6195. }
  6196. break;
  6197. case Js::OpCode::Conv_Num:
  6198. case Js::OpCode::LdC_A_I4:
  6199. value = intConstantValue;
  6200. if(isUsingOriginalSrc1Value)
  6201. {
  6202. doSetDstVal = false; // Let OptDst do it by copying src1Val
  6203. }
  6204. break;
  6205. case Js::OpCode::Incr_A:
  6206. if (Int32Math::Inc(intConstantValue, &value))
  6207. {
  6208. return false;
  6209. }
  6210. break;
  6211. case Js::OpCode::Decr_A:
  6212. if (Int32Math::Dec(intConstantValue, &value))
  6213. {
  6214. return false;
  6215. }
  6216. break;
  6217. case Js::OpCode::InlineMathAcos:
  6218. fValue = Js::Math::Acos((double)intConstantValue);
  6219. isInt = false;
  6220. break;
  6221. case Js::OpCode::InlineMathAsin:
  6222. fValue = Js::Math::Asin((double)intConstantValue);
  6223. isInt = false;
  6224. break;
  6225. case Js::OpCode::InlineMathAtan:
  6226. fValue = Js::Math::Atan((double)intConstantValue);
  6227. isInt = false;
  6228. break;
  6229. case Js::OpCode::InlineMathCos:
  6230. fValue = Js::Math::Cos((double)intConstantValue);
  6231. isInt = false;
  6232. break;
  6233. case Js::OpCode::InlineMathExp:
  6234. fValue = Js::Math::Exp((double)intConstantValue);
  6235. isInt = false;
  6236. break;
  6237. case Js::OpCode::InlineMathLog:
  6238. fValue = Js::Math::Log((double)intConstantValue);
  6239. isInt = false;
  6240. break;
  6241. case Js::OpCode::InlineMathSin:
  6242. fValue = Js::Math::Sin((double)intConstantValue);
  6243. isInt = false;
  6244. break;
  6245. case Js::OpCode::InlineMathSqrt:
  6246. fValue = ::sqrt((double)intConstantValue);
  6247. isInt = false;
  6248. break;
  6249. case Js::OpCode::InlineMathTan:
  6250. fValue = ::tan((double)intConstantValue);
  6251. isInt = false;
  6252. break;
  6253. case Js::OpCode::InlineMathFround:
  6254. fValue = (double) (float) intConstantValue;
  6255. isInt = false;
  6256. break;
  6257. case Js::OpCode::InlineMathAbs:
  6258. if (intConstantValue == INT32_MIN)
  6259. {
  6260. if (instr->GetDst()->IsInt32())
  6261. {
  6262. // if dst is an int (e.g. in asm.js), we should coerce it, not convert to float
  6263. value = static_cast<int32>(2147483648U);
  6264. }
  6265. else
  6266. {
  6267. // Rejit with AggressiveIntTypeSpecDisabled for Math.abs(INT32_MIN) because it causes dst
  6268. // to be float type which could be different with previous type spec result in LoopPrePass
  6269. throw Js::RejitException(RejitReason::AggressiveIntTypeSpecDisabled);
  6270. }
  6271. }
  6272. else
  6273. {
  6274. value = ::abs(intConstantValue);
  6275. }
  6276. break;
  6277. case Js::OpCode::InlineMathClz:
  6278. DWORD clz;
  6279. if (_BitScanReverse(&clz, intConstantValue))
  6280. {
  6281. value = 31 - clz;
  6282. }
  6283. else
  6284. {
  6285. value = 32;
  6286. }
  6287. instr->ClearBailOutInfo();
  6288. break;
  6289. case Js::OpCode::Ctz:
  6290. Assert(func->GetJITFunctionBody()->IsWasmFunction());
  6291. Assert(!instr->HasBailOutInfo());
  6292. DWORD ctz;
  6293. if (_BitScanForward(&ctz, intConstantValue))
  6294. {
  6295. value = ctz;
  6296. }
  6297. else
  6298. {
  6299. value = 32;
  6300. }
  6301. break;
  6302. case Js::OpCode::InlineMathFloor:
  6303. value = intConstantValue;
  6304. instr->ClearBailOutInfo();
  6305. break;
  6306. case Js::OpCode::InlineMathCeil:
  6307. value = intConstantValue;
  6308. instr->ClearBailOutInfo();
  6309. break;
  6310. case Js::OpCode::InlineMathRound:
  6311. value = intConstantValue;
  6312. instr->ClearBailOutInfo();
  6313. break;
  6314. case Js::OpCode::ToVar:
  6315. if (Js::TaggedInt::IsOverflow(intConstantValue))
  6316. {
  6317. return false;
  6318. }
  6319. else
  6320. {
  6321. value = intConstantValue;
  6322. instr->ClearBailOutInfo();
  6323. break;
  6324. }
  6325. default:
  6326. return false;
  6327. }
  6328. this->CaptureByteCodeSymUses(instr);
  6329. Assert(!instr->HasBailOutInfo()); // If we are, in fact, successful in constant folding the instruction, there is no point in having the bailoutinfo around anymore.
  6330. // Make sure that it is cleared if it was initially present.
  6331. if (!isInt)
  6332. {
  6333. value = (int32)fValue;
  6334. if (fValue == (double)value)
  6335. {
  6336. isInt = true;
  6337. }
  6338. }
  6339. if (isInt)
  6340. {
  6341. constOpnd = IR::IntConstOpnd::New(value, TyInt32, instr->m_func);
  6342. GOPT_TRACE(_u("Constant folding to %d\n"), value);
  6343. }
  6344. else
  6345. {
  6346. constOpnd = IR::FloatConstOpnd::New(fValue, TyFloat64, instr->m_func);
  6347. GOPT_TRACE(_u("Constant folding to %f\n"), fValue);
  6348. }
  6349. instr->ReplaceSrc1(constOpnd);
  6350. this->OptSrc(constOpnd, &instr);
  6351. IR::Opnd *dst = instr->GetDst();
  6352. Assert(dst->IsRegOpnd());
  6353. StackSym *dstSym = dst->AsRegOpnd()->m_sym;
  6354. if (isInt)
  6355. {
  6356. if (dstSym->IsSingleDef())
  6357. {
  6358. dstSym->SetIsIntConst(value);
  6359. }
  6360. if (doSetDstVal)
  6361. {
  6362. *pDstVal = GetIntConstantValue(value, instr, dst);
  6363. }
  6364. if (IsTypeSpecPhaseOff(this->func))
  6365. {
  6366. instr->m_opcode = Js::OpCode::LdC_A_I4;
  6367. this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock);
  6368. }
  6369. else
  6370. {
  6371. instr->m_opcode = Js::OpCode::Ld_I4;
  6372. this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  6373. StackSym * currDstSym = instr->GetDst()->AsRegOpnd()->m_sym;
  6374. if (currDstSym->IsSingleDef())
  6375. {
  6376. currDstSym->SetIsIntConst(value);
  6377. }
  6378. }
  6379. }
  6380. else
  6381. {
  6382. *pDstVal = NewFloatConstantValue(fValue, dst);
  6383. if (IsTypeSpecPhaseOff(this->func))
  6384. {
  6385. instr->m_opcode = Js::OpCode::LdC_A_R8;
  6386. this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock);
  6387. }
  6388. else
  6389. {
  6390. instr->m_opcode = Js::OpCode::LdC_F8_R8;
  6391. this->ToFloat64Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  6392. }
  6393. }
  6394. InvalidateInductionVariables(instr);
  6395. return true;
  6396. }
  6397. //------------------------------------------------------------------------------------------------------
  6398. // Type specialization
  6399. //------------------------------------------------------------------------------------------------------
  6400. bool
  6401. GlobOpt::IsWorthSpecializingToInt32DueToSrc(IR::Opnd *const src, Value *const val)
  6402. {
  6403. Assert(src);
  6404. Assert(val);
  6405. ValueInfo *valueInfo = val->GetValueInfo();
  6406. Assert(valueInfo->IsLikelyInt());
  6407. // If it is not known that the operand is definitely an int, the operand is not already type-specialized, and it's not live
  6408. // in the loop landing pad (if we're in a loop), it's probably not worth type-specializing this instruction. The common case
  6409. // where type-specializing this would be bad is where the operations are entirely on properties or array elements, where the
  6410. // ratio of FromVars and ToVars to the number of actual operations is high, and the conversions would dominate the time
  6411. // spent. On the other hand, if we're using a function formal parameter more than once, it would probably be worth
  6412. // type-specializing it, hence the IsDead check on the operands.
  6413. return
  6414. valueInfo->IsInt() ||
  6415. valueInfo->HasIntConstantValue(true) ||
  6416. !src->GetIsDead() ||
  6417. !src->IsRegOpnd() ||
  6418. CurrentBlockData()->IsInt32TypeSpecialized(src->AsRegOpnd()->m_sym) ||
  6419. (this->currentBlock->loop && this->currentBlock->loop->landingPad->globOptData.IsLive(src->AsRegOpnd()->m_sym));
  6420. }
  6421. bool
  6422. GlobOpt::IsWorthSpecializingToInt32DueToDst(IR::Opnd *const dst)
  6423. {
  6424. Assert(dst);
  6425. const auto sym = dst->AsRegOpnd()->m_sym;
  6426. return
  6427. CurrentBlockData()->IsInt32TypeSpecialized(sym) ||
  6428. (this->currentBlock->loop && this->currentBlock->loop->landingPad->globOptData.IsLive(sym));
  6429. }
  6430. bool
  6431. GlobOpt::IsWorthSpecializingToInt32(IR::Instr *const instr, Value *const src1Val, Value *const src2Val)
  6432. {
  6433. Assert(instr);
  6434. const auto src1 = instr->GetSrc1();
  6435. const auto src2 = instr->GetSrc2();
  6436. // In addition to checking each operand and the destination, if for any reason we only have to do a maximum of two
  6437. // conversions instead of the worst-case 3 conversions, it's probably worth specializing.
  6438. if (IsWorthSpecializingToInt32DueToSrc(src1, src1Val) ||
  6439. (src2Val && IsWorthSpecializingToInt32DueToSrc(src2, src2Val)))
  6440. {
  6441. return true;
  6442. }
  6443. IR::Opnd *dst = instr->GetDst();
  6444. if (!dst || IsWorthSpecializingToInt32DueToDst(dst))
  6445. {
  6446. return true;
  6447. }
  6448. if (dst->IsEqual(src1) || (src2Val && (dst->IsEqual(src2) || src1->IsEqual(src2))))
  6449. {
  6450. return true;
  6451. }
  6452. IR::Instr *instrNext = instr->GetNextRealInstrOrLabel();
  6453. // Skip useless Ld_A's
  6454. do
  6455. {
  6456. switch (instrNext->m_opcode)
  6457. {
  6458. case Js::OpCode::Ld_A:
  6459. if (!dst->IsEqual(instrNext->GetSrc1()))
  6460. {
  6461. goto done;
  6462. }
  6463. dst = instrNext->GetDst();
  6464. break;
  6465. case Js::OpCode::LdFld:
  6466. case Js::OpCode::LdRootFld:
  6467. case Js::OpCode::LdRootFldForTypeOf:
  6468. case Js::OpCode::LdFldForTypeOf:
  6469. case Js::OpCode::LdElemI_A:
  6470. case Js::OpCode::ByteCodeUses:
  6471. break;
  6472. default:
  6473. goto done;
  6474. }
  6475. instrNext = instrNext->GetNextRealInstrOrLabel();
  6476. } while (true);
  6477. done:
  6478. // If the next instr could also be type specialized, then it is probably worth it.
  6479. if ((instrNext->GetSrc1() && dst->IsEqual(instrNext->GetSrc1())) || (instrNext->GetSrc2() && dst->IsEqual(instrNext->GetSrc2())))
  6480. {
  6481. switch (instrNext->m_opcode)
  6482. {
  6483. case Js::OpCode::Add_A:
  6484. case Js::OpCode::Sub_A:
  6485. case Js::OpCode::Mul_A:
  6486. case Js::OpCode::Div_A:
  6487. case Js::OpCode::Rem_A:
  6488. case Js::OpCode::Xor_A:
  6489. case Js::OpCode::And_A:
  6490. case Js::OpCode::Or_A:
  6491. case Js::OpCode::Shl_A:
  6492. case Js::OpCode::Shr_A:
  6493. case Js::OpCode::Incr_A:
  6494. case Js::OpCode::Decr_A:
  6495. case Js::OpCode::Neg_A:
  6496. case Js::OpCode::Not_A:
  6497. case Js::OpCode::Conv_Num:
  6498. case Js::OpCode::BrEq_I4:
  6499. case Js::OpCode::BrTrue_I4:
  6500. case Js::OpCode::BrFalse_I4:
  6501. case Js::OpCode::BrGe_I4:
  6502. case Js::OpCode::BrGt_I4:
  6503. case Js::OpCode::BrLt_I4:
  6504. case Js::OpCode::BrLe_I4:
  6505. case Js::OpCode::BrNeq_I4:
  6506. return true;
  6507. }
  6508. }
  6509. return false;
  6510. }
  6511. bool
  6512. GlobOpt::TypeSpecializeNumberUnary(IR::Instr *instr, Value *src1Val, Value **pDstVal)
  6513. {
  6514. Assert(src1Val->GetValueInfo()->IsNumber());
  6515. if (this->IsLoopPrePass())
  6516. {
  6517. return false;
  6518. }
  6519. switch (instr->m_opcode)
  6520. {
  6521. case Js::OpCode::Conv_Num:
  6522. // Optimize Conv_Num away since we know this is a number
  6523. instr->m_opcode = Js::OpCode::Ld_A;
  6524. return false;
  6525. }
  6526. return false;
  6527. }
  6528. bool
  6529. GlobOpt::TypeSpecializeUnary(
  6530. IR::Instr **pInstr,
  6531. Value **pSrc1Val,
  6532. Value **pDstVal,
  6533. Value *const src1OriginalVal,
  6534. bool *redoTypeSpecRef,
  6535. bool *const forceInvariantHoistingRef)
  6536. {
  6537. Assert(pSrc1Val);
  6538. Value *&src1Val = *pSrc1Val;
  6539. Assert(src1Val);
  6540. // We don't need to do typespec for asmjs
  6541. if (IsTypeSpecPhaseOff(this->func) || GetIsAsmJSFunc())
  6542. {
  6543. return false;
  6544. }
  6545. IR::Instr *&instr = *pInstr;
  6546. int32 min, max;
  6547. // Inline built-ins explicitly specify how srcs/dst must be specialized.
  6548. if (OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  6549. {
  6550. TypeSpecializeInlineBuiltInUnary(pInstr, &src1Val, pDstVal, src1OriginalVal, redoTypeSpecRef);
  6551. return true;
  6552. }
  6553. // Consider: If type spec wasn't completely done, make sure that we don't type-spec the dst 2nd time.
  6554. if(instr->m_opcode == Js::OpCode::LdLen_A && TypeSpecializeLdLen(&instr, &src1Val, pDstVal, forceInvariantHoistingRef))
  6555. {
  6556. return true;
  6557. }
  6558. if (!src1Val->GetValueInfo()->GetIntValMinMax(&min, &max, this->DoAggressiveIntTypeSpec()))
  6559. {
  6560. src1Val = src1OriginalVal;
  6561. if (src1Val->GetValueInfo()->IsLikelyFloat())
  6562. {
  6563. // Try to type specialize to float
  6564. return this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal);
  6565. }
  6566. else if (src1Val->GetValueInfo()->IsNumber())
  6567. {
  6568. return TypeSpecializeNumberUnary(instr, src1Val, pDstVal);
  6569. }
  6570. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  6571. }
  6572. return this->TypeSpecializeIntUnary(pInstr, &src1Val, pDstVal, min, max, src1OriginalVal, redoTypeSpecRef);
  6573. }
  6574. // Returns true if the built-in requested type specialization, and no further action needed,
  6575. // otherwise returns false.
  6576. void
  6577. GlobOpt::TypeSpecializeInlineBuiltInUnary(IR::Instr **pInstr, Value **pSrc1Val, Value **pDstVal, Value *const src1OriginalVal, bool *redoTypeSpecRef)
  6578. {
  6579. IR::Instr *&instr = *pInstr;
  6580. Assert(pSrc1Val);
  6581. Value *&src1Val = *pSrc1Val;
  6582. Assert(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode));
  6583. Js::BuiltinFunction builtInId = Js::JavascriptLibrary::GetBuiltInInlineCandidateId(instr->m_opcode); // From actual instr, not profile based.
  6584. Assert(builtInId != Js::BuiltinFunction::None);
  6585. // Consider using different bailout for float/int FromVars, so that when the arg cannot be converted to number we don't disable
  6586. // type spec for other parts of the big function but rather just don't inline that built-in instr.
  6587. // E.g. could do that if the value is not likelyInt/likelyFloat.
  6588. Js::BuiltInFlags builtInFlags = Js::JavascriptLibrary::GetFlagsForBuiltIn(builtInId);
  6589. bool areAllArgsAlwaysFloat = (builtInFlags & Js::BuiltInFlags::BIF_Args) == Js::BuiltInFlags::BIF_TypeSpecUnaryToFloat;
  6590. if (areAllArgsAlwaysFloat)
  6591. {
  6592. // InlineMathAcos, InlineMathAsin, InlineMathAtan, InlineMathCos, InlineMathExp, InlineMathLog, InlineMathSin, InlineMathSqrt, InlineMathTan.
  6593. Assert(this->DoFloatTypeSpec());
  6594. // Type-spec the src.
  6595. src1Val = src1OriginalVal;
  6596. bool retVal = this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal, /* skipDst = */ true);
  6597. AssertMsg(retVal, "For inline built-ins the args have to be type-specialized to float, but something failed during the process.");
  6598. // Type-spec the dst.
  6599. this->TypeSpecializeFloatDst(instr, nullptr, src1Val, nullptr, pDstVal);
  6600. }
  6601. else if (instr->m_opcode == Js::OpCode::InlineMathAbs)
  6602. {
  6603. // Consider the case when the value is unknown - because of bailout in abs we may disable type spec for the whole function which is too much.
  6604. // First, try int.
  6605. int minVal, maxVal;
  6606. bool shouldTypeSpecToInt = src1Val->GetValueInfo()->GetIntValMinMax(&minVal, &maxVal, /* doAggressiveIntTypeSpec = */ true);
  6607. if (shouldTypeSpecToInt)
  6608. {
  6609. Assert(this->DoAggressiveIntTypeSpec());
  6610. bool retVal = this->TypeSpecializeIntUnary(pInstr, &src1Val, pDstVal, minVal, maxVal, src1OriginalVal, redoTypeSpecRef, true);
  6611. AssertMsg(retVal, "For inline built-ins the args have to be type-specialized (int), but something failed during the process.");
  6612. if (!this->IsLoopPrePass())
  6613. {
  6614. // Create bailout for INT_MIN which does not have corresponding int value on the positive side.
  6615. // Check int range: if we know the range is out of overflow, we do not need the bail out at all.
  6616. if (minVal == INT32_MIN)
  6617. {
  6618. GenerateBailAtOperation(&instr, IR::BailOnIntMin);
  6619. }
  6620. }
  6621. // Account for ::abs(INT_MIN) == INT_MIN (which is less than 0).
  6622. maxVal = ::max(
  6623. ::abs(Int32Math::NearestInRangeTo(minVal, INT_MIN + 1, INT_MAX)),
  6624. ::abs(Int32Math::NearestInRangeTo(maxVal, INT_MIN + 1, INT_MAX)));
  6625. minVal = minVal >= 0 ? minVal : 0;
  6626. this->TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, src1Val, nullptr, IR::BailOutInvalid, minVal, maxVal, pDstVal);
  6627. }
  6628. else
  6629. {
  6630. // If we couldn't do int, do float.
  6631. Assert(this->DoFloatTypeSpec());
  6632. src1Val = src1OriginalVal;
  6633. bool retVal = this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal, true);
  6634. AssertMsg(retVal, "For inline built-ins the args have to be type-specialized (float), but something failed during the process.");
  6635. this->TypeSpecializeFloatDst(instr, nullptr, src1Val, nullptr, pDstVal);
  6636. }
  6637. }
  6638. else if (instr->m_opcode == Js::OpCode::InlineMathFloor || instr->m_opcode == Js::OpCode::InlineMathCeil || instr->m_opcode == Js::OpCode::InlineMathRound)
  6639. {
  6640. // Type specialize src to float
  6641. src1Val = src1OriginalVal;
  6642. bool retVal = this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal, /* skipDst = */ true);
  6643. AssertMsg(retVal, "For inline Math.floor and Math.ceil the src has to be type-specialized to float, but something failed during the process.");
  6644. // Type specialize dst to int
  6645. this->TypeSpecializeIntDst(
  6646. instr,
  6647. instr->m_opcode,
  6648. nullptr,
  6649. src1Val,
  6650. nullptr,
  6651. IR::BailOutInvalid,
  6652. INT32_MIN,
  6653. INT32_MAX,
  6654. pDstVal);
  6655. }
  6656. else if(instr->m_opcode == Js::OpCode::InlineArrayPop)
  6657. {
  6658. IR::Opnd *const thisOpnd = instr->GetSrc1();
  6659. Assert(thisOpnd);
  6660. // Ensure src1 (Array) is a var
  6661. this->ToVarUses(instr, thisOpnd, false, src1Val);
  6662. if(!this->IsLoopPrePass() && thisOpnd->GetValueType().IsLikelyNativeArray())
  6663. {
  6664. // We bail out, if there is illegal access or a mismatch in the Native array type that is optimized for, during the run time.
  6665. GenerateBailAtOperation(&instr, IR::BailOutConventionalNativeArrayAccessOnly);
  6666. }
  6667. if(!instr->GetDst())
  6668. {
  6669. return;
  6670. }
  6671. // Try Type Specializing the element (return item from Pop) based on the array's profile data.
  6672. if(thisOpnd->GetValueType().IsLikelyNativeIntArray())
  6673. {
  6674. this->TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, nullptr, nullptr, IR::BailOutInvalid, INT32_MIN, INT32_MAX, pDstVal);
  6675. }
  6676. else if(thisOpnd->GetValueType().IsLikelyNativeFloatArray())
  6677. {
  6678. this->TypeSpecializeFloatDst(instr, nullptr, nullptr, nullptr, pDstVal);
  6679. }
  6680. else
  6681. {
  6682. // We reached here so the Element is not yet type specialized. Ensure element is a var
  6683. if(instr->GetDst()->IsRegOpnd())
  6684. {
  6685. this->ToVarRegOpnd(instr->GetDst()->AsRegOpnd(), currentBlock);
  6686. }
  6687. }
  6688. }
  6689. else if (instr->m_opcode == Js::OpCode::InlineMathClz)
  6690. {
  6691. Assert(this->DoAggressiveIntTypeSpec());
  6692. Assert(this->DoLossyIntTypeSpec());
  6693. //Type specialize to int
  6694. bool retVal = this->TypeSpecializeIntUnary(pInstr, &src1Val, pDstVal, INT32_MIN, INT32_MAX, src1OriginalVal, redoTypeSpecRef);
  6695. AssertMsg(retVal, "For clz32, the arg has to be type-specialized to int.");
  6696. }
  6697. else
  6698. {
  6699. AssertMsg(FALSE, "Unsupported built-in!");
  6700. }
  6701. }
  6702. void
  6703. GlobOpt::TypeSpecializeInlineBuiltInBinary(IR::Instr **pInstr, Value *src1Val, Value* src2Val, Value **pDstVal, Value *const src1OriginalVal, Value *const src2OriginalVal)
  6704. {
  6705. IR::Instr *&instr = *pInstr;
  6706. Assert(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode));
  6707. switch(instr->m_opcode)
  6708. {
  6709. case Js::OpCode::InlineMathAtan2:
  6710. {
  6711. Js::BuiltinFunction builtInId = Js::JavascriptLibrary::GetBuiltInInlineCandidateId(instr->m_opcode); // From actual instr, not profile based.
  6712. Js::BuiltInFlags builtInFlags = Js::JavascriptLibrary::GetFlagsForBuiltIn(builtInId);
  6713. bool areAllArgsAlwaysFloat = (builtInFlags & Js::BuiltInFlags::BIF_TypeSpecAllToFloat) != 0;
  6714. Assert(areAllArgsAlwaysFloat);
  6715. Assert(this->DoFloatTypeSpec());
  6716. // Type-spec the src1, src2 and dst.
  6717. src1Val = src1OriginalVal;
  6718. src2Val = src2OriginalVal;
  6719. bool retVal = this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  6720. AssertMsg(retVal, "For pow and atnan2 the args have to be type-specialized to float, but something failed during the process.");
  6721. break;
  6722. }
  6723. case Js::OpCode::InlineMathPow:
  6724. {
  6725. #ifndef _M_ARM32_OR_ARM64
  6726. if (src2Val->GetValueInfo()->IsLikelyInt())
  6727. {
  6728. bool lossy = false;
  6729. this->ToInt32(instr, instr->GetSrc2(), this->currentBlock, src2Val, nullptr, lossy);
  6730. IR::Opnd* src1 = instr->GetSrc1();
  6731. int32 valueMin, valueMax;
  6732. if (src1Val->GetValueInfo()->IsLikelyInt() &&
  6733. this->DoPowIntIntTypeSpec() &&
  6734. src2Val->GetValueInfo()->GetIntValMinMax(&valueMin, &valueMax, this->DoAggressiveIntTypeSpec()) &&
  6735. valueMin >= 0)
  6736. {
  6737. this->ToInt32(instr, src1, this->currentBlock, src1Val, nullptr, lossy);
  6738. this->TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, src1Val, src2Val, IR::BailOutInvalid, INT32_MIN, INT32_MAX, pDstVal);
  6739. if(!this->IsLoopPrePass())
  6740. {
  6741. GenerateBailAtOperation(&instr, IR::BailOutOnPowIntIntOverflow);
  6742. }
  6743. }
  6744. else
  6745. {
  6746. this->ToFloat64(instr, src1, this->currentBlock, src1Val, nullptr, IR::BailOutPrimitiveButString);
  6747. TypeSpecializeFloatDst(instr, nullptr, src1Val, src2Val, pDstVal);
  6748. }
  6749. }
  6750. else
  6751. {
  6752. #endif
  6753. this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  6754. #ifndef _M_ARM32_OR_ARM64
  6755. }
  6756. #endif
  6757. break;
  6758. }
  6759. case Js::OpCode::InlineMathImul:
  6760. {
  6761. Assert(this->DoAggressiveIntTypeSpec());
  6762. Assert(this->DoLossyIntTypeSpec());
  6763. //Type specialize to int
  6764. bool retVal = this->TypeSpecializeIntBinary(pInstr, src1Val, src2Val, pDstVal, INT32_MIN, INT32_MAX, false /* skipDst */);
  6765. AssertMsg(retVal, "For imul, the args have to be type-specialized to int but something failed during the process.");
  6766. break;
  6767. }
  6768. case Js::OpCode::InlineMathMin:
  6769. case Js::OpCode::InlineMathMax:
  6770. {
  6771. if(src1Val->GetValueInfo()->IsLikelyInt() && src2Val->GetValueInfo()->IsLikelyInt())
  6772. {
  6773. // Compute resulting range info
  6774. int32 min1 = INT32_MIN;
  6775. int32 max1 = INT32_MAX;
  6776. int32 min2 = INT32_MIN;
  6777. int32 max2 = INT32_MAX;
  6778. int32 newMin, newMax;
  6779. Assert(this->DoAggressiveIntTypeSpec());
  6780. src1Val->GetValueInfo()->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec());
  6781. src2Val->GetValueInfo()->GetIntValMinMax(&min2, &max2, this->DoAggressiveIntTypeSpec());
  6782. if (instr->m_opcode == Js::OpCode::InlineMathMin)
  6783. {
  6784. newMin = min(min1, min2);
  6785. newMax = min(max1, max2);
  6786. }
  6787. else
  6788. {
  6789. Assert(instr->m_opcode == Js::OpCode::InlineMathMax);
  6790. newMin = max(min1, min2);
  6791. newMax = max(max1, max2);
  6792. }
  6793. // Type specialize to int
  6794. bool retVal = this->TypeSpecializeIntBinary(pInstr, src1Val, src2Val, pDstVal, newMin, newMax, false /* skipDst */);
  6795. AssertMsg(retVal, "For min and max, the args have to be type-specialized to int if any one of the sources is an int, but something failed during the process.");
  6796. }
  6797. // Couldn't type specialize to int, type specialize to float
  6798. else
  6799. {
  6800. Assert(this->DoFloatTypeSpec());
  6801. src1Val = src1OriginalVal;
  6802. src2Val = src2OriginalVal;
  6803. bool retVal = this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  6804. AssertMsg(retVal, "For min and max, the args have to be type-specialized to float if any one of the sources is a float, but something failed during the process.");
  6805. }
  6806. break;
  6807. }
  6808. case Js::OpCode::InlineArrayPush:
  6809. {
  6810. IR::Opnd *const thisOpnd = instr->GetSrc1();
  6811. Assert(thisOpnd);
  6812. if(instr->GetDst() && instr->GetDst()->IsRegOpnd())
  6813. {
  6814. // Set the dst as live here, as the built-ins return early from the TypeSpecialization functions - before the dst is marked as live.
  6815. // Also, we are not specializing the dst separately and we are skipping the dst to be handled when we specialize the instruction above.
  6816. this->ToVarRegOpnd(instr->GetDst()->AsRegOpnd(), currentBlock);
  6817. }
  6818. // Ensure src1 (Array) is a var
  6819. this->ToVarUses(instr, thisOpnd, false, src1Val);
  6820. if(!this->IsLoopPrePass())
  6821. {
  6822. if(thisOpnd->GetValueType().IsLikelyNativeArray())
  6823. {
  6824. // We bail out, if there is illegal access or a mismatch in the Native array type that is optimized for, during run time.
  6825. GenerateBailAtOperation(&instr, IR::BailOutConventionalNativeArrayAccessOnly);
  6826. }
  6827. else
  6828. {
  6829. GenerateBailAtOperation(&instr, IR::BailOutOnImplicitCallsPreOp);
  6830. }
  6831. }
  6832. // Try Type Specializing the element based on the array's profile data.
  6833. if(thisOpnd->GetValueType().IsLikelyNativeFloatArray())
  6834. {
  6835. src1Val = src1OriginalVal;
  6836. src2Val = src2OriginalVal;
  6837. }
  6838. if((thisOpnd->GetValueType().IsLikelyNativeIntArray() && this->TypeSpecializeIntBinary(pInstr, src1Val, src2Val, pDstVal, INT32_MIN, INT32_MAX, true))
  6839. || (thisOpnd->GetValueType().IsLikelyNativeFloatArray() && this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal)))
  6840. {
  6841. break;
  6842. }
  6843. // The Element is not yet type specialized. Ensure element is a var
  6844. this->ToVarUses(instr, instr->GetSrc2(), false, src2Val);
  6845. break;
  6846. }
  6847. }
  6848. }
  6849. void
  6850. GlobOpt::TypeSpecializeInlineBuiltInDst(IR::Instr **pInstr, Value **pDstVal)
  6851. {
  6852. IR::Instr *&instr = *pInstr;
  6853. Assert(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode));
  6854. if (instr->m_opcode == Js::OpCode::InlineMathRandom)
  6855. {
  6856. Assert(this->DoFloatTypeSpec());
  6857. // Type specialize dst to float
  6858. this->TypeSpecializeFloatDst(instr, nullptr, nullptr, nullptr, pDstVal);
  6859. }
  6860. }
  6861. bool
  6862. GlobOpt::TryTypeSpecializeUnaryToFloatHelper(IR::Instr** pInstr, Value** pSrc1Val, Value* const src1OriginalVal, Value **pDstVal)
  6863. {
  6864. // It has been determined that this instruction cannot be int-specialized. We need to determine whether to attempt to
  6865. // float-specialize the instruction, or leave it unspecialized.
  6866. #if !INT32VAR
  6867. Value*& src1Val = *pSrc1Val;
  6868. if(src1Val->GetValueInfo()->IsLikelyUntaggedInt())
  6869. {
  6870. // An input range is completely outside the range of an int31. Even if the operation may overflow, it is
  6871. // unlikely to overflow on these operations, so we leave it unspecialized on 64-bit platforms. However, on
  6872. // 32-bit platforms, the value is untaggable and will be a JavascriptNumber, which is significantly slower to
  6873. // use in an unspecialized operation compared to a tagged int. So, try to float-specialize the instruction.
  6874. src1Val = src1OriginalVal;
  6875. return this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal);
  6876. }
  6877. #endif
  6878. return false;
  6879. }
  6880. bool
  6881. GlobOpt::TypeSpecializeIntBinary(IR::Instr **pInstr, Value *src1Val, Value *src2Val, Value **pDstVal, int32 min, int32 max, bool skipDst /* = false */)
  6882. {
  6883. // Consider moving the code for int type spec-ing binary functions here.
  6884. IR::Instr *&instr = *pInstr;
  6885. bool lossy = false;
  6886. if(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  6887. {
  6888. if(instr->m_opcode == Js::OpCode::InlineArrayPush)
  6889. {
  6890. int32 intConstantValue;
  6891. bool isIntConstMissingItem = src2Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue);
  6892. if(isIntConstMissingItem)
  6893. {
  6894. isIntConstMissingItem = Js::SparseArraySegment<int>::IsMissingItem(&intConstantValue);
  6895. }
  6896. // Don't specialize if the element is not likelyInt or an IntConst which is a missing item value.
  6897. if(!(src2Val->GetValueInfo()->IsLikelyInt()) || isIntConstMissingItem)
  6898. {
  6899. return false;
  6900. }
  6901. // We don't want to specialize both the source operands, though it is a binary instr.
  6902. IR::Opnd * elementOpnd = instr->GetSrc2();
  6903. this->ToInt32(instr, elementOpnd, this->currentBlock, src2Val, nullptr, lossy);
  6904. }
  6905. else
  6906. {
  6907. IR::Opnd *src1 = instr->GetSrc1();
  6908. this->ToInt32(instr, src1, this->currentBlock, src1Val, nullptr, lossy);
  6909. IR::Opnd *src2 = instr->GetSrc2();
  6910. this->ToInt32(instr, src2, this->currentBlock, src2Val, nullptr, lossy);
  6911. }
  6912. if(!skipDst)
  6913. {
  6914. IR::Opnd *dst = instr->GetDst();
  6915. if (dst)
  6916. {
  6917. TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, src1Val, src2Val, IR::BailOutInvalid, min, max, pDstVal);
  6918. }
  6919. }
  6920. return true;
  6921. }
  6922. else
  6923. {
  6924. AssertMsg(false, "Yet to move code for other binary functions here");
  6925. return false;
  6926. }
  6927. }
  6928. bool
  6929. GlobOpt::TypeSpecializeIntUnary(
  6930. IR::Instr **pInstr,
  6931. Value **pSrc1Val,
  6932. Value **pDstVal,
  6933. int32 min,
  6934. int32 max,
  6935. Value *const src1OriginalVal,
  6936. bool *redoTypeSpecRef,
  6937. bool skipDst /* = false */)
  6938. {
  6939. IR::Instr *&instr = *pInstr;
  6940. Assert(pSrc1Val);
  6941. Value *&src1Val = *pSrc1Val;
  6942. bool isTransfer = false;
  6943. Js::OpCode opcode;
  6944. int32 newMin, newMax;
  6945. bool lossy = false;
  6946. IR::BailOutKind bailOutKind = IR::BailOutInvalid;
  6947. bool ignoredIntOverflow = this->ignoredIntOverflowForCurrentInstr;
  6948. bool ignoredNegativeZero = false;
  6949. bool checkTypeSpecWorth = false;
  6950. if(instr->GetSrc1()->IsRegOpnd() && instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber)
  6951. {
  6952. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  6953. }
  6954. AddSubConstantInfo addSubConstantInfo;
  6955. switch(instr->m_opcode)
  6956. {
  6957. case Js::OpCode::Ld_A:
  6958. if (instr->GetSrc1()->IsRegOpnd())
  6959. {
  6960. StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  6961. if (CurrentBlockData()->IsInt32TypeSpecialized(sym) == false)
  6962. {
  6963. // Type specializing an Ld_A isn't worth it, unless the src
  6964. // is already type specialized.
  6965. return false;
  6966. }
  6967. }
  6968. newMin = min;
  6969. newMax = max;
  6970. opcode = Js::OpCode::Ld_I4;
  6971. isTransfer = true;
  6972. break;
  6973. case Js::OpCode::Conv_Num:
  6974. newMin = min;
  6975. newMax = max;
  6976. opcode = Js::OpCode::Ld_I4;
  6977. isTransfer = true;
  6978. break;
  6979. case Js::OpCode::LdC_A_I4:
  6980. newMin = newMax = instr->GetSrc1()->AsIntConstOpnd()->AsInt32();
  6981. opcode = Js::OpCode::Ld_I4;
  6982. break;
  6983. case Js::OpCode::Neg_A:
  6984. if (min <= 0 && max >= 0)
  6985. {
  6986. if(instr->ShouldCheckForNegativeZero())
  6987. {
  6988. // -0 matters since the sym is not a local, or is used in a way in which -0 would differ from +0
  6989. if(!DoAggressiveIntTypeSpec())
  6990. {
  6991. // May result in -0
  6992. // Consider adding a dynamic check for src1 == 0
  6993. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  6994. }
  6995. if(min == 0 && max == 0)
  6996. {
  6997. // Always results in -0
  6998. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  6999. }
  7000. bailOutKind |= IR::BailOutOnNegativeZero;
  7001. }
  7002. else
  7003. {
  7004. ignoredNegativeZero = true;
  7005. }
  7006. }
  7007. if (Int32Math::Neg(min, &newMax))
  7008. {
  7009. if(instr->ShouldCheckForIntOverflow())
  7010. {
  7011. if(!DoAggressiveIntTypeSpec())
  7012. {
  7013. // May overflow
  7014. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7015. }
  7016. if(min == max)
  7017. {
  7018. // Always overflows
  7019. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7020. }
  7021. bailOutKind |= IR::BailOutOnOverflow;
  7022. newMax = INT32_MAX;
  7023. }
  7024. else
  7025. {
  7026. ignoredIntOverflow = true;
  7027. }
  7028. }
  7029. if (Int32Math::Neg(max, &newMin))
  7030. {
  7031. if(instr->ShouldCheckForIntOverflow())
  7032. {
  7033. if(!DoAggressiveIntTypeSpec())
  7034. {
  7035. // May overflow
  7036. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7037. }
  7038. bailOutKind |= IR::BailOutOnOverflow;
  7039. newMin = INT32_MAX;
  7040. }
  7041. else
  7042. {
  7043. ignoredIntOverflow = true;
  7044. }
  7045. }
  7046. if(!instr->ShouldCheckForIntOverflow() && newMin > newMax)
  7047. {
  7048. // When ignoring overflow, the range needs to account for overflow. Since MIN_INT is the only int32 value that
  7049. // overflows on Neg, and the value resulting from overflow is also MIN_INT, if calculating only the new min or new
  7050. // max overflowed but not both, then the new min will be greater than the new max. In that case we need to consider
  7051. // the full range of int32s as possible resulting values.
  7052. newMin = INT32_MIN;
  7053. newMax = INT32_MAX;
  7054. }
  7055. opcode = Js::OpCode::Neg_I4;
  7056. checkTypeSpecWorth = true;
  7057. break;
  7058. case Js::OpCode::Not_A:
  7059. if(!DoLossyIntTypeSpec())
  7060. {
  7061. return false;
  7062. }
  7063. this->PropagateIntRangeForNot(min, max, &newMin, &newMax);
  7064. opcode = Js::OpCode::Not_I4;
  7065. lossy = true;
  7066. break;
  7067. case Js::OpCode::Incr_A:
  7068. do // while(false)
  7069. {
  7070. const auto CannotOverflowBasedOnRelativeBounds = [&]()
  7071. {
  7072. const ValueInfo *const src1ValueInfo = src1Val->GetValueInfo();
  7073. return
  7074. (src1ValueInfo->IsInt() || DoAggressiveIntTypeSpec()) &&
  7075. src1ValueInfo->IsIntBounded() &&
  7076. src1ValueInfo->AsIntBounded()->Bounds()->AddCannotOverflowBasedOnRelativeBounds(1);
  7077. };
  7078. if (Int32Math::Inc(min, &newMin))
  7079. {
  7080. if(CannotOverflowBasedOnRelativeBounds())
  7081. {
  7082. newMin = INT32_MAX;
  7083. }
  7084. else if(instr->ShouldCheckForIntOverflow())
  7085. {
  7086. // Always overflows
  7087. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7088. }
  7089. else
  7090. {
  7091. // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since overflow
  7092. // causes the value to wrap around, and we don't have a way to specify a lower and upper range of ints,
  7093. // we use the full range of int32s.
  7094. ignoredIntOverflow = true;
  7095. newMin = INT32_MIN;
  7096. newMax = INT32_MAX;
  7097. break;
  7098. }
  7099. }
  7100. if (Int32Math::Inc(max, &newMax))
  7101. {
  7102. if(CannotOverflowBasedOnRelativeBounds())
  7103. {
  7104. newMax = INT32_MAX;
  7105. }
  7106. else if(instr->ShouldCheckForIntOverflow())
  7107. {
  7108. if(!DoAggressiveIntTypeSpec())
  7109. {
  7110. // May overflow
  7111. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7112. }
  7113. bailOutKind |= IR::BailOutOnOverflow;
  7114. newMax = INT32_MAX;
  7115. }
  7116. else
  7117. {
  7118. // See comment about ignoring overflow above
  7119. ignoredIntOverflow = true;
  7120. newMin = INT32_MIN;
  7121. newMax = INT32_MAX;
  7122. break;
  7123. }
  7124. }
  7125. } while(false);
  7126. if(!ignoredIntOverflow && instr->GetSrc1()->IsRegOpnd())
  7127. {
  7128. addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min == max, 1);
  7129. }
  7130. opcode = Js::OpCode::Add_I4;
  7131. if (!this->IsLoopPrePass())
  7132. {
  7133. instr->SetSrc2(IR::IntConstOpnd::New(1, TyInt32, instr->m_func));
  7134. }
  7135. checkTypeSpecWorth = true;
  7136. break;
  7137. case Js::OpCode::Decr_A:
  7138. do // while(false)
  7139. {
  7140. const auto CannotOverflowBasedOnRelativeBounds = [&]()
  7141. {
  7142. const ValueInfo *const src1ValueInfo = src1Val->GetValueInfo();
  7143. return
  7144. (src1ValueInfo->IsInt() || DoAggressiveIntTypeSpec()) &&
  7145. src1ValueInfo->IsIntBounded() &&
  7146. src1ValueInfo->AsIntBounded()->Bounds()->SubCannotOverflowBasedOnRelativeBounds(1);
  7147. };
  7148. if (Int32Math::Dec(max, &newMax))
  7149. {
  7150. if(CannotOverflowBasedOnRelativeBounds())
  7151. {
  7152. newMax = INT32_MIN;
  7153. }
  7154. else if(instr->ShouldCheckForIntOverflow())
  7155. {
  7156. // Always overflows
  7157. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7158. }
  7159. else
  7160. {
  7161. // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since overflow
  7162. // causes the value to wrap around, and we don't have a way to specify a lower and upper range of ints, we
  7163. // use the full range of int32s.
  7164. ignoredIntOverflow = true;
  7165. newMin = INT32_MIN;
  7166. newMax = INT32_MAX;
  7167. break;
  7168. }
  7169. }
  7170. if (Int32Math::Dec(min, &newMin))
  7171. {
  7172. if(CannotOverflowBasedOnRelativeBounds())
  7173. {
  7174. newMin = INT32_MIN;
  7175. }
  7176. else if(instr->ShouldCheckForIntOverflow())
  7177. {
  7178. if(!DoAggressiveIntTypeSpec())
  7179. {
  7180. // May overflow
  7181. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7182. }
  7183. bailOutKind |= IR::BailOutOnOverflow;
  7184. newMin = INT32_MIN;
  7185. }
  7186. else
  7187. {
  7188. // See comment about ignoring overflow above
  7189. ignoredIntOverflow = true;
  7190. newMin = INT32_MIN;
  7191. newMax = INT32_MAX;
  7192. break;
  7193. }
  7194. }
  7195. } while(false);
  7196. if(!ignoredIntOverflow && instr->GetSrc1()->IsRegOpnd())
  7197. {
  7198. addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min == max, -1);
  7199. }
  7200. opcode = Js::OpCode::Sub_I4;
  7201. if (!this->IsLoopPrePass())
  7202. {
  7203. instr->SetSrc2(IR::IntConstOpnd::New(1, TyInt32, instr->m_func));
  7204. }
  7205. checkTypeSpecWorth = true;
  7206. break;
  7207. case Js::OpCode::BrFalse_A:
  7208. case Js::OpCode::BrTrue_A:
  7209. {
  7210. if(DoConstFold() && !IsLoopPrePass() && TryOptConstFoldBrFalse(instr, src1Val, min, max))
  7211. {
  7212. return true;
  7213. }
  7214. bool specialize = true;
  7215. if (!src1Val->GetValueInfo()->HasIntConstantValue() && instr->GetSrc1()->IsRegOpnd())
  7216. {
  7217. StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  7218. if (CurrentBlockData()->IsInt32TypeSpecialized(sym) == false)
  7219. {
  7220. // Type specializing a BrTrue_A/BrFalse_A isn't worth it, unless the src
  7221. // is already type specialized
  7222. specialize = false;
  7223. }
  7224. }
  7225. if(instr->m_opcode == Js::OpCode::BrTrue_A)
  7226. {
  7227. UpdateIntBoundsForNotEqualBranch(src1Val, nullptr, 0);
  7228. opcode = Js::OpCode::BrTrue_I4;
  7229. }
  7230. else
  7231. {
  7232. UpdateIntBoundsForEqualBranch(src1Val, nullptr, 0);
  7233. opcode = Js::OpCode::BrFalse_I4;
  7234. }
  7235. if(!specialize)
  7236. {
  7237. return false;
  7238. }
  7239. newMin = 2; newMax = 1; // We'll assert if we make a range where min > max
  7240. break;
  7241. }
  7242. case Js::OpCode::MultiBr:
  7243. newMin = min;
  7244. newMax = max;
  7245. opcode = instr->m_opcode;
  7246. break;
  7247. case Js::OpCode::StElemI_A:
  7248. case Js::OpCode::StElemI_A_Strict:
  7249. case Js::OpCode::StElemC:
  7250. if(instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsLikelyAnyArrayWithNativeFloatValues())
  7251. {
  7252. src1Val = src1OriginalVal;
  7253. }
  7254. return TypeSpecializeStElem(pInstr, src1Val, pDstVal);
  7255. case Js::OpCode::NewScArray:
  7256. case Js::OpCode::NewScArrayWithMissingValues:
  7257. case Js::OpCode::InitFld:
  7258. case Js::OpCode::InitRootFld:
  7259. case Js::OpCode::StSlot:
  7260. case Js::OpCode::StSlotChkUndecl:
  7261. #if !FLOATVAR
  7262. case Js::OpCode::StSlotBoxTemp:
  7263. #endif
  7264. case Js::OpCode::StFld:
  7265. case Js::OpCode::StRootFld:
  7266. case Js::OpCode::StFldStrict:
  7267. case Js::OpCode::StRootFldStrict:
  7268. case Js::OpCode::ArgOut_A:
  7269. case Js::OpCode::ArgOut_A_Inline:
  7270. case Js::OpCode::ArgOut_A_FixupForStackArgs:
  7271. case Js::OpCode::ArgOut_A_Dynamic:
  7272. case Js::OpCode::ArgOut_A_FromStackArgs:
  7273. case Js::OpCode::ArgOut_A_SpreadArg:
  7274. // For this one we need to implement type specialization
  7275. //case Js::OpCode::ArgOut_A_InlineBuiltIn:
  7276. case Js::OpCode::Ret:
  7277. case Js::OpCode::LdElemUndef:
  7278. case Js::OpCode::LdElemUndefScoped:
  7279. return false;
  7280. default:
  7281. if (OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  7282. {
  7283. newMin = min;
  7284. newMax = max;
  7285. opcode = instr->m_opcode;
  7286. break; // Note: we must keep checkTypeSpecWorth = false to make sure we never return false from this function.
  7287. }
  7288. return false;
  7289. }
  7290. // If this instruction is in a range of instructions where int overflow does not matter, we will still specialize it (won't
  7291. // leave it unspecialized based on heuristics), since it is most likely worth specializing, and the dst value needs to be
  7292. // guaranteed to be an int
  7293. if(checkTypeSpecWorth &&
  7294. !ignoredIntOverflow &&
  7295. !ignoredNegativeZero &&
  7296. instr->ShouldCheckForIntOverflow() &&
  7297. !IsWorthSpecializingToInt32(instr, src1Val))
  7298. {
  7299. // Even though type specialization is being skipped since it may not be worth it, the proper value should still be
  7300. // maintained so that the result may be type specialized later. An int value is not created for the dst in any of
  7301. // the following cases.
  7302. // - A bailout check is necessary to specialize this instruction. The bailout check is what guarantees the result to be
  7303. // an int, but since we're not going to specialize this instruction, there won't be a bailout check.
  7304. // - Aggressive int type specialization is disabled and we're in a loop prepass. We're conservative on dst values in
  7305. // that case, especially if the dst sym is live on the back-edge.
  7306. if(bailOutKind == IR::BailOutInvalid &&
  7307. instr->GetDst() &&
  7308. (DoAggressiveIntTypeSpec() || !this->IsLoopPrePass()))
  7309. {
  7310. *pDstVal = CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, nullptr);
  7311. }
  7312. if(instr->GetSrc2())
  7313. {
  7314. instr->FreeSrc2();
  7315. }
  7316. return false;
  7317. }
  7318. this->ignoredIntOverflowForCurrentInstr = ignoredIntOverflow;
  7319. this->ignoredNegativeZeroForCurrentInstr = ignoredNegativeZero;
  7320. {
  7321. // Try CSE again before modifying the IR, in case some attributes are required for successful CSE
  7322. Value *src1IndirIndexVal = nullptr;
  7323. Value *src2Val = nullptr;
  7324. if(CSEOptimize(currentBlock, &instr, &src1Val, &src2Val, &src1IndirIndexVal, true /* intMathExprOnly */))
  7325. {
  7326. *redoTypeSpecRef = true;
  7327. return false;
  7328. }
  7329. }
  7330. const Js::OpCode originalOpCode = instr->m_opcode;
  7331. if (!this->IsLoopPrePass())
  7332. {
  7333. // No re-write on prepass
  7334. instr->m_opcode = opcode;
  7335. }
  7336. Value *src1ValueToSpecialize = src1Val;
  7337. if(lossy)
  7338. {
  7339. // Lossy conversions to int32 must be done based on the original source values. For instance, if one of the values is a
  7340. // float constant with a value that fits in a uint32 but not an int32, and the instruction can ignore int overflow, the
  7341. // source value for the purposes of int specialization would have been changed to an int constant value by ignoring
  7342. // overflow. If we were to specialize the sym using the int constant value, it would be treated as a lossless
  7343. // conversion, but since there may be subsequent uses of the same float constant value that may not ignore overflow,
  7344. // this must be treated as a lossy conversion by specializing the sym using the original float constant value.
  7345. src1ValueToSpecialize = src1OriginalVal;
  7346. }
  7347. // Make sure the srcs are specialized
  7348. IR::Opnd *src1 = instr->GetSrc1();
  7349. this->ToInt32(instr, src1, this->currentBlock, src1ValueToSpecialize, nullptr, lossy);
  7350. if(bailOutKind != IR::BailOutInvalid && !this->IsLoopPrePass())
  7351. {
  7352. GenerateBailAtOperation(&instr, bailOutKind);
  7353. }
  7354. if (!skipDst)
  7355. {
  7356. IR::Opnd *dst = instr->GetDst();
  7357. if (dst)
  7358. {
  7359. AssertMsg(!(isTransfer && !this->IsLoopPrePass()) || min == newMin && max == newMax, "If this is just a copy, old/new min/max should be the same");
  7360. TypeSpecializeIntDst(
  7361. instr,
  7362. originalOpCode,
  7363. isTransfer ? src1Val : nullptr,
  7364. src1Val,
  7365. nullptr,
  7366. bailOutKind,
  7367. newMin,
  7368. newMax,
  7369. pDstVal,
  7370. addSubConstantInfo.HasInfo() ? &addSubConstantInfo : nullptr);
  7371. }
  7372. }
  7373. if(bailOutKind == IR::BailOutInvalid)
  7374. {
  7375. GOPT_TRACE(_u("Type specialized to INT\n"));
  7376. #if ENABLE_DEBUG_CONFIG_OPTIONS
  7377. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  7378. {
  7379. Output::Print(_u("Type specialized to INT: "));
  7380. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  7381. }
  7382. #endif
  7383. }
  7384. else
  7385. {
  7386. GOPT_TRACE(_u("Type specialized to INT with bailout on:\n"));
  7387. if(bailOutKind & IR::BailOutOnOverflow)
  7388. {
  7389. GOPT_TRACE(_u(" Overflow\n"));
  7390. #if ENABLE_DEBUG_CONFIG_OPTIONS
  7391. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  7392. {
  7393. Output::Print(_u("Type specialized to INT with bailout (%S): "), "Overflow");
  7394. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  7395. }
  7396. #endif
  7397. }
  7398. if(bailOutKind & IR::BailOutOnNegativeZero)
  7399. {
  7400. GOPT_TRACE(_u(" Zero\n"));
  7401. #if ENABLE_DEBUG_CONFIG_OPTIONS
  7402. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  7403. {
  7404. Output::Print(_u("Type specialized to INT with bailout (%S): "), "Zero");
  7405. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  7406. }
  7407. #endif
  7408. }
  7409. }
  7410. return true;
  7411. }
  7412. void
  7413. GlobOpt::TypeSpecializeIntDst(IR::Instr* instr, Js::OpCode originalOpCode, Value* valToTransfer, Value *const src1Value, Value *const src2Value, const IR::BailOutKind bailOutKind, int32 newMin, int32 newMax, Value** pDstVal, const AddSubConstantInfo *const addSubConstantInfo)
  7414. {
  7415. this->TypeSpecializeIntDst(instr, originalOpCode, valToTransfer, src1Value, src2Value, bailOutKind, ValueType::GetInt(IntConstantBounds(newMin, newMax).IsLikelyTaggable()), newMin, newMax, pDstVal, addSubConstantInfo);
  7416. }
  7417. void
  7418. GlobOpt::TypeSpecializeIntDst(IR::Instr* instr, Js::OpCode originalOpCode, Value* valToTransfer, Value *const src1Value, Value *const src2Value, const IR::BailOutKind bailOutKind, ValueType valueType, Value** pDstVal, const AddSubConstantInfo *const addSubConstantInfo)
  7419. {
  7420. this->TypeSpecializeIntDst(instr, originalOpCode, valToTransfer, src1Value, src2Value, bailOutKind, valueType, 0, 0, pDstVal, addSubConstantInfo);
  7421. }
  7422. void
  7423. GlobOpt::TypeSpecializeIntDst(IR::Instr* instr, Js::OpCode originalOpCode, Value* valToTransfer, Value *const src1Value, Value *const src2Value, const IR::BailOutKind bailOutKind, ValueType valueType, int32 newMin, int32 newMax, Value** pDstVal, const AddSubConstantInfo *const addSubConstantInfo)
  7424. {
  7425. Assert(valueType.IsInt() || (valueType.IsNumber() && valueType.IsLikelyInt() && newMin == 0 && newMax == 0));
  7426. Assert(!valToTransfer || valToTransfer == src1Value);
  7427. Assert(!addSubConstantInfo || addSubConstantInfo->HasInfo());
  7428. IR::Opnd *dst = instr->GetDst();
  7429. Assert(dst);
  7430. bool isValueInfoPrecise;
  7431. if(IsLoopPrePass())
  7432. {
  7433. isValueInfoPrecise = IsPrepassSrcValueInfoPrecise(instr, src1Value, src2Value);
  7434. valueType = GetPrepassValueTypeForDst(valueType, instr, src1Value, src2Value, isValueInfoPrecise);
  7435. }
  7436. else
  7437. {
  7438. isValueInfoPrecise = true;
  7439. }
  7440. // If dst has a circular reference in a loop, it probably won't get specialized. Don't mark the dst as type-specialized on
  7441. // the pre-pass. With aggressive int spec though, it will take care of bailing out if necessary so there's no need to assume
  7442. // that the dst will be a var even if it's live on the back-edge. Also if the op always produces an int32, then there's no
  7443. // ambiguity in the dst's value type even in the prepass.
  7444. if (!DoAggressiveIntTypeSpec() && this->IsLoopPrePass() && !valueType.IsInt())
  7445. {
  7446. if (dst->IsRegOpnd())
  7447. {
  7448. this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock);
  7449. }
  7450. return;
  7451. }
  7452. const IntBounds *dstBounds = nullptr;
  7453. if(addSubConstantInfo && !addSubConstantInfo->SrcValueIsLikelyConstant() && DoTrackRelativeIntBounds())
  7454. {
  7455. Assert(!ignoredIntOverflowForCurrentInstr);
  7456. // Track bounds for add or sub with a constant. For instance, consider (b = a + 2). The value of 'b' should track that
  7457. // it is equal to (the value of 'a') + 2. Additionally, the value of 'b' should inherit the bounds of 'a', offset by
  7458. // the constant value.
  7459. if(!valueType.IsInt() || !isValueInfoPrecise)
  7460. {
  7461. newMin = INT32_MIN;
  7462. newMax = INT32_MAX;
  7463. }
  7464. dstBounds =
  7465. IntBounds::Add(
  7466. addSubConstantInfo->SrcValue(),
  7467. addSubConstantInfo->Offset(),
  7468. isValueInfoPrecise,
  7469. IntConstantBounds(newMin, newMax),
  7470. alloc);
  7471. }
  7472. // Src1's value could change later in the loop, so the value wouldn't be the same for each
  7473. // iteration. Since we don't iterate over loops "while (!changed)", go conservative on the
  7474. // pre-pass.
  7475. if (valToTransfer)
  7476. {
  7477. // If this is just a copy, no need for creating a new value.
  7478. Assert(!addSubConstantInfo);
  7479. *pDstVal = this->ValueNumberTransferDst(instr, valToTransfer);
  7480. CurrentBlockData()->InsertNewValue(*pDstVal, dst);
  7481. }
  7482. else if (valueType.IsInt() && isValueInfoPrecise)
  7483. {
  7484. bool wasNegativeZeroPreventedByBailout = false;
  7485. if(newMin <= 0 && newMax >= 0)
  7486. {
  7487. switch(originalOpCode)
  7488. {
  7489. case Js::OpCode::Add_A:
  7490. // -0 + -0 == -0
  7491. Assert(src1Value);
  7492. Assert(src2Value);
  7493. wasNegativeZeroPreventedByBailout =
  7494. src1Value->GetValueInfo()->WasNegativeZeroPreventedByBailout() &&
  7495. src2Value->GetValueInfo()->WasNegativeZeroPreventedByBailout();
  7496. break;
  7497. case Js::OpCode::Sub_A:
  7498. // -0 - 0 == -0
  7499. Assert(src1Value);
  7500. wasNegativeZeroPreventedByBailout = src1Value->GetValueInfo()->WasNegativeZeroPreventedByBailout();
  7501. break;
  7502. case Js::OpCode::Neg_A:
  7503. case Js::OpCode::Mul_A:
  7504. case Js::OpCode::Div_A:
  7505. case Js::OpCode::Rem_A:
  7506. wasNegativeZeroPreventedByBailout = !!(bailOutKind & IR::BailOutOnNegativeZero);
  7507. break;
  7508. }
  7509. }
  7510. *pDstVal =
  7511. dstBounds
  7512. ? NewIntBoundedValue(valueType, dstBounds, wasNegativeZeroPreventedByBailout, nullptr)
  7513. : NewIntRangeValue(newMin, newMax, wasNegativeZeroPreventedByBailout, nullptr);
  7514. }
  7515. else
  7516. {
  7517. *pDstVal = dstBounds ? NewIntBoundedValue(valueType, dstBounds, false, nullptr) : NewGenericValue(valueType);
  7518. }
  7519. if(addSubConstantInfo || updateInductionVariableValueNumber)
  7520. {
  7521. TrackIntSpecializedAddSubConstant(instr, addSubConstantInfo, *pDstVal, !!dstBounds);
  7522. }
  7523. CurrentBlockData()->SetValue(*pDstVal, dst);
  7524. AssertMsg(dst->IsRegOpnd(), "What else?");
  7525. this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  7526. }
  7527. bool
  7528. GlobOpt::TypeSpecializeBinary(IR::Instr **pInstr, Value **pSrc1Val, Value **pSrc2Val, Value **pDstVal, Value *const src1OriginalVal, Value *const src2OriginalVal, bool *redoTypeSpecRef)
  7529. {
  7530. IR::Instr *&instr = *pInstr;
  7531. int32 min1 = INT32_MIN, max1 = INT32_MAX, min2 = INT32_MIN, max2 = INT32_MAX, newMin, newMax, tmp;
  7532. Js::OpCode opcode;
  7533. Value *&src1Val = *pSrc1Val;
  7534. Value *&src2Val = *pSrc2Val;
  7535. // We don't need to do typespec for asmjs
  7536. if (IsTypeSpecPhaseOff(this->func) || GetIsAsmJSFunc())
  7537. {
  7538. return false;
  7539. }
  7540. if (OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  7541. {
  7542. this->TypeSpecializeInlineBuiltInBinary(pInstr, src1Val, src2Val, pDstVal, src1OriginalVal, src2OriginalVal);
  7543. return true;
  7544. }
  7545. if (src1Val)
  7546. {
  7547. src1Val->GetValueInfo()->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec());
  7548. }
  7549. if (src2Val)
  7550. {
  7551. src2Val->GetValueInfo()->GetIntValMinMax(&min2, &max2, this->DoAggressiveIntTypeSpec());
  7552. }
  7553. // Type specialize binary operators to int32
  7554. bool src1Lossy = true;
  7555. bool src2Lossy = true;
  7556. IR::BailOutKind bailOutKind = IR::BailOutInvalid;
  7557. bool ignoredIntOverflow = this->ignoredIntOverflowForCurrentInstr;
  7558. bool ignoredNegativeZero = false;
  7559. bool skipSrc2 = false;
  7560. bool skipDst = false;
  7561. bool needsBoolConv = false;
  7562. AddSubConstantInfo addSubConstantInfo;
  7563. switch (instr->m_opcode)
  7564. {
  7565. case Js::OpCode::Or_A:
  7566. if (!DoLossyIntTypeSpec())
  7567. {
  7568. return false;
  7569. }
  7570. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7571. opcode = Js::OpCode::Or_I4;
  7572. break;
  7573. case Js::OpCode::And_A:
  7574. if (!DoLossyIntTypeSpec())
  7575. {
  7576. return false;
  7577. }
  7578. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7579. opcode = Js::OpCode::And_I4;
  7580. break;
  7581. case Js::OpCode::Xor_A:
  7582. if (!DoLossyIntTypeSpec())
  7583. {
  7584. return false;
  7585. }
  7586. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7587. opcode = Js::OpCode::Xor_I4;
  7588. break;
  7589. case Js::OpCode::Shl_A:
  7590. if (!DoLossyIntTypeSpec())
  7591. {
  7592. return false;
  7593. }
  7594. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7595. opcode = Js::OpCode::Shl_I4;
  7596. break;
  7597. case Js::OpCode::Shr_A:
  7598. if (!DoLossyIntTypeSpec())
  7599. {
  7600. return false;
  7601. }
  7602. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7603. opcode = Js::OpCode::Shr_I4;
  7604. break;
  7605. case Js::OpCode::ShrU_A:
  7606. if (!DoLossyIntTypeSpec())
  7607. {
  7608. return false;
  7609. }
  7610. if (min1 < 0 && IntConstantBounds(min2, max2).And_0x1f().Contains(0))
  7611. {
  7612. // Src1 may be too large to represent as a signed int32, and src2 may be zero. Unless the resulting value is only
  7613. // used as a signed int32 (hence allowing us to ignore the result's sign), don't specialize the instruction.
  7614. if (!instr->ignoreIntOverflow)
  7615. return false;
  7616. ignoredIntOverflow = true;
  7617. }
  7618. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7619. opcode = Js::OpCode::ShrU_I4;
  7620. break;
  7621. case Js::OpCode::BrUnLe_A:
  7622. // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely
  7623. // int already, so require that both sources are likely int for folding.
  7624. if (DoConstFold() &&
  7625. !IsLoopPrePass() &&
  7626. TryOptConstFoldBrUnsignedGreaterThan(instr, false, src1Val, min1, max1, src2Val, min2, max2))
  7627. {
  7628. return true;
  7629. }
  7630. if (min1 >= 0 && min2 >= 0)
  7631. {
  7632. // Only handle positive values since this is unsigned...
  7633. // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults
  7634. // (INT32_MIN, INT32_MAX), so we're good.
  7635. Assert(src1Val);
  7636. Assert(src1Val->GetValueInfo()->IsLikelyInt());
  7637. Assert(src2Val);
  7638. Assert(src2Val->GetValueInfo()->IsLikelyInt());
  7639. UpdateIntBoundsForLessThanOrEqualBranch(src1Val, src2Val);
  7640. }
  7641. if (!DoLossyIntTypeSpec())
  7642. {
  7643. return false;
  7644. }
  7645. newMin = newMax = 0;
  7646. opcode = Js::OpCode::BrUnLe_I4;
  7647. break;
  7648. case Js::OpCode::BrUnLt_A:
  7649. // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely
  7650. // int already, so require that both sources are likely int for folding.
  7651. if (DoConstFold() &&
  7652. !IsLoopPrePass() &&
  7653. TryOptConstFoldBrUnsignedLessThan(instr, true, src1Val, min1, max1, src2Val, min2, max2))
  7654. {
  7655. return true;
  7656. }
  7657. if (min1 >= 0 && min2 >= 0)
  7658. {
  7659. // Only handle positive values since this is unsigned...
  7660. // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults
  7661. // (INT32_MIN, INT32_MAX), so we're good.
  7662. Assert(src1Val);
  7663. Assert(src1Val->GetValueInfo()->IsLikelyInt());
  7664. Assert(src2Val);
  7665. Assert(src2Val->GetValueInfo()->IsLikelyInt());
  7666. UpdateIntBoundsForLessThanBranch(src1Val, src2Val);
  7667. }
  7668. if (!DoLossyIntTypeSpec())
  7669. {
  7670. return false;
  7671. }
  7672. newMin = newMax = 0;
  7673. opcode = Js::OpCode::BrUnLt_I4;
  7674. break;
  7675. case Js::OpCode::BrUnGe_A:
  7676. // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely
  7677. // int already, so require that both sources are likely int for folding.
  7678. if (DoConstFold() &&
  7679. !IsLoopPrePass() &&
  7680. TryOptConstFoldBrUnsignedLessThan(instr, false, src1Val, min1, max1, src2Val, min2, max2))
  7681. {
  7682. return true;
  7683. }
  7684. if (min1 >= 0 && min2 >= 0)
  7685. {
  7686. // Only handle positive values since this is unsigned...
  7687. // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults
  7688. // (INT32_MIN, INT32_MAX), so we're good.
  7689. Assert(src1Val);
  7690. Assert(src1Val->GetValueInfo()->IsLikelyInt());
  7691. Assert(src2Val);
  7692. Assert(src2Val->GetValueInfo()->IsLikelyInt());
  7693. UpdateIntBoundsForGreaterThanOrEqualBranch(src1Val, src2Val);
  7694. }
  7695. if (!DoLossyIntTypeSpec())
  7696. {
  7697. return false;
  7698. }
  7699. newMin = newMax = 0;
  7700. opcode = Js::OpCode::BrUnGe_I4;
  7701. break;
  7702. case Js::OpCode::BrUnGt_A:
  7703. // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely
  7704. // int already, so require that both sources are likely int for folding.
  7705. if (DoConstFold() &&
  7706. !IsLoopPrePass() &&
  7707. TryOptConstFoldBrUnsignedGreaterThan(instr, true, src1Val, min1, max1, src2Val, min2, max2))
  7708. {
  7709. return true;
  7710. }
  7711. if (min1 >= 0 && min2 >= 0)
  7712. {
  7713. // Only handle positive values since this is unsigned...
  7714. // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults
  7715. // (INT32_MIN, INT32_MAX), so we're good.
  7716. Assert(src1Val);
  7717. Assert(src1Val->GetValueInfo()->IsLikelyInt());
  7718. Assert(src2Val);
  7719. Assert(src2Val->GetValueInfo()->IsLikelyInt());
  7720. UpdateIntBoundsForGreaterThanBranch(src1Val, src2Val);
  7721. }
  7722. if (!DoLossyIntTypeSpec())
  7723. {
  7724. return false;
  7725. }
  7726. newMin = newMax = 0;
  7727. opcode = Js::OpCode::BrUnGt_I4;
  7728. break;
  7729. case Js::OpCode::CmUnLe_A:
  7730. if (!DoLossyIntTypeSpec())
  7731. {
  7732. return false;
  7733. }
  7734. newMin = 0;
  7735. newMax = 1;
  7736. opcode = Js::OpCode::CmUnLe_I4;
  7737. needsBoolConv = true;
  7738. break;
  7739. case Js::OpCode::CmUnLt_A:
  7740. if (!DoLossyIntTypeSpec())
  7741. {
  7742. return false;
  7743. }
  7744. newMin = 0;
  7745. newMax = 1;
  7746. opcode = Js::OpCode::CmUnLt_I4;
  7747. needsBoolConv = true;
  7748. break;
  7749. case Js::OpCode::CmUnGe_A:
  7750. if (!DoLossyIntTypeSpec())
  7751. {
  7752. return false;
  7753. }
  7754. newMin = 0;
  7755. newMax = 1;
  7756. opcode = Js::OpCode::CmUnGe_I4;
  7757. needsBoolConv = true;
  7758. break;
  7759. case Js::OpCode::CmUnGt_A:
  7760. if (!DoLossyIntTypeSpec())
  7761. {
  7762. return false;
  7763. }
  7764. newMin = 0;
  7765. newMax = 1;
  7766. opcode = Js::OpCode::CmUnGt_I4;
  7767. needsBoolConv = true;
  7768. break;
  7769. case Js::OpCode::Expo_A:
  7770. {
  7771. src1Val = src1OriginalVal;
  7772. src2Val = src2OriginalVal;
  7773. return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  7774. }
  7775. case Js::OpCode::Div_A:
  7776. {
  7777. ValueType specializedValueType = GetDivValueType(instr, src1Val, src2Val, true);
  7778. if (specializedValueType.IsFloat())
  7779. {
  7780. // Either result is float or 1/x or cst1/cst2 where cst1%cst2 != 0
  7781. // Note: We should really constant fold cst1%cst2...
  7782. src1Val = src1OriginalVal;
  7783. src2Val = src2OriginalVal;
  7784. return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  7785. }
  7786. #ifdef _M_ARM
  7787. if (!AutoSystemInfo::Data.ArmDivAvailable())
  7788. {
  7789. return false;
  7790. }
  7791. #endif
  7792. if (specializedValueType.IsInt())
  7793. {
  7794. if (max2 == 0x80000000 || (min2 == 0 && max2 == 00))
  7795. {
  7796. return false;
  7797. }
  7798. if (min1 == 0x80000000 && min2 <= -1 && max2 >= -1)
  7799. {
  7800. // Prevent integer overflow, as div by zero or MIN_INT / -1 will throw an exception
  7801. // Or we know we are dividing by zero (which is weird to have because the profile data
  7802. // say we got an int)
  7803. bailOutKind = IR::BailOutOnDivOfMinInt;
  7804. }
  7805. src1Lossy = false; // Detect -0 on the sources
  7806. src2Lossy = false;
  7807. opcode = Js::OpCode::Div_I4;
  7808. Assert(!instr->GetSrc1()->IsUnsigned());
  7809. bailOutKind |= IR::BailOnDivResultNotInt;
  7810. if (max2 >= 0 && min2 <= 0)
  7811. {
  7812. // Need to check for divide by zero if the denominator range includes 0
  7813. bailOutKind |= IR::BailOutOnDivByZero;
  7814. }
  7815. if (max1 >= 0 && min1 <= 0)
  7816. {
  7817. // Numerator contains 0 so the result contains 0
  7818. newMin = 0;
  7819. newMax = 0;
  7820. if (min2 < 0)
  7821. {
  7822. // Denominator may be negative, so the result could be negative 0
  7823. if (instr->ShouldCheckForNegativeZero())
  7824. {
  7825. bailOutKind |= IR::BailOutOnNegativeZero;
  7826. }
  7827. else
  7828. {
  7829. ignoredNegativeZero = true;
  7830. }
  7831. }
  7832. }
  7833. else
  7834. {
  7835. // Initialize to invalid value, one of the condition below will update it correctly
  7836. newMin = INT_MAX;
  7837. newMax = INT_MIN;
  7838. }
  7839. // Deal with the positive and negative range separately for both the numerator and the denominator,
  7840. // and integrate to the overall min and max.
  7841. // If the result is positive (positive/positive or negative/negative):
  7842. // The min should be the smallest magnitude numerator (positive_Min1 | negative_Max1)
  7843. // divided by ---------------------------------------------------------------
  7844. // largest magnitude denominator (positive_Max2 | negative_Min2)
  7845. //
  7846. // The max should be the largest magnitude numerator (positive_Max1 | negative_Max1)
  7847. // divided by ---------------------------------------------------------------
  7848. // smallest magnitude denominator (positive_Min2 | negative_Max2)
  7849. // If the result is negative (positive/negative or positive/negative):
  7850. // The min should be the largest magnitude numerator (positive_Max1 | negative_Min1)
  7851. // divided by ---------------------------------------------------------------
  7852. // smallest magnitude denominator (negative_Max2 | positive_Min2)
  7853. //
  7854. // The max should be the smallest magnitude numerator (positive_Min1 | negative_Max1)
  7855. // divided by ---------------------------------------------------------------
  7856. // largest magnitude denominator (negative_Min2 | positive_Max2)
  7857. // Consider: The range can be slightly more precise if we take care of the rounding
  7858. if (max1 > 0)
  7859. {
  7860. // Take only the positive numerator range
  7861. int32 positive_Min1 = max(1, min1);
  7862. int32 positive_Max1 = max1;
  7863. if (max2 > 0)
  7864. {
  7865. // Take only the positive denominator range
  7866. int32 positive_Min2 = max(1, min2);
  7867. int32 positive_Max2 = max2;
  7868. // Positive / Positive
  7869. int32 quadrant1_Min = positive_Min1 <= positive_Max2? 1 : positive_Min1 / positive_Max2;
  7870. int32 quadrant1_Max = positive_Max1 <= positive_Min2? 1 : positive_Max1 / positive_Min2;
  7871. Assert(1 <= quadrant1_Min && quadrant1_Min <= quadrant1_Max);
  7872. // The result should positive
  7873. newMin = min(newMin, quadrant1_Min);
  7874. newMax = max(newMax, quadrant1_Max);
  7875. }
  7876. if (min2 < 0)
  7877. {
  7878. // Take only the negative denominator range
  7879. int32 negative_Min2 = min2;
  7880. int32 negative_Max2 = min(-1, max2);
  7881. // Positive / Negative
  7882. int32 quadrant2_Min = -positive_Max1 >= negative_Max2? -1 : positive_Max1 / negative_Max2;
  7883. int32 quadrant2_Max = -positive_Min1 >= negative_Min2? -1 : positive_Min1 / negative_Min2;
  7884. // The result should negative
  7885. Assert(quadrant2_Min <= quadrant2_Max && quadrant2_Max <= -1);
  7886. newMin = min(newMin, quadrant2_Min);
  7887. newMax = max(newMax, quadrant2_Max);
  7888. }
  7889. }
  7890. if (min1 < 0)
  7891. {
  7892. // Take only the native numerator range
  7893. int32 negative_Min1 = min1;
  7894. int32 negative_Max1 = min(-1, max1);
  7895. if (max2 > 0)
  7896. {
  7897. // Take only the positive denominator range
  7898. int32 positive_Min2 = max(1, min2);
  7899. int32 positive_Max2 = max2;
  7900. // Negative / Positive
  7901. int32 quadrant4_Min = negative_Min1 >= -positive_Min2? -1 : negative_Min1 / positive_Min2;
  7902. int32 quadrant4_Max = negative_Max1 >= -positive_Max2? -1 : negative_Max1 / positive_Max2;
  7903. // The result should negative
  7904. Assert(quadrant4_Min <= quadrant4_Max && quadrant4_Max <= -1);
  7905. newMin = min(newMin, quadrant4_Min);
  7906. newMax = max(newMax, quadrant4_Max);
  7907. }
  7908. if (min2 < 0)
  7909. {
  7910. // Take only the negative denominator range
  7911. int32 negative_Min2 = min2;
  7912. int32 negative_Max2 = min(-1, max2);
  7913. int32 quadrant3_Min;
  7914. int32 quadrant3_Max;
  7915. // Negative / Negative
  7916. if (negative_Max1 == 0x80000000 && negative_Min2 == -1)
  7917. {
  7918. quadrant3_Min = negative_Max1 >= negative_Min2? 1 : (negative_Max1+1) / negative_Min2;
  7919. }
  7920. else
  7921. {
  7922. quadrant3_Min = negative_Max1 >= negative_Min2? 1 : negative_Max1 / negative_Min2;
  7923. }
  7924. if (negative_Min1 == 0x80000000 && negative_Max2 == -1)
  7925. {
  7926. quadrant3_Max = negative_Min1 >= negative_Max2? 1 : (negative_Min1+1) / negative_Max2;
  7927. }
  7928. else
  7929. {
  7930. quadrant3_Max = negative_Min1 >= negative_Max2? 1 : negative_Min1 / negative_Max2;
  7931. }
  7932. // The result should positive
  7933. Assert(1 <= quadrant3_Min && quadrant3_Min <= quadrant3_Max);
  7934. newMin = min(newMin, quadrant3_Min);
  7935. newMax = max(newMax, quadrant3_Max);
  7936. }
  7937. }
  7938. Assert(newMin <= newMax);
  7939. // Continue to int type spec
  7940. break;
  7941. }
  7942. }
  7943. // fall-through
  7944. default:
  7945. {
  7946. const bool involesLargeInt32 =
  7947. (src1Val && src1Val->GetValueInfo()->IsLikelyUntaggedInt()) ||
  7948. (src2Val && src2Val->GetValueInfo()->IsLikelyUntaggedInt());
  7949. const auto trySpecializeToFloat =
  7950. [&](const bool mayOverflow) -> bool
  7951. {
  7952. // It has been determined that this instruction cannot be int-specialized. Need to determine whether to attempt
  7953. // to float-specialize the instruction, or leave it unspecialized.
  7954. if((involesLargeInt32
  7955. #if INT32VAR
  7956. && mayOverflow
  7957. #endif
  7958. ) || (instr->m_opcode == Js::OpCode::Mul_A && !this->DoAggressiveMulIntTypeSpec())
  7959. )
  7960. {
  7961. // An input range is completely outside the range of an int31 and the operation is likely to overflow.
  7962. // Additionally, on 32-bit platforms, the value is untaggable and will be a JavascriptNumber, which is
  7963. // significantly slower to use in an unspecialized operation compared to a tagged int. So, try to
  7964. // float-specialize the instruction.
  7965. src1Val = src1OriginalVal;
  7966. src2Val = src2OriginalVal;
  7967. return TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  7968. }
  7969. return false;
  7970. };
  7971. if (instr->m_opcode != Js::OpCode::ArgOut_A_InlineBuiltIn)
  7972. {
  7973. if ((src1Val && src1Val->GetValueInfo()->IsLikelyFloat()) || (src2Val && src2Val->GetValueInfo()->IsLikelyFloat()))
  7974. {
  7975. // Try to type specialize to float
  7976. src1Val = src1OriginalVal;
  7977. src2Val = src2OriginalVal;
  7978. return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  7979. }
  7980. if (src1Val == nullptr ||
  7981. src2Val == nullptr ||
  7982. !src1Val->GetValueInfo()->IsLikelyInt() ||
  7983. !src2Val->GetValueInfo()->IsLikelyInt() ||
  7984. (
  7985. !DoAggressiveIntTypeSpec() &&
  7986. (
  7987. !(src1Val->GetValueInfo()->IsInt() || CurrentBlockData()->IsSwitchInt32TypeSpecialized(instr)) ||
  7988. !src2Val->GetValueInfo()->IsInt()
  7989. )
  7990. ) ||
  7991. (instr->GetSrc1()->IsRegOpnd() && instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber) ||
  7992. (instr->GetSrc2()->IsRegOpnd() && instr->GetSrc2()->AsRegOpnd()->m_sym->m_isNotNumber))
  7993. {
  7994. return trySpecializeToFloat(true);
  7995. }
  7996. }
  7997. // Try to type specialize to int32
  7998. // If one of the values is a float constant with a value that fits in a uint32 but not an int32,
  7999. // and the instruction can ignore int overflow, the source value for the purposes of int specialization
  8000. // would have been changed to an int constant value by ignoring overflow. But, the conversion is still lossy.
  8001. if (!(src1OriginalVal && src1OriginalVal->GetValueInfo()->IsFloatConstant() && src1Val && src1Val->GetValueInfo()->HasIntConstantValue()))
  8002. {
  8003. src1Lossy = false;
  8004. }
  8005. if (!(src2OriginalVal && src2OriginalVal->GetValueInfo()->IsFloatConstant() && src2Val && src2Val->GetValueInfo()->HasIntConstantValue()))
  8006. {
  8007. src2Lossy = false;
  8008. }
  8009. switch(instr->m_opcode)
  8010. {
  8011. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  8012. // If the src is already type-specialized, if we don't type-specialize ArgOut_A_InlineBuiltIn instr, we'll get additional ToVar.
  8013. // So, to avoid that, type-specialize the ArgOut_A_InlineBuiltIn instr.
  8014. // Else we don't need to type-specialize the instr, we are fine with src being Var.
  8015. if (instr->GetSrc1()->IsRegOpnd())
  8016. {
  8017. StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  8018. if (CurrentBlockData()->IsInt32TypeSpecialized(sym))
  8019. {
  8020. opcode = instr->m_opcode;
  8021. skipDst = true; // We should keep dst as is, otherwise the link opnd for next ArgOut/InlineBuiltInStart would be broken.
  8022. skipSrc2 = true; // src2 is linkOpnd. We don't need to type-specialize it.
  8023. newMin = min1; newMax = max1; // Values don't matter, these are unused.
  8024. goto LOutsideSwitch; // Continue to int-type-specialize.
  8025. }
  8026. else if (CurrentBlockData()->IsFloat64TypeSpecialized(sym))
  8027. {
  8028. src1Val = src1OriginalVal;
  8029. src2Val = src2OriginalVal;
  8030. return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  8031. }
  8032. }
  8033. return false;
  8034. case Js::OpCode::Add_A:
  8035. do // while(false)
  8036. {
  8037. const auto CannotOverflowBasedOnRelativeBounds = [&](int32 *const constantValueRef)
  8038. {
  8039. Assert(constantValueRef);
  8040. if(min2 == max2 &&
  8041. src1Val->GetValueInfo()->IsIntBounded() &&
  8042. src1Val->GetValueInfo()->AsIntBounded()->Bounds()->AddCannotOverflowBasedOnRelativeBounds(min2))
  8043. {
  8044. *constantValueRef = min2;
  8045. return true;
  8046. }
  8047. else if(
  8048. min1 == max1 &&
  8049. src2Val->GetValueInfo()->IsIntBounded() &&
  8050. src2Val->GetValueInfo()->AsIntBounded()->Bounds()->AddCannotOverflowBasedOnRelativeBounds(min1))
  8051. {
  8052. *constantValueRef = min1;
  8053. return true;
  8054. }
  8055. return false;
  8056. };
  8057. if (Int32Math::Add(min1, min2, &newMin))
  8058. {
  8059. int32 constantSrcValue;
  8060. if(CannotOverflowBasedOnRelativeBounds(&constantSrcValue))
  8061. {
  8062. newMin = constantSrcValue >= 0 ? INT32_MAX : INT32_MIN;
  8063. }
  8064. else if(instr->ShouldCheckForIntOverflow())
  8065. {
  8066. if(involesLargeInt32 || !DoAggressiveIntTypeSpec())
  8067. {
  8068. // May overflow
  8069. return trySpecializeToFloat(true);
  8070. }
  8071. bailOutKind |= IR::BailOutOnOverflow;
  8072. newMin = min1 < 0 ? INT32_MIN : INT32_MAX;
  8073. }
  8074. else
  8075. {
  8076. // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since
  8077. // overflow causes the value to wrap around, and we don't have a way to specify a lower and upper
  8078. // range of ints, we use the full range of int32s.
  8079. ignoredIntOverflow = true;
  8080. newMin = INT32_MIN;
  8081. newMax = INT32_MAX;
  8082. break;
  8083. }
  8084. }
  8085. if (Int32Math::Add(max1, max2, &newMax))
  8086. {
  8087. int32 constantSrcValue;
  8088. if(CannotOverflowBasedOnRelativeBounds(&constantSrcValue))
  8089. {
  8090. newMax = constantSrcValue >= 0 ? INT32_MAX : INT32_MIN;
  8091. }
  8092. else if(instr->ShouldCheckForIntOverflow())
  8093. {
  8094. if(involesLargeInt32 || !DoAggressiveIntTypeSpec())
  8095. {
  8096. // May overflow
  8097. return trySpecializeToFloat(true);
  8098. }
  8099. bailOutKind |= IR::BailOutOnOverflow;
  8100. newMax = max1 < 0 ? INT32_MIN : INT32_MAX;
  8101. }
  8102. else
  8103. {
  8104. // See comment about ignoring overflow above
  8105. ignoredIntOverflow = true;
  8106. newMin = INT32_MIN;
  8107. newMax = INT32_MAX;
  8108. break;
  8109. }
  8110. }
  8111. if(bailOutKind & IR::BailOutOnOverflow)
  8112. {
  8113. Assert(bailOutKind == IR::BailOutOnOverflow);
  8114. Assert(instr->ShouldCheckForIntOverflow());
  8115. int32 temp;
  8116. if(Int32Math::Add(
  8117. Int32Math::NearestInRangeTo(0, min1, max1),
  8118. Int32Math::NearestInRangeTo(0, min2, max2),
  8119. &temp))
  8120. {
  8121. // Always overflows
  8122. return trySpecializeToFloat(true);
  8123. }
  8124. }
  8125. } while(false);
  8126. if (!this->IsLoopPrePass() && newMin == newMax && bailOutKind == IR::BailOutInvalid)
  8127. {
  8128. // Take care of Add with zero here, since we know we're dealing with 2 numbers.
  8129. this->CaptureByteCodeSymUses(instr);
  8130. IR::Opnd *src;
  8131. bool isAddZero = true;
  8132. int32 intConstantValue;
  8133. if (src1Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue) && intConstantValue == 0)
  8134. {
  8135. src = instr->UnlinkSrc2();
  8136. instr->FreeSrc1();
  8137. }
  8138. else if (src2Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue) && intConstantValue == 0)
  8139. {
  8140. src = instr->UnlinkSrc1();
  8141. instr->FreeSrc2();
  8142. }
  8143. else
  8144. {
  8145. // This should have been handled by const folding, unless:
  8146. // - A source's value was substituted with a different value here, which is after const folding happened
  8147. // - A value is not definitely int, but once converted to definite int, it would be zero due to a
  8148. // condition in the source code such as if(a === 0). Ideally, we would specialize the sources and
  8149. // remove the add, but doesn't seem too important for now.
  8150. Assert(
  8151. !DoConstFold() ||
  8152. src1Val != src1OriginalVal ||
  8153. src2Val != src2OriginalVal ||
  8154. !src1Val->GetValueInfo()->IsInt() ||
  8155. !src2Val->GetValueInfo()->IsInt());
  8156. isAddZero = false;
  8157. src = nullptr;
  8158. }
  8159. if (isAddZero)
  8160. {
  8161. IR::Instr *newInstr = IR::Instr::New(Js::OpCode::Ld_A, instr->UnlinkDst(), src, instr->m_func);
  8162. newInstr->SetByteCodeOffset(instr);
  8163. instr->m_opcode = Js::OpCode::Nop;
  8164. this->currentBlock->InsertInstrAfter(newInstr, instr);
  8165. return true;
  8166. }
  8167. }
  8168. if(!ignoredIntOverflow)
  8169. {
  8170. if(min2 == max2 &&
  8171. (!IsLoopPrePass() || IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Val)) &&
  8172. instr->GetSrc1()->IsRegOpnd())
  8173. {
  8174. addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min1 == max1, min2);
  8175. }
  8176. else if(
  8177. min1 == max1 &&
  8178. (!IsLoopPrePass() || IsPrepassSrcValueInfoPrecise(instr->GetSrc1(), src1Val)) &&
  8179. instr->GetSrc2()->IsRegOpnd())
  8180. {
  8181. addSubConstantInfo.Set(instr->GetSrc2()->AsRegOpnd()->m_sym, src2Val, min2 == max2, min1);
  8182. }
  8183. }
  8184. opcode = Js::OpCode::Add_I4;
  8185. break;
  8186. case Js::OpCode::Sub_A:
  8187. do // while(false)
  8188. {
  8189. const auto CannotOverflowBasedOnRelativeBounds = [&]()
  8190. {
  8191. return
  8192. min2 == max2 &&
  8193. src1Val->GetValueInfo()->IsIntBounded() &&
  8194. src1Val->GetValueInfo()->AsIntBounded()->Bounds()->SubCannotOverflowBasedOnRelativeBounds(min2);
  8195. };
  8196. if (Int32Math::Sub(min1, max2, &newMin))
  8197. {
  8198. if(CannotOverflowBasedOnRelativeBounds())
  8199. {
  8200. Assert(min2 == max2);
  8201. newMin = min2 >= 0 ? INT32_MIN : INT32_MAX;
  8202. }
  8203. else if(instr->ShouldCheckForIntOverflow())
  8204. {
  8205. if(involesLargeInt32 || !DoAggressiveIntTypeSpec())
  8206. {
  8207. // May overflow
  8208. return trySpecializeToFloat(true);
  8209. }
  8210. bailOutKind |= IR::BailOutOnOverflow;
  8211. newMin = min1 < 0 ? INT32_MIN : INT32_MAX;
  8212. }
  8213. else
  8214. {
  8215. // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since overflow
  8216. // causes the value to wrap around, and we don't have a way to specify a lower and upper range of ints,
  8217. // we use the full range of int32s.
  8218. ignoredIntOverflow = true;
  8219. newMin = INT32_MIN;
  8220. newMax = INT32_MAX;
  8221. break;
  8222. }
  8223. }
  8224. if (Int32Math::Sub(max1, min2, &newMax))
  8225. {
  8226. if(CannotOverflowBasedOnRelativeBounds())
  8227. {
  8228. Assert(min2 == max2);
  8229. newMax = min2 >= 0 ? INT32_MIN: INT32_MAX;
  8230. }
  8231. else if(instr->ShouldCheckForIntOverflow())
  8232. {
  8233. if(involesLargeInt32 || !DoAggressiveIntTypeSpec())
  8234. {
  8235. // May overflow
  8236. return trySpecializeToFloat(true);
  8237. }
  8238. bailOutKind |= IR::BailOutOnOverflow;
  8239. newMax = max1 < 0 ? INT32_MIN : INT32_MAX;
  8240. }
  8241. else
  8242. {
  8243. // See comment about ignoring overflow above
  8244. ignoredIntOverflow = true;
  8245. newMin = INT32_MIN;
  8246. newMax = INT32_MAX;
  8247. break;
  8248. }
  8249. }
  8250. if(bailOutKind & IR::BailOutOnOverflow)
  8251. {
  8252. Assert(bailOutKind == IR::BailOutOnOverflow);
  8253. Assert(instr->ShouldCheckForIntOverflow());
  8254. int32 temp;
  8255. if(Int32Math::Sub(
  8256. Int32Math::NearestInRangeTo(-1, min1, max1),
  8257. Int32Math::NearestInRangeTo(0, min2, max2),
  8258. &temp))
  8259. {
  8260. // Always overflows
  8261. return trySpecializeToFloat(true);
  8262. }
  8263. }
  8264. } while(false);
  8265. if(!ignoredIntOverflow &&
  8266. min2 == max2 &&
  8267. min2 != INT32_MIN &&
  8268. (!IsLoopPrePass() || IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Val)) &&
  8269. instr->GetSrc1()->IsRegOpnd())
  8270. {
  8271. addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min1 == max1, -min2);
  8272. }
  8273. opcode = Js::OpCode::Sub_I4;
  8274. break;
  8275. case Js::OpCode::Mul_A:
  8276. {
  8277. bool isConservativeMulInt = !DoAggressiveMulIntTypeSpec() || !DoAggressiveIntTypeSpec();
  8278. // Be conservative about predicting Mul overflow in prepass.
  8279. // Operands that are live on back edge may be denied lossless-conversion to int32 and
  8280. // trigger rejit with AggressiveIntTypeSpec off.
  8281. // Besides multiplying a variable in a loop can overflow in just a few iterations even in simple cases like v *= 2
  8282. // So, make sure we definitely know the source max/min values, otherwise assume the full range.
  8283. if (isConservativeMulInt && IsLoopPrePass())
  8284. {
  8285. if (!IsPrepassSrcValueInfoPrecise(instr->GetSrc1(), src1Val))
  8286. {
  8287. max1 = INT32_MAX;
  8288. min1 = INT32_MIN;
  8289. }
  8290. if (!IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Val))
  8291. {
  8292. max2 = INT32_MAX;
  8293. min2 = INT32_MIN;
  8294. }
  8295. }
  8296. if (Int32Math::Mul(min1, min2, &newMin))
  8297. {
  8298. if (involesLargeInt32 || isConservativeMulInt)
  8299. {
  8300. // May overflow
  8301. return trySpecializeToFloat(true);
  8302. }
  8303. bailOutKind |= IR::BailOutOnMulOverflow;
  8304. newMin = (min1 < 0) ^ (min2 < 0) ? INT32_MIN : INT32_MAX;
  8305. }
  8306. newMax = newMin;
  8307. if (Int32Math::Mul(max1, max2, &tmp))
  8308. {
  8309. if (involesLargeInt32 || isConservativeMulInt)
  8310. {
  8311. // May overflow
  8312. return trySpecializeToFloat(true);
  8313. }
  8314. bailOutKind |= IR::BailOutOnMulOverflow;
  8315. tmp = (max1 < 0) ^ (max2 < 0) ? INT32_MIN : INT32_MAX;
  8316. }
  8317. newMin = min(newMin, tmp);
  8318. newMax = max(newMax, tmp);
  8319. if (Int32Math::Mul(min1, max2, &tmp))
  8320. {
  8321. if (involesLargeInt32 || isConservativeMulInt)
  8322. {
  8323. // May overflow
  8324. return trySpecializeToFloat(true);
  8325. }
  8326. bailOutKind |= IR::BailOutOnMulOverflow;
  8327. tmp = (min1 < 0) ^ (max2 < 0) ? INT32_MIN : INT32_MAX;
  8328. }
  8329. newMin = min(newMin, tmp);
  8330. newMax = max(newMax, tmp);
  8331. if (Int32Math::Mul(max1, min2, &tmp))
  8332. {
  8333. if (involesLargeInt32 || isConservativeMulInt)
  8334. {
  8335. // May overflow
  8336. return trySpecializeToFloat(true);
  8337. }
  8338. bailOutKind |= IR::BailOutOnMulOverflow;
  8339. tmp = (max1 < 0) ^ (min2 < 0) ? INT32_MIN : INT32_MAX;
  8340. }
  8341. newMin = min(newMin, tmp);
  8342. newMax = max(newMax, tmp);
  8343. if (bailOutKind & IR::BailOutOnMulOverflow)
  8344. {
  8345. // CSE only if two MULs have the same overflow check behavior.
  8346. // Currently this is set to be ignore int32 overflow, but not 53-bit, or int32 overflow matters.
  8347. if (!instr->ShouldCheckFor32BitOverflow() && instr->ShouldCheckForNon32BitOverflow())
  8348. {
  8349. // If we allow int to overflow then there can be anything in the resulting int
  8350. newMin = IntConstMin;
  8351. newMax = IntConstMax;
  8352. ignoredIntOverflow = true;
  8353. }
  8354. int32 temp, overflowValue;
  8355. if (Int32Math::Mul(
  8356. Int32Math::NearestInRangeTo(0, min1, max1),
  8357. Int32Math::NearestInRangeTo(0, min2, max2),
  8358. &temp,
  8359. &overflowValue))
  8360. {
  8361. Assert(instr->ignoreOverflowBitCount >= 32);
  8362. int overflowMatters = 64 - instr->ignoreOverflowBitCount;
  8363. if (!ignoredIntOverflow ||
  8364. // Use shift to check high bits in case its negative
  8365. ((overflowValue << overflowMatters) >> overflowMatters) != overflowValue
  8366. )
  8367. {
  8368. // Always overflows
  8369. return trySpecializeToFloat(true);
  8370. }
  8371. }
  8372. }
  8373. if (newMin <= 0 && newMax >= 0 && // New range crosses zero
  8374. (min1 < 0 || min2 < 0) && // An operand's range contains a negative integer
  8375. !(min1 > 0 || min2 > 0) && // Neither operand's range contains only positive integers
  8376. !instr->GetSrc1()->IsEqual(instr->GetSrc2())) // The operands don't have the same value
  8377. {
  8378. if (instr->ShouldCheckForNegativeZero())
  8379. {
  8380. // -0 matters since the sym is not a local, or is used in a way in which -0 would differ from +0
  8381. if (!DoAggressiveIntTypeSpec())
  8382. {
  8383. // May result in -0
  8384. return trySpecializeToFloat(false);
  8385. }
  8386. if (((min1 == 0 && max1 == 0) || (min2 == 0 && max2 == 0)) && (max1 < 0 || max2 < 0))
  8387. {
  8388. // Always results in -0
  8389. return trySpecializeToFloat(false);
  8390. }
  8391. bailOutKind |= IR::BailOutOnNegativeZero;
  8392. }
  8393. else
  8394. {
  8395. ignoredNegativeZero = true;
  8396. }
  8397. }
  8398. opcode = Js::OpCode::Mul_I4;
  8399. break;
  8400. }
  8401. case Js::OpCode::Rem_A:
  8402. {
  8403. IR::Opnd* src2 = instr->GetSrc2();
  8404. if (!this->IsLoopPrePass() && min2 == max2 && min1 >= 0)
  8405. {
  8406. int32 value = min2;
  8407. if (value == (1 << Math::Log2(value)) && src2->IsAddrOpnd())
  8408. {
  8409. Assert(src2->AsAddrOpnd()->IsVar());
  8410. instr->m_opcode = Js::OpCode::And_A;
  8411. src2->AsAddrOpnd()->SetAddress(Js::TaggedInt::ToVarUnchecked(value - 1),
  8412. IR::AddrOpndKindConstantVar);
  8413. *pSrc2Val = GetIntConstantValue(value - 1, instr);
  8414. src2Val = *pSrc2Val;
  8415. return this->TypeSpecializeBinary(&instr, pSrc1Val, pSrc2Val, pDstVal, src1OriginalVal, src2Val, redoTypeSpecRef);
  8416. }
  8417. }
  8418. #ifdef _M_ARM
  8419. if (!AutoSystemInfo::Data.ArmDivAvailable())
  8420. {
  8421. return false;
  8422. }
  8423. #endif
  8424. if (min1 < 0)
  8425. {
  8426. // The most negative it can be is min1, unless limited by min2/max2
  8427. int32 negMaxAbs2;
  8428. if (min2 == INT32_MIN)
  8429. {
  8430. negMaxAbs2 = INT32_MIN;
  8431. }
  8432. else
  8433. {
  8434. negMaxAbs2 = -max(abs(min2), abs(max2)) + 1;
  8435. }
  8436. newMin = max(min1, negMaxAbs2);
  8437. }
  8438. else
  8439. {
  8440. newMin = 0;
  8441. }
  8442. bool isModByPowerOf2 = (instr->IsProfiledInstr() && instr->m_func->HasProfileInfo() &&
  8443. instr->m_func->GetReadOnlyProfileInfo()->IsModulusOpByPowerOf2(static_cast<Js::ProfileId>(instr->AsProfiledInstr()->u.profileId)));
  8444. if(isModByPowerOf2)
  8445. {
  8446. Assert(bailOutKind == IR::BailOutInvalid);
  8447. bailOutKind = IR::BailOnModByPowerOf2;
  8448. newMin = 0;
  8449. }
  8450. else
  8451. {
  8452. if (min2 <= 0 && max2 >= 0)
  8453. {
  8454. // Consider: We could handle the zero case with a check and bailout...
  8455. return false;
  8456. }
  8457. if (min1 == 0x80000000 && (min2 <= -1 && max2 >= -1))
  8458. {
  8459. // Prevent integer overflow, as div by zero or MIN_INT / -1 will throw an exception
  8460. return false;
  8461. }
  8462. if (min1 < 0)
  8463. {
  8464. if(instr->ShouldCheckForNegativeZero())
  8465. {
  8466. if (!DoAggressiveIntTypeSpec())
  8467. {
  8468. return false;
  8469. }
  8470. bailOutKind |= IR::BailOutOnNegativeZero;
  8471. }
  8472. else
  8473. {
  8474. ignoredNegativeZero = true;
  8475. }
  8476. }
  8477. }
  8478. {
  8479. int32 absMax2;
  8480. if (min2 == INT32_MIN)
  8481. {
  8482. // abs(INT32_MIN) == INT32_MAX because of overflow
  8483. absMax2 = INT32_MAX;
  8484. }
  8485. else
  8486. {
  8487. absMax2 = max(abs(min2), abs(max2)) - 1;
  8488. }
  8489. newMax = min(absMax2, max(max1, 0));
  8490. newMax = max(newMin, newMax);
  8491. }
  8492. opcode = Js::OpCode::Rem_I4;
  8493. Assert(!instr->GetSrc1()->IsUnsigned());
  8494. break;
  8495. }
  8496. case Js::OpCode::CmEq_A:
  8497. case Js::OpCode::CmSrEq_A:
  8498. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8499. {
  8500. return false;
  8501. }
  8502. newMin = 0;
  8503. newMax = 1;
  8504. opcode = Js::OpCode::CmEq_I4;
  8505. needsBoolConv = true;
  8506. break;
  8507. case Js::OpCode::CmNeq_A:
  8508. case Js::OpCode::CmSrNeq_A:
  8509. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8510. {
  8511. return false;
  8512. }
  8513. newMin = 0;
  8514. newMax = 1;
  8515. opcode = Js::OpCode::CmNeq_I4;
  8516. needsBoolConv = true;
  8517. break;
  8518. case Js::OpCode::CmLe_A:
  8519. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8520. {
  8521. return false;
  8522. }
  8523. newMin = 0;
  8524. newMax = 1;
  8525. opcode = Js::OpCode::CmLe_I4;
  8526. needsBoolConv = true;
  8527. break;
  8528. case Js::OpCode::CmLt_A:
  8529. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8530. {
  8531. return false;
  8532. }
  8533. newMin = 0;
  8534. newMax = 1;
  8535. opcode = Js::OpCode::CmLt_I4;
  8536. needsBoolConv = true;
  8537. break;
  8538. case Js::OpCode::CmGe_A:
  8539. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8540. {
  8541. return false;
  8542. }
  8543. newMin = 0;
  8544. newMax = 1;
  8545. opcode = Js::OpCode::CmGe_I4;
  8546. needsBoolConv = true;
  8547. break;
  8548. case Js::OpCode::CmGt_A:
  8549. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8550. {
  8551. return false;
  8552. }
  8553. newMin = 0;
  8554. newMax = 1;
  8555. opcode = Js::OpCode::CmGt_I4;
  8556. needsBoolConv = true;
  8557. break;
  8558. case Js::OpCode::BrSrEq_A:
  8559. case Js::OpCode::BrEq_A:
  8560. case Js::OpCode::BrNotNeq_A:
  8561. case Js::OpCode::BrSrNotNeq_A:
  8562. {
  8563. if(DoConstFold() &&
  8564. !IsLoopPrePass() &&
  8565. TryOptConstFoldBrEqual(instr, true, src1Val, min1, max1, src2Val, min2, max2))
  8566. {
  8567. return true;
  8568. }
  8569. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8570. UpdateIntBoundsForEqualBranch(src1Val, src2Val);
  8571. if(!specialize)
  8572. {
  8573. return false;
  8574. }
  8575. opcode = Js::OpCode::BrEq_I4;
  8576. // We'll get a warning if we don't assign a value to these...
  8577. // We'll assert if we use them and make a range where min > max
  8578. newMin = 2; newMax = 1;
  8579. break;
  8580. }
  8581. case Js::OpCode::BrSrNeq_A:
  8582. case Js::OpCode::BrNeq_A:
  8583. case Js::OpCode::BrSrNotEq_A:
  8584. case Js::OpCode::BrNotEq_A:
  8585. {
  8586. if(DoConstFold() &&
  8587. !IsLoopPrePass() &&
  8588. TryOptConstFoldBrEqual(instr, false, src1Val, min1, max1, src2Val, min2, max2))
  8589. {
  8590. return true;
  8591. }
  8592. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8593. UpdateIntBoundsForNotEqualBranch(src1Val, src2Val);
  8594. if(!specialize)
  8595. {
  8596. return false;
  8597. }
  8598. opcode = Js::OpCode::BrNeq_I4;
  8599. // We'll get a warning if we don't assign a value to these...
  8600. // We'll assert if we use them and make a range where min > max
  8601. newMin = 2; newMax = 1;
  8602. break;
  8603. }
  8604. case Js::OpCode::BrGt_A:
  8605. case Js::OpCode::BrNotLe_A:
  8606. {
  8607. if(DoConstFold() &&
  8608. !IsLoopPrePass() &&
  8609. TryOptConstFoldBrGreaterThan(instr, true, src1Val, min1, max1, src2Val, min2, max2))
  8610. {
  8611. return true;
  8612. }
  8613. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8614. UpdateIntBoundsForGreaterThanBranch(src1Val, src2Val);
  8615. if(!specialize)
  8616. {
  8617. return false;
  8618. }
  8619. opcode = Js::OpCode::BrGt_I4;
  8620. // We'll get a warning if we don't assign a value to these...
  8621. // We'll assert if we use them and make a range where min > max
  8622. newMin = 2; newMax = 1;
  8623. break;
  8624. }
  8625. case Js::OpCode::BrGe_A:
  8626. case Js::OpCode::BrNotLt_A:
  8627. {
  8628. if(DoConstFold() &&
  8629. !IsLoopPrePass() &&
  8630. TryOptConstFoldBrGreaterThanOrEqual(instr, true, src1Val, min1, max1, src2Val, min2, max2))
  8631. {
  8632. return true;
  8633. }
  8634. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8635. UpdateIntBoundsForGreaterThanOrEqualBranch(src1Val, src2Val);
  8636. if(!specialize)
  8637. {
  8638. return false;
  8639. }
  8640. opcode = Js::OpCode::BrGe_I4;
  8641. // We'll get a warning if we don't assign a value to these...
  8642. // We'll assert if we use them and make a range where min > max
  8643. newMin = 2; newMax = 1;
  8644. break;
  8645. }
  8646. case Js::OpCode::BrLt_A:
  8647. case Js::OpCode::BrNotGe_A:
  8648. {
  8649. if(DoConstFold() &&
  8650. !IsLoopPrePass() &&
  8651. TryOptConstFoldBrGreaterThanOrEqual(instr, false, src1Val, min1, max1, src2Val, min2, max2))
  8652. {
  8653. return true;
  8654. }
  8655. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8656. UpdateIntBoundsForLessThanBranch(src1Val, src2Val);
  8657. if(!specialize)
  8658. {
  8659. return false;
  8660. }
  8661. opcode = Js::OpCode::BrLt_I4;
  8662. // We'll get a warning if we don't assign a value to these...
  8663. // We'll assert if we use them and make a range where min > max
  8664. newMin = 2; newMax = 1;
  8665. break;
  8666. }
  8667. case Js::OpCode::BrLe_A:
  8668. case Js::OpCode::BrNotGt_A:
  8669. {
  8670. if(DoConstFold() &&
  8671. !IsLoopPrePass() &&
  8672. TryOptConstFoldBrGreaterThan(instr, false, src1Val, min1, max1, src2Val, min2, max2))
  8673. {
  8674. return true;
  8675. }
  8676. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8677. UpdateIntBoundsForLessThanOrEqualBranch(src1Val, src2Val);
  8678. if(!specialize)
  8679. {
  8680. return false;
  8681. }
  8682. opcode = Js::OpCode::BrLe_I4;
  8683. // We'll get a warning if we don't assign a value to these...
  8684. // We'll assert if we use them and make a range where min > max
  8685. newMin = 2; newMax = 1;
  8686. break;
  8687. }
  8688. default:
  8689. return false;
  8690. }
  8691. // If this instruction is in a range of instructions where int overflow does not matter, we will still specialize it
  8692. // (won't leave it unspecialized based on heuristics), since it is most likely worth specializing, and the dst value
  8693. // needs to be guaranteed to be an int
  8694. if(!ignoredIntOverflow &&
  8695. !ignoredNegativeZero &&
  8696. !needsBoolConv &&
  8697. instr->ShouldCheckForIntOverflow() &&
  8698. !IsWorthSpecializingToInt32(instr, src1Val, src2Val))
  8699. {
  8700. // Even though type specialization is being skipped since it may not be worth it, the proper value should still be
  8701. // maintained so that the result may be type specialized later. An int value is not created for the dst in any of
  8702. // the following cases.
  8703. // - A bailout check is necessary to specialize this instruction. The bailout check is what guarantees the result to
  8704. // be an int, but since we're not going to specialize this instruction, there won't be a bailout check.
  8705. // - Aggressive int type specialization is disabled and we're in a loop prepass. We're conservative on dst values in
  8706. // that case, especially if the dst sym is live on the back-edge.
  8707. if(bailOutKind == IR::BailOutInvalid &&
  8708. instr->GetDst() &&
  8709. src1Val->GetValueInfo()->IsInt() &&
  8710. src2Val->GetValueInfo()->IsInt() &&
  8711. (DoAggressiveIntTypeSpec() || !this->IsLoopPrePass()))
  8712. {
  8713. *pDstVal = CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, src2Val);
  8714. }
  8715. return false;
  8716. }
  8717. } // case default
  8718. } // switch
  8719. LOutsideSwitch:
  8720. this->ignoredIntOverflowForCurrentInstr = ignoredIntOverflow;
  8721. this->ignoredNegativeZeroForCurrentInstr = ignoredNegativeZero;
  8722. {
  8723. // Try CSE again before modifying the IR, in case some attributes are required for successful CSE
  8724. Value *src1IndirIndexVal = nullptr;
  8725. if(CSEOptimize(currentBlock, &instr, &src1Val, &src2Val, &src1IndirIndexVal, true /* intMathExprOnly */))
  8726. {
  8727. *redoTypeSpecRef = true;
  8728. return false;
  8729. }
  8730. }
  8731. const Js::OpCode originalOpCode = instr->m_opcode;
  8732. if (!this->IsLoopPrePass())
  8733. {
  8734. // No re-write on prepass
  8735. instr->m_opcode = opcode;
  8736. }
  8737. Value *src1ValueToSpecialize = src1Val, *src2ValueToSpecialize = src2Val;
  8738. // Lossy conversions to int32 must be done based on the original source values. For instance, if one of the values is a
  8739. // float constant with a value that fits in a uint32 but not an int32, and the instruction can ignore int overflow, the
  8740. // source value for the purposes of int specialization would have been changed to an int constant value by ignoring
  8741. // overflow. If we were to specialize the sym using the int constant value, it would be treated as a lossless
  8742. // conversion, but since there may be subsequent uses of the same float constant value that may not ignore overflow,
  8743. // this must be treated as a lossy conversion by specializing the sym using the original float constant value.
  8744. if(src1Lossy)
  8745. {
  8746. src1ValueToSpecialize = src1OriginalVal;
  8747. }
  8748. if (src2Lossy)
  8749. {
  8750. src2ValueToSpecialize = src2OriginalVal;
  8751. }
  8752. // Make sure the srcs are specialized
  8753. IR::Opnd* src1 = instr->GetSrc1();
  8754. this->ToInt32(instr, src1, this->currentBlock, src1ValueToSpecialize, nullptr, src1Lossy);
  8755. if (!skipSrc2)
  8756. {
  8757. IR::Opnd* src2 = instr->GetSrc2();
  8758. this->ToInt32(instr, src2, this->currentBlock, src2ValueToSpecialize, nullptr, src2Lossy);
  8759. }
  8760. if(bailOutKind != IR::BailOutInvalid && !this->IsLoopPrePass())
  8761. {
  8762. GenerateBailAtOperation(&instr, bailOutKind);
  8763. }
  8764. if (!skipDst && instr->GetDst())
  8765. {
  8766. if (needsBoolConv)
  8767. {
  8768. IR::RegOpnd *varDst;
  8769. if (this->IsLoopPrePass())
  8770. {
  8771. varDst = instr->GetDst()->AsRegOpnd();
  8772. this->ToVarRegOpnd(varDst, this->currentBlock);
  8773. }
  8774. else
  8775. {
  8776. // Generate:
  8777. // t1.i = CmCC t2.i, t3.i
  8778. // t1.v = Conv_bool t1.i
  8779. //
  8780. // If the only uses of t1 are ints, the conv_bool will get dead-stored
  8781. TypeSpecializeIntDst(instr, originalOpCode, nullptr, src1Val, src2Val, bailOutKind, newMin, newMax, pDstVal);
  8782. IR::RegOpnd *intDst = instr->GetDst()->AsRegOpnd();
  8783. intDst->SetIsJITOptimizedReg(true);
  8784. varDst = IR::RegOpnd::New(intDst->m_sym->GetVarEquivSym(this->func), TyVar, this->func);
  8785. IR::Instr *convBoolInstr = IR::Instr::New(Js::OpCode::Conv_Bool, varDst, intDst, this->func);
  8786. // In some cases (e.g. unsigned compare peep code), a comparison will use variables
  8787. // other than the ones initially intended for it, if we can determine that we would
  8788. // arrive at the same result. This means that we get a ByteCodeUses operation after
  8789. // the actual comparison. Since Inserting the Conv_bool just after the compare, and
  8790. // just before the ByteCodeUses, would cause issues later on with register lifetime
  8791. // calculation, we want to insert the Conv_bool after the whole compare instruction
  8792. // block.
  8793. IR::Instr *putAfter = instr;
  8794. while (putAfter->m_next && putAfter->m_next->IsByteCodeUsesInstrFor(instr))
  8795. {
  8796. putAfter = putAfter->m_next;
  8797. }
  8798. putAfter->InsertAfter(convBoolInstr);
  8799. convBoolInstr->SetByteCodeOffset(instr);
  8800. this->ToVarRegOpnd(varDst, this->currentBlock);
  8801. CurrentBlockData()->liveInt32Syms->Set(varDst->m_sym->m_id);
  8802. CurrentBlockData()->liveLossyInt32Syms->Set(varDst->m_sym->m_id);
  8803. }
  8804. *pDstVal = this->NewGenericValue(ValueType::Boolean, varDst);
  8805. }
  8806. else
  8807. {
  8808. TypeSpecializeIntDst(
  8809. instr,
  8810. originalOpCode,
  8811. nullptr,
  8812. src1Val,
  8813. src2Val,
  8814. bailOutKind,
  8815. newMin,
  8816. newMax,
  8817. pDstVal,
  8818. addSubConstantInfo.HasInfo() ? &addSubConstantInfo : nullptr);
  8819. }
  8820. }
  8821. if(bailOutKind == IR::BailOutInvalid)
  8822. {
  8823. GOPT_TRACE(_u("Type specialized to INT\n"));
  8824. #if ENABLE_DEBUG_CONFIG_OPTIONS
  8825. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  8826. {
  8827. Output::Print(_u("Type specialized to INT: "));
  8828. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  8829. }
  8830. #endif
  8831. }
  8832. else
  8833. {
  8834. GOPT_TRACE(_u("Type specialized to INT with bailout on:\n"));
  8835. if(bailOutKind & (IR::BailOutOnOverflow | IR::BailOutOnMulOverflow) )
  8836. {
  8837. GOPT_TRACE(_u(" Overflow\n"));
  8838. #if ENABLE_DEBUG_CONFIG_OPTIONS
  8839. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  8840. {
  8841. Output::Print(_u("Type specialized to INT with bailout (%S): "), "Overflow");
  8842. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  8843. }
  8844. #endif
  8845. }
  8846. if(bailOutKind & IR::BailOutOnNegativeZero)
  8847. {
  8848. GOPT_TRACE(_u(" Zero\n"));
  8849. #if ENABLE_DEBUG_CONFIG_OPTIONS
  8850. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  8851. {
  8852. Output::Print(_u("Type specialized to INT with bailout (%S): "), "Zero");
  8853. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  8854. }
  8855. #endif
  8856. }
  8857. }
  8858. return true;
  8859. }
  8860. bool
  8861. GlobOpt::IsWorthSpecializingToInt32Branch(IR::Instr const * instr, Value const * src1Val, Value const * src2Val) const
  8862. {
  8863. if (!src1Val->GetValueInfo()->HasIntConstantValue() && instr->GetSrc1()->IsRegOpnd())
  8864. {
  8865. StackSym const *sym1 = instr->GetSrc1()->AsRegOpnd()->m_sym;
  8866. if (CurrentBlockData()->IsInt32TypeSpecialized(sym1) == false)
  8867. {
  8868. if (!src2Val->GetValueInfo()->HasIntConstantValue() && instr->GetSrc2()->IsRegOpnd())
  8869. {
  8870. StackSym const *sym2 = instr->GetSrc2()->AsRegOpnd()->m_sym;
  8871. if (CurrentBlockData()->IsInt32TypeSpecialized(sym2) == false)
  8872. {
  8873. // Type specializing a Br itself isn't worth it, unless one src
  8874. // is already type specialized
  8875. return false;
  8876. }
  8877. }
  8878. }
  8879. }
  8880. return true;
  8881. }
  8882. bool
  8883. GlobOpt::TryOptConstFoldBrFalse(
  8884. IR::Instr *const instr,
  8885. Value *const srcValue,
  8886. const int32 min,
  8887. const int32 max)
  8888. {
  8889. Assert(instr);
  8890. Assert(instr->m_opcode == Js::OpCode::BrFalse_A || instr->m_opcode == Js::OpCode::BrTrue_A);
  8891. Assert(srcValue);
  8892. if(!(DoAggressiveIntTypeSpec() ? srcValue->GetValueInfo()->IsLikelyInt() : srcValue->GetValueInfo()->IsInt()))
  8893. {
  8894. return false;
  8895. }
  8896. if(ValueInfo::IsEqualTo(srcValue, min, max, nullptr, 0, 0))
  8897. {
  8898. OptConstFoldBr(instr->m_opcode == Js::OpCode::BrFalse_A, instr, srcValue);
  8899. return true;
  8900. }
  8901. if(ValueInfo::IsNotEqualTo(srcValue, min, max, nullptr, 0, 0))
  8902. {
  8903. OptConstFoldBr(instr->m_opcode == Js::OpCode::BrTrue_A, instr, srcValue);
  8904. return true;
  8905. }
  8906. return false;
  8907. }
  8908. bool
  8909. GlobOpt::TryOptConstFoldBrEqual(
  8910. IR::Instr *const instr,
  8911. const bool branchOnEqual,
  8912. Value *const src1Value,
  8913. const int32 min1,
  8914. const int32 max1,
  8915. Value *const src2Value,
  8916. const int32 min2,
  8917. const int32 max2)
  8918. {
  8919. Assert(instr);
  8920. Assert(src1Value);
  8921. Assert(DoAggressiveIntTypeSpec() ? src1Value->GetValueInfo()->IsLikelyInt() : src1Value->GetValueInfo()->IsInt());
  8922. Assert(src2Value);
  8923. Assert(DoAggressiveIntTypeSpec() ? src2Value->GetValueInfo()->IsLikelyInt() : src2Value->GetValueInfo()->IsInt());
  8924. if(ValueInfo::IsEqualTo(src1Value, min1, max1, src2Value, min2, max2))
  8925. {
  8926. OptConstFoldBr(branchOnEqual, instr, src1Value, src2Value);
  8927. return true;
  8928. }
  8929. if(ValueInfo::IsNotEqualTo(src1Value, min1, max1, src2Value, min2, max2))
  8930. {
  8931. OptConstFoldBr(!branchOnEqual, instr, src1Value, src2Value);
  8932. return true;
  8933. }
  8934. return false;
  8935. }
  8936. bool
  8937. GlobOpt::TryOptConstFoldBrGreaterThan(
  8938. IR::Instr *const instr,
  8939. const bool branchOnGreaterThan,
  8940. Value *const src1Value,
  8941. const int32 min1,
  8942. const int32 max1,
  8943. Value *const src2Value,
  8944. const int32 min2,
  8945. const int32 max2)
  8946. {
  8947. Assert(instr);
  8948. Assert(src1Value);
  8949. Assert(DoAggressiveIntTypeSpec() ? src1Value->GetValueInfo()->IsLikelyInt() : src1Value->GetValueInfo()->IsInt());
  8950. Assert(src2Value);
  8951. Assert(DoAggressiveIntTypeSpec() ? src2Value->GetValueInfo()->IsLikelyInt() : src2Value->GetValueInfo()->IsInt());
  8952. if(ValueInfo::IsGreaterThan(src1Value, min1, max1, src2Value, min2, max2))
  8953. {
  8954. OptConstFoldBr(branchOnGreaterThan, instr, src1Value, src2Value);
  8955. return true;
  8956. }
  8957. if(ValueInfo::IsLessThanOrEqualTo(src1Value, min1, max1, src2Value, min2, max2))
  8958. {
  8959. OptConstFoldBr(!branchOnGreaterThan, instr, src1Value, src2Value);
  8960. return true;
  8961. }
  8962. return false;
  8963. }
  8964. bool
  8965. GlobOpt::TryOptConstFoldBrGreaterThanOrEqual(
  8966. IR::Instr *const instr,
  8967. const bool branchOnGreaterThanOrEqual,
  8968. Value *const src1Value,
  8969. const int32 min1,
  8970. const int32 max1,
  8971. Value *const src2Value,
  8972. const int32 min2,
  8973. const int32 max2)
  8974. {
  8975. Assert(instr);
  8976. Assert(src1Value);
  8977. Assert(DoAggressiveIntTypeSpec() ? src1Value->GetValueInfo()->IsLikelyInt() : src1Value->GetValueInfo()->IsInt());
  8978. Assert(src2Value);
  8979. Assert(DoAggressiveIntTypeSpec() ? src2Value->GetValueInfo()->IsLikelyInt() : src2Value->GetValueInfo()->IsInt());
  8980. if(ValueInfo::IsGreaterThanOrEqualTo(src1Value, min1, max1, src2Value, min2, max2))
  8981. {
  8982. OptConstFoldBr(branchOnGreaterThanOrEqual, instr, src1Value, src2Value);
  8983. return true;
  8984. }
  8985. if(ValueInfo::IsLessThan(src1Value, min1, max1, src2Value, min2, max2))
  8986. {
  8987. OptConstFoldBr(!branchOnGreaterThanOrEqual, instr, src1Value, src2Value);
  8988. return true;
  8989. }
  8990. return false;
  8991. }
  8992. bool
  8993. GlobOpt::TryOptConstFoldBrUnsignedLessThan(
  8994. IR::Instr *const instr,
  8995. const bool branchOnLessThan,
  8996. Value *const src1Value,
  8997. const int32 min1,
  8998. const int32 max1,
  8999. Value *const src2Value,
  9000. const int32 min2,
  9001. const int32 max2)
  9002. {
  9003. Assert(DoConstFold());
  9004. Assert(!IsLoopPrePass());
  9005. if(!src1Value ||
  9006. !src2Value ||
  9007. !(
  9008. DoAggressiveIntTypeSpec()
  9009. ? src1Value->GetValueInfo()->IsLikelyInt() && src2Value->GetValueInfo()->IsLikelyInt()
  9010. : src1Value->GetValueInfo()->IsInt() && src2Value->GetValueInfo()->IsInt()
  9011. ))
  9012. {
  9013. return false;
  9014. }
  9015. uint uMin1 = (min1 < 0 ? (max1 < 0 ? min((uint)min1, (uint)max1) : 0) : min1);
  9016. uint uMax1 = max((uint)min1, (uint)max1);
  9017. uint uMin2 = (min2 < 0 ? (max2 < 0 ? min((uint)min2, (uint)max2) : 0) : min2);
  9018. uint uMax2 = max((uint)min2, (uint)max2);
  9019. if (uMax1 < uMin2)
  9020. {
  9021. // Range 1 is always lesser than Range 2
  9022. OptConstFoldBr(branchOnLessThan, instr, src1Value, src2Value);
  9023. return true;
  9024. }
  9025. if (uMin1 >= uMax2)
  9026. {
  9027. // Range 2 is always lesser than Range 1
  9028. OptConstFoldBr(!branchOnLessThan, instr, src1Value, src2Value);
  9029. return true;
  9030. }
  9031. return false;
  9032. }
  9033. bool
  9034. GlobOpt::TryOptConstFoldBrUnsignedGreaterThan(
  9035. IR::Instr *const instr,
  9036. const bool branchOnGreaterThan,
  9037. Value *const src1Value,
  9038. const int32 min1,
  9039. const int32 max1,
  9040. Value *const src2Value,
  9041. const int32 min2,
  9042. const int32 max2)
  9043. {
  9044. Assert(DoConstFold());
  9045. Assert(!IsLoopPrePass());
  9046. if(!src1Value ||
  9047. !src2Value ||
  9048. !(
  9049. DoAggressiveIntTypeSpec()
  9050. ? src1Value->GetValueInfo()->IsLikelyInt() && src2Value->GetValueInfo()->IsLikelyInt()
  9051. : src1Value->GetValueInfo()->IsInt() && src2Value->GetValueInfo()->IsInt()
  9052. ))
  9053. {
  9054. return false;
  9055. }
  9056. uint uMin1 = (min1 < 0 ? (max1 < 0 ? min((uint)min1, (uint)max1) : 0) : min1);
  9057. uint uMax1 = max((uint)min1, (uint)max1);
  9058. uint uMin2 = (min2 < 0 ? (max2 < 0 ? min((uint)min2, (uint)max2) : 0) : min2);
  9059. uint uMax2 = max((uint)min2, (uint)max2);
  9060. if (uMin1 > uMax2)
  9061. {
  9062. // Range 1 is always greater than Range 2
  9063. OptConstFoldBr(branchOnGreaterThan, instr, src1Value, src2Value);
  9064. return true;
  9065. }
  9066. if (uMax1 <= uMin2)
  9067. {
  9068. // Range 2 is always greater than Range 1
  9069. OptConstFoldBr(!branchOnGreaterThan, instr, src1Value, src2Value);
  9070. return true;
  9071. }
  9072. return false;
  9073. }
  9074. void
  9075. GlobOpt::SetPathDependentInfo(const bool conditionToBranch, const PathDependentInfo &info)
  9076. {
  9077. Assert(this->currentBlock->GetSuccList()->Count() == 2);
  9078. IR::Instr * fallthrough = this->currentBlock->GetNext()->GetFirstInstr();
  9079. FOREACH_SLISTBASECOUNTED_ENTRY(FlowEdge*, edge, this->currentBlock->GetSuccList())
  9080. {
  9081. if (conditionToBranch == (edge->GetSucc()->GetFirstInstr() != fallthrough))
  9082. {
  9083. edge->SetPathDependentInfo(info, alloc);
  9084. return;
  9085. }
  9086. }
  9087. NEXT_SLISTBASECOUNTED_ENTRY;
  9088. // In case flowgraph peeps is disabled, we could have conditional branch to next instr
  9089. Assert(this->func->HasTry() || PHASE_OFF(Js::FGPeepsPhase, this->func));
  9090. }
  9091. PathDependentInfoToRestore
  9092. GlobOpt::UpdatePathDependentInfo(PathDependentInfo *const info)
  9093. {
  9094. Assert(info);
  9095. if(!info->HasInfo())
  9096. {
  9097. return PathDependentInfoToRestore();
  9098. }
  9099. decltype(&GlobOpt::UpdateIntBoundsForEqual) UpdateIntBoundsForLeftValue, UpdateIntBoundsForRightValue;
  9100. switch(info->Relationship())
  9101. {
  9102. case PathDependentRelationship::Equal:
  9103. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForEqual;
  9104. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForEqual;
  9105. break;
  9106. case PathDependentRelationship::NotEqual:
  9107. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForNotEqual;
  9108. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForNotEqual;
  9109. break;
  9110. case PathDependentRelationship::GreaterThanOrEqual:
  9111. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForGreaterThanOrEqual;
  9112. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForLessThanOrEqual;
  9113. break;
  9114. case PathDependentRelationship::GreaterThan:
  9115. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForGreaterThan;
  9116. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForLessThan;
  9117. break;
  9118. case PathDependentRelationship::LessThanOrEqual:
  9119. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForLessThanOrEqual;
  9120. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForGreaterThanOrEqual;
  9121. break;
  9122. case PathDependentRelationship::LessThan:
  9123. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForLessThan;
  9124. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForGreaterThan;
  9125. break;
  9126. default:
  9127. Assert(false);
  9128. __assume(false);
  9129. }
  9130. ValueInfo *leftValueInfo = info->LeftValue()->GetValueInfo();
  9131. IntConstantBounds leftConstantBounds;
  9132. AssertVerify(leftValueInfo->TryGetIntConstantBounds(&leftConstantBounds, true));
  9133. ValueInfo *rightValueInfo;
  9134. IntConstantBounds rightConstantBounds;
  9135. if(info->RightValue())
  9136. {
  9137. rightValueInfo = info->RightValue()->GetValueInfo();
  9138. AssertVerify(rightValueInfo->TryGetIntConstantBounds(&rightConstantBounds, true));
  9139. }
  9140. else
  9141. {
  9142. rightValueInfo = nullptr;
  9143. rightConstantBounds = IntConstantBounds(info->RightConstantValue(), info->RightConstantValue());
  9144. }
  9145. ValueInfo *const newLeftValueInfo =
  9146. (this->*UpdateIntBoundsForLeftValue)(
  9147. info->LeftValue(),
  9148. leftConstantBounds,
  9149. info->RightValue(),
  9150. rightConstantBounds,
  9151. true);
  9152. if(newLeftValueInfo)
  9153. {
  9154. ChangeValueInfo(nullptr, info->LeftValue(), newLeftValueInfo);
  9155. AssertVerify(newLeftValueInfo->TryGetIntConstantBounds(&leftConstantBounds, true));
  9156. }
  9157. else
  9158. {
  9159. leftValueInfo = nullptr;
  9160. }
  9161. ValueInfo *const newRightValueInfo =
  9162. (this->*UpdateIntBoundsForRightValue)(
  9163. info->RightValue(),
  9164. rightConstantBounds,
  9165. info->LeftValue(),
  9166. leftConstantBounds,
  9167. true);
  9168. if(newRightValueInfo)
  9169. {
  9170. ChangeValueInfo(nullptr, info->RightValue(), newRightValueInfo);
  9171. }
  9172. else
  9173. {
  9174. rightValueInfo = nullptr;
  9175. }
  9176. return PathDependentInfoToRestore(leftValueInfo, rightValueInfo);
  9177. }
  9178. void
  9179. GlobOpt::RestorePathDependentInfo(PathDependentInfo *const info, const PathDependentInfoToRestore infoToRestore)
  9180. {
  9181. Assert(info);
  9182. if(infoToRestore.LeftValueInfo())
  9183. {
  9184. Assert(info->LeftValue());
  9185. ChangeValueInfo(nullptr, info->LeftValue(), infoToRestore.LeftValueInfo());
  9186. }
  9187. if(infoToRestore.RightValueInfo())
  9188. {
  9189. Assert(info->RightValue());
  9190. ChangeValueInfo(nullptr, info->RightValue(), infoToRestore.RightValueInfo());
  9191. }
  9192. }
  9193. bool
  9194. GlobOpt::TypeSpecializeFloatUnary(IR::Instr **pInstr, Value *src1Val, Value **pDstVal, bool skipDst /* = false */)
  9195. {
  9196. IR::Instr *&instr = *pInstr;
  9197. IR::Opnd *src1;
  9198. IR::Opnd *dst;
  9199. Js::OpCode opcode = instr->m_opcode;
  9200. Value *valueToTransfer = nullptr;
  9201. Assert(src1Val && src1Val->GetValueInfo()->IsLikelyNumber() || OpCodeAttr::IsInlineBuiltIn(instr->m_opcode));
  9202. if (!this->DoFloatTypeSpec())
  9203. {
  9204. return false;
  9205. }
  9206. // For inline built-ins we need to do type specialization. Check upfront to avoid duplicating same case labels.
  9207. if (!OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  9208. {
  9209. switch (opcode)
  9210. {
  9211. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  9212. skipDst = true;
  9213. // fall-through
  9214. case Js::OpCode::Ld_A:
  9215. case Js::OpCode::BrTrue_A:
  9216. case Js::OpCode::BrFalse_A:
  9217. if (instr->GetSrc1()->IsRegOpnd())
  9218. {
  9219. StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  9220. if (CurrentBlockData()->IsFloat64TypeSpecialized(sym) == false)
  9221. {
  9222. // Type specializing an Ld_A isn't worth it, unless the src
  9223. // is already type specialized
  9224. return false;
  9225. }
  9226. }
  9227. if (instr->m_opcode == Js::OpCode::Ld_A)
  9228. {
  9229. valueToTransfer = src1Val;
  9230. }
  9231. break;
  9232. case Js::OpCode::Neg_A:
  9233. break;
  9234. case Js::OpCode::Conv_Num:
  9235. Assert(src1Val);
  9236. opcode = Js::OpCode::Ld_A;
  9237. valueToTransfer = src1Val;
  9238. if (!src1Val->GetValueInfo()->IsNumber())
  9239. {
  9240. StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  9241. valueToTransfer = NewGenericValue(ValueType::Float, instr->GetDst()->GetStackSym());
  9242. if (CurrentBlockData()->IsFloat64TypeSpecialized(sym) == false)
  9243. {
  9244. // Set the dst as a nonDeadStore. We want to keep the Ld_A to prevent the FromVar from
  9245. // being dead-stored, as it could cause implicit calls.
  9246. dst = instr->GetDst();
  9247. dst->AsRegOpnd()->m_dontDeadStore = true;
  9248. }
  9249. }
  9250. break;
  9251. case Js::OpCode::StElemI_A:
  9252. case Js::OpCode::StElemI_A_Strict:
  9253. case Js::OpCode::StElemC:
  9254. return TypeSpecializeStElem(pInstr, src1Val, pDstVal);
  9255. default:
  9256. return false;
  9257. }
  9258. }
  9259. // Make sure the srcs are specialized
  9260. src1 = instr->GetSrc1();
  9261. // Use original val when calling toFloat64 as this is what we'll use to try hoisting the fromVar if we're in a loop.
  9262. this->ToFloat64(instr, src1, this->currentBlock, src1Val, nullptr, IR::BailOutPrimitiveButString);
  9263. if (!skipDst)
  9264. {
  9265. dst = instr->GetDst();
  9266. if (dst)
  9267. {
  9268. this->TypeSpecializeFloatDst(instr, valueToTransfer, src1Val, nullptr, pDstVal);
  9269. if (!this->IsLoopPrePass())
  9270. {
  9271. instr->m_opcode = opcode;
  9272. }
  9273. }
  9274. }
  9275. GOPT_TRACE_INSTR(instr, _u("Type specialized to FLOAT: "));
  9276. #if ENABLE_DEBUG_CONFIG_OPTIONS
  9277. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FloatTypeSpecPhase))
  9278. {
  9279. Output::Print(_u("Type specialized to FLOAT: "));
  9280. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  9281. }
  9282. #endif
  9283. return true;
  9284. }
  9285. // Unconditionally type-spec dst to float.
  9286. void
  9287. GlobOpt::TypeSpecializeFloatDst(IR::Instr *instr, Value *valToTransfer, Value *const src1Value, Value *const src2Value, Value **pDstVal)
  9288. {
  9289. IR::Opnd* dst = instr->GetDst();
  9290. Assert(dst);
  9291. AssertMsg(dst->IsRegOpnd(), "What else?");
  9292. this->ToFloat64Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  9293. if(valToTransfer)
  9294. {
  9295. *pDstVal = this->ValueNumberTransferDst(instr, valToTransfer);
  9296. CurrentBlockData()->InsertNewValue(*pDstVal, dst);
  9297. }
  9298. else
  9299. {
  9300. *pDstVal = CreateDstUntransferredValue(ValueType::Float, instr, src1Value, src2Value);
  9301. }
  9302. }
  9303. bool
  9304. GlobOpt::TypeSpecializeLdLen(
  9305. IR::Instr * *const instrRef,
  9306. Value * *const src1ValueRef,
  9307. Value * *const dstValueRef,
  9308. bool *const forceInvariantHoistingRef)
  9309. {
  9310. Assert(instrRef);
  9311. IR::Instr *&instr = *instrRef;
  9312. Assert(instr);
  9313. Assert(instr->m_opcode == Js::OpCode::LdLen_A);
  9314. Assert(src1ValueRef);
  9315. Value *&src1Value = *src1ValueRef;
  9316. Assert(dstValueRef);
  9317. Value *&dstValue = *dstValueRef;
  9318. Assert(forceInvariantHoistingRef);
  9319. bool &forceInvariantHoisting = *forceInvariantHoistingRef;
  9320. if(!DoLdLenIntSpec(instr, instr->GetSrc1()->GetValueType()))
  9321. {
  9322. return false;
  9323. }
  9324. IR::BailOutKind bailOutKind = IR::BailOutOnIrregularLength;
  9325. if(!IsLoopPrePass())
  9326. {
  9327. IR::RegOpnd *const baseOpnd = instr->GetSrc1()->AsRegOpnd();
  9328. if(baseOpnd->IsArrayRegOpnd())
  9329. {
  9330. StackSym *const lengthSym = baseOpnd->AsArrayRegOpnd()->LengthSym();
  9331. if(lengthSym)
  9332. {
  9333. CaptureByteCodeSymUses(instr);
  9334. instr->m_opcode = Js::OpCode::Ld_I4;
  9335. instr->ReplaceSrc1(IR::RegOpnd::New(lengthSym, lengthSym->GetType(), func));
  9336. instr->ClearBailOutInfo();
  9337. // Find the hoisted length value
  9338. Value *const lengthValue = CurrentBlockData()->FindValue(lengthSym);
  9339. Assert(lengthValue);
  9340. src1Value = lengthValue;
  9341. ValueInfo *const lengthValueInfo = lengthValue->GetValueInfo();
  9342. IntConstantBounds lengthConstantBounds;
  9343. AssertVerify(lengthValueInfo->TryGetIntConstantBounds(&lengthConstantBounds));
  9344. Assert(lengthConstantBounds.LowerBound() >= 0);
  9345. if (lengthValueInfo->GetSymStore() == lengthSym)
  9346. {
  9347. // When type specializing the dst below, we will end up inserting lengthSym.u32 as symstore for a var
  9348. // Clear the symstore here, so that we dont end up with problems with copyprop later on
  9349. lengthValueInfo->SetSymStore(nullptr);
  9350. }
  9351. // Int-specialize, and transfer the value to the dst
  9352. TypeSpecializeIntDst(
  9353. instr,
  9354. Js::OpCode::LdLen_A,
  9355. src1Value,
  9356. src1Value,
  9357. nullptr,
  9358. bailOutKind,
  9359. lengthConstantBounds.LowerBound(),
  9360. lengthConstantBounds.UpperBound(),
  9361. &dstValue);
  9362. // Try to force hoisting the Ld_I4 so that the length will have an invariant sym store that can be
  9363. // copy-propped. Invariant hoisting does not automatically hoist Ld_I4.
  9364. forceInvariantHoisting = true;
  9365. return true;
  9366. }
  9367. }
  9368. if (instr->HasBailOutInfo())
  9369. {
  9370. Assert(instr->GetBailOutKind() == IR::BailOutMarkTempObject);
  9371. bailOutKind = IR::BailOutOnIrregularLength | IR::BailOutMarkTempObject;
  9372. instr->SetBailOutKind(bailOutKind);
  9373. }
  9374. else
  9375. {
  9376. Assert(bailOutKind == IR::BailOutOnIrregularLength);
  9377. GenerateBailAtOperation(&instr, bailOutKind);
  9378. }
  9379. }
  9380. TypeSpecializeIntDst(
  9381. instr,
  9382. Js::OpCode::LdLen_A,
  9383. nullptr,
  9384. nullptr,
  9385. nullptr,
  9386. bailOutKind,
  9387. 0,
  9388. INT32_MAX,
  9389. &dstValue);
  9390. return true;
  9391. }
  9392. bool
  9393. GlobOpt::TypeSpecializeFloatBinary(IR::Instr *instr, Value *src1Val, Value *src2Val, Value **pDstVal)
  9394. {
  9395. IR::Opnd *src1;
  9396. IR::Opnd *src2;
  9397. IR::Opnd *dst;
  9398. bool allowUndefinedOrNullSrc1 = true;
  9399. bool allowUndefinedOrNullSrc2 = true;
  9400. bool skipSrc1 = false;
  9401. bool skipSrc2 = false;
  9402. bool skipDst = false;
  9403. if (!this->DoFloatTypeSpec())
  9404. {
  9405. return false;
  9406. }
  9407. // For inline built-ins we need to do type specialization. Check upfront to avoid duplicating same case labels.
  9408. if (!OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  9409. {
  9410. switch (instr->m_opcode)
  9411. {
  9412. case Js::OpCode::Sub_A:
  9413. case Js::OpCode::Mul_A:
  9414. case Js::OpCode::Div_A:
  9415. case Js::OpCode::Expo_A:
  9416. // Avoid if one source is known not to be a number.
  9417. if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber())
  9418. {
  9419. return false;
  9420. }
  9421. break;
  9422. case Js::OpCode::BrSrEq_A:
  9423. case Js::OpCode::BrSrNeq_A:
  9424. case Js::OpCode::BrEq_A:
  9425. case Js::OpCode::BrNeq_A:
  9426. case Js::OpCode::BrSrNotEq_A:
  9427. case Js::OpCode::BrNotEq_A:
  9428. case Js::OpCode::BrSrNotNeq_A:
  9429. case Js::OpCode::BrNotNeq_A:
  9430. // Avoid if one source is known not to be a number.
  9431. if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber())
  9432. {
  9433. return false;
  9434. }
  9435. // Undef == Undef, but +Undef != +Undef
  9436. // 0.0 != null, but 0.0 == +null
  9437. //
  9438. // So Bailout on anything but numbers for both src1 and src2
  9439. allowUndefinedOrNullSrc1 = false;
  9440. allowUndefinedOrNullSrc2 = false;
  9441. break;
  9442. case Js::OpCode::BrGt_A:
  9443. case Js::OpCode::BrGe_A:
  9444. case Js::OpCode::BrLt_A:
  9445. case Js::OpCode::BrLe_A:
  9446. case Js::OpCode::BrNotGt_A:
  9447. case Js::OpCode::BrNotGe_A:
  9448. case Js::OpCode::BrNotLt_A:
  9449. case Js::OpCode::BrNotLe_A:
  9450. // Avoid if one source is known not to be a number.
  9451. if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber())
  9452. {
  9453. return false;
  9454. }
  9455. break;
  9456. case Js::OpCode::Add_A:
  9457. // For Add, we need both sources to be Numbers, otherwise it could be a string concat
  9458. if (!src1Val || !src2Val || !(src1Val->GetValueInfo()->IsLikelyNumber() && src2Val->GetValueInfo()->IsLikelyNumber()))
  9459. {
  9460. return false;
  9461. }
  9462. break;
  9463. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  9464. skipSrc2 = true;
  9465. skipDst = true;
  9466. break;
  9467. default:
  9468. return false;
  9469. }
  9470. }
  9471. else
  9472. {
  9473. switch (instr->m_opcode)
  9474. {
  9475. case Js::OpCode::InlineArrayPush:
  9476. bool isFloatConstMissingItem = src2Val->GetValueInfo()->IsFloatConstant();
  9477. if(isFloatConstMissingItem)
  9478. {
  9479. FloatConstType floatValue = src2Val->GetValueInfo()->AsFloatConstant()->FloatValue();
  9480. isFloatConstMissingItem = Js::SparseArraySegment<double>::IsMissingItem(&floatValue);
  9481. }
  9482. // Don't specialize if the element is not likelyNumber - we will surely bailout
  9483. if(!(src2Val->GetValueInfo()->IsLikelyNumber()) || isFloatConstMissingItem)
  9484. {
  9485. return false;
  9486. }
  9487. // Only specialize the Second source - element
  9488. skipSrc1 = true;
  9489. skipDst = true;
  9490. allowUndefinedOrNullSrc2 = false;
  9491. break;
  9492. }
  9493. }
  9494. // Make sure the srcs are specialized
  9495. if(!skipSrc1)
  9496. {
  9497. src1 = instr->GetSrc1();
  9498. this->ToFloat64(instr, src1, this->currentBlock, src1Val, nullptr, (allowUndefinedOrNullSrc1 ? IR::BailOutPrimitiveButString : IR::BailOutNumberOnly));
  9499. }
  9500. if (!skipSrc2)
  9501. {
  9502. src2 = instr->GetSrc2();
  9503. this->ToFloat64(instr, src2, this->currentBlock, src2Val, nullptr, (allowUndefinedOrNullSrc2 ? IR::BailOutPrimitiveButString : IR::BailOutNumberOnly));
  9504. }
  9505. if (!skipDst)
  9506. {
  9507. dst = instr->GetDst();
  9508. if (dst)
  9509. {
  9510. *pDstVal = CreateDstUntransferredValue(ValueType::Float, instr, src1Val, src2Val);
  9511. AssertMsg(dst->IsRegOpnd(), "What else?");
  9512. this->ToFloat64Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  9513. }
  9514. }
  9515. GOPT_TRACE_INSTR(instr, _u("Type specialized to FLOAT: "));
  9516. #if ENABLE_DEBUG_CONFIG_OPTIONS
  9517. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FloatTypeSpecPhase))
  9518. {
  9519. Output::Print(_u("Type specialized to FLOAT: "));
  9520. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  9521. }
  9522. #endif
  9523. return true;
  9524. }
  9525. bool
  9526. GlobOpt::TypeSpecializeStElem(IR::Instr ** pInstr, Value *src1Val, Value **pDstVal)
  9527. {
  9528. IR::Instr *&instr = *pInstr;
  9529. IR::RegOpnd *baseOpnd = instr->GetDst()->AsIndirOpnd()->GetBaseOpnd();
  9530. ValueType baseValueType(baseOpnd->GetValueType());
  9531. if (instr->DoStackArgsOpt(this->func) ||
  9532. (!this->DoTypedArrayTypeSpec() && baseValueType.IsLikelyOptimizedTypedArray()) ||
  9533. (!this->DoNativeArrayTypeSpec() && baseValueType.IsLikelyNativeArray()) ||
  9534. !(baseValueType.IsLikelyOptimizedTypedArray() || baseValueType.IsLikelyNativeArray()))
  9535. {
  9536. GOPT_TRACE_INSTR(instr, _u("Didn't type specialize array access, because typed array type specialization is disabled, or base is not an optimized typed array.\n"));
  9537. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  9538. {
  9539. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  9540. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  9541. baseValueType.ToString(baseValueTypeStr);
  9542. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not specialize because %s.\n"),
  9543. this->func->GetJITFunctionBody()->GetDisplayName(),
  9544. this->func->GetDebugNumberSet(debugStringBuffer),
  9545. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  9546. baseValueTypeStr,
  9547. instr->DoStackArgsOpt(this->func) ?
  9548. _u("instruction uses the arguments object") :
  9549. _u("typed array type specialization is disabled, or base is not an optimized typed array"));
  9550. Output::Flush();
  9551. }
  9552. return false;
  9553. }
  9554. Assert(instr->GetSrc1()->IsRegOpnd() || (src1Val && src1Val->GetValueInfo()->HasIntConstantValue()));
  9555. StackSym *sym = instr->GetSrc1()->IsRegOpnd() ? instr->GetSrc1()->AsRegOpnd()->m_sym : nullptr;
  9556. // Only type specialize the source of store element if the source symbol is already type specialized to int or float.
  9557. if (sym)
  9558. {
  9559. if (baseValueType.IsLikelyNativeArray())
  9560. {
  9561. // Gently coerce these src's into native if it seems likely to work.
  9562. // Otherwise we can't use the fast path to store.
  9563. // But don't try to put a float-specialized number into an int array this way.
  9564. if (!(
  9565. CurrentBlockData()->IsInt32TypeSpecialized(sym) ||
  9566. (
  9567. src1Val &&
  9568. (
  9569. DoAggressiveIntTypeSpec()
  9570. ? src1Val->GetValueInfo()->IsLikelyInt()
  9571. : src1Val->GetValueInfo()->IsInt()
  9572. )
  9573. )
  9574. ))
  9575. {
  9576. if (!(
  9577. CurrentBlockData()->IsFloat64TypeSpecialized(sym) ||
  9578. (src1Val && src1Val->GetValueInfo()->IsLikelyNumber())
  9579. ) ||
  9580. baseValueType.HasIntElements())
  9581. {
  9582. return false;
  9583. }
  9584. }
  9585. }
  9586. else if (!CurrentBlockData()->IsInt32TypeSpecialized(sym) && !CurrentBlockData()->IsFloat64TypeSpecialized(sym))
  9587. {
  9588. GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access, because src is not type specialized.\n"));
  9589. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  9590. {
  9591. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  9592. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  9593. baseValueType.ToString(baseValueTypeStr);
  9594. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not specialize because src is not specialized.\n"),
  9595. this->func->GetJITFunctionBody()->GetDisplayName(),
  9596. this->func->GetDebugNumberSet(debugStringBuffer),
  9597. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  9598. baseValueTypeStr);
  9599. Output::Flush();
  9600. }
  9601. return false;
  9602. }
  9603. }
  9604. int32 src1IntConstantValue;
  9605. if(baseValueType.IsLikelyNativeIntArray() && src1Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&src1IntConstantValue))
  9606. {
  9607. if(Js::SparseArraySegment<int32>::IsMissingItem(&src1IntConstantValue))
  9608. {
  9609. return false;
  9610. }
  9611. }
  9612. // Note: doing ToVarUses to make sure we do get the int32 version of the index before trying to access its value in
  9613. // ShouldExpectConventionalArrayIndexValue. Not sure why that never gave us a problem before.
  9614. Assert(instr->GetDst()->IsIndirOpnd());
  9615. IR::IndirOpnd *dst = instr->GetDst()->AsIndirOpnd();
  9616. // Make sure we use the int32 version of the index operand symbol, if available. Otherwise, ensure the var symbol is live (by
  9617. // potentially inserting a ToVar).
  9618. this->ToVarUses(instr, dst, /* isDst = */ true, nullptr);
  9619. if (!ShouldExpectConventionalArrayIndexValue(dst))
  9620. {
  9621. GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access, because index is negative or likely not int.\n"));
  9622. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  9623. {
  9624. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  9625. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  9626. baseValueType.ToString(baseValueTypeStr);
  9627. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not specialize because index is negative or likely not int.\n"),
  9628. this->func->GetJITFunctionBody()->GetDisplayName(),
  9629. this->func->GetDebugNumberSet(debugStringBuffer),
  9630. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  9631. baseValueTypeStr);
  9632. Output::Flush();
  9633. }
  9634. return false;
  9635. }
  9636. IRType toType = TyVar;
  9637. bool isLossyAllowed = true;
  9638. IR::BailOutKind arrayBailOutKind = IR::BailOutConventionalTypedArrayAccessOnly;
  9639. switch(baseValueType.GetObjectType())
  9640. {
  9641. case ObjectType::Int8Array:
  9642. case ObjectType::Uint8Array:
  9643. case ObjectType::Int16Array:
  9644. case ObjectType::Uint16Array:
  9645. case ObjectType::Int32Array:
  9646. case ObjectType::Int8VirtualArray:
  9647. case ObjectType::Uint8VirtualArray:
  9648. case ObjectType::Int16VirtualArray:
  9649. case ObjectType::Uint16VirtualArray:
  9650. case ObjectType::Int32VirtualArray:
  9651. case ObjectType::Int8MixedArray:
  9652. case ObjectType::Uint8MixedArray:
  9653. case ObjectType::Int16MixedArray:
  9654. case ObjectType::Uint16MixedArray:
  9655. case ObjectType::Int32MixedArray:
  9656. Int32Array:
  9657. if (this->DoAggressiveIntTypeSpec() || this->DoFloatTypeSpec())
  9658. {
  9659. toType = TyInt32;
  9660. }
  9661. break;
  9662. case ObjectType::Uint32Array:
  9663. case ObjectType::Uint32VirtualArray:
  9664. case ObjectType::Uint32MixedArray:
  9665. // Uint32Arrays may store values that overflow int32. If the value being stored comes from a symbol that's
  9666. // already losslessly type specialized to int32, we'll use it. Otherwise, if we only have a float64 specialized
  9667. // value, we don't want to force bailout if it doesn't fit in int32. Instead, we'll emit conversion in the
  9668. // lowerer, and handle overflow, if necessary.
  9669. if (!sym || CurrentBlockData()->IsInt32TypeSpecialized(sym))
  9670. {
  9671. toType = TyInt32;
  9672. }
  9673. else if (CurrentBlockData()->IsFloat64TypeSpecialized(sym))
  9674. {
  9675. toType = TyFloat64;
  9676. }
  9677. break;
  9678. case ObjectType::Float32Array:
  9679. case ObjectType::Float64Array:
  9680. case ObjectType::Float32VirtualArray:
  9681. case ObjectType::Float32MixedArray:
  9682. case ObjectType::Float64VirtualArray:
  9683. case ObjectType::Float64MixedArray:
  9684. Float64Array:
  9685. if (this->DoFloatTypeSpec())
  9686. {
  9687. toType = TyFloat64;
  9688. }
  9689. break;
  9690. case ObjectType::Uint8ClampedArray:
  9691. case ObjectType::Uint8ClampedVirtualArray:
  9692. case ObjectType::Uint8ClampedMixedArray:
  9693. // Uint8ClampedArray requires rounding (as opposed to truncation) of floating point values. If source symbol is
  9694. // float type specialized, type specialize this instruction to float as well, and handle rounding in the
  9695. // lowerer.
  9696. if (!sym || CurrentBlockData()->IsInt32TypeSpecialized(sym))
  9697. {
  9698. toType = TyInt32;
  9699. isLossyAllowed = false;
  9700. }
  9701. else if (CurrentBlockData()->IsFloat64TypeSpecialized(sym))
  9702. {
  9703. toType = TyFloat64;
  9704. }
  9705. break;
  9706. default:
  9707. Assert(baseValueType.IsLikelyNativeArray());
  9708. isLossyAllowed = false;
  9709. arrayBailOutKind = IR::BailOutConventionalNativeArrayAccessOnly;
  9710. if(baseValueType.HasIntElements())
  9711. {
  9712. goto Int32Array;
  9713. }
  9714. Assert(baseValueType.HasFloatElements());
  9715. goto Float64Array;
  9716. }
  9717. if (toType != TyVar)
  9718. {
  9719. GOPT_TRACE_INSTR(instr, _u("Type specialized array access.\n"));
  9720. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  9721. {
  9722. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  9723. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  9724. baseValueType.ToString(baseValueTypeStr);
  9725. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, type specialized to %s.\n"),
  9726. this->func->GetJITFunctionBody()->GetDisplayName(),
  9727. this->func->GetDebugNumberSet(debugStringBuffer),
  9728. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  9729. baseValueTypeStr,
  9730. toType == TyInt32 ? _u("int32") : _u("float64"));
  9731. Output::Flush();
  9732. }
  9733. IR::BailOutKind bailOutKind = ((toType == TyInt32) ? IR::BailOutIntOnly : IR::BailOutNumberOnly);
  9734. this->ToTypeSpecUse(instr, instr->GetSrc1(), this->currentBlock, src1Val, nullptr, toType, bailOutKind, /* lossy = */ isLossyAllowed);
  9735. if (!this->IsLoopPrePass())
  9736. {
  9737. bool bConvertToBailoutInstr = true;
  9738. // Definite StElemC doesn't need bailout, because it can't fail or cause conversion.
  9739. if (instr->m_opcode == Js::OpCode::StElemC && baseValueType.IsObject())
  9740. {
  9741. if (baseValueType.HasIntElements())
  9742. {
  9743. //Native int array requires a missing element check & bailout
  9744. int32 min = INT32_MIN;
  9745. int32 max = INT32_MAX;
  9746. if (src1Val->GetValueInfo()->GetIntValMinMax(&min, &max, false))
  9747. {
  9748. bConvertToBailoutInstr = ((min <= Js::JavascriptNativeIntArray::MissingItem) && (max >= Js::JavascriptNativeIntArray::MissingItem));
  9749. }
  9750. }
  9751. else
  9752. {
  9753. bConvertToBailoutInstr = false;
  9754. }
  9755. }
  9756. if (bConvertToBailoutInstr)
  9757. {
  9758. if(instr->HasBailOutInfo())
  9759. {
  9760. const IR::BailOutKind oldBailOutKind = instr->GetBailOutKind();
  9761. Assert(
  9762. (
  9763. !(oldBailOutKind & ~IR::BailOutKindBits) ||
  9764. (oldBailOutKind & ~IR::BailOutKindBits) == IR::BailOutOnImplicitCallsPreOp
  9765. ) &&
  9766. !(oldBailOutKind & IR::BailOutKindBits & ~(IR::BailOutOnArrayAccessHelperCall | IR::BailOutMarkTempObject)));
  9767. if(arrayBailOutKind == IR::BailOutConventionalTypedArrayAccessOnly)
  9768. {
  9769. // BailOutConventionalTypedArrayAccessOnly also bails out if the array access is outside the head
  9770. // segment bounds, and guarantees no implicit calls. Override the bailout kind so that the instruction
  9771. // bails out for the right reason.
  9772. instr->SetBailOutKind(
  9773. arrayBailOutKind | (oldBailOutKind & (IR::BailOutKindBits - IR::BailOutOnArrayAccessHelperCall)));
  9774. }
  9775. else
  9776. {
  9777. // BailOutConventionalNativeArrayAccessOnly by itself may generate a helper call, and may cause implicit
  9778. // calls to occur, so it must be merged in to eliminate generating the helper call.
  9779. Assert(arrayBailOutKind == IR::BailOutConventionalNativeArrayAccessOnly);
  9780. instr->SetBailOutKind(oldBailOutKind | arrayBailOutKind);
  9781. }
  9782. }
  9783. else
  9784. {
  9785. GenerateBailAtOperation(&instr, arrayBailOutKind);
  9786. }
  9787. }
  9788. }
  9789. }
  9790. else
  9791. {
  9792. GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access, because the source was not already specialized.\n"));
  9793. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  9794. {
  9795. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  9796. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  9797. baseValueType.ToString(baseValueTypeStr);
  9798. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not type specialize, because of array type.\n"),
  9799. this->func->GetJITFunctionBody()->GetDisplayName(),
  9800. this->func->GetDebugNumberSet(debugStringBuffer),
  9801. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  9802. baseValueTypeStr);
  9803. Output::Flush();
  9804. }
  9805. }
  9806. return toType != TyVar;
  9807. }
  9808. IR::Instr *
  9809. GlobOpt::ToVarUses(IR::Instr *instr, IR::Opnd *opnd, bool isDst, Value *val)
  9810. {
  9811. Sym *sym;
  9812. switch (opnd->GetKind())
  9813. {
  9814. case IR::OpndKindReg:
  9815. if (!isDst && !CurrentBlockData()->liveVarSyms->Test(opnd->AsRegOpnd()->m_sym->m_id))
  9816. {
  9817. instr = this->ToVar(instr, opnd->AsRegOpnd(), this->currentBlock, val, true);
  9818. }
  9819. break;
  9820. case IR::OpndKindSym:
  9821. sym = opnd->AsSymOpnd()->m_sym;
  9822. if (sym->IsPropertySym() && !CurrentBlockData()->liveVarSyms->Test(sym->AsPropertySym()->m_stackSym->m_id)
  9823. && sym->AsPropertySym()->m_stackSym->IsVar())
  9824. {
  9825. StackSym *propertyBase = sym->AsPropertySym()->m_stackSym;
  9826. IR::RegOpnd *newOpnd = IR::RegOpnd::New(propertyBase, TyVar, instr->m_func);
  9827. instr = this->ToVar(instr, newOpnd, this->currentBlock, CurrentBlockData()->FindValue(propertyBase), true);
  9828. }
  9829. break;
  9830. case IR::OpndKindIndir:
  9831. IR::RegOpnd *baseOpnd = opnd->AsIndirOpnd()->GetBaseOpnd();
  9832. if (!CurrentBlockData()->liveVarSyms->Test(baseOpnd->m_sym->m_id))
  9833. {
  9834. instr = this->ToVar(instr, baseOpnd, this->currentBlock, CurrentBlockData()->FindValue(baseOpnd->m_sym), true);
  9835. }
  9836. IR::RegOpnd *indexOpnd = opnd->AsIndirOpnd()->GetIndexOpnd();
  9837. if (indexOpnd && !indexOpnd->m_sym->IsTypeSpec())
  9838. {
  9839. instr = ToTypeSpecIndex(instr, indexOpnd, opnd->AsIndirOpnd());
  9840. }
  9841. break;
  9842. }
  9843. return instr;
  9844. }
  9845. IR::Instr *
  9846. GlobOpt::ToTypeSpecIndex(IR::Instr * instr, IR::RegOpnd * indexOpnd, IR::IndirOpnd * indirOpnd)
  9847. {
  9848. Assert(indirOpnd != nullptr || indexOpnd == instr->GetSrc1());
  9849. bool isGetterOrSetter = instr->m_opcode == Js::OpCode::InitGetElemI ||
  9850. instr->m_opcode == Js::OpCode::InitSetElemI ||
  9851. instr->m_opcode == Js::OpCode::InitClassMemberGetComputedName ||
  9852. instr->m_opcode == Js::OpCode::InitClassMemberSetComputedName;
  9853. if (!isGetterOrSetter // typespec is disabled for getters, setters
  9854. && (indexOpnd->GetValueType().IsInt()
  9855. ? !IsTypeSpecPhaseOff(func)
  9856. : indexOpnd->GetValueType().IsLikelyInt() && DoAggressiveIntTypeSpec())
  9857. && !GetIsAsmJSFunc()) // typespec is disabled for asmjs
  9858. {
  9859. StackSym *const indexVarSym = indexOpnd->m_sym;
  9860. Value *const indexValue = CurrentBlockData()->FindValue(indexVarSym);
  9861. Assert(indexValue);
  9862. Assert(indexValue->GetValueInfo()->IsLikelyInt());
  9863. ToInt32(instr, indexOpnd, currentBlock, indexValue, indirOpnd, false);
  9864. Assert(indexValue->GetValueInfo()->IsInt() || IsLoopPrePass());
  9865. if (!IsLoopPrePass())
  9866. {
  9867. IR::Opnd * intOpnd = indirOpnd ? indirOpnd->GetIndexOpnd() : instr->GetSrc1();
  9868. if (intOpnd != nullptr)
  9869. {
  9870. Assert(!intOpnd->IsRegOpnd() || intOpnd->AsRegOpnd()->m_sym->IsTypeSpec());
  9871. IntConstantBounds indexConstantBounds;
  9872. AssertVerify(indexValue->GetValueInfo()->TryGetIntConstantBounds(&indexConstantBounds));
  9873. if (ValueInfo::IsGreaterThanOrEqualTo(
  9874. indexValue,
  9875. indexConstantBounds.LowerBound(),
  9876. indexConstantBounds.UpperBound(),
  9877. nullptr,
  9878. 0,
  9879. 0))
  9880. {
  9881. intOpnd->SetType(TyUint32);
  9882. }
  9883. }
  9884. }
  9885. }
  9886. else if (!CurrentBlockData()->liveVarSyms->Test(indexOpnd->m_sym->m_id))
  9887. {
  9888. instr = this->ToVar(instr, indexOpnd, this->currentBlock, CurrentBlockData()->FindValue(indexOpnd->m_sym), true);
  9889. }
  9890. return instr;
  9891. }
  9892. IR::Instr *
  9893. GlobOpt::ToVar(IR::Instr *instr, IR::RegOpnd *regOpnd, BasicBlock *block, Value *value, bool needsUpdate)
  9894. {
  9895. IR::Instr *newInstr;
  9896. StackSym *varSym = regOpnd->m_sym;
  9897. if (IsTypeSpecPhaseOff(this->func))
  9898. {
  9899. return instr;
  9900. }
  9901. if (this->IsLoopPrePass())
  9902. {
  9903. block->globOptData.liveVarSyms->Set(varSym->m_id);
  9904. return instr;
  9905. }
  9906. if (block->globOptData.liveVarSyms->Test(varSym->m_id))
  9907. {
  9908. // Already live, nothing to do
  9909. return instr;
  9910. }
  9911. if (!varSym->IsVar())
  9912. {
  9913. Assert(!varSym->IsTypeSpec());
  9914. // Leave non-vars alone.
  9915. return instr;
  9916. }
  9917. Assert(block->globOptData.IsTypeSpecialized(varSym));
  9918. if (!value)
  9919. {
  9920. value = block->globOptData.FindValue(varSym);
  9921. }
  9922. ValueInfo *valueInfo = value ? value->GetValueInfo() : nullptr;
  9923. if(valueInfo && valueInfo->IsInt())
  9924. {
  9925. // If two syms have the same value, one is lossy-int-specialized, and then the other is int-specialized, the value
  9926. // would have been updated to definitely int. Upon using the lossy-int-specialized sym later, it would be flagged as
  9927. // lossy while the value is definitely int. Since the bit-vectors are based on the sym and not the value, update the
  9928. // lossy state.
  9929. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  9930. }
  9931. IRType fromType = TyIllegal;
  9932. StackSym *typeSpecSym = nullptr;
  9933. if (block->globOptData.liveInt32Syms->Test(varSym->m_id) && !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id))
  9934. {
  9935. fromType = TyInt32;
  9936. typeSpecSym = varSym->GetInt32EquivSym(this->func);
  9937. Assert(valueInfo);
  9938. Assert(valueInfo->IsInt());
  9939. }
  9940. else if (block->globOptData.liveFloat64Syms->Test(varSym->m_id))
  9941. {
  9942. fromType = TyFloat64;
  9943. typeSpecSym = varSym->GetFloat64EquivSym(this->func);
  9944. // Ensure that all bailout FromVars that generate a value for this type-specialized sym will bail out on any non-number
  9945. // value, even ones that have already been generated before. Float-specialized non-number values cannot be converted
  9946. // back to Var since they will not go back to the original non-number value. The dead-store pass will update the bailout
  9947. // kind on already-generated FromVars based on this bit.
  9948. typeSpecSym->m_requiresBailOnNotNumber = true;
  9949. // A previous float conversion may have used BailOutPrimitiveButString, which does not change the value type to say
  9950. // definitely float, since it can also be a non-string primitive. The convert back to Var though, will cause that
  9951. // bailout kind to be changed to BailOutNumberOnly in the dead-store phase, so from the point of the initial conversion
  9952. // to float, that the value is definitely number. Since we don't know where the FromVar is, change the value type here.
  9953. if(valueInfo)
  9954. {
  9955. if(!valueInfo->IsNumber())
  9956. {
  9957. valueInfo = valueInfo->SpecializeToFloat64(alloc);
  9958. ChangeValueInfo(block, value, valueInfo);
  9959. regOpnd->SetValueType(valueInfo->Type());
  9960. }
  9961. }
  9962. else
  9963. {
  9964. value = NewGenericValue(ValueType::Float);
  9965. valueInfo = value->GetValueInfo();
  9966. block->globOptData.SetValue(value, varSym);
  9967. regOpnd->SetValueType(valueInfo->Type());
  9968. }
  9969. }
  9970. else
  9971. {
  9972. Assert(UNREACHED);
  9973. }
  9974. AssertOrFailFast(valueInfo);
  9975. int32 intConstantValue;
  9976. if (valueInfo->TryGetIntConstantValue(&intConstantValue))
  9977. {
  9978. // Lower will tag or create a number directly
  9979. newInstr = IR::Instr::New(Js::OpCode::LdC_A_I4, regOpnd,
  9980. IR::IntConstOpnd::New(intConstantValue, TyInt32, instr->m_func), instr->m_func);
  9981. }
  9982. else
  9983. {
  9984. IR::RegOpnd * regNew = IR::RegOpnd::New(typeSpecSym, fromType, instr->m_func);
  9985. Js::OpCode opcode = Js::OpCode::ToVar;
  9986. regNew->SetIsJITOptimizedReg(true);
  9987. newInstr = IR::Instr::New(opcode, regOpnd, regNew, instr->m_func);
  9988. }
  9989. newInstr->SetByteCodeOffset(instr);
  9990. newInstr->GetDst()->AsRegOpnd()->SetIsJITOptimizedReg(true);
  9991. ValueType valueType = valueInfo->Type();
  9992. if(fromType == TyInt32)
  9993. {
  9994. #if !INT32VAR // All 32-bit ints are taggable on 64-bit architectures
  9995. IntConstantBounds constantBounds;
  9996. AssertVerify(valueInfo->TryGetIntConstantBounds(&constantBounds));
  9997. if(constantBounds.IsTaggable())
  9998. #endif
  9999. {
  10000. // The value is within the taggable range, so set the opnd value types to TaggedInt to avoid the overflow check
  10001. valueType = ValueType::GetTaggedInt();
  10002. }
  10003. }
  10004. newInstr->GetDst()->SetValueType(valueType);
  10005. newInstr->GetSrc1()->SetValueType(valueType);
  10006. IR::Instr *insertAfterInstr = instr->m_prev;
  10007. if (instr == block->GetLastInstr() &&
  10008. (instr->IsBranchInstr() || instr->m_opcode == Js::OpCode::BailTarget))
  10009. {
  10010. // Don't insert code between the branch and the preceding ByteCodeUses instrs...
  10011. while(insertAfterInstr->m_opcode == Js::OpCode::ByteCodeUses)
  10012. {
  10013. insertAfterInstr = insertAfterInstr->m_prev;
  10014. }
  10015. }
  10016. block->InsertInstrAfter(newInstr, insertAfterInstr);
  10017. block->globOptData.liveVarSyms->Set(varSym->m_id);
  10018. GOPT_TRACE_OPND(regOpnd, _u("Converting to var\n"));
  10019. if (block->loop)
  10020. {
  10021. Assert(!this->IsLoopPrePass());
  10022. this->TryHoistInvariant(newInstr, block, value, value, nullptr, false);
  10023. }
  10024. if (needsUpdate)
  10025. {
  10026. // Make sure that the kill effect of the ToVar instruction is tracked and that the kill of a property
  10027. // type is reflected in the current instruction.
  10028. this->ProcessKills(newInstr);
  10029. this->ValueNumberObjectType(newInstr->GetDst(), newInstr);
  10030. if (instr->GetSrc1() && instr->GetSrc1()->IsSymOpnd() && instr->GetSrc1()->AsSymOpnd()->IsPropertySymOpnd())
  10031. {
  10032. // Reprocess the load source. We need to reset the PropertySymOpnd fields first.
  10033. IR::PropertySymOpnd *propertySymOpnd = instr->GetSrc1()->AsPropertySymOpnd();
  10034. if (propertySymOpnd->IsTypeCheckSeqCandidate())
  10035. {
  10036. propertySymOpnd->SetTypeChecked(false);
  10037. propertySymOpnd->SetTypeAvailable(false);
  10038. propertySymOpnd->SetWriteGuardChecked(false);
  10039. }
  10040. this->FinishOptPropOp(instr, propertySymOpnd);
  10041. instr = this->SetTypeCheckBailOut(instr->GetSrc1(), instr, nullptr);
  10042. }
  10043. }
  10044. return instr;
  10045. }
  10046. IR::Instr *
  10047. GlobOpt::ToInt32(IR::Instr *instr, IR::Opnd *opnd, BasicBlock *block, Value *val, IR::IndirOpnd *indir, bool lossy)
  10048. {
  10049. return this->ToTypeSpecUse(instr, opnd, block, val, indir, TyInt32, IR::BailOutIntOnly, lossy);
  10050. }
  10051. IR::Instr *
  10052. GlobOpt::ToFloat64(IR::Instr *instr, IR::Opnd *opnd, BasicBlock *block, Value *val, IR::IndirOpnd *indir, IR::BailOutKind bailOutKind)
  10053. {
  10054. return this->ToTypeSpecUse(instr, opnd, block, val, indir, TyFloat64, bailOutKind);
  10055. }
  10056. IR::Instr *
  10057. GlobOpt::ToTypeSpecUse(IR::Instr *instr, IR::Opnd *opnd, BasicBlock *block, Value *val, IR::IndirOpnd *indir, IRType toType, IR::BailOutKind bailOutKind, bool lossy, IR::Instr *insertBeforeInstr)
  10058. {
  10059. Assert(bailOutKind != IR::BailOutInvalid);
  10060. IR::Instr *newInstr;
  10061. if (!val && opnd->IsRegOpnd())
  10062. {
  10063. val = block->globOptData.FindValue(opnd->AsRegOpnd()->m_sym);
  10064. }
  10065. ValueInfo *valueInfo = val ? val->GetValueInfo() : nullptr;
  10066. bool needReplaceSrc = false;
  10067. bool updateBlockLastInstr = false;
  10068. if (instr)
  10069. {
  10070. needReplaceSrc = true;
  10071. if (!insertBeforeInstr)
  10072. {
  10073. insertBeforeInstr = instr;
  10074. }
  10075. }
  10076. else if (!insertBeforeInstr)
  10077. {
  10078. // Insert it at the end of the block
  10079. insertBeforeInstr = block->GetLastInstr();
  10080. if (insertBeforeInstr->IsBranchInstr() || insertBeforeInstr->m_opcode == Js::OpCode::BailTarget)
  10081. {
  10082. // Don't insert code between the branch and the preceding ByteCodeUses instrs...
  10083. while(insertBeforeInstr->m_prev->m_opcode == Js::OpCode::ByteCodeUses)
  10084. {
  10085. insertBeforeInstr = insertBeforeInstr->m_prev;
  10086. }
  10087. }
  10088. else
  10089. {
  10090. insertBeforeInstr = insertBeforeInstr->m_next;
  10091. updateBlockLastInstr = true;
  10092. }
  10093. }
  10094. // Int constant values will be propagated into the instruction. For ArgOut_A_InlineBuiltIn, there's no benefit from
  10095. // const-propping, so those are excluded.
  10096. if (opnd->IsRegOpnd() &&
  10097. !(
  10098. valueInfo &&
  10099. (valueInfo->HasIntConstantValue() || valueInfo->IsFloatConstant()) &&
  10100. (!instr || instr->m_opcode != Js::OpCode::ArgOut_A_InlineBuiltIn)
  10101. ))
  10102. {
  10103. IR::RegOpnd *regSrc = opnd->AsRegOpnd();
  10104. StackSym *varSym = regSrc->m_sym;
  10105. Js::OpCode opcode = Js::OpCode::FromVar;
  10106. if (varSym->IsTypeSpec() || !block->globOptData.liveVarSyms->Test(varSym->m_id))
  10107. {
  10108. // Conversion between int32 and float64
  10109. if (varSym->IsTypeSpec())
  10110. {
  10111. varSym = varSym->GetVarEquivSym(this->func);
  10112. }
  10113. opcode = Js::OpCode::Conv_Prim;
  10114. }
  10115. Assert(block->globOptData.liveVarSyms->Test(varSym->m_id) || block->globOptData.IsTypeSpecialized(varSym));
  10116. StackSym *typeSpecSym = nullptr;
  10117. BOOL isLive = FALSE;
  10118. BVSparse<JitArenaAllocator> *livenessBv = nullptr;
  10119. if(valueInfo && valueInfo->IsInt())
  10120. {
  10121. // If two syms have the same value, one is lossy-int-specialized, and then the other is int-specialized, the value
  10122. // would have been updated to definitely int. Upon using the lossy-int-specialized sym later, it would be flagged as
  10123. // lossy while the value is definitely int. Since the bit-vectors are based on the sym and not the value, update the
  10124. // lossy state.
  10125. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  10126. }
  10127. if (toType == TyInt32)
  10128. {
  10129. // Need to determine whether the conversion is actually lossy or lossless. If the value is an int, then it's a
  10130. // lossless conversion despite the type of conversion requested. The liveness of the converted int32 sym needs to be
  10131. // set to reflect the actual type of conversion done. Also, a lossless conversion needs the value to determine
  10132. // whether the conversion may need to bail out.
  10133. Assert(valueInfo);
  10134. if(valueInfo->IsInt())
  10135. {
  10136. lossy = false;
  10137. }
  10138. else
  10139. {
  10140. Assert(IsLoopPrePass() || !block->globOptData.IsInt32TypeSpecialized(varSym));
  10141. }
  10142. livenessBv = block->globOptData.liveInt32Syms;
  10143. isLive = livenessBv->Test(varSym->m_id) && (lossy || !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id));
  10144. if (this->IsLoopPrePass())
  10145. {
  10146. if (!isLive)
  10147. {
  10148. livenessBv->Set(varSym->m_id);
  10149. if (lossy)
  10150. {
  10151. block->globOptData.liveLossyInt32Syms->Set(varSym->m_id);
  10152. }
  10153. else
  10154. {
  10155. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  10156. }
  10157. }
  10158. return instr;
  10159. }
  10160. typeSpecSym = varSym->GetInt32EquivSym(this->func);
  10161. if (!isLive)
  10162. {
  10163. if (!opnd->IsVar() ||
  10164. !block->globOptData.liveVarSyms->Test(varSym->m_id) ||
  10165. (block->globOptData.liveFloat64Syms->Test(varSym->m_id) && valueInfo && valueInfo->IsLikelyFloat()))
  10166. {
  10167. Assert(block->globOptData.liveFloat64Syms->Test(varSym->m_id));
  10168. if(!lossy && !valueInfo->IsInt())
  10169. {
  10170. // Shouldn't try to do a lossless conversion from float64 to int32 when the value is not known to be an
  10171. // int. There are cases where we need more than two passes over loops to flush out all dependencies.
  10172. // It's possible for the loop prepass to think that a sym s1 remains an int because it acquires the
  10173. // value of another sym s2 that is an int in the prepass at that time. However, s2 can become a float
  10174. // later in the loop body, in which case s1 would become a float on the second iteration of the loop. By
  10175. // that time, we would have already committed to having s1 live as a lossless int on entry into the
  10176. // loop, and we end up having to compensate by doing a lossless conversion from float to int, which will
  10177. // need a bailout and will most likely bail out.
  10178. //
  10179. // If s2 becomes a var instead of a float, then the compensation is legal although not ideal. After
  10180. // enough bailouts, rejit would be triggered with aggressive int type spec turned off. For the
  10181. // float-to-int conversion though, there's no point in emitting a bailout because we already know that
  10182. // the value is a float and has high probability of bailing out (whereas a var has a chance to be a
  10183. // tagged int), and so currently lossless conversion from float to int with bailout is not supported.
  10184. //
  10185. // So, treating this case as a compile-time bailout. The exception will trigger the jit work item to be
  10186. // restarted with aggressive int type specialization disabled.
  10187. if(bailOutKind == IR::BailOutExpectingInteger)
  10188. {
  10189. Assert(IsSwitchOptEnabledForIntTypeSpec());
  10190. throw Js::RejitException(RejitReason::DisableSwitchOptExpectingInteger);
  10191. }
  10192. else
  10193. {
  10194. Assert(DoAggressiveIntTypeSpec());
  10195. if(PHASE_TRACE(Js::BailOutPhase, this->func))
  10196. {
  10197. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  10198. Output::Print(
  10199. _u("BailOut (compile-time): function: %s (%s) varSym: "),
  10200. this->func->GetJITFunctionBody()->GetDisplayName(),
  10201. this->func->GetDebugNumberSet(debugStringBuffer),
  10202. varSym->m_id);
  10203. #if DBG_DUMP
  10204. varSym->Dump();
  10205. #else
  10206. Output::Print(_u("s%u"), varSym->m_id);
  10207. #endif
  10208. if(varSym->HasByteCodeRegSlot())
  10209. {
  10210. Output::Print(_u(" byteCodeReg: R%u"), varSym->GetByteCodeRegSlot());
  10211. }
  10212. Output::Print(_u(" (lossless conversion from float64 to int32)\n"));
  10213. Output::Flush();
  10214. }
  10215. if(!DoAggressiveIntTypeSpec())
  10216. {
  10217. // Aggressive int type specialization is already off for some reason. Prevent trying to rejit again
  10218. // because it won't help and the same thing will happen again. Just abort jitting this function.
  10219. if(PHASE_TRACE(Js::BailOutPhase, this->func))
  10220. {
  10221. Output::Print(_u(" Aborting JIT because AggressiveIntTypeSpec is already off\n"));
  10222. Output::Flush();
  10223. }
  10224. throw Js::OperationAbortedException();
  10225. }
  10226. throw Js::RejitException(RejitReason::AggressiveIntTypeSpecDisabled);
  10227. }
  10228. }
  10229. if(opnd->IsVar())
  10230. {
  10231. regSrc->SetType(TyFloat64);
  10232. regSrc->m_sym = varSym->GetFloat64EquivSym(this->func);
  10233. opcode = Js::OpCode::Conv_Prim;
  10234. }
  10235. else
  10236. {
  10237. Assert(regSrc->IsFloat64());
  10238. Assert(regSrc->m_sym->IsFloat64());
  10239. Assert(opcode == Js::OpCode::Conv_Prim);
  10240. }
  10241. }
  10242. }
  10243. GOPT_TRACE_OPND(regSrc, _u("Converting to int32\n"));
  10244. }
  10245. else if (toType == TyFloat64)
  10246. {
  10247. // float64
  10248. typeSpecSym = varSym->GetFloat64EquivSym(this->func);
  10249. if(!IsLoopPrePass() && typeSpecSym->m_requiresBailOnNotNumber && block->globOptData.IsFloat64TypeSpecialized(varSym))
  10250. {
  10251. // This conversion is already protected by a BailOutNumberOnly bailout (or at least it will be after the
  10252. // dead-store phase). Since 'requiresBailOnNotNumber' is not flow-based, change the value to definitely float.
  10253. if(valueInfo)
  10254. {
  10255. if(!valueInfo->IsNumber())
  10256. {
  10257. valueInfo = valueInfo->SpecializeToFloat64(alloc);
  10258. ChangeValueInfo(block, val, valueInfo);
  10259. opnd->SetValueType(valueInfo->Type());
  10260. }
  10261. }
  10262. else
  10263. {
  10264. val = NewGenericValue(ValueType::Float);
  10265. valueInfo = val->GetValueInfo();
  10266. block->globOptData.SetValue(val, varSym);
  10267. opnd->SetValueType(valueInfo->Type());
  10268. }
  10269. }
  10270. if(bailOutKind == IR::BailOutNumberOnly)
  10271. {
  10272. if(!IsLoopPrePass())
  10273. {
  10274. // Ensure that all bailout FromVars that generate a value for this type-specialized sym will bail out on any
  10275. // non-number value, even ones that have already been generated before. The dead-store pass will update the
  10276. // bailout kind on already-generated FromVars based on this bit.
  10277. typeSpecSym->m_requiresBailOnNotNumber = true;
  10278. }
  10279. }
  10280. else if(typeSpecSym->m_requiresBailOnNotNumber)
  10281. {
  10282. Assert(bailOutKind == IR::BailOutPrimitiveButString);
  10283. bailOutKind = IR::BailOutNumberOnly;
  10284. }
  10285. livenessBv = block->globOptData.liveFloat64Syms;
  10286. isLive = livenessBv->Test(varSym->m_id);
  10287. if (this->IsLoopPrePass())
  10288. {
  10289. if(!isLive)
  10290. {
  10291. livenessBv->Set(varSym->m_id);
  10292. }
  10293. if (this->OptIsInvariant(opnd, block, this->prePassLoop, val, false, true))
  10294. {
  10295. this->prePassLoop->forceFloat64SymsOnEntry->Set(varSym->m_id);
  10296. }
  10297. else
  10298. {
  10299. Sym *symStore = (valueInfo ? valueInfo->GetSymStore() : NULL);
  10300. if (symStore && symStore != varSym
  10301. && this->OptIsInvariant(symStore, block, this->prePassLoop, block->globOptData.FindValue(symStore), false, true))
  10302. {
  10303. // If symStore is assigned to sym and we want sym to be type-specialized, for symStore to be specialized
  10304. // outside the loop.
  10305. this->prePassLoop->forceFloat64SymsOnEntry->Set(symStore->m_id);
  10306. }
  10307. }
  10308. return instr;
  10309. }
  10310. if (!isLive && regSrc->IsVar())
  10311. {
  10312. if (!block->globOptData.liveVarSyms->Test(varSym->m_id) ||
  10313. (
  10314. block->globOptData.liveInt32Syms->Test(varSym->m_id) &&
  10315. !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id) &&
  10316. valueInfo &&
  10317. valueInfo->IsLikelyInt()
  10318. ))
  10319. {
  10320. Assert(block->globOptData.liveInt32Syms->Test(varSym->m_id));
  10321. Assert(!block->globOptData.liveLossyInt32Syms->Test(varSym->m_id)); // Shouldn't try to convert a lossy int32 to anything
  10322. regSrc->SetType(TyInt32);
  10323. regSrc->m_sym = varSym->GetInt32EquivSym(this->func);
  10324. opcode = Js::OpCode::Conv_Prim;
  10325. }
  10326. }
  10327. GOPT_TRACE_OPND(regSrc, _u("Converting to float64\n"));
  10328. }
  10329. bool needLoad = false;
  10330. if (needReplaceSrc)
  10331. {
  10332. bool wasDead = regSrc->GetIsDead();
  10333. // needReplaceSrc means we are type specializing a use, and need to replace the src on the instr
  10334. if (!isLive)
  10335. {
  10336. needLoad = true;
  10337. // ReplaceSrc will delete it.
  10338. regSrc = regSrc->Copy(instr->m_func)->AsRegOpnd();
  10339. }
  10340. IR::RegOpnd * regNew = IR::RegOpnd::New(typeSpecSym, toType, instr->m_func);
  10341. if(valueInfo)
  10342. {
  10343. regNew->SetValueType(valueInfo->Type());
  10344. regNew->m_wasNegativeZeroPreventedByBailout = valueInfo->WasNegativeZeroPreventedByBailout();
  10345. }
  10346. regNew->SetIsDead(wasDead);
  10347. regNew->SetIsJITOptimizedReg(true);
  10348. this->CaptureByteCodeSymUses(instr);
  10349. if (indir == nullptr)
  10350. {
  10351. instr->ReplaceSrc(opnd, regNew);
  10352. }
  10353. else
  10354. {
  10355. indir->ReplaceIndexOpnd(regNew);
  10356. }
  10357. opnd = regNew;
  10358. if (!needLoad)
  10359. {
  10360. Assert(isLive);
  10361. return instr;
  10362. }
  10363. }
  10364. else
  10365. {
  10366. // We just need to insert a load of a type spec sym
  10367. if(isLive)
  10368. {
  10369. return instr;
  10370. }
  10371. // Insert it before the specified instruction
  10372. instr = insertBeforeInstr;
  10373. }
  10374. IR::RegOpnd *regDst = IR::RegOpnd::New(typeSpecSym, toType, instr->m_func);
  10375. bool isBailout = false;
  10376. bool isHoisted = false;
  10377. bool isInLandingPad = (block->next && !block->next->isDeleted && block->next->isLoopHeader);
  10378. if (isInLandingPad)
  10379. {
  10380. Loop *loop = block->next->loop;
  10381. Assert(loop && loop->landingPad == block);
  10382. Assert(loop->bailOutInfo);
  10383. }
  10384. if (opcode == Js::OpCode::FromVar)
  10385. {
  10386. if (toType == TyInt32)
  10387. {
  10388. Assert(valueInfo);
  10389. if (lossy)
  10390. {
  10391. if (!valueInfo->IsPrimitive() && !block->globOptData.IsTypeSpecialized(varSym))
  10392. {
  10393. // Lossy conversions to int32 on non-primitive values may have implicit calls to toString or valueOf, which
  10394. // may be overridden to have a side effect. The side effect needs to happen every time the conversion is
  10395. // supposed to happen, so the resulting lossy int32 value cannot be reused. Bail out on implicit calls.
  10396. Assert(DoLossyIntTypeSpec());
  10397. bailOutKind = IR::BailOutOnNotPrimitive;
  10398. isBailout = true;
  10399. }
  10400. }
  10401. else if (!valueInfo->IsInt())
  10402. {
  10403. // The operand is likely an int (hence the request to convert to int), so bail out if it's not an int. Only
  10404. // bail out if a lossless conversion to int is requested. Lossy conversions to int such as in (a | 0) don't
  10405. // need to bail out.
  10406. if (bailOutKind == IR::BailOutExpectingInteger)
  10407. {
  10408. Assert(IsSwitchOptEnabledForIntTypeSpec());
  10409. }
  10410. else
  10411. {
  10412. Assert(DoAggressiveIntTypeSpec());
  10413. }
  10414. isBailout = true;
  10415. }
  10416. }
  10417. else if (toType == TyFloat64 &&
  10418. (!valueInfo || !valueInfo->IsNumber()))
  10419. {
  10420. // Bailout if converting vars to float if we can't prove they are floats:
  10421. // x = str + float; -> need to bailout if str is a string
  10422. //
  10423. // x = obj * 0.1;
  10424. // y = obj * 0.2; -> if obj has valueof, we'll only call valueof once on the FromVar conversion...
  10425. Assert(bailOutKind != IR::BailOutInvalid);
  10426. isBailout = true;
  10427. }
  10428. }
  10429. if (isBailout)
  10430. {
  10431. if (isInLandingPad)
  10432. {
  10433. Loop *loop = block->next->loop;
  10434. this->EnsureBailTarget(loop);
  10435. instr = loop->bailOutInfo->bailOutInstr;
  10436. updateBlockLastInstr = false;
  10437. newInstr = IR::BailOutInstr::New(opcode, bailOutKind, loop->bailOutInfo, instr->m_func);
  10438. newInstr->SetDst(regDst);
  10439. newInstr->SetSrc1(regSrc);
  10440. }
  10441. else
  10442. {
  10443. newInstr = IR::BailOutInstr::New(opcode, regDst, regSrc, bailOutKind, instr, instr->m_func);
  10444. }
  10445. }
  10446. else
  10447. {
  10448. newInstr = IR::Instr::New(opcode, regDst, regSrc, instr->m_func);
  10449. }
  10450. newInstr->SetByteCodeOffset(instr);
  10451. instr->InsertBefore(newInstr);
  10452. if (updateBlockLastInstr)
  10453. {
  10454. block->SetLastInstr(newInstr);
  10455. }
  10456. regDst->SetIsJITOptimizedReg(true);
  10457. newInstr->GetSrc1()->AsRegOpnd()->SetIsJITOptimizedReg(true);
  10458. ValueInfo *const oldValueInfo = valueInfo;
  10459. if(valueInfo)
  10460. {
  10461. newInstr->GetSrc1()->SetValueType(valueInfo->Type());
  10462. }
  10463. if(isBailout)
  10464. {
  10465. Assert(opcode == Js::OpCode::FromVar);
  10466. if(toType == TyInt32)
  10467. {
  10468. Assert(valueInfo);
  10469. if(!lossy)
  10470. {
  10471. Assert(bailOutKind == IR::BailOutIntOnly || bailOutKind == IR::BailOutExpectingInteger);
  10472. valueInfo = valueInfo->SpecializeToInt32(alloc, isPerformingLoopBackEdgeCompensation);
  10473. ChangeValueInfo(nullptr, val, valueInfo);
  10474. int32 intConstantValue;
  10475. if(indir && needReplaceSrc && valueInfo->TryGetIntConstantValue(&intConstantValue))
  10476. {
  10477. // A likely-int value can have constant bounds due to conditional branches narrowing its range. Now that
  10478. // the sym has been proven to be an int, the likely-int value, after specialization, will be constant.
  10479. // Replace the index opnd in the indir with an offset.
  10480. Assert(opnd == indir->GetIndexOpnd());
  10481. Assert(indir->GetScale() == 0);
  10482. indir->UnlinkIndexOpnd()->Free(instr->m_func);
  10483. opnd = nullptr;
  10484. indir->SetOffset(intConstantValue);
  10485. }
  10486. }
  10487. }
  10488. else if (toType == TyFloat64)
  10489. {
  10490. if(bailOutKind == IR::BailOutNumberOnly)
  10491. {
  10492. if(valueInfo)
  10493. {
  10494. valueInfo = valueInfo->SpecializeToFloat64(alloc);
  10495. ChangeValueInfo(block, val, valueInfo);
  10496. }
  10497. else
  10498. {
  10499. val = NewGenericValue(ValueType::Float);
  10500. valueInfo = val->GetValueInfo();
  10501. block->globOptData.SetValue(val, varSym);
  10502. }
  10503. }
  10504. }
  10505. else
  10506. {
  10507. Assert(UNREACHED);
  10508. }
  10509. }
  10510. if(valueInfo)
  10511. {
  10512. newInstr->GetDst()->SetValueType(valueInfo->Type());
  10513. if(needReplaceSrc && opnd)
  10514. {
  10515. opnd->SetValueType(valueInfo->Type());
  10516. }
  10517. }
  10518. if (block->loop)
  10519. {
  10520. Assert(!this->IsLoopPrePass());
  10521. isHoisted = this->TryHoistInvariant(newInstr, block, val, val, nullptr, false, lossy, false, bailOutKind);
  10522. }
  10523. if (isBailout)
  10524. {
  10525. if (!isHoisted && !isInLandingPad)
  10526. {
  10527. if(valueInfo)
  10528. {
  10529. // Since this is a pre-op bailout, the old value info should be used for the purposes of bailout. For
  10530. // instance, the value info could be LikelyInt but with a constant range. Once specialized to int, the value
  10531. // info would be an int constant. However, the int constant is only guaranteed if the value is actually an
  10532. // int, which this conversion is verifying, so bailout cannot assume the constant value.
  10533. if(oldValueInfo)
  10534. {
  10535. val->SetValueInfo(oldValueInfo);
  10536. }
  10537. else
  10538. {
  10539. block->globOptData.ClearSymValue(varSym);
  10540. }
  10541. }
  10542. // Fill in bail out info if the FromVar is a bailout instr, and it wasn't hoisted as invariant.
  10543. // If it was hoisted, the invariant code will fill out the bailout info with the loop landing pad bailout info.
  10544. this->FillBailOutInfo(block, newInstr);
  10545. if(valueInfo)
  10546. {
  10547. // Restore the new value info after filling the bailout info
  10548. if(oldValueInfo)
  10549. {
  10550. val->SetValueInfo(valueInfo);
  10551. }
  10552. else
  10553. {
  10554. block->globOptData.SetValue(val, varSym);
  10555. }
  10556. }
  10557. }
  10558. }
  10559. // Now that we've captured the liveness in the bailout info, we can mark this as live.
  10560. // This type specialized sym isn't live if the FromVar bails out.
  10561. livenessBv->Set(varSym->m_id);
  10562. if(toType == TyInt32)
  10563. {
  10564. if(lossy)
  10565. {
  10566. block->globOptData.liveLossyInt32Syms->Set(varSym->m_id);
  10567. }
  10568. else
  10569. {
  10570. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  10571. }
  10572. }
  10573. }
  10574. else
  10575. {
  10576. Assert(valueInfo);
  10577. if(opnd->IsRegOpnd() && valueInfo->IsInt())
  10578. {
  10579. // If two syms have the same value, one is lossy-int-specialized, and then the other is int-specialized, the value
  10580. // would have been updated to definitely int. Upon using the lossy-int-specialized sym later, it would be flagged as
  10581. // lossy while the value is definitely int. Since the bit-vectors are based on the sym and not the value, update the
  10582. // lossy state.
  10583. block->globOptData.liveLossyInt32Syms->Clear(opnd->AsRegOpnd()->m_sym->m_id);
  10584. if(toType == TyInt32)
  10585. {
  10586. lossy = false;
  10587. }
  10588. }
  10589. if (this->IsLoopPrePass())
  10590. {
  10591. if(opnd->IsRegOpnd())
  10592. {
  10593. StackSym *const sym = opnd->AsRegOpnd()->m_sym;
  10594. if(toType == TyInt32)
  10595. {
  10596. Assert(!sym->IsTypeSpec());
  10597. block->globOptData.liveInt32Syms->Set(sym->m_id);
  10598. if(lossy)
  10599. {
  10600. block->globOptData.liveLossyInt32Syms->Set(sym->m_id);
  10601. }
  10602. else
  10603. {
  10604. block->globOptData.liveLossyInt32Syms->Clear(sym->m_id);
  10605. }
  10606. }
  10607. else
  10608. {
  10609. Assert(toType == TyFloat64);
  10610. AnalysisAssert(instr);
  10611. StackSym *const varSym = sym->IsTypeSpec() ? sym->GetVarEquivSym(instr->m_func) : sym;
  10612. block->globOptData.liveFloat64Syms->Set(varSym->m_id);
  10613. }
  10614. }
  10615. return instr;
  10616. }
  10617. if (!needReplaceSrc)
  10618. {
  10619. instr = insertBeforeInstr;
  10620. }
  10621. IR::Opnd *constOpnd;
  10622. int32 intConstantValue;
  10623. if(valueInfo->TryGetIntConstantValue(&intConstantValue))
  10624. {
  10625. if(toType == TyInt32)
  10626. {
  10627. constOpnd = IR::IntConstOpnd::New(intConstantValue, TyInt32, instr->m_func);
  10628. }
  10629. else
  10630. {
  10631. Assert(toType == TyFloat64);
  10632. constOpnd = IR::FloatConstOpnd::New(static_cast<FloatConstType>(intConstantValue), TyFloat64, instr->m_func);
  10633. }
  10634. }
  10635. else if(valueInfo->IsFloatConstant())
  10636. {
  10637. const FloatConstType floatValue = valueInfo->AsFloatConstant()->FloatValue();
  10638. if(toType == TyInt32)
  10639. {
  10640. Assert(lossy);
  10641. constOpnd =
  10642. IR::IntConstOpnd::New(
  10643. Js::JavascriptMath::ToInt32(floatValue),
  10644. TyInt32,
  10645. instr->m_func);
  10646. }
  10647. else
  10648. {
  10649. Assert(toType == TyFloat64);
  10650. constOpnd = IR::FloatConstOpnd::New(floatValue, TyFloat64, instr->m_func);
  10651. }
  10652. }
  10653. else
  10654. {
  10655. Assert(opnd->IsVar());
  10656. Assert(opnd->IsAddrOpnd());
  10657. AssertMsg(opnd->AsAddrOpnd()->IsVar(), "We only expect to see addr that are var before lower.");
  10658. // Don't need to capture uses, we are only replacing an addr opnd
  10659. if(toType == TyInt32)
  10660. {
  10661. constOpnd = IR::IntConstOpnd::New(Js::TaggedInt::ToInt32(opnd->AsAddrOpnd()->m_address), TyInt32, instr->m_func);
  10662. }
  10663. else
  10664. {
  10665. Assert(toType == TyFloat64);
  10666. constOpnd = IR::FloatConstOpnd::New(Js::TaggedInt::ToDouble(opnd->AsAddrOpnd()->m_address), TyFloat64, instr->m_func);
  10667. }
  10668. }
  10669. if (toType == TyInt32)
  10670. {
  10671. if (needReplaceSrc)
  10672. {
  10673. CaptureByteCodeSymUses(instr);
  10674. if(indir)
  10675. {
  10676. Assert(opnd == indir->GetIndexOpnd());
  10677. Assert(indir->GetScale() == 0);
  10678. indir->UnlinkIndexOpnd()->Free(instr->m_func);
  10679. indir->SetOffset(constOpnd->AsIntConstOpnd()->AsInt32());
  10680. }
  10681. else
  10682. {
  10683. instr->ReplaceSrc(opnd, constOpnd);
  10684. }
  10685. }
  10686. else
  10687. {
  10688. StackSym *varSym = opnd->AsRegOpnd()->m_sym;
  10689. if(varSym->IsTypeSpec())
  10690. {
  10691. varSym = varSym->GetVarEquivSym(nullptr);
  10692. Assert(varSym);
  10693. }
  10694. if(block->globOptData.liveInt32Syms->TestAndSet(varSym->m_id))
  10695. {
  10696. Assert(!!block->globOptData.liveLossyInt32Syms->Test(varSym->m_id) == lossy);
  10697. }
  10698. else
  10699. {
  10700. if(lossy)
  10701. {
  10702. block->globOptData.liveLossyInt32Syms->Set(varSym->m_id);
  10703. }
  10704. StackSym *int32Sym = varSym->GetInt32EquivSym(instr->m_func);
  10705. IR::RegOpnd *int32Reg = IR::RegOpnd::New(int32Sym, TyInt32, instr->m_func);
  10706. int32Reg->SetIsJITOptimizedReg(true);
  10707. newInstr = IR::Instr::New(Js::OpCode::Ld_I4, int32Reg, constOpnd, instr->m_func);
  10708. newInstr->SetByteCodeOffset(instr);
  10709. instr->InsertBefore(newInstr);
  10710. if (updateBlockLastInstr)
  10711. {
  10712. block->SetLastInstr(newInstr);
  10713. }
  10714. }
  10715. }
  10716. }
  10717. else
  10718. {
  10719. StackSym *floatSym;
  10720. bool newFloatSym = false;
  10721. StackSym* varSym;
  10722. if (opnd->IsRegOpnd())
  10723. {
  10724. varSym = opnd->AsRegOpnd()->m_sym;
  10725. if (varSym->IsTypeSpec())
  10726. {
  10727. varSym = varSym->GetVarEquivSym(nullptr);
  10728. Assert(varSym);
  10729. }
  10730. floatSym = varSym->GetFloat64EquivSym(instr->m_func);
  10731. }
  10732. else
  10733. {
  10734. varSym = block->globOptData.GetCopyPropSym(nullptr, val);
  10735. if(!varSym)
  10736. {
  10737. // Clear the symstore to ensure it's set below to this new symbol
  10738. this->SetSymStoreDirect(val->GetValueInfo(), nullptr);
  10739. varSym = StackSym::New(TyVar, instr->m_func);
  10740. newFloatSym = true;
  10741. }
  10742. floatSym = varSym->GetFloat64EquivSym(instr->m_func);
  10743. }
  10744. IR::RegOpnd *floatReg = IR::RegOpnd::New(floatSym, TyFloat64, instr->m_func);
  10745. floatReg->SetIsJITOptimizedReg(true);
  10746. // If the value is not live - let's load it.
  10747. if(!block->globOptData.liveFloat64Syms->TestAndSet(varSym->m_id))
  10748. {
  10749. newInstr = IR::Instr::New(Js::OpCode::LdC_F8_R8, floatReg, constOpnd, instr->m_func);
  10750. newInstr->SetByteCodeOffset(instr);
  10751. instr->InsertBefore(newInstr);
  10752. if (updateBlockLastInstr)
  10753. {
  10754. block->SetLastInstr(newInstr);
  10755. }
  10756. if(newFloatSym)
  10757. {
  10758. block->globOptData.SetValue(val, varSym);
  10759. }
  10760. // Src is always invariant, but check if the dst is, and then hoist.
  10761. if (block->loop &&
  10762. (
  10763. (newFloatSym && block->loop->CanHoistInvariants()) ||
  10764. this->OptIsInvariant(floatReg, block, block->loop, val, false, false)
  10765. ))
  10766. {
  10767. Assert(!this->IsLoopPrePass());
  10768. this->OptHoistInvariant(newInstr, block, block->loop, val, val, nullptr, false);
  10769. }
  10770. }
  10771. if (needReplaceSrc)
  10772. {
  10773. CaptureByteCodeSymUses(instr);
  10774. instr->ReplaceSrc(opnd, floatReg);
  10775. }
  10776. }
  10777. return instr;
  10778. }
  10779. return newInstr;
  10780. }
  10781. void
  10782. GlobOpt::ToVarRegOpnd(IR::RegOpnd *dst, BasicBlock *block)
  10783. {
  10784. ToVarStackSym(dst->m_sym, block);
  10785. }
  10786. void
  10787. GlobOpt::ToVarStackSym(StackSym *varSym, BasicBlock *block)
  10788. {
  10789. //added another check for sym , in case of asmjs there is mostly no var syms and hence added a new check to see if it is the primary sym
  10790. Assert(!varSym->IsTypeSpec());
  10791. block->globOptData.liveVarSyms->Set(varSym->m_id);
  10792. block->globOptData.liveInt32Syms->Clear(varSym->m_id);
  10793. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  10794. block->globOptData.liveFloat64Syms->Clear(varSym->m_id);
  10795. }
  10796. void
  10797. GlobOpt::ToInt32Dst(IR::Instr *instr, IR::RegOpnd *dst, BasicBlock *block)
  10798. {
  10799. StackSym *varSym = dst->m_sym;
  10800. Assert(!varSym->IsTypeSpec());
  10801. if (!this->IsLoopPrePass() && varSym->IsVar())
  10802. {
  10803. StackSym *int32Sym = varSym->GetInt32EquivSym(instr->m_func);
  10804. // Use UnlinkDst / SetDst to make sure isSingleDef is tracked properly,
  10805. // since we'll just be hammering the symbol.
  10806. dst = instr->UnlinkDst()->AsRegOpnd();
  10807. dst->m_sym = int32Sym;
  10808. dst->SetType(TyInt32);
  10809. instr->SetDst(dst);
  10810. }
  10811. block->globOptData.liveInt32Syms->Set(varSym->m_id);
  10812. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id); // The store makes it lossless
  10813. block->globOptData.liveVarSyms->Clear(varSym->m_id);
  10814. block->globOptData.liveFloat64Syms->Clear(varSym->m_id);
  10815. }
  10816. void
  10817. GlobOpt::ToUInt32Dst(IR::Instr *instr, IR::RegOpnd *dst, BasicBlock *block)
  10818. {
  10819. // We should be calling only for asmjs function
  10820. Assert(GetIsAsmJSFunc());
  10821. StackSym *varSym = dst->m_sym;
  10822. Assert(!varSym->IsTypeSpec());
  10823. block->globOptData.liveInt32Syms->Set(varSym->m_id);
  10824. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id); // The store makes it lossless
  10825. block->globOptData.liveVarSyms->Clear(varSym->m_id);
  10826. block->globOptData.liveFloat64Syms->Clear(varSym->m_id);
  10827. }
  10828. void
  10829. GlobOpt::ToFloat64Dst(IR::Instr *instr, IR::RegOpnd *dst, BasicBlock *block)
  10830. {
  10831. StackSym *varSym = dst->m_sym;
  10832. Assert(!varSym->IsTypeSpec());
  10833. if (!this->IsLoopPrePass() && varSym->IsVar())
  10834. {
  10835. StackSym *float64Sym = varSym->GetFloat64EquivSym(this->func);
  10836. // Use UnlinkDst / SetDst to make sure isSingleDef is tracked properly,
  10837. // since we'll just be hammering the symbol.
  10838. dst = instr->UnlinkDst()->AsRegOpnd();
  10839. dst->m_sym = float64Sym;
  10840. dst->SetType(TyFloat64);
  10841. instr->SetDst(dst);
  10842. }
  10843. block->globOptData.liveFloat64Syms->Set(varSym->m_id);
  10844. block->globOptData.liveVarSyms->Clear(varSym->m_id);
  10845. block->globOptData.liveInt32Syms->Clear(varSym->m_id);
  10846. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  10847. }
  10848. static void SetIsConstFlag(StackSym* dstSym, int64 value)
  10849. {
  10850. Assert(dstSym);
  10851. dstSym->SetIsInt64Const();
  10852. }
  10853. static void SetIsConstFlag(StackSym* dstSym, int value)
  10854. {
  10855. Assert(dstSym);
  10856. dstSym->SetIsIntConst(value);
  10857. }
  10858. static IR::Opnd* CreateIntConstOpnd(IR::Instr* instr, int64 value)
  10859. {
  10860. return (IR::Opnd*)IR::Int64ConstOpnd::New(value, instr->GetDst()->GetType(), instr->m_func);
  10861. }
  10862. static IR::Opnd* CreateIntConstOpnd(IR::Instr* instr, int value)
  10863. {
  10864. IntConstType constVal;
  10865. if (instr->GetDst()->IsUnsigned())
  10866. {
  10867. // we should zero extend in case of uint
  10868. constVal = (uint32)value;
  10869. }
  10870. else
  10871. {
  10872. constVal = value;
  10873. }
  10874. return (IR::Opnd*)IR::IntConstOpnd::New(constVal, instr->GetDst()->GetType(), instr->m_func);
  10875. }
  10876. template <typename T>
  10877. IR::Opnd* GlobOpt::ReplaceWConst(IR::Instr **pInstr, T value, Value **pDstVal)
  10878. {
  10879. IR::Instr * &instr = *pInstr;
  10880. IR::Opnd * constOpnd = CreateIntConstOpnd(instr, value);
  10881. instr->ReplaceSrc1(constOpnd);
  10882. instr->FreeSrc2();
  10883. this->OptSrc(constOpnd, &instr);
  10884. IR::Opnd *dst = instr->GetDst();
  10885. StackSym *dstSym = dst->AsRegOpnd()->m_sym;
  10886. if (dstSym->IsSingleDef())
  10887. {
  10888. SetIsConstFlag(dstSym, value);
  10889. }
  10890. GOPT_TRACE_INSTR(instr, _u("Constant folding to %d: \n"), value);
  10891. *pDstVal = GetIntConstantValue(value, instr, dst);
  10892. return dst;
  10893. }
  10894. template <typename T>
  10895. bool GlobOpt::OptConstFoldBinaryWasm(
  10896. IR::Instr** pInstr,
  10897. const Value* src1,
  10898. const Value* src2,
  10899. Value **pDstVal)
  10900. {
  10901. IR::Instr* &instr = *pInstr;
  10902. if (!DoConstFold())
  10903. {
  10904. return false;
  10905. }
  10906. T src1IntConstantValue, src2IntConstantValue;
  10907. if (!src1 || !src1->GetValueInfo()->TryGetIntConstantValue(&src1IntConstantValue, false) || //a bit sketchy: false for int32 means likelyInt = false
  10908. !src2 || !src2->GetValueInfo()->TryGetIntConstantValue(&src2IntConstantValue, false) //and unsigned = false for int64
  10909. )
  10910. {
  10911. return false;
  10912. }
  10913. int64 tmpValueOut;
  10914. if (!instr->BinaryCalculatorT<T>(src1IntConstantValue, src2IntConstantValue, &tmpValueOut, func->GetJITFunctionBody()->IsWasmFunction()))
  10915. {
  10916. return false;
  10917. }
  10918. this->CaptureByteCodeSymUses(instr);
  10919. IR::Opnd *dst = (instr->GetDst()->IsInt64()) ? //dst can be int32 for int64 comparison operators
  10920. ReplaceWConst(pInstr, tmpValueOut, pDstVal) :
  10921. ReplaceWConst(pInstr, (int)tmpValueOut, pDstVal);
  10922. instr->m_opcode = Js::OpCode::Ld_I4;
  10923. this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  10924. return true;
  10925. }
  10926. bool
  10927. GlobOpt::OptConstFoldBinary(
  10928. IR::Instr * *pInstr,
  10929. const IntConstantBounds &src1IntConstantBounds,
  10930. const IntConstantBounds &src2IntConstantBounds,
  10931. Value **pDstVal)
  10932. {
  10933. IR::Instr * &instr = *pInstr;
  10934. int32 value;
  10935. IR::IntConstOpnd *constOpnd;
  10936. if (!DoConstFold())
  10937. {
  10938. return false;
  10939. }
  10940. int32 src1IntConstantValue = -1;
  10941. int32 src2IntConstantValue = -1;
  10942. int32 src1MaxIntConstantValue = -1;
  10943. int32 src2MaxIntConstantValue = -1;
  10944. int32 src1MinIntConstantValue = -1;
  10945. int32 src2MinIntConstantValue = -1;
  10946. if (instr->IsBranchInstr())
  10947. {
  10948. src1MinIntConstantValue = src1IntConstantBounds.LowerBound();
  10949. src1MaxIntConstantValue = src1IntConstantBounds.UpperBound();
  10950. src2MinIntConstantValue = src2IntConstantBounds.LowerBound();
  10951. src2MaxIntConstantValue = src2IntConstantBounds.UpperBound();
  10952. }
  10953. else if (src1IntConstantBounds.IsConstant() && src2IntConstantBounds.IsConstant())
  10954. {
  10955. src1IntConstantValue = src1IntConstantBounds.LowerBound();
  10956. src2IntConstantValue = src2IntConstantBounds.LowerBound();
  10957. }
  10958. else
  10959. {
  10960. return false;
  10961. }
  10962. IntConstType tmpValueOut;
  10963. if (!instr->BinaryCalculator(src1IntConstantValue, src2IntConstantValue, &tmpValueOut, TyInt32)
  10964. || !Math::FitsInDWord(tmpValueOut))
  10965. {
  10966. return false;
  10967. }
  10968. value = (int32)tmpValueOut;
  10969. this->CaptureByteCodeSymUses(instr);
  10970. constOpnd = IR::IntConstOpnd::New(value, TyInt32, instr->m_func);
  10971. instr->ReplaceSrc1(constOpnd);
  10972. instr->FreeSrc2();
  10973. this->OptSrc(constOpnd, &instr);
  10974. IR::Opnd *dst = instr->GetDst();
  10975. Assert(dst->IsRegOpnd());
  10976. StackSym *dstSym = dst->AsRegOpnd()->m_sym;
  10977. if (dstSym->IsSingleDef())
  10978. {
  10979. dstSym->SetIsIntConst(value);
  10980. }
  10981. GOPT_TRACE_INSTR(instr, _u("Constant folding to %d: \n"), value);
  10982. *pDstVal = GetIntConstantValue(value, instr, dst);
  10983. if (IsTypeSpecPhaseOff(this->func))
  10984. {
  10985. instr->m_opcode = Js::OpCode::LdC_A_I4;
  10986. this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock);
  10987. }
  10988. else
  10989. {
  10990. instr->m_opcode = Js::OpCode::Ld_I4;
  10991. this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  10992. }
  10993. InvalidateInductionVariables(instr);
  10994. return true;
  10995. }
  10996. void
  10997. GlobOpt::OptConstFoldBr(bool test, IR::Instr *instr, Value * src1Val, Value * src2Val)
  10998. {
  10999. GOPT_TRACE_INSTR(instr, _u("Constant folding to branch: "));
  11000. BasicBlock *deadBlock;
  11001. if (src1Val)
  11002. {
  11003. this->ToInt32(instr, instr->GetSrc1(), this->currentBlock, src1Val, nullptr, false);
  11004. }
  11005. if (src2Val)
  11006. {
  11007. this->ToInt32(instr, instr->GetSrc2(), this->currentBlock, src2Val, nullptr, false);
  11008. }
  11009. this->CaptureByteCodeSymUses(instr);
  11010. if (test)
  11011. {
  11012. instr->m_opcode = Js::OpCode::Br;
  11013. instr->FreeSrc1();
  11014. if(instr->GetSrc2())
  11015. {
  11016. instr->FreeSrc2();
  11017. }
  11018. deadBlock = instr->m_next->AsLabelInstr()->GetBasicBlock();
  11019. }
  11020. else
  11021. {
  11022. AssertMsg(instr->m_next->IsLabelInstr(), "Next instr of branch should be a label...");
  11023. if(instr->AsBranchInstr()->IsMultiBranch())
  11024. {
  11025. return;
  11026. }
  11027. deadBlock = instr->AsBranchInstr()->GetTarget()->GetBasicBlock();
  11028. instr->FreeSrc1();
  11029. if(instr->GetSrc2())
  11030. {
  11031. instr->FreeSrc2();
  11032. }
  11033. instr->m_opcode = Js::OpCode::Nop;
  11034. }
  11035. // Loop back edge: we would have already decremented data use count for the tail block when we processed the loop header.
  11036. if (!(this->currentBlock->loop && this->currentBlock->loop->GetHeadBlock() == deadBlock))
  11037. {
  11038. this->currentBlock->DecrementDataUseCount();
  11039. }
  11040. this->currentBlock->RemoveDeadSucc(deadBlock, this->func->m_fg);
  11041. if (deadBlock->GetPredList()->Count() == 0)
  11042. {
  11043. deadBlock->SetDataUseCount(0);
  11044. }
  11045. }
  11046. void
  11047. GlobOpt::ChangeValueType(
  11048. BasicBlock *const block,
  11049. Value *const value,
  11050. const ValueType newValueType,
  11051. const bool preserveSubclassInfo,
  11052. const bool allowIncompatibleType) const
  11053. {
  11054. Assert(value);
  11055. // Why are we trying to change the value type of the type sym value? Asserting here to make sure we don't deep copy the type sym's value info.
  11056. Assert(!value->GetValueInfo()->IsJsType());
  11057. ValueInfo *const valueInfo = value->GetValueInfo();
  11058. const ValueType valueType(valueInfo->Type());
  11059. if(valueType == newValueType && (preserveSubclassInfo || valueInfo->IsGeneric()))
  11060. {
  11061. return;
  11062. }
  11063. // ArrayValueInfo has information specific to the array type, so make sure that doesn't change
  11064. Assert(
  11065. !preserveSubclassInfo ||
  11066. !valueInfo->IsArrayValueInfo() ||
  11067. newValueType.IsObject() && newValueType.GetObjectType() == valueInfo->GetObjectType());
  11068. Assert(!valueInfo->GetSymStore() || !valueInfo->GetSymStore()->IsStackSym() || !valueInfo->GetSymStore()->AsStackSym()->IsFromByteCodeConstantTable());
  11069. ValueInfo *const newValueInfo =
  11070. preserveSubclassInfo
  11071. ? valueInfo->Copy(alloc)
  11072. : valueInfo->CopyWithGenericStructureKind(alloc);
  11073. newValueInfo->Type() = newValueType;
  11074. ChangeValueInfo(block, value, newValueInfo, allowIncompatibleType);
  11075. }
  11076. void
  11077. GlobOpt::ChangeValueInfo(BasicBlock *const block, Value *const value, ValueInfo *const newValueInfo, const bool allowIncompatibleType, const bool compensated) const
  11078. {
  11079. Assert(value);
  11080. Assert(newValueInfo);
  11081. // The value type must be changed to something more specific or something more generic. For instance, it would be changed to
  11082. // something more specific if the current value type is LikelyArray and checks have been done to ensure that it's an array,
  11083. // and it would be changed to something more generic if a call kills the Array value type and it must be treated as
  11084. // LikelyArray going forward.
  11085. // There are cases where we change the type because of different profile information, and because of rejit, these profile information
  11086. // may conflict. Need to allow incompatible type in those cause. However, the old type should be indefinite.
  11087. Assert((allowIncompatibleType && !value->GetValueInfo()->IsDefinite()) ||
  11088. AreValueInfosCompatible(newValueInfo, value->GetValueInfo()));
  11089. // ArrayValueInfo has information specific to the array type, so make sure that doesn't change
  11090. Assert(
  11091. !value->GetValueInfo()->IsArrayValueInfo() ||
  11092. !newValueInfo->IsArrayValueInfo() ||
  11093. newValueInfo->GetObjectType() == value->GetValueInfo()->GetObjectType());
  11094. if(block)
  11095. {
  11096. TrackValueInfoChangeForKills(block, value, newValueInfo, compensated);
  11097. }
  11098. value->SetValueInfo(newValueInfo);
  11099. }
  11100. bool
  11101. GlobOpt::AreValueInfosCompatible(const ValueInfo *const v0, const ValueInfo *const v1) const
  11102. {
  11103. Assert(v0);
  11104. Assert(v1);
  11105. if(v0->IsUninitialized() || v1->IsUninitialized())
  11106. {
  11107. return true;
  11108. }
  11109. const bool doAggressiveIntTypeSpec = DoAggressiveIntTypeSpec();
  11110. if(doAggressiveIntTypeSpec && (v0->IsInt() || v1->IsInt()))
  11111. {
  11112. // Int specialization in some uncommon loop cases involving dependencies, needs to allow specializing values of
  11113. // arbitrary types, even values that are definitely not int, to compensate for aggressive assumptions made by a loop
  11114. // prepass
  11115. return true;
  11116. }
  11117. if ((v0->Type()).IsMixedTypedArrayPair(v1->Type()) || (v1->Type()).IsMixedTypedArrayPair(v0->Type()))
  11118. {
  11119. return true;
  11120. }
  11121. const bool doFloatTypeSpec = DoFloatTypeSpec();
  11122. if(doFloatTypeSpec && (v0->IsFloat() || v1->IsFloat()))
  11123. {
  11124. // Float specialization allows specializing values of arbitrary types, even values that are definitely not float
  11125. return true;
  11126. }
  11127. const bool doArrayMissingValueCheckHoist = DoArrayMissingValueCheckHoist();
  11128. const bool doNativeArrayTypeSpec = DoNativeArrayTypeSpec();
  11129. const auto AreValueTypesCompatible = [=](const ValueType t0, const ValueType t1)
  11130. {
  11131. return
  11132. t0.IsSubsetOf(t1, doAggressiveIntTypeSpec, doFloatTypeSpec, doArrayMissingValueCheckHoist, doNativeArrayTypeSpec) ||
  11133. t1.IsSubsetOf(t0, doAggressiveIntTypeSpec, doFloatTypeSpec, doArrayMissingValueCheckHoist, doNativeArrayTypeSpec);
  11134. };
  11135. const ValueType t0(v0->Type().ToDefinite()), t1(v1->Type().ToDefinite());
  11136. if(t0.IsLikelyObject() && t1.IsLikelyObject())
  11137. {
  11138. // Check compatibility for the primitive portions and the object portions of the value types separately
  11139. if(AreValueTypesCompatible(t0.ToDefiniteObject(), t1.ToDefiniteObject()) &&
  11140. (
  11141. !t0.HasBeenPrimitive() ||
  11142. !t1.HasBeenPrimitive() ||
  11143. AreValueTypesCompatible(t0.ToDefinitePrimitiveSubset(), t1.ToDefinitePrimitiveSubset())
  11144. ))
  11145. {
  11146. return true;
  11147. }
  11148. }
  11149. else if(AreValueTypesCompatible(t0, t1))
  11150. {
  11151. return true;
  11152. }
  11153. const FloatConstantValueInfo *floatConstantValueInfo;
  11154. const ValueInfo *likelyIntValueinfo;
  11155. if(v0->IsFloatConstant() && v1->IsLikelyInt())
  11156. {
  11157. floatConstantValueInfo = v0->AsFloatConstant();
  11158. likelyIntValueinfo = v1;
  11159. }
  11160. else if(v0->IsLikelyInt() && v1->IsFloatConstant())
  11161. {
  11162. floatConstantValueInfo = v1->AsFloatConstant();
  11163. likelyIntValueinfo = v0;
  11164. }
  11165. else
  11166. {
  11167. return false;
  11168. }
  11169. // A float constant value with a value that is actually an int is a subset of a likely-int value.
  11170. // Ideally, we should create an int constant value for this up front, such that IsInt() also returns true. There
  11171. // were other issues with that, should see if that can be done.
  11172. int32 int32Value;
  11173. return
  11174. Js::JavascriptNumber::TryGetInt32Value(floatConstantValueInfo->FloatValue(), &int32Value) &&
  11175. (!likelyIntValueinfo->IsLikelyTaggedInt() || !Js::TaggedInt::IsOverflow(int32Value));
  11176. }
  11177. #if DBG
  11178. void
  11179. GlobOpt::VerifyArrayValueInfoForTracking(
  11180. const ValueInfo *const valueInfo,
  11181. const bool isJsArray,
  11182. const BasicBlock *const block,
  11183. const bool ignoreKnownImplicitCalls) const
  11184. {
  11185. Assert(valueInfo);
  11186. Assert(valueInfo->IsAnyOptimizedArray());
  11187. Assert(isJsArray == valueInfo->IsArrayOrObjectWithArray());
  11188. Assert(!isJsArray == valueInfo->IsOptimizedTypedArray());
  11189. Assert(block);
  11190. Loop *implicitCallsLoop;
  11191. if(block->next && !block->next->isDeleted && block->next->isLoopHeader)
  11192. {
  11193. // Since a loop's landing pad does not have user code, determine whether disabling implicit calls is allowed in the
  11194. // landing pad based on the loop for which this block is the landing pad.
  11195. implicitCallsLoop = block->next->loop;
  11196. Assert(implicitCallsLoop);
  11197. Assert(implicitCallsLoop->landingPad == block);
  11198. }
  11199. else
  11200. {
  11201. implicitCallsLoop = block->loop;
  11202. }
  11203. Assert(
  11204. !isJsArray ||
  11205. DoArrayCheckHoist(valueInfo->Type(), implicitCallsLoop) ||
  11206. (
  11207. ignoreKnownImplicitCalls &&
  11208. !(implicitCallsLoop ? ImplicitCallFlagsAllowOpts(implicitCallsLoop) : ImplicitCallFlagsAllowOpts(func))
  11209. ));
  11210. Assert(!(isJsArray && valueInfo->HasNoMissingValues() && !DoArrayMissingValueCheckHoist()));
  11211. Assert(
  11212. !(
  11213. valueInfo->IsArrayValueInfo() &&
  11214. (
  11215. valueInfo->AsArrayValueInfo()->HeadSegmentSym() ||
  11216. valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()
  11217. ) &&
  11218. !DoArraySegmentHoist(valueInfo->Type())
  11219. ));
  11220. #if 0
  11221. // We can't assert here that there is only a head segment length sym if hoisting is allowed in the current block,
  11222. // because we may have propagated the sym forward out of a loop, and hoisting may be allowed inside but not
  11223. // outside the loop.
  11224. Assert(
  11225. isJsArray ||
  11226. !valueInfo->IsArrayValueInfo() ||
  11227. !valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym() ||
  11228. DoTypedArraySegmentLengthHoist(implicitCallsLoop) ||
  11229. ignoreKnownImplicitCalls ||
  11230. (implicitCallsLoop ? ImplicitCallFlagsAllowOpts(implicitCallsLoop) : ImplicitCallFlagsAllowOpts(func))
  11231. );
  11232. #endif
  11233. Assert(
  11234. !(
  11235. isJsArray &&
  11236. valueInfo->IsArrayValueInfo() &&
  11237. valueInfo->AsArrayValueInfo()->LengthSym() &&
  11238. !DoArrayLengthHoist()
  11239. ));
  11240. }
  11241. #endif
  11242. void
  11243. GlobOpt::TrackNewValueForKills(Value *const value)
  11244. {
  11245. Assert(value);
  11246. if(!value->GetValueInfo()->IsAnyOptimizedArray())
  11247. {
  11248. return;
  11249. }
  11250. DoTrackNewValueForKills(value);
  11251. }
  11252. void
  11253. GlobOpt::DoTrackNewValueForKills(Value *const value)
  11254. {
  11255. Assert(value);
  11256. ValueInfo *const valueInfo = value->GetValueInfo();
  11257. Assert(valueInfo->IsAnyOptimizedArray());
  11258. Assert(!valueInfo->IsArrayValueInfo());
  11259. // The value and value info here are new, so it's okay to modify the value info in-place
  11260. Assert(!valueInfo->GetSymStore());
  11261. const bool isJsArray = valueInfo->IsArrayOrObjectWithArray();
  11262. Assert(!isJsArray == valueInfo->IsOptimizedTypedArray());
  11263. const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray();
  11264. Loop *implicitCallsLoop;
  11265. if(currentBlock->next && !currentBlock->next->isDeleted && currentBlock->next->isLoopHeader)
  11266. {
  11267. // Since a loop's landing pad does not have user code, determine whether disabling implicit calls is allowed in the
  11268. // landing pad based on the loop for which this block is the landing pad.
  11269. implicitCallsLoop = currentBlock->next->loop;
  11270. Assert(implicitCallsLoop);
  11271. Assert(implicitCallsLoop->landingPad == currentBlock);
  11272. }
  11273. else
  11274. {
  11275. implicitCallsLoop = currentBlock->loop;
  11276. }
  11277. if(isJsArray || isVirtualTypedArray)
  11278. {
  11279. if(!DoArrayCheckHoist(valueInfo->Type(), implicitCallsLoop))
  11280. {
  11281. // Array opts are disabled for this value type, so treat it as an indefinite value type going forward
  11282. valueInfo->Type() = valueInfo->Type().ToLikely();
  11283. return;
  11284. }
  11285. if(isJsArray && valueInfo->HasNoMissingValues() && !DoArrayMissingValueCheckHoist())
  11286. {
  11287. valueInfo->Type() = valueInfo->Type().SetHasNoMissingValues(false);
  11288. }
  11289. }
  11290. #if DBG
  11291. VerifyArrayValueInfoForTracking(valueInfo, isJsArray, currentBlock);
  11292. #endif
  11293. if(!isJsArray && !isVirtualTypedArray)
  11294. {
  11295. return;
  11296. }
  11297. // Can't assume going forward that it will definitely be an array without disabling implicit calls, because the
  11298. // array may be transformed into an ES5 array. Since array opts are enabled, implicit calls can be disabled, and we can
  11299. // treat it as a definite value type going forward, but the value needs to be tracked so that something like a call can
  11300. // revert the value type to a likely version.
  11301. CurrentBlockData()->valuesToKillOnCalls->Add(value);
  11302. }
  11303. void
  11304. GlobOpt::TrackCopiedValueForKills(Value *const value)
  11305. {
  11306. Assert(value);
  11307. if(!value->GetValueInfo()->IsAnyOptimizedArray())
  11308. {
  11309. return;
  11310. }
  11311. DoTrackCopiedValueForKills(value);
  11312. }
  11313. void
  11314. GlobOpt::DoTrackCopiedValueForKills(Value *const value)
  11315. {
  11316. Assert(value);
  11317. ValueInfo *const valueInfo = value->GetValueInfo();
  11318. Assert(valueInfo->IsAnyOptimizedArray());
  11319. const bool isJsArray = valueInfo->IsArrayOrObjectWithArray();
  11320. Assert(!isJsArray == valueInfo->IsOptimizedTypedArray());
  11321. const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray();
  11322. #if DBG
  11323. VerifyArrayValueInfoForTracking(valueInfo, isJsArray, currentBlock);
  11324. #endif
  11325. if(!isJsArray && !isVirtualTypedArray && !(valueInfo->IsArrayValueInfo() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()))
  11326. {
  11327. return;
  11328. }
  11329. // Can't assume going forward that it will definitely be an array without disabling implicit calls, because the
  11330. // array may be transformed into an ES5 array. Since array opts are enabled, implicit calls can be disabled, and we can
  11331. // treat it as a definite value type going forward, but the value needs to be tracked so that something like a call can
  11332. // revert the value type to a likely version.
  11333. CurrentBlockData()->valuesToKillOnCalls->Add(value);
  11334. }
  11335. void
  11336. GlobOpt::TrackMergedValueForKills(
  11337. Value *const value,
  11338. GlobOptBlockData *const blockData,
  11339. BVSparse<JitArenaAllocator> *const mergedValueTypesTrackedForKills) const
  11340. {
  11341. Assert(value);
  11342. if(!value->GetValueInfo()->IsAnyOptimizedArray())
  11343. {
  11344. return;
  11345. }
  11346. DoTrackMergedValueForKills(value, blockData, mergedValueTypesTrackedForKills);
  11347. }
  11348. void
  11349. GlobOpt::DoTrackMergedValueForKills(
  11350. Value *const value,
  11351. GlobOptBlockData *const blockData,
  11352. BVSparse<JitArenaAllocator> *const mergedValueTypesTrackedForKills) const
  11353. {
  11354. Assert(value);
  11355. Assert(blockData);
  11356. ValueInfo *valueInfo = value->GetValueInfo();
  11357. Assert(valueInfo->IsAnyOptimizedArray());
  11358. const bool isJsArray = valueInfo->IsArrayOrObjectWithArray();
  11359. Assert(!isJsArray == valueInfo->IsOptimizedTypedArray());
  11360. const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray();
  11361. #if DBG
  11362. VerifyArrayValueInfoForTracking(valueInfo, isJsArray, currentBlock, true);
  11363. #endif
  11364. if(!isJsArray && !isVirtualTypedArray && !(valueInfo->IsArrayValueInfo() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()))
  11365. {
  11366. return;
  11367. }
  11368. // Can't assume going forward that it will definitely be an array without disabling implicit calls, because the
  11369. // array may be transformed into an ES5 array. Since array opts are enabled, implicit calls can be disabled, and we can
  11370. // treat it as a definite value type going forward, but the value needs to be tracked so that something like a call can
  11371. // revert the value type to a likely version.
  11372. if(!mergedValueTypesTrackedForKills || !mergedValueTypesTrackedForKills->TestAndSet(value->GetValueNumber()))
  11373. {
  11374. blockData->valuesToKillOnCalls->Add(value);
  11375. }
  11376. }
  11377. void
  11378. GlobOpt::TrackValueInfoChangeForKills(BasicBlock *const block, Value *const value, ValueInfo *const newValueInfo, const bool compensated) const
  11379. {
  11380. Assert(block);
  11381. Assert(value);
  11382. Assert(newValueInfo);
  11383. ValueInfo *const oldValueInfo = value->GetValueInfo();
  11384. #if DBG
  11385. if(oldValueInfo->IsAnyOptimizedArray())
  11386. {
  11387. VerifyArrayValueInfoForTracking(oldValueInfo, oldValueInfo->IsArrayOrObjectWithArray(), block, compensated);
  11388. }
  11389. #endif
  11390. const bool trackOldValueInfo =
  11391. oldValueInfo->IsArrayOrObjectWithArray() ||
  11392. oldValueInfo->IsOptimizedVirtualTypedArray() ||
  11393. (
  11394. oldValueInfo->IsOptimizedTypedArray() &&
  11395. oldValueInfo->IsArrayValueInfo() &&
  11396. oldValueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()
  11397. );
  11398. Assert(trackOldValueInfo == block->globOptData.valuesToKillOnCalls->ContainsKey(value));
  11399. #if DBG
  11400. if(newValueInfo->IsAnyOptimizedArray())
  11401. {
  11402. VerifyArrayValueInfoForTracking(newValueInfo, newValueInfo->IsArrayOrObjectWithArray(), block, compensated);
  11403. }
  11404. #endif
  11405. const bool trackNewValueInfo =
  11406. newValueInfo->IsArrayOrObjectWithArray() ||
  11407. newValueInfo->IsOptimizedVirtualTypedArray() ||
  11408. (
  11409. newValueInfo->IsOptimizedTypedArray() &&
  11410. newValueInfo->IsArrayValueInfo() &&
  11411. newValueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()
  11412. );
  11413. if(trackOldValueInfo == trackNewValueInfo)
  11414. {
  11415. return;
  11416. }
  11417. if(trackNewValueInfo)
  11418. {
  11419. block->globOptData.valuesToKillOnCalls->Add(value);
  11420. }
  11421. else
  11422. {
  11423. block->globOptData.valuesToKillOnCalls->Remove(value);
  11424. }
  11425. }
  11426. void
  11427. GlobOpt::ProcessValueKills(IR::Instr *const instr)
  11428. {
  11429. Assert(instr);
  11430. ValueSet *const valuesToKillOnCalls = CurrentBlockData()->valuesToKillOnCalls;
  11431. if(!IsLoopPrePass() && valuesToKillOnCalls->Count() == 0)
  11432. {
  11433. return;
  11434. }
  11435. const JsArrayKills kills = CheckJsArrayKills(instr);
  11436. Assert(!kills.KillsArrayHeadSegments() || kills.KillsArrayHeadSegmentLengths());
  11437. if(IsLoopPrePass())
  11438. {
  11439. rootLoopPrePass->jsArrayKills = rootLoopPrePass->jsArrayKills.Merge(kills);
  11440. Assert(
  11441. !rootLoopPrePass->parent ||
  11442. rootLoopPrePass->jsArrayKills.AreSubsetOf(rootLoopPrePass->parent->jsArrayKills));
  11443. if(kills.KillsAllArrays())
  11444. {
  11445. rootLoopPrePass->needImplicitCallBailoutChecksForJsArrayCheckHoist = false;
  11446. }
  11447. if(valuesToKillOnCalls->Count() == 0)
  11448. {
  11449. return;
  11450. }
  11451. }
  11452. if(kills.KillsAllArrays())
  11453. {
  11454. Assert(kills.KillsTypedArrayHeadSegmentLengths());
  11455. // - Calls need to kill the value types of values in the following list. For instance, calls can transform a JS array
  11456. // into an ES5 array, so any definitely-array value types need to be killed. Also, VirtualTypeArrays do not have
  11457. // bounds checks; this can be problematic if the array is detached, so check to ensure that it is a virtual array.
  11458. // Update the value types to likley to ensure a bailout that asserts Array type is generated.
  11459. // - Calls also need to kill typed array head segment lengths. A typed array's array buffer may be transferred to a web
  11460. // worker, in which case the typed array's length is set to zero.
  11461. for(auto it = valuesToKillOnCalls->GetIterator(); it.IsValid(); it.MoveNext())
  11462. {
  11463. Value *const value = it.CurrentValue();
  11464. ValueInfo *const valueInfo = value->GetValueInfo();
  11465. Assert(
  11466. valueInfo->IsArrayOrObjectWithArray() ||
  11467. valueInfo->IsOptimizedVirtualTypedArray() ||
  11468. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11469. if (valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsOptimizedVirtualTypedArray())
  11470. {
  11471. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11472. continue;
  11473. }
  11474. ChangeValueInfo(
  11475. nullptr,
  11476. value,
  11477. valueInfo->AsArrayValueInfo()->Copy(alloc, true, false /* copyHeadSegmentLength */, true));
  11478. }
  11479. valuesToKillOnCalls->Clear();
  11480. return;
  11481. }
  11482. if(kills.KillsArraysWithNoMissingValues())
  11483. {
  11484. // Some operations may kill arrays with no missing values in unlikely circumstances. Convert their value types to likely
  11485. // versions so that the checks have to be redone.
  11486. for(auto it = valuesToKillOnCalls->GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext())
  11487. {
  11488. Value *const value = it.CurrentValue();
  11489. ValueInfo *const valueInfo = value->GetValueInfo();
  11490. Assert(
  11491. valueInfo->IsArrayOrObjectWithArray() ||
  11492. valueInfo->IsOptimizedVirtualTypedArray() ||
  11493. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11494. if(!valueInfo->IsArrayOrObjectWithArray() || !valueInfo->HasNoMissingValues())
  11495. {
  11496. continue;
  11497. }
  11498. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11499. it.RemoveCurrent();
  11500. }
  11501. }
  11502. if(kills.KillsNativeArrays())
  11503. {
  11504. // Some operations may kill native arrays in (what should be) unlikely circumstances. Convert their value types to
  11505. // likely versions so that the checks have to be redone.
  11506. for(auto it = valuesToKillOnCalls->GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext())
  11507. {
  11508. Value *const value = it.CurrentValue();
  11509. ValueInfo *const valueInfo = value->GetValueInfo();
  11510. Assert(
  11511. valueInfo->IsArrayOrObjectWithArray() ||
  11512. valueInfo->IsOptimizedVirtualTypedArray() ||
  11513. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11514. if(!valueInfo->IsArrayOrObjectWithArray() || valueInfo->HasVarElements())
  11515. {
  11516. continue;
  11517. }
  11518. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11519. it.RemoveCurrent();
  11520. }
  11521. }
  11522. const bool likelyKillsJsArraysWithNoMissingValues = IsOperationThatLikelyKillsJsArraysWithNoMissingValues(instr);
  11523. if(!kills.KillsArrayHeadSegmentLengths())
  11524. {
  11525. Assert(!kills.KillsArrayHeadSegments());
  11526. if(!likelyKillsJsArraysWithNoMissingValues && !kills.KillsArrayLengths())
  11527. {
  11528. return;
  11529. }
  11530. }
  11531. for(auto it = valuesToKillOnCalls->GetIterator(); it.IsValid(); it.MoveNext())
  11532. {
  11533. Value *const value = it.CurrentValue();
  11534. ValueInfo *valueInfo = value->GetValueInfo();
  11535. Assert(
  11536. valueInfo->IsArrayOrObjectWithArray() ||
  11537. valueInfo->IsOptimizedVirtualTypedArray() ||
  11538. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11539. if(!valueInfo->IsArrayOrObjectWithArray())
  11540. {
  11541. continue;
  11542. }
  11543. if(likelyKillsJsArraysWithNoMissingValues && valueInfo->HasNoMissingValues())
  11544. {
  11545. ChangeValueType(nullptr, value, valueInfo->Type().SetHasNoMissingValues(false), true);
  11546. valueInfo = value->GetValueInfo();
  11547. }
  11548. if(!valueInfo->IsArrayValueInfo())
  11549. {
  11550. continue;
  11551. }
  11552. ArrayValueInfo *const arrayValueInfo = valueInfo->AsArrayValueInfo();
  11553. const bool removeHeadSegment = kills.KillsArrayHeadSegments() && arrayValueInfo->HeadSegmentSym();
  11554. const bool removeHeadSegmentLength = kills.KillsArrayHeadSegmentLengths() && arrayValueInfo->HeadSegmentLengthSym();
  11555. const bool removeLength = kills.KillsArrayLengths() && arrayValueInfo->LengthSym();
  11556. if(removeHeadSegment || removeHeadSegmentLength || removeLength)
  11557. {
  11558. ChangeValueInfo(
  11559. nullptr,
  11560. value,
  11561. arrayValueInfo->Copy(alloc, !removeHeadSegment, !removeHeadSegmentLength, !removeLength));
  11562. valueInfo = value->GetValueInfo();
  11563. }
  11564. }
  11565. }
  11566. void
  11567. GlobOpt::ProcessValueKills(BasicBlock *const block, GlobOptBlockData *const blockData)
  11568. {
  11569. Assert(block);
  11570. Assert(blockData);
  11571. ValueSet *const valuesToKillOnCalls = blockData->valuesToKillOnCalls;
  11572. if(!IsLoopPrePass() && valuesToKillOnCalls->Count() == 0)
  11573. {
  11574. return;
  11575. }
  11576. // If the current block or loop has implicit calls, kill all definitely-array value types, as using that info will cause
  11577. // implicit calls to be disabled, resulting in unnecessary bailouts
  11578. const bool killValuesOnImplicitCalls =
  11579. (block->loop ? !this->ImplicitCallFlagsAllowOpts(block->loop) : !this->ImplicitCallFlagsAllowOpts(func));
  11580. if (!killValuesOnImplicitCalls)
  11581. {
  11582. return;
  11583. }
  11584. if(IsLoopPrePass() && block->loop == rootLoopPrePass)
  11585. {
  11586. AnalysisAssert(rootLoopPrePass);
  11587. for (Loop * loop = rootLoopPrePass; loop != nullptr; loop = loop->parent)
  11588. {
  11589. loop->jsArrayKills.SetKillsAllArrays();
  11590. }
  11591. Assert(!rootLoopPrePass->parent || rootLoopPrePass->jsArrayKills.AreSubsetOf(rootLoopPrePass->parent->jsArrayKills));
  11592. if(valuesToKillOnCalls->Count() == 0)
  11593. {
  11594. return;
  11595. }
  11596. }
  11597. for(auto it = valuesToKillOnCalls->GetIterator(); it.IsValid(); it.MoveNext())
  11598. {
  11599. Value *const value = it.CurrentValue();
  11600. ValueInfo *const valueInfo = value->GetValueInfo();
  11601. Assert(
  11602. valueInfo->IsArrayOrObjectWithArray() ||
  11603. valueInfo->IsOptimizedVirtualTypedArray() ||
  11604. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11605. if(valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsOptimizedVirtualTypedArray())
  11606. {
  11607. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11608. continue;
  11609. }
  11610. ChangeValueInfo(
  11611. nullptr,
  11612. value,
  11613. valueInfo->AsArrayValueInfo()->Copy(alloc, true, false /* copyHeadSegmentLength */, true));
  11614. }
  11615. valuesToKillOnCalls->Clear();
  11616. }
  11617. void
  11618. GlobOpt::ProcessValueKillsForLoopHeaderAfterBackEdgeMerge(BasicBlock *const block, GlobOptBlockData *const blockData)
  11619. {
  11620. Assert(block);
  11621. Assert(block->isLoopHeader);
  11622. Assert(blockData);
  11623. ValueSet *const valuesToKillOnCalls = blockData->valuesToKillOnCalls;
  11624. if(valuesToKillOnCalls->Count() == 0)
  11625. {
  11626. return;
  11627. }
  11628. const JsArrayKills loopKills(block->loop->jsArrayKills);
  11629. for(auto it = valuesToKillOnCalls->GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext())
  11630. {
  11631. Value *const value = it.CurrentValue();
  11632. ValueInfo *valueInfo = value->GetValueInfo();
  11633. Assert(
  11634. valueInfo->IsArrayOrObjectWithArray() ||
  11635. valueInfo->IsOptimizedVirtualTypedArray() ||
  11636. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11637. const bool isJsArray = valueInfo->IsArrayOrObjectWithArray();
  11638. Assert(!isJsArray == valueInfo->IsOptimizedTypedArray());
  11639. const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray();
  11640. if((isJsArray || isVirtualTypedArray) ? loopKills.KillsValueType(valueInfo->Type()) : loopKills.KillsTypedArrayHeadSegmentLengths())
  11641. {
  11642. // Hoisting array checks and other related things for this type is disabled for the loop due to the kill, as
  11643. // compensation code is currently not added on back-edges. When merging values from a back-edge, the array value
  11644. // type cannot be definite, as that may require adding compensation code on the back-edge if the optimization pass
  11645. // chooses to not optimize the array.
  11646. if(isJsArray || isVirtualTypedArray)
  11647. {
  11648. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11649. }
  11650. else
  11651. {
  11652. ChangeValueInfo(
  11653. nullptr,
  11654. value,
  11655. valueInfo->AsArrayValueInfo()->Copy(alloc, true, false /* copyHeadSegmentLength */, true));
  11656. }
  11657. it.RemoveCurrent();
  11658. continue;
  11659. }
  11660. if(!isJsArray || !valueInfo->IsArrayValueInfo())
  11661. {
  11662. continue;
  11663. }
  11664. // Similarly, if the loop contains an operation that kills JS array segments, don't make the segment or other related
  11665. // syms available initially inside the loop
  11666. ArrayValueInfo *const arrayValueInfo = valueInfo->AsArrayValueInfo();
  11667. const bool removeHeadSegment = loopKills.KillsArrayHeadSegments() && arrayValueInfo->HeadSegmentSym();
  11668. const bool removeHeadSegmentLength = loopKills.KillsArrayHeadSegmentLengths() && arrayValueInfo->HeadSegmentLengthSym();
  11669. const bool removeLength = loopKills.KillsArrayLengths() && arrayValueInfo->LengthSym();
  11670. if(removeHeadSegment || removeHeadSegmentLength || removeLength)
  11671. {
  11672. ChangeValueInfo(
  11673. nullptr,
  11674. value,
  11675. arrayValueInfo->Copy(alloc, !removeHeadSegment, !removeHeadSegmentLength, !removeLength));
  11676. valueInfo = value->GetValueInfo();
  11677. }
  11678. }
  11679. }
  11680. bool
  11681. GlobOpt::NeedBailOnImplicitCallForLiveValues(BasicBlock const * const block, const bool isForwardPass) const
  11682. {
  11683. if(isForwardPass)
  11684. {
  11685. return block->globOptData.valuesToKillOnCalls->Count() != 0;
  11686. }
  11687. if(block->noImplicitCallUses->IsEmpty())
  11688. {
  11689. Assert(block->noImplicitCallNoMissingValuesUses->IsEmpty());
  11690. Assert(block->noImplicitCallNativeArrayUses->IsEmpty());
  11691. Assert(block->noImplicitCallJsArrayHeadSegmentSymUses->IsEmpty());
  11692. Assert(block->noImplicitCallArrayLengthSymUses->IsEmpty());
  11693. return false;
  11694. }
  11695. return true;
  11696. }
  11697. IR::Instr*
  11698. GlobOpt::CreateBoundsCheckInstr(IR::Opnd* lowerBound, IR::Opnd* upperBound, int offset, Func* func)
  11699. {
  11700. IR::Instr* instr = IR::Instr::New(Js::OpCode::BoundCheck, func);
  11701. return AttachBoundsCheckData(instr, lowerBound, upperBound, offset);
  11702. }
  11703. IR::Instr*
  11704. GlobOpt::CreateBoundsCheckInstr(IR::Opnd* lowerBound, IR::Opnd* upperBound, int offset, IR::BailOutKind bailoutkind, BailOutInfo* bailoutInfo, Func * func)
  11705. {
  11706. IR::Instr* instr = IR::BailOutInstr::New(Js::OpCode::BoundCheck, bailoutkind, bailoutInfo, func);
  11707. return AttachBoundsCheckData(instr, lowerBound, upperBound, offset);
  11708. }
  11709. IR::Instr*
  11710. GlobOpt::AttachBoundsCheckData(IR::Instr* instr, IR::Opnd* lowerBound, IR::Opnd* upperBound, int offset)
  11711. {
  11712. instr->SetSrc1(lowerBound);
  11713. instr->SetSrc2(upperBound);
  11714. if (offset != 0)
  11715. {
  11716. instr->SetDst(IR::IntConstOpnd::New(offset, TyInt32, instr->m_func));
  11717. }
  11718. return instr;
  11719. }
  11720. void
  11721. GlobOpt::OptArraySrc(IR::Instr ** const instrRef, Value ** src1Val, Value ** src2Val)
  11722. {
  11723. Assert(instrRef != nullptr);
  11724. ArraySrcOpt arraySrcOpt(this, instrRef, src1Val, src2Val);
  11725. arraySrcOpt.Optimize();
  11726. }
  11727. void
  11728. GlobOpt::CaptureNoImplicitCallUses(
  11729. IR::Opnd *opnd,
  11730. const bool usesNoMissingValuesInfo,
  11731. IR::Instr *const includeCurrentInstr)
  11732. {
  11733. Assert(!IsLoopPrePass());
  11734. Assert(noImplicitCallUsesToInsert);
  11735. Assert(opnd);
  11736. // The opnd may be deleted later, so make a copy to ensure it is alive for inserting NoImplicitCallUses later
  11737. opnd = opnd->Copy(func);
  11738. if(!usesNoMissingValuesInfo)
  11739. {
  11740. const ValueType valueType(opnd->GetValueType());
  11741. if(valueType.IsArrayOrObjectWithArray() && valueType.HasNoMissingValues())
  11742. {
  11743. // Inserting NoImplicitCallUses for an opnd with a definitely-array-with-no-missing-values value type means that the
  11744. // instruction following it uses the information that the array has no missing values in some way, for instance, it
  11745. // may omit missing value checks. Based on that, the dead-store phase in turn ensures that the necessary bailouts
  11746. // are inserted to ensure that the array still has no missing values until the following instruction. Since
  11747. // 'usesNoMissingValuesInfo' is false, change the value type to indicate to the dead-store phase that the following
  11748. // instruction does not use the no-missing-values information.
  11749. opnd->SetValueType(valueType.SetHasNoMissingValues(false));
  11750. }
  11751. }
  11752. if(includeCurrentInstr)
  11753. {
  11754. IR::Instr *const noImplicitCallUses =
  11755. IR::PragmaInstr::New(Js::OpCode::NoImplicitCallUses, 0, includeCurrentInstr->m_func);
  11756. noImplicitCallUses->SetSrc1(opnd);
  11757. noImplicitCallUses->GetSrc1()->SetIsJITOptimizedReg(true);
  11758. includeCurrentInstr->InsertAfter(noImplicitCallUses);
  11759. return;
  11760. }
  11761. noImplicitCallUsesToInsert->Add(opnd);
  11762. }
  11763. void
  11764. GlobOpt::InsertNoImplicitCallUses(IR::Instr *const instr)
  11765. {
  11766. Assert(noImplicitCallUsesToInsert);
  11767. const int n = noImplicitCallUsesToInsert->Count();
  11768. if(n == 0)
  11769. {
  11770. return;
  11771. }
  11772. IR::Instr *const insertBeforeInstr = instr->GetInsertBeforeByteCodeUsesInstr();
  11773. for(int i = 0; i < n;)
  11774. {
  11775. IR::Instr *const noImplicitCallUses = IR::PragmaInstr::New(Js::OpCode::NoImplicitCallUses, 0, instr->m_func);
  11776. noImplicitCallUses->SetSrc1(noImplicitCallUsesToInsert->Item(i));
  11777. noImplicitCallUses->GetSrc1()->SetIsJITOptimizedReg(true);
  11778. ++i;
  11779. if(i < n)
  11780. {
  11781. noImplicitCallUses->SetSrc2(noImplicitCallUsesToInsert->Item(i));
  11782. noImplicitCallUses->GetSrc2()->SetIsJITOptimizedReg(true);
  11783. ++i;
  11784. }
  11785. noImplicitCallUses->SetByteCodeOffset(instr);
  11786. insertBeforeInstr->InsertBefore(noImplicitCallUses);
  11787. }
  11788. noImplicitCallUsesToInsert->Clear();
  11789. }
  11790. void
  11791. GlobOpt::PrepareLoopArrayCheckHoist()
  11792. {
  11793. if(IsLoopPrePass() || !currentBlock->loop || !currentBlock->isLoopHeader || !currentBlock->loop->parent)
  11794. {
  11795. return;
  11796. }
  11797. if(currentBlock->loop->parent->needImplicitCallBailoutChecksForJsArrayCheckHoist)
  11798. {
  11799. // If the parent loop is an array check elimination candidate, so is the current loop. Even though the current loop may
  11800. // not have array accesses, if the parent loop hoists array checks, the current loop also needs implicit call checks.
  11801. currentBlock->loop->needImplicitCallBailoutChecksForJsArrayCheckHoist = true;
  11802. }
  11803. }
  11804. JsArrayKills
  11805. GlobOpt::CheckJsArrayKills(IR::Instr *const instr)
  11806. {
  11807. Assert(instr);
  11808. JsArrayKills kills;
  11809. if(instr->UsesAllFields())
  11810. {
  11811. // Calls can (but are unlikely to) change a javascript array into an ES5 array, which may have different behavior for
  11812. // index properties.
  11813. kills.SetKillsAllArrays();
  11814. return kills;
  11815. }
  11816. const bool doArrayMissingValueCheckHoist = DoArrayMissingValueCheckHoist();
  11817. const bool doNativeArrayTypeSpec = DoNativeArrayTypeSpec();
  11818. const bool doArraySegmentHoist = DoArraySegmentHoist(ValueType::GetObject(ObjectType::Array));
  11819. Assert(doArraySegmentHoist == DoArraySegmentHoist(ValueType::GetObject(ObjectType::ObjectWithArray)));
  11820. const bool doArrayLengthHoist = DoArrayLengthHoist();
  11821. if(!doArrayMissingValueCheckHoist && !doNativeArrayTypeSpec && !doArraySegmentHoist && !doArrayLengthHoist)
  11822. {
  11823. return kills;
  11824. }
  11825. // The following operations may create missing values in an array in an unlikely circumstance. Even though they don't kill
  11826. // the fact that the 'this' parameter is an array (when implicit calls are disabled), we don't have a way to say the value
  11827. // type is definitely array but it likely has no missing values. So, these will kill the definite value type as well, making
  11828. // it likely array, such that the array checks will have to be redone.
  11829. const bool useValueTypes = !IsLoopPrePass(); // Source value types are not guaranteed to be correct in a loop prepass
  11830. switch(instr->m_opcode)
  11831. {
  11832. case Js::OpCode::StElemI_A:
  11833. case Js::OpCode::StElemI_A_Strict:
  11834. {
  11835. Assert(instr->GetDst());
  11836. if(!instr->GetDst()->IsIndirOpnd())
  11837. {
  11838. break;
  11839. }
  11840. const ValueType baseValueType =
  11841. useValueTypes ? instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType() : ValueType::Uninitialized;
  11842. if(useValueTypes && baseValueType.IsNotArrayOrObjectWithArray())
  11843. {
  11844. break;
  11845. }
  11846. if(instr->IsProfiledInstr())
  11847. {
  11848. const Js::StElemInfo *const stElemInfo = instr->AsProfiledInstr()->u.stElemInfo;
  11849. if(doArraySegmentHoist && stElemInfo->LikelyStoresOutsideHeadSegmentBounds())
  11850. {
  11851. kills.SetKillsArrayHeadSegments();
  11852. kills.SetKillsArrayHeadSegmentLengths();
  11853. }
  11854. if(doArrayLengthHoist &&
  11855. !(useValueTypes && baseValueType.IsNotArray()) &&
  11856. stElemInfo->LikelyStoresOutsideArrayBounds())
  11857. {
  11858. kills.SetKillsArrayLengths();
  11859. }
  11860. }
  11861. break;
  11862. }
  11863. case Js::OpCode::DeleteElemI_A:
  11864. case Js::OpCode::DeleteElemIStrict_A:
  11865. Assert(instr->GetSrc1());
  11866. if(!instr->GetSrc1()->IsIndirOpnd() ||
  11867. (useValueTypes && instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsNotArrayOrObjectWithArray()))
  11868. {
  11869. break;
  11870. }
  11871. if(doArrayMissingValueCheckHoist)
  11872. {
  11873. kills.SetKillsArraysWithNoMissingValues();
  11874. }
  11875. if(doArraySegmentHoist)
  11876. {
  11877. kills.SetKillsArrayHeadSegmentLengths();
  11878. }
  11879. break;
  11880. case Js::OpCode::StFld:
  11881. case Js::OpCode::StFldStrict:
  11882. {
  11883. Assert(instr->GetDst());
  11884. if(!doArraySegmentHoist && !doArrayLengthHoist)
  11885. {
  11886. break;
  11887. }
  11888. IR::SymOpnd *const symDst = instr->GetDst()->AsSymOpnd();
  11889. if(!symDst->IsPropertySymOpnd())
  11890. {
  11891. break;
  11892. }
  11893. IR::PropertySymOpnd *const dst = symDst->AsPropertySymOpnd();
  11894. if(dst->m_sym->AsPropertySym()->m_propertyId != Js::PropertyIds::length)
  11895. {
  11896. break;
  11897. }
  11898. if(useValueTypes && dst->GetPropertyOwnerValueType().IsNotArray())
  11899. {
  11900. // Setting the 'length' property of an object that is not an array, even if it has an internal array, does
  11901. // not kill the head segment or head segment length of any arrays.
  11902. break;
  11903. }
  11904. if(doArraySegmentHoist)
  11905. {
  11906. kills.SetKillsArrayHeadSegmentLengths();
  11907. }
  11908. if(doArrayLengthHoist)
  11909. {
  11910. kills.SetKillsArrayLengths();
  11911. }
  11912. break;
  11913. }
  11914. case Js::OpCode::InlineArrayPush:
  11915. {
  11916. Assert(instr->GetSrc2());
  11917. IR::Opnd *const arrayOpnd = instr->GetSrc1();
  11918. Assert(arrayOpnd);
  11919. const ValueType arrayValueType(arrayOpnd->GetValueType());
  11920. if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray()))
  11921. {
  11922. break;
  11923. }
  11924. if(doArrayMissingValueCheckHoist)
  11925. {
  11926. kills.SetKillsArraysWithNoMissingValues();
  11927. }
  11928. if(doArraySegmentHoist)
  11929. {
  11930. kills.SetKillsArrayHeadSegments();
  11931. kills.SetKillsArrayHeadSegmentLengths();
  11932. }
  11933. if(doArrayLengthHoist && !(useValueTypes && arrayValueType.IsNotArray()))
  11934. {
  11935. kills.SetKillsArrayLengths();
  11936. }
  11937. // Don't kill NativeArray, if there is no mismatch between array's type and element's type.
  11938. if(doNativeArrayTypeSpec &&
  11939. !(useValueTypes && arrayValueType.IsNativeArray() &&
  11940. ((arrayValueType.IsLikelyNativeIntArray() && instr->GetSrc2()->IsInt32()) ||
  11941. (arrayValueType.IsLikelyNativeFloatArray() && instr->GetSrc2()->IsFloat()))
  11942. ) &&
  11943. !(useValueTypes && arrayValueType.IsNotNativeArray()))
  11944. {
  11945. kills.SetKillsNativeArrays();
  11946. }
  11947. break;
  11948. }
  11949. case Js::OpCode::InlineArrayPop:
  11950. {
  11951. IR::Opnd *const arrayOpnd = instr->GetSrc1();
  11952. Assert(arrayOpnd);
  11953. const ValueType arrayValueType(arrayOpnd->GetValueType());
  11954. if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray()))
  11955. {
  11956. break;
  11957. }
  11958. if(doArraySegmentHoist)
  11959. {
  11960. kills.SetKillsArrayHeadSegmentLengths();
  11961. }
  11962. if(doArrayLengthHoist && !(useValueTypes && arrayValueType.IsNotArray()))
  11963. {
  11964. kills.SetKillsArrayLengths();
  11965. }
  11966. break;
  11967. }
  11968. case Js::OpCode::CallDirect:
  11969. {
  11970. Assert(instr->GetSrc1());
  11971. // Find the 'this' parameter and check if it's possible for it to be an array
  11972. IR::Opnd *const arrayOpnd = instr->FindCallArgumentOpnd(1);
  11973. Assert(arrayOpnd);
  11974. const ValueType arrayValueType(arrayOpnd->GetValueType());
  11975. if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray()))
  11976. {
  11977. break;
  11978. }
  11979. const IR::JnHelperMethod helperMethod = instr->GetSrc1()->AsHelperCallOpnd()->m_fnHelper;
  11980. if(doArrayMissingValueCheckHoist)
  11981. {
  11982. switch(helperMethod)
  11983. {
  11984. case IR::HelperArray_Reverse:
  11985. case IR::HelperArray_Shift:
  11986. case IR::HelperArray_Splice:
  11987. case IR::HelperArray_Unshift:
  11988. kills.SetKillsArraysWithNoMissingValues();
  11989. break;
  11990. }
  11991. }
  11992. if(doArraySegmentHoist)
  11993. {
  11994. switch(helperMethod)
  11995. {
  11996. case IR::HelperArray_Reverse:
  11997. case IR::HelperArray_Shift:
  11998. case IR::HelperArray_Splice:
  11999. case IR::HelperArray_Unshift:
  12000. case IR::HelperArray_Concat:
  12001. kills.SetKillsArrayHeadSegments();
  12002. kills.SetKillsArrayHeadSegmentLengths();
  12003. break;
  12004. }
  12005. }
  12006. if(doArrayLengthHoist && !(useValueTypes && arrayValueType.IsNotArray()))
  12007. {
  12008. switch(helperMethod)
  12009. {
  12010. case IR::HelperArray_Shift:
  12011. case IR::HelperArray_Splice:
  12012. case IR::HelperArray_Unshift:
  12013. kills.SetKillsArrayLengths();
  12014. break;
  12015. }
  12016. }
  12017. if(doNativeArrayTypeSpec && !(useValueTypes && arrayValueType.IsNotNativeArray()))
  12018. {
  12019. switch(helperMethod)
  12020. {
  12021. case IR::HelperArray_Reverse:
  12022. case IR::HelperArray_Shift:
  12023. case IR::HelperArray_Slice:
  12024. // Currently not inlined.
  12025. //case IR::HelperArray_Sort:
  12026. case IR::HelperArray_Splice:
  12027. case IR::HelperArray_Unshift:
  12028. case IR::HelperArray_Concat:
  12029. kills.SetKillsNativeArrays();
  12030. break;
  12031. }
  12032. }
  12033. break;
  12034. }
  12035. case Js::OpCode::InitProto:
  12036. {
  12037. // Find the 'this' parameter and check if it's possible for it to be an array
  12038. IR::Opnd *const arrayOpnd = instr->GetSrc1();
  12039. Assert(arrayOpnd);
  12040. const ValueType arrayValueType(arrayOpnd->GetValueType());
  12041. if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray()))
  12042. {
  12043. break;
  12044. }
  12045. if(doNativeArrayTypeSpec && !(useValueTypes && arrayValueType.IsNotNativeArray()))
  12046. {
  12047. kills.SetKillsNativeArrays();
  12048. }
  12049. break;
  12050. }
  12051. case Js::OpCode::InitClass:
  12052. Assert(instr->GetSrc1());
  12053. if (instr->GetSrc2() == nullptr)
  12054. {
  12055. // No extends operand, so the InitClass will not make something into a prototype
  12056. break;
  12057. }
  12058. if(doNativeArrayTypeSpec)
  12059. {
  12060. // Class/object construction can make something a prototype
  12061. kills.SetKillsNativeArrays();
  12062. }
  12063. break;
  12064. case Js::OpCode::NewScObjectNoCtor:
  12065. if(doNativeArrayTypeSpec)
  12066. {
  12067. // Class/object construction can make something a prototype
  12068. kills.SetKillsNativeArrays();
  12069. }
  12070. break;
  12071. }
  12072. return kills;
  12073. }
  12074. GlobOptBlockData const * GlobOpt::CurrentBlockData() const
  12075. {
  12076. return &this->currentBlock->globOptData;
  12077. }
  12078. GlobOptBlockData * GlobOpt::CurrentBlockData()
  12079. {
  12080. return &this->currentBlock->globOptData;
  12081. }
  12082. void GlobOpt::CommitCapturedValuesCandidate()
  12083. {
  12084. GlobOptBlockData * globOptData = CurrentBlockData();
  12085. globOptData->changedSyms->ClearAll();
  12086. if (!this->changedSymsAfterIncBailoutCandidate->IsEmpty())
  12087. {
  12088. //
  12089. // some symbols are changed after the values for current bailout have been
  12090. // captured (GlobOpt::CapturedValues), need to restore such symbols as changed
  12091. // for following incremental bailout construction, or we will miss capturing
  12092. // values for later bailout
  12093. //
  12094. // swap changedSyms and changedSymsAfterIncBailoutCandidate
  12095. // because both are from this->alloc
  12096. BVSparse<JitArenaAllocator> * tempBvSwap = globOptData->changedSyms;
  12097. globOptData->changedSyms = this->changedSymsAfterIncBailoutCandidate;
  12098. this->changedSymsAfterIncBailoutCandidate = tempBvSwap;
  12099. }
  12100. if (globOptData->capturedValues)
  12101. {
  12102. globOptData->capturedValues->DecrementRefCount();
  12103. }
  12104. globOptData->capturedValues = globOptData->capturedValuesCandidate;
  12105. // null out capturedValuesCandidate to stop tracking symbols change for it
  12106. globOptData->capturedValuesCandidate = nullptr;
  12107. }
  12108. bool
  12109. GlobOpt::IsOperationThatLikelyKillsJsArraysWithNoMissingValues(IR::Instr *const instr)
  12110. {
  12111. // StElem is profiled with information indicating whether it will likely create a missing value in the array. In that case,
  12112. // we prefer to kill the no-missing-values information in the value so that we don't bail out in a likely circumstance.
  12113. return
  12114. (instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict) &&
  12115. DoArrayMissingValueCheckHoist() &&
  12116. instr->IsProfiledInstr() &&
  12117. instr->AsProfiledInstr()->u.stElemInfo->LikelyCreatesMissingValue();
  12118. }
  12119. bool
  12120. GlobOpt::NeedBailOnImplicitCallForArrayCheckHoist(BasicBlock const * const block, const bool isForwardPass) const
  12121. {
  12122. Assert(block);
  12123. return isForwardPass && block->loop && block->loop->needImplicitCallBailoutChecksForJsArrayCheckHoist;
  12124. }
  12125. bool
  12126. GlobOpt::PrepareForIgnoringIntOverflow(IR::Instr *const instr)
  12127. {
  12128. Assert(instr);
  12129. const bool isBoundary = instr->m_opcode == Js::OpCode::NoIntOverflowBoundary;
  12130. // Update the instruction's "int overflow matters" flag based on whether we are currently allowing ignoring int overflows.
  12131. // Some operations convert their srcs to int32s, those can still ignore int overflow.
  12132. if(instr->ignoreIntOverflowInRange)
  12133. {
  12134. instr->ignoreIntOverflowInRange = !intOverflowCurrentlyMattersInRange || OpCodeAttr::IsInt32(instr->m_opcode);
  12135. }
  12136. if(!intOverflowDoesNotMatterRange)
  12137. {
  12138. Assert(intOverflowCurrentlyMattersInRange);
  12139. // There are no more ranges of instructions where int overflow does not matter, in this block.
  12140. return isBoundary;
  12141. }
  12142. if(instr == intOverflowDoesNotMatterRange->LastInstr())
  12143. {
  12144. Assert(isBoundary);
  12145. // Reached the last instruction in the range
  12146. intOverflowCurrentlyMattersInRange = true;
  12147. intOverflowDoesNotMatterRange = intOverflowDoesNotMatterRange->Next();
  12148. return isBoundary;
  12149. }
  12150. if(!intOverflowCurrentlyMattersInRange)
  12151. {
  12152. return isBoundary;
  12153. }
  12154. if(instr != intOverflowDoesNotMatterRange->FirstInstr())
  12155. {
  12156. // Have not reached the next range
  12157. return isBoundary;
  12158. }
  12159. Assert(isBoundary);
  12160. // This is the first instruction in a range of instructions where int overflow does not matter. There can be many inputs to
  12161. // instructions in the range, some of which are inputs to the range itself (that is, the values are not defined in the
  12162. // range). Ignoring int overflow is only valid for int operations, so we need to ensure that all inputs to the range are
  12163. // int (not "likely int") before ignoring any overflows in the range. Ensuring that a sym with a "likely int" value is an
  12164. // int requires a bail-out. These bail-out check need to happen before any overflows are ignored, otherwise it's too late.
  12165. // The backward pass tracked all inputs into the range. Iterate over them and verify the values, and insert lossless
  12166. // conversions to int as necessary, before the first instruction in the range. If for any reason all values cannot be
  12167. // guaranteed to be ints, the optimization will be disabled for this range.
  12168. intOverflowCurrentlyMattersInRange = false;
  12169. {
  12170. BVSparse<JitArenaAllocator> tempBv1(tempAlloc);
  12171. BVSparse<JitArenaAllocator> tempBv2(tempAlloc);
  12172. {
  12173. // Just renaming the temp BVs for this section to indicate how they're used so that it makes sense
  12174. BVSparse<JitArenaAllocator> &symsToExclude = tempBv1;
  12175. BVSparse<JitArenaAllocator> &symsToInclude = tempBv2;
  12176. #if DBG_DUMP
  12177. SymID couldNotConvertSymId = 0;
  12178. #endif
  12179. FOREACH_BITSET_IN_SPARSEBV(id, intOverflowDoesNotMatterRange->SymsRequiredToBeInt())
  12180. {
  12181. Sym *const sym = func->m_symTable->Find(id);
  12182. Assert(sym);
  12183. // Some instructions with property syms are also tracked by the backward pass, and may be included in the range
  12184. // (LdSlot for instance). These property syms don't get their values until either copy-prop resolves a value for
  12185. // them, or a new value is created once the use of the property sym is reached. In either case, we're not that
  12186. // far yet, so we need to find the future value of the property sym by evaluating copy-prop in reverse.
  12187. Value *const value = sym->IsStackSym() ? CurrentBlockData()->FindValue(sym) : CurrentBlockData()->FindFuturePropertyValue(sym->AsPropertySym());
  12188. if(!value)
  12189. {
  12190. #if DBG_DUMP
  12191. couldNotConvertSymId = id;
  12192. #endif
  12193. intOverflowCurrentlyMattersInRange = true;
  12194. BREAK_BITSET_IN_SPARSEBV;
  12195. }
  12196. const bool isInt32OrUInt32Float =
  12197. value->GetValueInfo()->IsFloatConstant() &&
  12198. Js::JavascriptNumber::IsInt32OrUInt32(value->GetValueInfo()->AsFloatConstant()->FloatValue());
  12199. if(value->GetValueInfo()->IsInt() || isInt32OrUInt32Float)
  12200. {
  12201. if(!IsLoopPrePass())
  12202. {
  12203. // Input values that are already int can be excluded from int-specialization. We can treat unsigned
  12204. // int32 values as int32 values (ignoring the overflow), since the values will only be used inside the
  12205. // range where overflow does not matter.
  12206. symsToExclude.Set(sym->m_id);
  12207. }
  12208. continue;
  12209. }
  12210. if(!DoAggressiveIntTypeSpec() || !value->GetValueInfo()->IsLikelyInt())
  12211. {
  12212. // When aggressive int specialization is off, syms with "likely int" values cannot be forced to int since
  12213. // int bail-out checks are not allowed in that mode. Similarly, with aggressive int specialization on, it
  12214. // wouldn't make sense to force non-"likely int" values to int since it would almost guarantee a bail-out at
  12215. // runtime. In both cases, just disable ignoring overflow for this range.
  12216. #if DBG_DUMP
  12217. couldNotConvertSymId = id;
  12218. #endif
  12219. intOverflowCurrentlyMattersInRange = true;
  12220. BREAK_BITSET_IN_SPARSEBV;
  12221. }
  12222. if(IsLoopPrePass())
  12223. {
  12224. // The loop prepass does not modify bit-vectors. Since it doesn't add bail-out checks, it also does not need
  12225. // to specialize anything up-front. It only needs to be consistent in how it determines whether to allow
  12226. // ignoring overflow for a range, based on the values of inputs into the range.
  12227. continue;
  12228. }
  12229. // Since input syms are tracked in the backward pass, where there is no value tracking, it will not be aware of
  12230. // copy-prop. If a copy-prop sym is available, it will be used instead, so exclude the original sym and include
  12231. // the copy-prop sym for specialization.
  12232. StackSym *const copyPropSym = CurrentBlockData()->GetCopyPropSym(sym, value);
  12233. if(copyPropSym)
  12234. {
  12235. symsToExclude.Set(sym->m_id);
  12236. Assert(!symsToExclude.Test(copyPropSym->m_id));
  12237. const bool needsToBeLossless =
  12238. !intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Test(sym->m_id);
  12239. if(intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Test(copyPropSym->m_id) ||
  12240. symsToInclude.TestAndSet(copyPropSym->m_id))
  12241. {
  12242. // The copy-prop sym is already included
  12243. if(needsToBeLossless)
  12244. {
  12245. // The original sym needs to be lossless, so make the copy-prop sym lossless as well.
  12246. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Clear(copyPropSym->m_id);
  12247. }
  12248. }
  12249. else if(!needsToBeLossless)
  12250. {
  12251. // The copy-prop sym was not included before, and the original sym can be lossy, so make it lossy.
  12252. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Set(copyPropSym->m_id);
  12253. }
  12254. }
  12255. else if(!sym->IsStackSym())
  12256. {
  12257. // Only stack syms can be converted to int, and copy-prop syms are stack syms. If a copy-prop sym was not
  12258. // found for the property sym, we can't ignore overflows in this range.
  12259. #if DBG_DUMP
  12260. couldNotConvertSymId = id;
  12261. #endif
  12262. intOverflowCurrentlyMattersInRange = true;
  12263. BREAK_BITSET_IN_SPARSEBV;
  12264. }
  12265. } NEXT_BITSET_IN_SPARSEBV;
  12266. if(intOverflowCurrentlyMattersInRange)
  12267. {
  12268. #if DBG_DUMP
  12269. if(PHASE_TRACE(Js::TrackCompoundedIntOverflowPhase, func) && !IsLoopPrePass())
  12270. {
  12271. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  12272. Output::Print(
  12273. _u("TrackCompoundedIntOverflow - Top function: %s (%s), Phase: %s, Block: %u, Disabled ignoring overflows\n"),
  12274. func->GetJITFunctionBody()->GetDisplayName(),
  12275. func->GetDebugNumberSet(debugStringBuffer),
  12276. Js::PhaseNames[Js::ForwardPhase],
  12277. currentBlock->GetBlockNum());
  12278. Output::Print(_u(" Input sym could not be turned into an int: %u\n"), couldNotConvertSymId);
  12279. Output::Print(_u(" First instr: "));
  12280. instr->m_next->Dump();
  12281. Output::Flush();
  12282. }
  12283. #endif
  12284. intOverflowDoesNotMatterRange = intOverflowDoesNotMatterRange->Next();
  12285. return isBoundary;
  12286. }
  12287. if(IsLoopPrePass())
  12288. {
  12289. return isBoundary;
  12290. }
  12291. // Update the syms to specialize after enumeration
  12292. intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Minus(&symsToExclude);
  12293. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Minus(&symsToExclude);
  12294. intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Or(&symsToInclude);
  12295. }
  12296. {
  12297. // Exclude syms that are already live as lossless int32, and exclude lossy conversions of syms that are already live
  12298. // as lossy int32.
  12299. // symsToExclude = liveInt32Syms - liveLossyInt32Syms // syms live as lossless int
  12300. // lossySymsToExclude = symsRequiredToBeLossyInt & liveLossyInt32Syms; // syms we want as lossy int that are already live as lossy int
  12301. // symsToExclude |= lossySymsToExclude
  12302. // symsRequiredToBeInt -= symsToExclude
  12303. // symsRequiredToBeLossyInt -= symsToExclude
  12304. BVSparse<JitArenaAllocator> &symsToExclude = tempBv1;
  12305. BVSparse<JitArenaAllocator> &lossySymsToExclude = tempBv2;
  12306. symsToExclude.Minus(CurrentBlockData()->liveInt32Syms, CurrentBlockData()->liveLossyInt32Syms);
  12307. lossySymsToExclude.And(
  12308. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt(),
  12309. CurrentBlockData()->liveLossyInt32Syms);
  12310. symsToExclude.Or(&lossySymsToExclude);
  12311. intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Minus(&symsToExclude);
  12312. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Minus(&symsToExclude);
  12313. }
  12314. #if DBG
  12315. {
  12316. // Verify that the syms to be converted are live
  12317. // liveSyms = liveInt32Syms | liveFloat64Syms | liveVarSyms
  12318. // deadSymsRequiredToBeInt = symsRequiredToBeInt - liveSyms
  12319. BVSparse<JitArenaAllocator> &liveSyms = tempBv1;
  12320. BVSparse<JitArenaAllocator> &deadSymsRequiredToBeInt = tempBv2;
  12321. liveSyms.Or(CurrentBlockData()->liveInt32Syms, CurrentBlockData()->liveFloat64Syms);
  12322. liveSyms.Or(CurrentBlockData()->liveVarSyms);
  12323. deadSymsRequiredToBeInt.Minus(intOverflowDoesNotMatterRange->SymsRequiredToBeInt(), &liveSyms);
  12324. Assert(deadSymsRequiredToBeInt.IsEmpty());
  12325. }
  12326. #endif
  12327. }
  12328. // Int-specialize the syms before the first instruction of the range (the current instruction)
  12329. intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Minus(intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt());
  12330. #if DBG_DUMP
  12331. if(PHASE_TRACE(Js::TrackCompoundedIntOverflowPhase, func))
  12332. {
  12333. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  12334. Output::Print(
  12335. _u("TrackCompoundedIntOverflow - Top function: %s (%s), Phase: %s, Block: %u\n"),
  12336. func->GetJITFunctionBody()->GetDisplayName(),
  12337. func->GetDebugNumberSet(debugStringBuffer),
  12338. Js::PhaseNames[Js::ForwardPhase],
  12339. currentBlock->GetBlockNum());
  12340. Output::Print(_u(" Input syms to be int-specialized (lossless): "));
  12341. intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Dump();
  12342. Output::Print(_u(" Input syms to be converted to int (lossy): "));
  12343. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Dump();
  12344. Output::Print(_u(" First instr: "));
  12345. instr->m_next->Dump();
  12346. Output::Flush();
  12347. }
  12348. #endif
  12349. ToInt32(intOverflowDoesNotMatterRange->SymsRequiredToBeInt(), currentBlock, false /* lossy */, instr);
  12350. ToInt32(intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt(), currentBlock, true /* lossy */, instr);
  12351. return isBoundary;
  12352. }
  12353. void
  12354. GlobOpt::VerifyIntSpecForIgnoringIntOverflow(IR::Instr *const instr)
  12355. {
  12356. if(intOverflowCurrentlyMattersInRange || IsLoopPrePass())
  12357. {
  12358. return;
  12359. }
  12360. Assert(instr->m_opcode != Js::OpCode::Mul_I4 ||
  12361. (instr->m_opcode == Js::OpCode::Mul_I4 && !instr->ShouldCheckFor32BitOverflow() && instr->ShouldCheckForNon32BitOverflow() ));
  12362. // Instructions that are marked as "overflow doesn't matter" in the range must guarantee that they operate on int values and
  12363. // result in int values, for ignoring overflow to be valid. So, int-specialization is required for such instructions in the
  12364. // range. Ld_A is an exception because it only specializes if the src sym is available as a required specialized sym, and it
  12365. // doesn't generate bailouts or cause ignoring int overflow to be invalid.
  12366. // MULs are allowed to start a region and have BailOutInfo since they will bailout on non-32 bit overflow.
  12367. if(instr->m_opcode == Js::OpCode::Ld_A ||
  12368. ((!instr->HasBailOutInfo() || instr->m_opcode == Js::OpCode::Mul_I4) &&
  12369. (!instr->GetDst() || instr->GetDst()->IsInt32()) &&
  12370. (!instr->GetSrc1() || instr->GetSrc1()->IsInt32()) &&
  12371. (!instr->GetSrc2() || instr->GetSrc2()->IsInt32())))
  12372. {
  12373. return;
  12374. }
  12375. if (!instr->HasBailOutInfo() && !instr->HasAnySideEffects())
  12376. {
  12377. return;
  12378. }
  12379. // This can happen for Neg_A if it needs to bail out on negative zero, and perhaps other cases as well. It's too late to fix
  12380. // the problem (overflows may already be ignored), so handle it by bailing out at compile-time and disabling tracking int
  12381. // overflow.
  12382. Assert(!func->IsTrackCompoundedIntOverflowDisabled());
  12383. if(PHASE_TRACE(Js::BailOutPhase, this->func))
  12384. {
  12385. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  12386. Output::Print(
  12387. _u("BailOut (compile-time): function: %s (%s) instr: "),
  12388. func->GetJITFunctionBody()->GetDisplayName(),
  12389. func->GetDebugNumberSet(debugStringBuffer));
  12390. #if DBG_DUMP
  12391. instr->Dump();
  12392. #else
  12393. Output::Print(_u("%s "), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  12394. #endif
  12395. Output::Print(_u("(overflow does not matter but could not int-spec or needed bailout)\n"));
  12396. Output::Flush();
  12397. }
  12398. if(func->IsTrackCompoundedIntOverflowDisabled())
  12399. {
  12400. // Tracking int overflows is already off for some reason. Prevent trying to rejit again because it won't help and the
  12401. // same thing will happen again and cause an infinite loop. Just abort jitting this function.
  12402. if(PHASE_TRACE(Js::BailOutPhase, this->func))
  12403. {
  12404. Output::Print(_u(" Aborting JIT because TrackIntOverflow is already off\n"));
  12405. Output::Flush();
  12406. }
  12407. throw Js::OperationAbortedException();
  12408. }
  12409. throw Js::RejitException(RejitReason::TrackIntOverflowDisabled);
  12410. }
  12411. // It makes lowering easier if it can assume that the first src is never a constant,
  12412. // at least for commutative operators. For non-commutative, just hoist the constant.
  12413. void
  12414. GlobOpt::PreLowerCanonicalize(IR::Instr *instr, Value **pSrc1Val, Value **pSrc2Val)
  12415. {
  12416. IR::Opnd *dst = instr->GetDst();
  12417. IR::Opnd *src1 = instr->GetSrc1();
  12418. IR::Opnd *src2 = instr->GetSrc2();
  12419. if (src1->IsImmediateOpnd())
  12420. {
  12421. // Swap for dst, src
  12422. }
  12423. else if (src2 && dst && src2->IsRegOpnd())
  12424. {
  12425. if (src2->GetIsDead() && !src1->GetIsDead() && !src1->IsEqual(dst))
  12426. {
  12427. // Swap if src2 is dead, as the reg can be reuse for the dst for opEqs like on x86 (ADD r1, r2)
  12428. }
  12429. else if (src2->IsEqual(dst))
  12430. {
  12431. // Helps lowering of opEqs
  12432. }
  12433. else
  12434. {
  12435. return;
  12436. }
  12437. // Make sure we don't swap 2 srcs with valueOf calls.
  12438. if (OpCodeAttr::OpndHasImplicitCall(instr->m_opcode))
  12439. {
  12440. if (instr->IsBranchInstr())
  12441. {
  12442. if (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive())
  12443. {
  12444. return;
  12445. }
  12446. }
  12447. else if (!src1->GetValueType().IsPrimitive() && !src2->GetValueType().IsPrimitive())
  12448. {
  12449. return;
  12450. }
  12451. }
  12452. }
  12453. else
  12454. {
  12455. return;
  12456. }
  12457. Js::OpCode opcode = instr->m_opcode;
  12458. switch (opcode)
  12459. {
  12460. case Js::OpCode::And_A:
  12461. case Js::OpCode::Mul_A:
  12462. case Js::OpCode::Or_A:
  12463. case Js::OpCode::Xor_A:
  12464. case Js::OpCode::And_I4:
  12465. case Js::OpCode::Mul_I4:
  12466. case Js::OpCode::Or_I4:
  12467. case Js::OpCode::Xor_I4:
  12468. case Js::OpCode::Add_I4:
  12469. swap_srcs:
  12470. if (!instr->GetSrc2()->IsImmediateOpnd())
  12471. {
  12472. instr->m_opcode = opcode;
  12473. instr->SwapOpnds();
  12474. Value *tempVal = *pSrc1Val;
  12475. *pSrc1Val = *pSrc2Val;
  12476. *pSrc2Val = tempVal;
  12477. return;
  12478. }
  12479. break;
  12480. case Js::OpCode::BrSrEq_A:
  12481. case Js::OpCode::BrSrNotNeq_A:
  12482. case Js::OpCode::BrEq_I4:
  12483. goto swap_srcs;
  12484. case Js::OpCode::BrSrNeq_A:
  12485. case Js::OpCode::BrNeq_A:
  12486. case Js::OpCode::BrSrNotEq_A:
  12487. case Js::OpCode::BrNotEq_A:
  12488. case Js::OpCode::BrNeq_I4:
  12489. goto swap_srcs;
  12490. case Js::OpCode::BrGe_A:
  12491. opcode = Js::OpCode::BrLe_A;
  12492. goto swap_srcs;
  12493. case Js::OpCode::BrNotGe_A:
  12494. opcode = Js::OpCode::BrNotLe_A;
  12495. goto swap_srcs;
  12496. case Js::OpCode::BrGe_I4:
  12497. opcode = Js::OpCode::BrLe_I4;
  12498. goto swap_srcs;
  12499. case Js::OpCode::BrGt_A:
  12500. opcode = Js::OpCode::BrLt_A;
  12501. goto swap_srcs;
  12502. case Js::OpCode::BrNotGt_A:
  12503. opcode = Js::OpCode::BrNotLt_A;
  12504. goto swap_srcs;
  12505. case Js::OpCode::BrGt_I4:
  12506. opcode = Js::OpCode::BrLt_I4;
  12507. goto swap_srcs;
  12508. case Js::OpCode::BrLe_A:
  12509. opcode = Js::OpCode::BrGe_A;
  12510. goto swap_srcs;
  12511. case Js::OpCode::BrNotLe_A:
  12512. opcode = Js::OpCode::BrNotGe_A;
  12513. goto swap_srcs;
  12514. case Js::OpCode::BrLe_I4:
  12515. opcode = Js::OpCode::BrGe_I4;
  12516. goto swap_srcs;
  12517. case Js::OpCode::BrLt_A:
  12518. opcode = Js::OpCode::BrGt_A;
  12519. goto swap_srcs;
  12520. case Js::OpCode::BrNotLt_A:
  12521. opcode = Js::OpCode::BrNotGt_A;
  12522. goto swap_srcs;
  12523. case Js::OpCode::BrLt_I4:
  12524. opcode = Js::OpCode::BrGt_I4;
  12525. goto swap_srcs;
  12526. case Js::OpCode::BrEq_A:
  12527. case Js::OpCode::BrNotNeq_A:
  12528. case Js::OpCode::CmEq_A:
  12529. case Js::OpCode::CmNeq_A:
  12530. // this == "" not the same as "" == this...
  12531. if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()))
  12532. {
  12533. return;
  12534. }
  12535. goto swap_srcs;
  12536. case Js::OpCode::CmGe_A:
  12537. if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()))
  12538. {
  12539. return;
  12540. }
  12541. opcode = Js::OpCode::CmLe_A;
  12542. goto swap_srcs;
  12543. case Js::OpCode::CmGt_A:
  12544. if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()))
  12545. {
  12546. return;
  12547. }
  12548. opcode = Js::OpCode::CmLt_A;
  12549. goto swap_srcs;
  12550. case Js::OpCode::CmLe_A:
  12551. if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()))
  12552. {
  12553. return;
  12554. }
  12555. opcode = Js::OpCode::CmGe_A;
  12556. goto swap_srcs;
  12557. case Js::OpCode::CmLt_A:
  12558. if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()))
  12559. {
  12560. return;
  12561. }
  12562. opcode = Js::OpCode::CmGt_A;
  12563. goto swap_srcs;
  12564. case Js::OpCode::CallI:
  12565. case Js::OpCode::CallIFixed:
  12566. case Js::OpCode::NewScObject:
  12567. case Js::OpCode::NewScObjectSpread:
  12568. case Js::OpCode::NewScObjArray:
  12569. case Js::OpCode::NewScObjArraySpread:
  12570. case Js::OpCode::NewScObjectNoCtor:
  12571. // Don't insert load to register if the function operand is a fixed function.
  12572. if (instr->HasFixedFunctionAddressTarget())
  12573. {
  12574. return;
  12575. }
  12576. break;
  12577. // Can't do add because <32 + "Hello"> isn't equal to <"Hello" + 32>
  12578. // Lower can do the swap. Other op-codes listed below don't need immediate source hoisting, as the fast paths handle it,
  12579. // or the lowering handles the hoisting.
  12580. case Js::OpCode::Add_A:
  12581. if (src1->IsFloat())
  12582. {
  12583. goto swap_srcs;
  12584. }
  12585. return;
  12586. case Js::OpCode::Sub_I4:
  12587. case Js::OpCode::Neg_I4:
  12588. case Js::OpCode::Not_I4:
  12589. case Js::OpCode::NewScFunc:
  12590. case Js::OpCode::NewScGenFunc:
  12591. case Js::OpCode::NewScFuncHomeObj:
  12592. case Js::OpCode::NewScGenFuncHomeObj:
  12593. case Js::OpCode::NewScArray:
  12594. case Js::OpCode::NewScIntArray:
  12595. case Js::OpCode::NewScFltArray:
  12596. case Js::OpCode::NewScArrayWithMissingValues:
  12597. case Js::OpCode::NewRegEx:
  12598. case Js::OpCode::Ld_A:
  12599. case Js::OpCode::Ld_I4:
  12600. case Js::OpCode::ThrowRuntimeError:
  12601. case Js::OpCode::TrapIfMinIntOverNegOne:
  12602. case Js::OpCode::TrapIfTruncOverflow:
  12603. case Js::OpCode::TrapIfZero:
  12604. case Js::OpCode::TrapIfUnalignedAccess:
  12605. case Js::OpCode::FromVar:
  12606. case Js::OpCode::Conv_Prim:
  12607. case Js::OpCode::Conv_Prim_Sat:
  12608. case Js::OpCode::LdC_A_I4:
  12609. case Js::OpCode::LdStr:
  12610. case Js::OpCode::InitFld:
  12611. case Js::OpCode::InitRootFld:
  12612. case Js::OpCode::StartCall:
  12613. case Js::OpCode::ArgOut_A:
  12614. case Js::OpCode::ArgOut_A_Inline:
  12615. case Js::OpCode::ArgOut_A_Dynamic:
  12616. case Js::OpCode::ArgOut_A_FromStackArgs:
  12617. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  12618. case Js::OpCode::ArgOut_A_InlineSpecialized:
  12619. case Js::OpCode::ArgOut_A_SpreadArg:
  12620. case Js::OpCode::InlineeEnd:
  12621. case Js::OpCode::EndCallForPolymorphicInlinee:
  12622. case Js::OpCode::InlineeMetaArg:
  12623. case Js::OpCode::InlineBuiltInEnd:
  12624. case Js::OpCode::InlineNonTrackingBuiltInEnd:
  12625. case Js::OpCode::CallHelper:
  12626. case Js::OpCode::LdElemUndef:
  12627. case Js::OpCode::LdElemUndefScoped:
  12628. case Js::OpCode::RuntimeTypeError:
  12629. case Js::OpCode::RuntimeReferenceError:
  12630. case Js::OpCode::Ret:
  12631. case Js::OpCode::NewScObjectSimple:
  12632. case Js::OpCode::NewScObjectLiteral:
  12633. case Js::OpCode::StFld:
  12634. case Js::OpCode::StRootFld:
  12635. case Js::OpCode::StSlot:
  12636. case Js::OpCode::StSlotChkUndecl:
  12637. case Js::OpCode::StElemC:
  12638. case Js::OpCode::StArrSegElemC:
  12639. case Js::OpCode::StElemI_A:
  12640. case Js::OpCode::StElemI_A_Strict:
  12641. case Js::OpCode::CallDirect:
  12642. case Js::OpCode::BrNotHasSideEffects:
  12643. case Js::OpCode::NewConcatStrMulti:
  12644. case Js::OpCode::NewConcatStrMultiBE:
  12645. case Js::OpCode::ExtendArg_A:
  12646. #ifdef ENABLE_DOM_FAST_PATH
  12647. case Js::OpCode::DOMFastPathGetter:
  12648. case Js::OpCode::DOMFastPathSetter:
  12649. #endif
  12650. case Js::OpCode::NewScopeSlots:
  12651. case Js::OpCode::NewScopeSlotsWithoutPropIds:
  12652. case Js::OpCode::NewStackScopeSlots:
  12653. case Js::OpCode::IsInst:
  12654. case Js::OpCode::BailOnEqual:
  12655. case Js::OpCode::BailOnNotEqual:
  12656. case Js::OpCode::StArrViewElem:
  12657. return;
  12658. }
  12659. if (!src1->IsImmediateOpnd())
  12660. {
  12661. return;
  12662. }
  12663. // The fast paths or lowering of the remaining instructions may not support handling immediate opnds for the first src. The
  12664. // immediate src1 is hoisted here into a separate instruction.
  12665. if (src1->IsIntConstOpnd())
  12666. {
  12667. IR::Instr *newInstr = instr->HoistSrc1(Js::OpCode::Ld_I4);
  12668. ToInt32Dst(newInstr, newInstr->GetDst()->AsRegOpnd(), this->currentBlock);
  12669. }
  12670. else if (src1->IsInt64ConstOpnd())
  12671. {
  12672. instr->HoistSrc1(Js::OpCode::Ld_I4);
  12673. }
  12674. else
  12675. {
  12676. instr->HoistSrc1(Js::OpCode::Ld_A);
  12677. }
  12678. src1 = instr->GetSrc1();
  12679. src1->AsRegOpnd()->m_sym->SetIsConst();
  12680. }
  12681. // Clear the ValueMap pf the values invalidated by this instr.
  12682. void
  12683. GlobOpt::ProcessKills(IR::Instr *instr)
  12684. {
  12685. this->ProcessFieldKills(instr);
  12686. this->ProcessValueKills(instr);
  12687. this->ProcessArrayValueKills(instr);
  12688. }
  12689. bool
  12690. GlobOpt::OptIsInvariant(IR::Opnd *src, BasicBlock *block, Loop *loop, Value *srcVal, bool isNotTypeSpecConv, bool allowNonPrimitives)
  12691. {
  12692. if(!loop->CanHoistInvariants())
  12693. {
  12694. return false;
  12695. }
  12696. Sym *sym;
  12697. switch(src->GetKind())
  12698. {
  12699. case IR::OpndKindAddr:
  12700. case IR::OpndKindFloatConst:
  12701. case IR::OpndKindIntConst:
  12702. return true;
  12703. case IR::OpndKindReg:
  12704. sym = src->AsRegOpnd()->m_sym;
  12705. break;
  12706. case IR::OpndKindSym:
  12707. sym = src->AsSymOpnd()->m_sym;
  12708. if (src->AsSymOpnd()->IsPropertySymOpnd())
  12709. {
  12710. if (src->AsSymOpnd()->AsPropertySymOpnd()->IsTypeChecked())
  12711. {
  12712. // We do not handle hoisting these yet. We might be hoisting this across the instr with the type check protecting this one.
  12713. // And somehow, the dead-store pass now removes the type check on that instr later on...
  12714. // For CheckFixedFld, there is no benefit hoisting these if they don't have a type check as they won't generate code.
  12715. return false;
  12716. }
  12717. }
  12718. break;
  12719. case IR::OpndKindHelperCall:
  12720. // Helper calls, like the private slot getter, can be invariant.
  12721. // Consider moving more math builtin to invariant?
  12722. return HelperMethodAttributes::IsInVariant(src->AsHelperCallOpnd()->m_fnHelper);
  12723. default:
  12724. return false;
  12725. }
  12726. return OptIsInvariant(sym, block, loop, srcVal, isNotTypeSpecConv, allowNonPrimitives);
  12727. }
  12728. bool
  12729. GlobOpt::OptIsInvariant(Sym *sym, BasicBlock *block, Loop *loop, Value *srcVal, bool isNotTypeSpecConv, bool allowNonPrimitives, Value **loopHeadValRef)
  12730. {
  12731. Value *localLoopHeadVal;
  12732. if(!loopHeadValRef)
  12733. {
  12734. loopHeadValRef = &localLoopHeadVal;
  12735. }
  12736. Value *&loopHeadVal = *loopHeadValRef;
  12737. loopHeadVal = nullptr;
  12738. if(!loop->CanHoistInvariants())
  12739. {
  12740. return false;
  12741. }
  12742. if (sym->IsStackSym())
  12743. {
  12744. if (sym->AsStackSym()->IsTypeSpec())
  12745. {
  12746. StackSym *varSym = sym->AsStackSym()->GetVarEquivSym(this->func);
  12747. // Make sure the int32/float64 version of this is available.
  12748. // Note: We could handle this by converting the src, but usually the
  12749. // conversion is hoistable if this is hoistable anyway.
  12750. // In some weird cases it may not be however, so we'll bail out.
  12751. if (sym->AsStackSym()->IsInt32())
  12752. {
  12753. Assert(block->globOptData.liveInt32Syms->Test(varSym->m_id));
  12754. if (!loop->landingPad->globOptData.liveInt32Syms->Test(varSym->m_id) ||
  12755. (loop->landingPad->globOptData.liveLossyInt32Syms->Test(varSym->m_id) &&
  12756. !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id)))
  12757. {
  12758. // Either the int32 sym is not live in the landing pad, or it's lossy in the landing pad and the
  12759. // instruction's block is using the lossless version. In either case, the instruction cannot be hoisted
  12760. // without doing a conversion of this operand.
  12761. return false;
  12762. }
  12763. }
  12764. else if (sym->AsStackSym()->IsFloat64())
  12765. {
  12766. if (!loop->landingPad->globOptData.liveFloat64Syms->Test(varSym->m_id))
  12767. {
  12768. return false;
  12769. }
  12770. }
  12771. sym = sym->AsStackSym()->GetVarEquivSym(this->func);
  12772. }
  12773. else
  12774. {
  12775. // Make sure the var version of this is available.
  12776. // Note: We could handle this by converting the src, but usually the
  12777. // conversion is hoistable if this is hoistable anyway.
  12778. // In some weird cases it may not be however, so we'll bail out.
  12779. if (!loop->landingPad->globOptData.liveVarSyms->Test(sym->m_id))
  12780. {
  12781. return false;
  12782. }
  12783. }
  12784. }
  12785. else if (sym->IsPropertySym())
  12786. {
  12787. if (!loop->landingPad->globOptData.liveVarSyms->Test(sym->AsPropertySym()->m_stackSym->m_id))
  12788. {
  12789. return false;
  12790. }
  12791. }
  12792. else
  12793. {
  12794. return false;
  12795. }
  12796. // We rely on having a value.
  12797. if (srcVal == NULL)
  12798. {
  12799. return false;
  12800. }
  12801. // A symbol is invariant if its current value is the same as it was upon entering the loop.
  12802. loopHeadVal = loop->landingPad->globOptData.FindValue(sym);
  12803. if (loopHeadVal == NULL || loopHeadVal->GetValueNumber() != srcVal->GetValueNumber())
  12804. {
  12805. return false;
  12806. }
  12807. // Can't hoist non-primitives, unless we have safeguards against valueof/tostring. Additionally, we need to consider
  12808. // the value annotations on the source *before* the loop: if we hoist this instruction outside the loop, we can't
  12809. // necessarily rely on type annotations added (and enforced) earlier in the loop's body.
  12810. //
  12811. // It might look as though !loopHeadVal->GetValueInfo()->IsPrimitive() implies
  12812. // !loop->landingPad->globOptData.IsTypeSpecialized(sym), but it turns out that this is not always the case. We
  12813. // encountered a test case in which we had previously hoisted a FromVar (to float 64) instruction, but its bailout code was
  12814. // BailoutPrimitiveButString, rather than BailoutNumberOnly, which would have allowed us to conclude that the dest was
  12815. // definitely a float64. Instead, it was only *likely* a float64, causing IsPrimitive to return false.
  12816. if (!allowNonPrimitives && !loopHeadVal->GetValueInfo()->IsPrimitive() && !loop->landingPad->globOptData.IsTypeSpecialized(sym))
  12817. {
  12818. return false;
  12819. }
  12820. if(!isNotTypeSpecConv && loop->symsDefInLoop->Test(sym->m_id))
  12821. {
  12822. // Typically, a sym is considered invariant if it has the same value in the current block and in the loop landing pad.
  12823. // The sym may have had a different value earlier in the loop or on the back-edge, but as long as it's reassigned to its
  12824. // value outside the loop, it would be considered invariant in this block. Consider that case:
  12825. // s1 = s2[invariant]
  12826. // <loop start>
  12827. // s1 = s2[invariant]
  12828. // // s1 now has the same value as in the landing pad, and is considered invariant
  12829. // s1 += s3
  12830. // // s1 is not invariant here, or on the back-edge
  12831. // ++s3 // s3 is not invariant, so the add above cannot be hoisted
  12832. // <loop end>
  12833. //
  12834. // A problem occurs at the point of (s1 += s3) when:
  12835. // - At (s1 = s2) inside the loop, s1 was made to be the sym store of that value. This by itself is legal, because
  12836. // after that transfer, s1 and s2 have the same value.
  12837. // - (s1 += s3) is type-specialized but s1 is not specialized in the loop header. This happens when s1 is not
  12838. // specialized entering the loop, and since s1 is not used before it's defined in the loop, it's not specialized
  12839. // on back-edges.
  12840. //
  12841. // With that, at (s1 += s3), the conversion of s1 to the type-specialized version would be hoisted because s1 is
  12842. // invariant just before that instruction. Since this add is specialized, the specialized version of the sym is modified
  12843. // in the loop without a reassignment at (s1 = s2) inside the loop, and (s1 += s3) would then use an incorrect value of
  12844. // s1 (it would use the value of s1 from the previous loop iteration, instead of using the value of s2).
  12845. //
  12846. // The problem here, is that we cannot hoist the conversion of s1 into its specialized version across the assignment
  12847. // (s1 = s2) inside the loop. So for the purposes of type specialization, don't consider a sym invariant if it has a def
  12848. // inside the loop.
  12849. return false;
  12850. }
  12851. // For values with an int range, require additionally that the range is the same as in the landing pad, as the range may
  12852. // have been changed on this path based on branches, and int specialization and invariant hoisting may rely on the range
  12853. // being the same. For type spec conversions, only require that if the value is an int constant in the current block, that
  12854. // it is also an int constant with the same value in the landing pad. Other range differences don't matter for type spec.
  12855. IntConstantBounds srcIntConstantBounds, loopHeadIntConstantBounds;
  12856. if(srcVal->GetValueInfo()->TryGetIntConstantBounds(&srcIntConstantBounds) &&
  12857. (isNotTypeSpecConv || srcIntConstantBounds.IsConstant()) &&
  12858. (
  12859. !loopHeadVal->GetValueInfo()->TryGetIntConstantBounds(&loopHeadIntConstantBounds) ||
  12860. loopHeadIntConstantBounds.LowerBound() != srcIntConstantBounds.LowerBound() ||
  12861. loopHeadIntConstantBounds.UpperBound() != srcIntConstantBounds.UpperBound()
  12862. ))
  12863. {
  12864. return false;
  12865. }
  12866. // Disabling this assert, because it does not hold true when we force specialize in the loop landing pad
  12867. //Assert((!loopHeadVal->GetValueInfo()->IsPrimitive()) || srcVal->GetValueInfo()->IsLikelyPrimitive());
  12868. return true;
  12869. }
  12870. bool
  12871. GlobOpt::OptIsInvariant(
  12872. IR::Instr *instr,
  12873. BasicBlock *block,
  12874. Loop *loop,
  12875. Value *src1Val,
  12876. Value *src2Val,
  12877. bool isNotTypeSpecConv,
  12878. const bool forceInvariantHoisting)
  12879. {
  12880. if (!loop->CanHoistInvariants())
  12881. {
  12882. return false;
  12883. }
  12884. if (!OpCodeAttr::CanCSE(instr->m_opcode))
  12885. {
  12886. return false;
  12887. }
  12888. bool allowNonPrimitives = !OpCodeAttr::OpndHasImplicitCall(instr->m_opcode);
  12889. switch(instr->m_opcode)
  12890. {
  12891. // Can't legally hoist these
  12892. case Js::OpCode::LdLen_A:
  12893. return false;
  12894. //Can't Hoist BailOnNotStackArgs, as it is necessary as InlineArgsOptimization relies on this opcode
  12895. //to decide whether to throw rejit exception or not.
  12896. case Js::OpCode::BailOnNotStackArgs:
  12897. return false;
  12898. // Usually not worth hoisting these
  12899. case Js::OpCode::Ld_A:
  12900. case Js::OpCode::Ld_I4:
  12901. case Js::OpCode::LdC_A_I4:
  12902. if(!forceInvariantHoisting)
  12903. {
  12904. return false;
  12905. }
  12906. break;
  12907. // Can't hoist these outside the function it's for. The LdArgumentsFromFrame for an inlinee depends on the inlinee meta arg
  12908. // that holds the arguments object, which is only initialized at the start of the inlinee. So, can't hoist this outside the
  12909. // inlinee.
  12910. case Js::OpCode::LdArgumentsFromFrame:
  12911. if(instr->m_func != loop->GetFunc())
  12912. {
  12913. return false;
  12914. }
  12915. break;
  12916. case Js::OpCode::FromVar:
  12917. if (instr->HasBailOutInfo())
  12918. {
  12919. allowNonPrimitives = true;
  12920. }
  12921. break;
  12922. case Js::OpCode::CheckObjType:
  12923. // Bug 11712101: If the operand is a field, ensure that its containing object type is invariant
  12924. // before hoisting -- that is, don't hoist a CheckObjType over a DeleteFld on that object.
  12925. // (CheckObjType only checks the operand and its immediate parent, so we don't need to go
  12926. // any farther up the object graph.)
  12927. Assert(instr->GetSrc1());
  12928. PropertySym *propertySym = instr->GetSrc1()->AsPropertySymOpnd()->GetPropertySym();
  12929. if (propertySym->HasObjectTypeSym()) {
  12930. StackSym *objectTypeSym = propertySym->GetObjectTypeSym();
  12931. if (!this->OptIsInvariant(objectTypeSym, block, loop, this->CurrentBlockData()->FindValue(objectTypeSym), true, true)) {
  12932. return false;
  12933. }
  12934. }
  12935. break;
  12936. }
  12937. IR::Opnd *dst = instr->GetDst();
  12938. if (dst && !dst->IsRegOpnd())
  12939. {
  12940. return false;
  12941. }
  12942. IR::Opnd *src1 = instr->GetSrc1();
  12943. if (src1)
  12944. {
  12945. if (!this->OptIsInvariant(src1, block, loop, src1Val, isNotTypeSpecConv, allowNonPrimitives))
  12946. {
  12947. return false;
  12948. }
  12949. IR::Opnd *src2 = instr->GetSrc2();
  12950. if (src2)
  12951. {
  12952. if (!this->OptIsInvariant(src2, block, loop, src2Val, isNotTypeSpecConv, allowNonPrimitives))
  12953. {
  12954. return false;
  12955. }
  12956. }
  12957. }
  12958. return true;
  12959. }
  12960. bool
  12961. GlobOpt::OptDstIsInvariant(IR::RegOpnd *dst)
  12962. {
  12963. StackSym *dstSym = dst->m_sym;
  12964. if (dstSym->IsTypeSpec())
  12965. {
  12966. // The type-specialized sym may be single def, but not the original...
  12967. dstSym = dstSym->GetVarEquivSym(this->func);
  12968. }
  12969. return (dstSym->m_isSingleDef);
  12970. }
  12971. void
  12972. GlobOpt::OptHoistUpdateValueType(
  12973. Loop* loop,
  12974. IR::Instr* instr,
  12975. IR::Opnd** srcOpndPtr /* All code paths that change src, should update srcOpndPtr*/,
  12976. Value* opndVal)
  12977. {
  12978. if (opndVal == nullptr || instr->m_opcode == Js::OpCode::FromVar || srcOpndPtr == nullptr || *srcOpndPtr == nullptr)
  12979. {
  12980. return;
  12981. }
  12982. IR::Opnd* srcOpnd = *srcOpndPtr;
  12983. Sym* opndSym = srcOpnd->GetSym();;
  12984. if (opndSym)
  12985. {
  12986. BasicBlock* landingPad = loop->landingPad;
  12987. Value* opndValueInLandingPad = landingPad->globOptData.FindValue(opndSym);
  12988. Assert(opndVal->GetValueNumber() == opndValueInLandingPad->GetValueNumber());
  12989. ValueType opndValueTypeInLandingPad = opndValueInLandingPad->GetValueInfo()->Type();
  12990. if (srcOpnd->GetValueType() != opndValueTypeInLandingPad)
  12991. {
  12992. srcOpnd->SetValueType(opndValueTypeInLandingPad);
  12993. if (instr->m_opcode == Js::OpCode::SetConcatStrMultiItemBE)
  12994. {
  12995. Assert(!opndSym->IsPropertySym());
  12996. Assert(!opndValueTypeInLandingPad.IsString());
  12997. Assert(instr->GetDst());
  12998. IR::RegOpnd* strOpnd = IR::RegOpnd::New(TyVar, instr->m_func);
  12999. strOpnd->SetValueType(ValueType::String);
  13000. strOpnd->SetValueTypeFixed();
  13001. IR::Instr* convPrimStrInstr =
  13002. IR::Instr::New(Js::OpCode::Conv_PrimStr, strOpnd, srcOpnd->Use(instr->m_func), instr->m_func);
  13003. instr->ReplaceSrc(srcOpnd, strOpnd);
  13004. // Replace above will free srcOpnd, so reassign it
  13005. *srcOpndPtr = srcOpnd = reinterpret_cast<IR::Opnd *>(strOpnd);
  13006. // We add ConvPrim_Str in the landingpad, and since this instruction doesn't go through the checks in OptInstr, the bailout is never added
  13007. // As we expand hoisting of instructions to new opcode, we need a better framework to handle such cases
  13008. if (IsImplicitCallBailOutCurrentlyNeeded(convPrimStrInstr, opndValueInLandingPad, nullptr, landingPad, landingPad->globOptData.liveFields->IsEmpty(), true, true))
  13009. {
  13010. EnsureBailTarget(loop);
  13011. loop->bailOutInfo->bailOutInstr->InsertBefore(convPrimStrInstr);
  13012. convPrimStrInstr = convPrimStrInstr->ConvertToBailOutInstr(convPrimStrInstr, IR::BailOutOnImplicitCallsPreOp, loop->bailOutInfo->bailOutOffset);
  13013. convPrimStrInstr->ReplaceBailOutInfo(loop->bailOutInfo);
  13014. }
  13015. else
  13016. {
  13017. if (loop->bailOutInfo->bailOutInstr)
  13018. {
  13019. loop->bailOutInfo->bailOutInstr->InsertBefore(convPrimStrInstr);
  13020. }
  13021. else
  13022. {
  13023. landingPad->InsertAfter(convPrimStrInstr);
  13024. }
  13025. }
  13026. // If we came here opndSym can't be PropertySym
  13027. return;
  13028. }
  13029. }
  13030. if (opndSym->IsPropertySym())
  13031. {
  13032. // Also fix valueInfo on objPtr
  13033. StackSym* opndObjPtrSym = opndSym->AsPropertySym()->m_stackSym;
  13034. Value* opndObjPtrSymValInLandingPad = landingPad->globOptData.FindValue(opndObjPtrSym);
  13035. ValueInfo* opndObjPtrSymValueInfoInLandingPad = opndObjPtrSymValInLandingPad->GetValueInfo();
  13036. srcOpnd->AsSymOpnd()->SetPropertyOwnerValueType(opndObjPtrSymValueInfoInLandingPad->Type());
  13037. }
  13038. }
  13039. }
  13040. void
  13041. GlobOpt::OptHoistInvariant(
  13042. IR::Instr *instr,
  13043. BasicBlock *block,
  13044. Loop *loop,
  13045. Value *dstVal,
  13046. Value *const src1Val,
  13047. Value *const src2Val,
  13048. bool isNotTypeSpecConv,
  13049. bool lossy,
  13050. IR::BailOutKind bailoutKind)
  13051. {
  13052. BasicBlock *landingPad = loop->landingPad;
  13053. IR::Opnd* src1 = instr->GetSrc1();
  13054. if (src1)
  13055. {
  13056. // We are hoisting this instruction possibly past other uses, which might invalidate the last use info. Clear it.
  13057. OptHoistUpdateValueType(loop, instr, &src1, src1Val);
  13058. if (src1->IsRegOpnd())
  13059. {
  13060. src1->AsRegOpnd()->m_isTempLastUse = false;
  13061. }
  13062. IR::Opnd* src2 = instr->GetSrc2();
  13063. if (src2)
  13064. {
  13065. OptHoistUpdateValueType(loop, instr, &src2, src2Val);
  13066. if (src2->IsRegOpnd())
  13067. {
  13068. src2->AsRegOpnd()->m_isTempLastUse = false;
  13069. }
  13070. }
  13071. }
  13072. IR::RegOpnd *dst = instr->GetDst() ? instr->GetDst()->AsRegOpnd() : nullptr;
  13073. if(dst)
  13074. {
  13075. switch (instr->m_opcode)
  13076. {
  13077. case Js::OpCode::CmEq_I4:
  13078. case Js::OpCode::CmNeq_I4:
  13079. case Js::OpCode::CmLt_I4:
  13080. case Js::OpCode::CmLe_I4:
  13081. case Js::OpCode::CmGt_I4:
  13082. case Js::OpCode::CmGe_I4:
  13083. case Js::OpCode::CmUnLt_I4:
  13084. case Js::OpCode::CmUnLe_I4:
  13085. case Js::OpCode::CmUnGt_I4:
  13086. case Js::OpCode::CmUnGe_I4:
  13087. // These operations are a special case. They generate a lossy int value, and the var sym is initialized using
  13088. // Conv_Bool. A sym cannot be live only as a lossy int sym, the var needs to be live as well since the lossy int
  13089. // sym cannot be used to convert to var. We don't know however, whether the Conv_Bool will be hoisted. The idea
  13090. // currently is that the sym is only used on the path in which it is initialized inside the loop. So, don't
  13091. // hoist any liveness info for the dst.
  13092. if (!this->GetIsAsmJSFunc())
  13093. {
  13094. lossy = true;
  13095. }
  13096. break;
  13097. case Js::OpCode::FromVar:
  13098. {
  13099. StackSym* src1StackSym = IR::RegOpnd::TryGetStackSym(instr->GetSrc1());
  13100. if (instr->HasBailOutInfo())
  13101. {
  13102. IR::BailOutKind instrBailoutKind = instr->GetBailOutKind();
  13103. Assert(instrBailoutKind == IR::BailOutIntOnly ||
  13104. instrBailoutKind == IR::BailOutExpectingInteger ||
  13105. instrBailoutKind == IR::BailOutOnNotPrimitive ||
  13106. instrBailoutKind == IR::BailOutNumberOnly ||
  13107. instrBailoutKind == IR::BailOutPrimitiveButString);
  13108. }
  13109. else if (src1StackSym && bailoutKind != IR::BailOutInvalid)
  13110. {
  13111. // We may be hoisting FromVar from a region where it didn't need a bailout (src1 had a definite value type) to a region
  13112. // where it would. In such cases, the FromVar needs a bailout based on the value type of src1 in its new position.
  13113. Assert(!src1StackSym->IsTypeSpec());
  13114. Value* landingPadSrc1val = landingPad->globOptData.FindValue(src1StackSym);
  13115. Assert(src1Val->GetValueNumber() == landingPadSrc1val->GetValueNumber());
  13116. ValueInfo *src1ValueInfo = src1Val->GetValueInfo();
  13117. ValueInfo *landingPadSrc1ValueInfo = landingPadSrc1val->GetValueInfo();
  13118. IRType dstType = dst->GetType();
  13119. const auto AddBailOutToFromVar = [&]()
  13120. {
  13121. instr->GetSrc1()->SetValueType(landingPadSrc1val->GetValueInfo()->Type());
  13122. EnsureBailTarget(loop);
  13123. if (block->IsLandingPad())
  13124. {
  13125. instr = instr->ConvertToBailOutInstr(instr, bailoutKind, loop->bailOutInfo->bailOutOffset);
  13126. }
  13127. else
  13128. {
  13129. instr = instr->ConvertToBailOutInstr(instr, bailoutKind);
  13130. }
  13131. };
  13132. // A definite type in the source position and not a definite type in the destination (landing pad)
  13133. // and no bailout on the instruction; we should put a bailout on the hoisted instruction.
  13134. if (dstType == TyInt32)
  13135. {
  13136. if (lossy)
  13137. {
  13138. if ((src1ValueInfo->IsPrimitive() || block->globOptData.IsTypeSpecialized(src1StackSym)) && // didn't need a lossy type spec bailout in the source block
  13139. (!landingPadSrc1ValueInfo->IsPrimitive() && !landingPad->globOptData.IsTypeSpecialized(src1StackSym))) // needs a lossy type spec bailout in the landing pad
  13140. {
  13141. bailoutKind = IR::BailOutOnNotPrimitive;
  13142. AddBailOutToFromVar();
  13143. }
  13144. }
  13145. else if (src1ValueInfo->IsInt() && !landingPadSrc1ValueInfo->IsInt())
  13146. {
  13147. AddBailOutToFromVar();
  13148. }
  13149. }
  13150. else if ((dstType == TyFloat64 && src1ValueInfo->IsNumber() && !landingPadSrc1ValueInfo->IsNumber()))
  13151. {
  13152. AddBailOutToFromVar();
  13153. }
  13154. }
  13155. break;
  13156. }
  13157. }
  13158. if (dstVal == NULL)
  13159. {
  13160. dstVal = this->NewGenericValue(ValueType::Uninitialized, dst);
  13161. }
  13162. // ToVar/FromVar don't need a new dst because it has to be invariant if their src is invariant.
  13163. bool dstDoesntNeedLoad = (!isNotTypeSpecConv && instr->m_opcode != Js::OpCode::LdC_A_I4);
  13164. StackSym *varSym = dst->m_sym;
  13165. if (varSym->IsTypeSpec())
  13166. {
  13167. varSym = varSym->GetVarEquivSym(this->func);
  13168. }
  13169. Value *const landingPadDstVal = loop->landingPad->globOptData.FindValue(varSym);
  13170. if(landingPadDstVal
  13171. ? dstVal->GetValueNumber() != landingPadDstVal->GetValueNumber()
  13172. : loop->symsDefInLoop->Test(varSym->m_id))
  13173. {
  13174. // We need a temp for FromVar/ToVar if dst changes in the loop.
  13175. dstDoesntNeedLoad = false;
  13176. }
  13177. if (!dstDoesntNeedLoad && this->OptDstIsInvariant(dst) == false)
  13178. {
  13179. // Keep dst in place, hoist instr using a new dst.
  13180. instr->UnlinkDst();
  13181. // Set type specialization info correctly for this new sym
  13182. StackSym *copyVarSym;
  13183. IR::RegOpnd *copyReg;
  13184. if (dst->m_sym->IsTypeSpec())
  13185. {
  13186. copyVarSym = StackSym::New(TyVar, instr->m_func);
  13187. StackSym *copySym = copyVarSym;
  13188. if (dst->m_sym->IsInt32())
  13189. {
  13190. if(lossy)
  13191. {
  13192. // The new sym would only be live as a lossy int since we're only hoisting the store to the int version
  13193. // of the sym, and cannot be converted to var. It is not legal to have a sym only live as a lossy int,
  13194. // so don't update liveness info for this sym.
  13195. }
  13196. else
  13197. {
  13198. block->globOptData.liveInt32Syms->Set(copyVarSym->m_id);
  13199. }
  13200. copySym = copySym->GetInt32EquivSym(instr->m_func);
  13201. }
  13202. else if (dst->m_sym->IsFloat64())
  13203. {
  13204. block->globOptData.liveFloat64Syms->Set(copyVarSym->m_id);
  13205. copySym = copySym->GetFloat64EquivSym(instr->m_func);
  13206. }
  13207. copyReg = IR::RegOpnd::New(copySym, copySym->GetType(), instr->m_func);
  13208. }
  13209. else
  13210. {
  13211. copyReg = IR::RegOpnd::New(dst->GetType(), instr->m_func);
  13212. copyVarSym = copyReg->m_sym;
  13213. block->globOptData.liveVarSyms->Set(copyVarSym->m_id);
  13214. }
  13215. copyReg->SetValueType(dst->GetValueType());
  13216. IR::Instr *copyInstr = IR::Instr::New(Js::OpCode::Ld_A, dst, copyReg, instr->m_func);
  13217. copyInstr->SetByteCodeOffset(instr);
  13218. instr->SetDst(copyReg);
  13219. instr->InsertBefore(copyInstr);
  13220. dst->m_sym->m_mayNotBeTempLastUse = true;
  13221. if (instr->GetSrc1() && instr->GetSrc1()->IsImmediateOpnd())
  13222. {
  13223. // Propagate IsIntConst if appropriate
  13224. switch(instr->m_opcode)
  13225. {
  13226. case Js::OpCode::Ld_A:
  13227. case Js::OpCode::Ld_I4:
  13228. case Js::OpCode::LdC_A_I4:
  13229. copyReg->m_sym->SetIsConst();
  13230. break;
  13231. }
  13232. }
  13233. ValueInfo *dstValueInfo = dstVal->GetValueInfo();
  13234. if((!dstValueInfo->GetSymStore() || dstValueInfo->GetSymStore() == varSym) && !lossy)
  13235. {
  13236. // The destination's value may have been transferred from one of the invariant sources, in which case we should
  13237. // keep the sym store intact, as that sym will likely have a better lifetime than this new copy sym. For
  13238. // instance, if we're inside a conditioned block, because we don't make the copy sym live and set its value in
  13239. // all preceding blocks, this sym would not be live after exiting this block, causing this value to not
  13240. // participate in copy-prop after this block.
  13241. this->SetSymStoreDirect(dstValueInfo, copyVarSym);
  13242. }
  13243. block->globOptData.InsertNewValue(dstVal, copyReg);
  13244. dst = copyReg;
  13245. }
  13246. }
  13247. // Move to landing pad
  13248. block->UnlinkInstr(instr);
  13249. if (loop->bailOutInfo->bailOutInstr)
  13250. {
  13251. loop->bailOutInfo->bailOutInstr->InsertBefore(instr);
  13252. }
  13253. else
  13254. {
  13255. landingPad->InsertAfter(instr);
  13256. }
  13257. GlobOpt::MarkNonByteCodeUsed(instr);
  13258. if (instr->HasBailOutInfo() || instr->HasAuxBailOut())
  13259. {
  13260. Assert(loop->bailOutInfo);
  13261. EnsureBailTarget(loop);
  13262. // Copy bailout info of loop top.
  13263. instr->ReplaceBailOutInfo(loop->bailOutInfo);
  13264. }
  13265. if(!dst)
  13266. {
  13267. return;
  13268. }
  13269. // The bailout info's liveness for the dst sym is not updated in loop landing pads because bailout instructions previously
  13270. // hoisted into the loop's landing pad may bail out before the current type of the dst sym became live (perhaps due to this
  13271. // instruction). Since the landing pad will have a shared bailout point, the bailout info cannot assume that the current
  13272. // type of the dst sym was live during every bailout hoisted into the landing pad.
  13273. StackSym *const dstSym = dst->m_sym;
  13274. StackSym *const dstVarSym = dstSym->IsTypeSpec() ? dstSym->GetVarEquivSym(nullptr) : dstSym;
  13275. Assert(dstVarSym);
  13276. if(isNotTypeSpecConv || !loop->landingPad->globOptData.IsLive(dstVarSym))
  13277. {
  13278. // A new dst is being hoisted, or the same single-def dst that would not be live before this block. So, make it live and
  13279. // update the value info with the same value info in this block.
  13280. if(lossy)
  13281. {
  13282. // This is a lossy conversion to int. The instruction was given a new dst specifically for hoisting, so this new dst
  13283. // will not be live as a var before this block. A sym cannot be live only as a lossy int sym, the var needs to be
  13284. // live as well since the lossy int sym cannot be used to convert to var. Since the var version of the sym is not
  13285. // going to be initialized, don't hoist any liveness info for the dst. The sym is only going to be used on the path
  13286. // in which it is initialized inside the loop.
  13287. Assert(dstSym->IsTypeSpec());
  13288. Assert(dstSym->IsInt32());
  13289. return;
  13290. }
  13291. // Check if the dst value was transferred from the src. If so, the value transfer needs to be replicated.
  13292. bool isTransfer = dstVal == src1Val;
  13293. StackSym *transferValueOfSym = nullptr;
  13294. if(isTransfer)
  13295. {
  13296. Assert(instr->GetSrc1());
  13297. if(instr->GetSrc1()->IsRegOpnd())
  13298. {
  13299. StackSym *src1Sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  13300. if(src1Sym->IsTypeSpec())
  13301. {
  13302. src1Sym = src1Sym->GetVarEquivSym(nullptr);
  13303. Assert(src1Sym);
  13304. }
  13305. if(dstVal == block->globOptData.FindValue(src1Sym))
  13306. {
  13307. transferValueOfSym = src1Sym;
  13308. }
  13309. }
  13310. }
  13311. // SIMD_JS
  13312. if (instr->m_opcode == Js::OpCode::ExtendArg_A)
  13313. {
  13314. // Check if we should have CSE'ed this EA
  13315. Assert(instr->GetSrc1());
  13316. // If the dstVal symstore is not the dst itself, then we copied the Value from another expression.
  13317. if (dstVal->GetValueInfo()->GetSymStore() != instr->GetDst()->GetStackSym())
  13318. {
  13319. isTransfer = true;
  13320. transferValueOfSym = dstVal->GetValueInfo()->GetSymStore()->AsStackSym();
  13321. }
  13322. }
  13323. const ValueNumber dstValueNumber = dstVal->GetValueNumber();
  13324. ValueNumber dstNewValueNumber = InvalidValueNumber;
  13325. for(InvariantBlockBackwardIterator it(this, block, loop->landingPad, nullptr); it.IsValid(); it.MoveNext())
  13326. {
  13327. BasicBlock *const hoistBlock = it.Block();
  13328. GlobOptBlockData &hoistBlockData = hoistBlock->globOptData;
  13329. Assert(!hoistBlockData.IsLive(dstVarSym));
  13330. hoistBlockData.MakeLive(dstSym, lossy);
  13331. Value *newDstValue;
  13332. do
  13333. {
  13334. if(isTransfer)
  13335. {
  13336. if(transferValueOfSym)
  13337. {
  13338. newDstValue = hoistBlockData.FindValue(transferValueOfSym);
  13339. if(newDstValue && newDstValue->GetValueNumber() == dstValueNumber)
  13340. {
  13341. break;
  13342. }
  13343. }
  13344. // It's a transfer, but we don't have a sym whose value number matches in the target block. Use a new value
  13345. // number since we don't know if there is already a value with the current number for the target block.
  13346. if(dstNewValueNumber == InvalidValueNumber)
  13347. {
  13348. dstNewValueNumber = NewValueNumber();
  13349. }
  13350. newDstValue = CopyValue(dstVal, dstNewValueNumber);
  13351. break;
  13352. }
  13353. newDstValue = CopyValue(dstVal, dstValueNumber);
  13354. } while(false);
  13355. hoistBlockData.SetValue(newDstValue, dstVarSym);
  13356. }
  13357. return;
  13358. }
  13359. #if DBG
  13360. if(instr->GetSrc1()->IsRegOpnd()) // Type spec conversion may load a constant into a dst sym
  13361. {
  13362. StackSym *const srcSym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  13363. Assert(srcSym != dstSym); // Type spec conversion must be changing the type, so the syms must be different
  13364. StackSym *const srcVarSym = srcSym->IsTypeSpec() ? srcSym->GetVarEquivSym(nullptr) : srcSym;
  13365. Assert(srcVarSym == dstVarSym); // Type spec conversion must be between variants of the same var sym
  13366. }
  13367. #endif
  13368. bool changeValueType = false, changeValueTypeToInt = false;
  13369. if(dstSym->IsTypeSpec())
  13370. {
  13371. if(dst->IsInt32())
  13372. {
  13373. if(!lossy)
  13374. {
  13375. Assert(
  13376. !instr->HasBailOutInfo() ||
  13377. instr->GetBailOutKind() == IR::BailOutIntOnly ||
  13378. instr->GetBailOutKind() == IR::BailOutExpectingInteger);
  13379. changeValueType = changeValueTypeToInt = true;
  13380. }
  13381. }
  13382. else if (dst->IsFloat64())
  13383. {
  13384. if(instr->HasBailOutInfo() && instr->GetBailOutKind() == IR::BailOutNumberOnly)
  13385. {
  13386. changeValueType = true;
  13387. }
  13388. }
  13389. }
  13390. ValueInfo *previousValueInfoBeforeUpdate = nullptr, *previousValueInfoAfterUpdate = nullptr;
  13391. for(InvariantBlockBackwardIterator it(
  13392. this,
  13393. block,
  13394. loop->landingPad,
  13395. dstVarSym,
  13396. dstVal->GetValueNumber());
  13397. it.IsValid();
  13398. it.MoveNext())
  13399. {
  13400. BasicBlock *const hoistBlock = it.Block();
  13401. GlobOptBlockData &hoistBlockData = hoistBlock->globOptData;
  13402. #if DBG
  13403. // TODO: There are some odd cases with field hoisting where the sym is invariant in only part of the loop and the info
  13404. // does not flow through all blocks. Un-comment the verification below after PRE replaces field hoisting.
  13405. //// Verify that the src sym is live as the required type, and that the conversion is valid
  13406. //Assert(IsLive(dstVarSym, &hoistBlockData));
  13407. //if(instr->GetSrc1()->IsRegOpnd())
  13408. //{
  13409. // IR::RegOpnd *const src = instr->GetSrc1()->AsRegOpnd();
  13410. // StackSym *const srcSym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  13411. // if(srcSym->IsTypeSpec())
  13412. // {
  13413. // if(src->IsInt32())
  13414. // {
  13415. // Assert(hoistBlockData.liveInt32Syms->Test(dstVarSym->m_id));
  13416. // Assert(!hoistBlockData.liveLossyInt32Syms->Test(dstVarSym->m_id)); // shouldn't try to convert a lossy int32 to anything
  13417. // }
  13418. // else
  13419. // {
  13420. // Assert(src->IsFloat64());
  13421. // Assert(hoistBlockData.liveFloat64Syms->Test(dstVarSym->m_id));
  13422. // if(dstSym->IsTypeSpec() && dst->IsInt32())
  13423. // {
  13424. // Assert(lossy); // shouldn't try to do a lossless conversion from float64 to int32
  13425. // }
  13426. // }
  13427. // }
  13428. // else
  13429. // {
  13430. // Assert(hoistBlockData.liveVarSyms->Test(dstVarSym->m_id));
  13431. // }
  13432. //}
  13433. //if(dstSym->IsTypeSpec() && dst->IsInt32())
  13434. //{
  13435. // // If the sym is already specialized as required in the block to which we are attempting to hoist the conversion,
  13436. // // that info should have flowed into this block
  13437. // if(lossy)
  13438. // {
  13439. // Assert(!hoistBlockData.liveInt32Syms->Test(dstVarSym->m_id));
  13440. // }
  13441. // else
  13442. // {
  13443. // Assert(!IsInt32TypeSpecialized(dstVarSym, hoistBlock));
  13444. // }
  13445. //}
  13446. #endif
  13447. hoistBlockData.MakeLive(dstSym, lossy);
  13448. if(!changeValueType)
  13449. {
  13450. continue;
  13451. }
  13452. Value *const hoistBlockValue = it.InvariantSymValue();
  13453. ValueInfo *const hoistBlockValueInfo = hoistBlockValue->GetValueInfo();
  13454. if(hoistBlockValueInfo == previousValueInfoBeforeUpdate)
  13455. {
  13456. if(hoistBlockValueInfo != previousValueInfoAfterUpdate)
  13457. {
  13458. HoistInvariantValueInfo(previousValueInfoAfterUpdate, hoistBlockValue, hoistBlock);
  13459. }
  13460. }
  13461. else
  13462. {
  13463. previousValueInfoBeforeUpdate = hoistBlockValueInfo;
  13464. ValueInfo *const newValueInfo =
  13465. changeValueTypeToInt
  13466. ? hoistBlockValueInfo->SpecializeToInt32(alloc)
  13467. : hoistBlockValueInfo->SpecializeToFloat64(alloc);
  13468. previousValueInfoAfterUpdate = newValueInfo;
  13469. ChangeValueInfo(changeValueTypeToInt ? nullptr : hoistBlock, hoistBlockValue, newValueInfo);
  13470. }
  13471. }
  13472. }
  13473. bool
  13474. GlobOpt::TryHoistInvariant(
  13475. IR::Instr *instr,
  13476. BasicBlock *block,
  13477. Value *dstVal,
  13478. Value *src1Val,
  13479. Value *src2Val,
  13480. bool isNotTypeSpecConv,
  13481. const bool lossy,
  13482. const bool forceInvariantHoisting,
  13483. IR::BailOutKind bailoutKind)
  13484. {
  13485. Assert(!this->IsLoopPrePass());
  13486. if (OptIsInvariant(instr, block, block->loop, src1Val, src2Val, isNotTypeSpecConv, forceInvariantHoisting))
  13487. {
  13488. #if DBG
  13489. if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::InvariantsPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId()))
  13490. {
  13491. Output::Print(_u(" **** INVARIANT *** "));
  13492. instr->Dump();
  13493. }
  13494. #endif
  13495. #if ENABLE_DEBUG_CONFIG_OPTIONS
  13496. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::InvariantsPhase))
  13497. {
  13498. Output::Print(_u(" **** INVARIANT *** "));
  13499. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  13500. }
  13501. #endif
  13502. Loop *loop = block->loop;
  13503. // Try hoisting from to outer most loop
  13504. while (loop->parent && OptIsInvariant(instr, block, loop->parent, src1Val, src2Val, isNotTypeSpecConv, forceInvariantHoisting))
  13505. {
  13506. loop = loop->parent;
  13507. }
  13508. // Record the byte code use here since we are going to move this instruction up
  13509. if (isNotTypeSpecConv)
  13510. {
  13511. InsertNoImplicitCallUses(instr);
  13512. this->CaptureByteCodeSymUses(instr);
  13513. this->InsertByteCodeUses(instr, true);
  13514. }
  13515. #if DBG
  13516. else
  13517. {
  13518. PropertySym *propertySymUse = NULL;
  13519. NoRecoverMemoryJitArenaAllocator tempAllocator(_u("BE-GlobOpt-Temp"), this->alloc->GetPageAllocator(), Js::Throw::OutOfMemory);
  13520. BVSparse<JitArenaAllocator> * tempByteCodeUse = JitAnew(&tempAllocator, BVSparse<JitArenaAllocator>, &tempAllocator);
  13521. GlobOpt::TrackByteCodeSymUsed(instr, tempByteCodeUse, &propertySymUse);
  13522. Assert(tempByteCodeUse->Count() == 0 && propertySymUse == NULL);
  13523. }
  13524. #endif
  13525. OptHoistInvariant(instr, block, loop, dstVal, src1Val, src2Val, isNotTypeSpecConv, lossy, bailoutKind);
  13526. return true;
  13527. }
  13528. return false;
  13529. }
  13530. InvariantBlockBackwardIterator::InvariantBlockBackwardIterator(
  13531. GlobOpt *const globOpt,
  13532. BasicBlock *const exclusiveBeginBlock,
  13533. BasicBlock *const inclusiveEndBlock,
  13534. StackSym *const invariantSym,
  13535. const ValueNumber invariantSymValueNumber,
  13536. bool followFlow)
  13537. : globOpt(globOpt),
  13538. exclusiveEndBlock(inclusiveEndBlock->prev),
  13539. invariantSym(invariantSym),
  13540. invariantSymValueNumber(invariantSymValueNumber),
  13541. block(exclusiveBeginBlock),
  13542. blockBV(globOpt->tempAlloc),
  13543. followFlow(followFlow)
  13544. #if DBG
  13545. ,
  13546. inclusiveEndBlock(inclusiveEndBlock)
  13547. #endif
  13548. {
  13549. Assert(exclusiveBeginBlock);
  13550. Assert(inclusiveEndBlock);
  13551. Assert(!inclusiveEndBlock->isDeleted);
  13552. Assert(exclusiveBeginBlock != inclusiveEndBlock);
  13553. Assert(!invariantSym == (invariantSymValueNumber == InvalidValueNumber));
  13554. MoveNext();
  13555. }
  13556. bool
  13557. InvariantBlockBackwardIterator::IsValid() const
  13558. {
  13559. return block != exclusiveEndBlock;
  13560. }
  13561. void
  13562. InvariantBlockBackwardIterator::MoveNext()
  13563. {
  13564. Assert(IsValid());
  13565. while(true)
  13566. {
  13567. #if DBG
  13568. BasicBlock *const previouslyIteratedBlock = block;
  13569. #endif
  13570. block = block->prev;
  13571. if(!IsValid())
  13572. {
  13573. Assert(previouslyIteratedBlock == inclusiveEndBlock);
  13574. break;
  13575. }
  13576. if (!this->UpdatePredBlockBV())
  13577. {
  13578. continue;
  13579. }
  13580. if (!this->UpdatePredBlockBV())
  13581. {
  13582. continue;
  13583. }
  13584. if(block->isDeleted)
  13585. {
  13586. continue;
  13587. }
  13588. if(!block->globOptData.HasData())
  13589. {
  13590. // This block's info has already been merged with all of its successors
  13591. continue;
  13592. }
  13593. if(!invariantSym)
  13594. {
  13595. break;
  13596. }
  13597. invariantSymValue = block->globOptData.FindValue(invariantSym);
  13598. if(!invariantSymValue || invariantSymValue->GetValueNumber() != invariantSymValueNumber)
  13599. {
  13600. // BailOnNoProfile and throw blocks are not moved outside loops. A sym table cleanup on these paths may delete the
  13601. // values. Field hoisting also has some odd cases where the hoisted stack sym is invariant in only part of the loop.
  13602. continue;
  13603. }
  13604. break;
  13605. }
  13606. }
  13607. bool
  13608. InvariantBlockBackwardIterator::UpdatePredBlockBV()
  13609. {
  13610. if (!this->followFlow)
  13611. {
  13612. return true;
  13613. }
  13614. // Track blocks we've visited to ensure that we only iterate over predecessor blocks
  13615. if (!this->blockBV.IsEmpty() && !this->blockBV.Test(this->block->GetBlockNum()))
  13616. {
  13617. return false;
  13618. }
  13619. FOREACH_SLISTBASECOUNTED_ENTRY(FlowEdge*, edge, this->block->GetPredList())
  13620. {
  13621. this->blockBV.Set(edge->GetPred()->GetBlockNum());
  13622. } NEXT_SLISTBASECOUNTED_ENTRY;
  13623. return true;
  13624. }
  13625. BasicBlock *
  13626. InvariantBlockBackwardIterator::Block() const
  13627. {
  13628. Assert(IsValid());
  13629. return block;
  13630. }
  13631. Value *
  13632. InvariantBlockBackwardIterator::InvariantSymValue() const
  13633. {
  13634. Assert(IsValid());
  13635. Assert(invariantSym);
  13636. return invariantSymValue;
  13637. }
  13638. void
  13639. GlobOpt::HoistInvariantValueInfo(
  13640. ValueInfo *const invariantValueInfoToHoist,
  13641. Value *const valueToUpdate,
  13642. BasicBlock *const targetBlock)
  13643. {
  13644. Assert(invariantValueInfoToHoist);
  13645. Assert(valueToUpdate);
  13646. Assert(targetBlock);
  13647. // Why are we trying to change the value type of the type sym value? Asserting here to make sure we don't deep copy the type sym's value info.
  13648. Assert(!invariantValueInfoToHoist->IsJsType());
  13649. Sym *const symStore = valueToUpdate->GetValueInfo()->GetSymStore();
  13650. ValueInfo *newValueInfo;
  13651. if(invariantValueInfoToHoist->GetSymStore() == symStore)
  13652. {
  13653. newValueInfo = invariantValueInfoToHoist;
  13654. }
  13655. else
  13656. {
  13657. newValueInfo = invariantValueInfoToHoist->Copy(alloc);
  13658. this->SetSymStoreDirect(newValueInfo, symStore);
  13659. }
  13660. ChangeValueInfo(targetBlock, valueToUpdate, newValueInfo, true);
  13661. }
  13662. // static
  13663. bool
  13664. GlobOpt::DoInlineArgsOpt(Func const * func)
  13665. {
  13666. Func const * topFunc = func->GetTopFunc();
  13667. Assert(topFunc != func);
  13668. bool doInlineArgsOpt =
  13669. !PHASE_OFF(Js::InlineArgsOptPhase, topFunc) &&
  13670. !func->GetHasCalls() &&
  13671. !func->GetHasUnoptimizedArgumentsAccess() &&
  13672. func->m_canDoInlineArgsOpt;
  13673. return doInlineArgsOpt;
  13674. }
  13675. bool
  13676. GlobOpt::IsSwitchOptEnabled(Func const * func)
  13677. {
  13678. Assert(func->IsTopFunc());
  13679. return !PHASE_OFF(Js::SwitchOptPhase, func) && !func->IsSwitchOptDisabled() && func->DoGlobOpt();
  13680. }
  13681. bool
  13682. GlobOpt::IsSwitchOptEnabledForIntTypeSpec(Func const * func)
  13683. {
  13684. return IsSwitchOptEnabled(func) && !IsTypeSpecPhaseOff(func) && DoAggressiveIntTypeSpec(func);
  13685. }
  13686. bool
  13687. GlobOpt::DoConstFold() const
  13688. {
  13689. return !PHASE_OFF(Js::ConstFoldPhase, func);
  13690. }
  13691. bool
  13692. GlobOpt::IsTypeSpecPhaseOff(Func const *func)
  13693. {
  13694. return PHASE_OFF(Js::TypeSpecPhase, func) || func->IsJitInDebugMode() || !func->DoGlobOptsForGeneratorFunc();
  13695. }
  13696. bool
  13697. GlobOpt::DoTypeSpec() const
  13698. {
  13699. return doTypeSpec;
  13700. }
  13701. bool
  13702. GlobOpt::DoAggressiveIntTypeSpec(Func const * func)
  13703. {
  13704. return
  13705. !PHASE_OFF(Js::AggressiveIntTypeSpecPhase, func) &&
  13706. !IsTypeSpecPhaseOff(func) &&
  13707. !func->IsAggressiveIntTypeSpecDisabled();
  13708. }
  13709. bool
  13710. GlobOpt::DoAggressiveIntTypeSpec() const
  13711. {
  13712. return doAggressiveIntTypeSpec;
  13713. }
  13714. bool
  13715. GlobOpt::DoAggressiveMulIntTypeSpec() const
  13716. {
  13717. return doAggressiveMulIntTypeSpec;
  13718. }
  13719. bool
  13720. GlobOpt::DoDivIntTypeSpec() const
  13721. {
  13722. return doDivIntTypeSpec;
  13723. }
  13724. // static
  13725. bool
  13726. GlobOpt::DoLossyIntTypeSpec(Func const * func)
  13727. {
  13728. return
  13729. !PHASE_OFF(Js::LossyIntTypeSpecPhase, func) &&
  13730. !IsTypeSpecPhaseOff(func) &&
  13731. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsLossyIntTypeSpecDisabled());
  13732. }
  13733. bool
  13734. GlobOpt::DoLossyIntTypeSpec() const
  13735. {
  13736. return doLossyIntTypeSpec;
  13737. }
  13738. // static
  13739. bool
  13740. GlobOpt::DoFloatTypeSpec(Func const * func)
  13741. {
  13742. return
  13743. !PHASE_OFF(Js::FloatTypeSpecPhase, func) &&
  13744. !IsTypeSpecPhaseOff(func) &&
  13745. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsFloatTypeSpecDisabled()) &&
  13746. AutoSystemInfo::Data.SSE2Available();
  13747. }
  13748. bool
  13749. GlobOpt::DoFloatTypeSpec() const
  13750. {
  13751. return doFloatTypeSpec;
  13752. }
  13753. bool
  13754. GlobOpt::DoStringTypeSpec(Func const * func)
  13755. {
  13756. return !PHASE_OFF(Js::StringTypeSpecPhase, func) && !IsTypeSpecPhaseOff(func);
  13757. }
  13758. // static
  13759. bool
  13760. GlobOpt::DoTypedArrayTypeSpec(Func const * func)
  13761. {
  13762. return !PHASE_OFF(Js::TypedArrayTypeSpecPhase, func) &&
  13763. !IsTypeSpecPhaseOff(func) &&
  13764. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsTypedArrayTypeSpecDisabled(func->IsLoopBody()))
  13765. #if defined(_M_IX86)
  13766. && AutoSystemInfo::Data.SSE2Available()
  13767. #endif
  13768. ;
  13769. }
  13770. // static
  13771. bool
  13772. GlobOpt::DoNativeArrayTypeSpec(Func const * func)
  13773. {
  13774. return !PHASE_OFF(Js::NativeArrayPhase, func) &&
  13775. !IsTypeSpecPhaseOff(func)
  13776. #if defined(_M_IX86)
  13777. && AutoSystemInfo::Data.SSE2Available()
  13778. #endif
  13779. ;
  13780. }
  13781. bool
  13782. GlobOpt::DoArrayCheckHoist(Func const * const func)
  13783. {
  13784. Assert(func->IsTopFunc());
  13785. return
  13786. !PHASE_OFF(Js::ArrayCheckHoistPhase, func) &&
  13787. !func->IsArrayCheckHoistDisabled() &&
  13788. !func->IsJitInDebugMode() && // StElemI fast path is not allowed when in debug mode, so it cannot have bailout
  13789. func->DoGlobOptsForGeneratorFunc();
  13790. }
  13791. bool
  13792. GlobOpt::DoArrayCheckHoist() const
  13793. {
  13794. return doArrayCheckHoist;
  13795. }
  13796. bool
  13797. GlobOpt::DoArrayCheckHoist(const ValueType baseValueType, Loop* loop, IR::Instr const * const instr) const
  13798. {
  13799. if(!DoArrayCheckHoist() || (instr && !IsLoopPrePass() && instr->DoStackArgsOpt(func)))
  13800. {
  13801. return false;
  13802. }
  13803. // This includes typed arrays, but not virtual typed arrays, whose vtable can change if the buffer goes away.
  13804. // Note that in the virtual case the vtable check is the only way to catch this, since there's no bound check.
  13805. if(!(baseValueType.IsLikelyArrayOrObjectWithArray() || baseValueType.IsLikelyOptimizedVirtualTypedArray()) ||
  13806. (loop ? ImplicitCallFlagsAllowOpts(loop) : ImplicitCallFlagsAllowOpts(func)))
  13807. {
  13808. return true;
  13809. }
  13810. // The function or loop does not allow disabling implicit calls, which is required to eliminate redundant JS array checks
  13811. #if DBG_DUMP
  13812. if((((loop ? loop->GetImplicitCallFlags() : func->m_fg->implicitCallFlags) & ~Js::ImplicitCall_External) == 0) &&
  13813. Js::Configuration::Global.flags.Trace.IsEnabled(Js::HostOptPhase))
  13814. {
  13815. Output::Print(_u("DoArrayCheckHoist disabled for JS arrays because of external: "));
  13816. func->DumpFullFunctionName();
  13817. Output::Print(_u("\n"));
  13818. Output::Flush();
  13819. }
  13820. #endif
  13821. return false;
  13822. }
  13823. bool
  13824. GlobOpt::DoArrayMissingValueCheckHoist(Func const * const func)
  13825. {
  13826. return
  13827. DoArrayCheckHoist(func) &&
  13828. !PHASE_OFF(Js::ArrayMissingValueCheckHoistPhase, func) &&
  13829. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsArrayMissingValueCheckHoistDisabled(func->IsLoopBody()));
  13830. }
  13831. bool
  13832. GlobOpt::DoArrayMissingValueCheckHoist() const
  13833. {
  13834. return doArrayMissingValueCheckHoist;
  13835. }
  13836. bool
  13837. GlobOpt::DoArraySegmentHoist(const ValueType baseValueType, Func const * const func)
  13838. {
  13839. Assert(baseValueType.IsLikelyAnyOptimizedArray());
  13840. if(!DoArrayCheckHoist(func) || PHASE_OFF(Js::ArraySegmentHoistPhase, func))
  13841. {
  13842. return false;
  13843. }
  13844. if(!baseValueType.IsLikelyArrayOrObjectWithArray())
  13845. {
  13846. return true;
  13847. }
  13848. return
  13849. !PHASE_OFF(Js::JsArraySegmentHoistPhase, func) &&
  13850. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsJsArraySegmentHoistDisabled(func->IsLoopBody()));
  13851. }
  13852. bool
  13853. GlobOpt::DoArraySegmentHoist(const ValueType baseValueType) const
  13854. {
  13855. Assert(baseValueType.IsLikelyAnyOptimizedArray());
  13856. return baseValueType.IsLikelyArrayOrObjectWithArray() ? doJsArraySegmentHoist : doArraySegmentHoist;
  13857. }
  13858. bool
  13859. GlobOpt::DoTypedArraySegmentLengthHoist(Loop *const loop) const
  13860. {
  13861. if(!DoArraySegmentHoist(ValueType::GetObject(ObjectType::Int32Array)))
  13862. {
  13863. return false;
  13864. }
  13865. if(loop ? ImplicitCallFlagsAllowOpts(loop) : ImplicitCallFlagsAllowOpts(func))
  13866. {
  13867. return true;
  13868. }
  13869. // The function or loop does not allow disabling implicit calls, which is required to eliminate redundant typed array
  13870. // segment length loads.
  13871. #if DBG_DUMP
  13872. if((((loop ? loop->GetImplicitCallFlags() : func->m_fg->implicitCallFlags) & ~Js::ImplicitCall_External) == 0) &&
  13873. Js::Configuration::Global.flags.Trace.IsEnabled(Js::HostOptPhase))
  13874. {
  13875. Output::Print(_u("DoArraySegmentLengthHoist disabled for typed arrays because of external: "));
  13876. func->DumpFullFunctionName();
  13877. Output::Print(_u("\n"));
  13878. Output::Flush();
  13879. }
  13880. #endif
  13881. return false;
  13882. }
  13883. bool
  13884. GlobOpt::DoArrayLengthHoist(Func const * const func)
  13885. {
  13886. return
  13887. DoArrayCheckHoist(func) &&
  13888. !PHASE_OFF(Js::Phase::ArrayLengthHoistPhase, func) &&
  13889. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsArrayLengthHoistDisabled(func->IsLoopBody()));
  13890. }
  13891. bool
  13892. GlobOpt::DoArrayLengthHoist() const
  13893. {
  13894. return doArrayLengthHoist;
  13895. }
  13896. bool
  13897. GlobOpt::DoEliminateArrayAccessHelperCall(Func *const func)
  13898. {
  13899. return DoArrayCheckHoist(func);
  13900. }
  13901. bool
  13902. GlobOpt::DoEliminateArrayAccessHelperCall() const
  13903. {
  13904. return doEliminateArrayAccessHelperCall;
  13905. }
  13906. bool
  13907. GlobOpt::DoLdLenIntSpec(IR::Instr * const instr, const ValueType baseValueType)
  13908. {
  13909. Assert(!instr || instr->m_opcode == Js::OpCode::LdLen_A);
  13910. Assert(!instr || instr->GetDst());
  13911. Assert(!instr || instr->GetSrc1());
  13912. if(PHASE_OFF(Js::LdLenIntSpecPhase, func) ||
  13913. IsTypeSpecPhaseOff(func) ||
  13914. (func->HasProfileInfo() && func->GetReadOnlyProfileInfo()->IsLdLenIntSpecDisabled()) ||
  13915. (instr && !IsLoopPrePass() && instr->DoStackArgsOpt(func)))
  13916. {
  13917. return false;
  13918. }
  13919. if(instr &&
  13920. instr->IsProfiledInstr() &&
  13921. (
  13922. !instr->AsProfiledInstr()->u.FldInfo().valueType.IsLikelyInt() ||
  13923. instr->GetDst()->AsRegOpnd()->m_sym->m_isNotNumber
  13924. ))
  13925. {
  13926. return false;
  13927. }
  13928. Assert(!instr || baseValueType == instr->GetSrc1()->GetValueType());
  13929. return
  13930. baseValueType.HasBeenString() ||
  13931. (baseValueType.IsLikelyAnyOptimizedArray() && baseValueType.GetObjectType() != ObjectType::ObjectWithArray);
  13932. }
  13933. bool
  13934. GlobOpt::DoPathDependentValues() const
  13935. {
  13936. return !PHASE_OFF(Js::Phase::PathDependentValuesPhase, func);
  13937. }
  13938. bool
  13939. GlobOpt::DoTrackRelativeIntBounds() const
  13940. {
  13941. return doTrackRelativeIntBounds;
  13942. }
  13943. bool
  13944. GlobOpt::DoBoundCheckElimination() const
  13945. {
  13946. return doBoundCheckElimination;
  13947. }
  13948. bool
  13949. GlobOpt::DoBoundCheckHoist() const
  13950. {
  13951. return doBoundCheckHoist;
  13952. }
  13953. bool
  13954. GlobOpt::DoLoopCountBasedBoundCheckHoist() const
  13955. {
  13956. return doLoopCountBasedBoundCheckHoist;
  13957. }
  13958. bool
  13959. GlobOpt::DoPowIntIntTypeSpec() const
  13960. {
  13961. return doPowIntIntTypeSpec;
  13962. }
  13963. bool
  13964. GlobOpt::DoTagChecks() const
  13965. {
  13966. return doTagChecks;
  13967. }
  13968. bool
  13969. GlobOpt::TrackArgumentsObject()
  13970. {
  13971. if (PHASE_OFF(Js::StackArgOptPhase, this->func))
  13972. {
  13973. this->CannotAllocateArgumentsObjectOnStack();
  13974. return false;
  13975. }
  13976. return func->GetHasStackArgs();
  13977. }
  13978. void
  13979. GlobOpt::CannotAllocateArgumentsObjectOnStack()
  13980. {
  13981. func->SetHasStackArgs(false);
  13982. #ifdef ENABLE_DEBUG_CONFIG_OPTIONS
  13983. if (PHASE_TESTTRACE(Js::StackArgOptPhase, this->func))
  13984. {
  13985. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  13986. Output::Print(_u("Stack args disabled for function %s(%s)\n"), func->GetJITFunctionBody()->GetDisplayName(), func->GetDebugNumberSet(debugStringBuffer));
  13987. Output::Flush();
  13988. }
  13989. #endif
  13990. }
  13991. IR::Instr *
  13992. GlobOpt::PreOptPeep(IR::Instr *instr)
  13993. {
  13994. if (OpCodeAttr::HasDeadFallThrough(instr->m_opcode))
  13995. {
  13996. switch (instr->m_opcode)
  13997. {
  13998. case Js::OpCode::BailOnNoProfile:
  13999. {
  14000. // Handle BailOnNoProfile
  14001. if (instr->HasBailOutInfo())
  14002. {
  14003. if (!this->prePassLoop)
  14004. {
  14005. FillBailOutInfo(this->currentBlock, instr);
  14006. }
  14007. // Already processed.
  14008. return instr;
  14009. }
  14010. // Convert to bailout instr
  14011. IR::Instr *nextBytecodeOffsetInstr = instr->GetNextRealInstrOrLabel();
  14012. while(nextBytecodeOffsetInstr->GetByteCodeOffset() == Js::Constants::NoByteCodeOffset)
  14013. {
  14014. nextBytecodeOffsetInstr = nextBytecodeOffsetInstr->GetNextRealInstrOrLabel();
  14015. Assert(!nextBytecodeOffsetInstr->IsLabelInstr());
  14016. }
  14017. instr = instr->ConvertToBailOutInstr(nextBytecodeOffsetInstr, IR::BailOutOnNoProfile);
  14018. instr->ClearByteCodeOffset();
  14019. instr->SetByteCodeOffset(nextBytecodeOffsetInstr);
  14020. if (!this->currentBlock->loop)
  14021. {
  14022. FillBailOutInfo(this->currentBlock, instr);
  14023. }
  14024. else
  14025. {
  14026. Assert(this->prePassLoop);
  14027. }
  14028. break;
  14029. }
  14030. case Js::OpCode::BailOnException:
  14031. {
  14032. Assert(
  14033. (
  14034. this->func->HasTry() && this->func->DoOptimizeTry() &&
  14035. instr->m_prev->m_opcode == Js::OpCode::Catch &&
  14036. instr->m_prev->m_prev->IsLabelInstr() &&
  14037. instr->m_prev->m_prev->AsLabelInstr()->GetRegion()->GetType() == RegionType::RegionTypeCatch
  14038. )
  14039. ||
  14040. (
  14041. this->func->HasFinally() && this->func->DoOptimizeTry() &&
  14042. instr->m_prev->AsLabelInstr() &&
  14043. instr->m_prev->AsLabelInstr()->GetRegion()->GetType() == RegionType::RegionTypeFinally
  14044. )
  14045. );
  14046. break;
  14047. }
  14048. case Js::OpCode::BailOnEarlyExit:
  14049. {
  14050. Assert(this->func->HasFinally() && this->func->DoOptimizeTry());
  14051. break;
  14052. }
  14053. default:
  14054. {
  14055. if(this->currentBlock->loop && !this->IsLoopPrePass())
  14056. {
  14057. return instr;
  14058. }
  14059. break;
  14060. }
  14061. }
  14062. RemoveCodeAfterNoFallthroughInstr(instr);
  14063. }
  14064. return instr;
  14065. }
  14066. void
  14067. GlobOpt::RemoveCodeAfterNoFallthroughInstr(IR::Instr *instr)
  14068. {
  14069. if (instr != this->currentBlock->GetLastInstr())
  14070. {
  14071. // Remove dead code after bailout
  14072. IR::Instr *instrDead = instr->m_next;
  14073. IR::Instr *instrNext;
  14074. for (; instrDead != this->currentBlock->GetLastInstr(); instrDead = instrNext)
  14075. {
  14076. instrNext = instrDead->m_next;
  14077. if (instrNext->m_opcode == Js::OpCode::FunctionExit)
  14078. {
  14079. break;
  14080. }
  14081. this->func->m_fg->RemoveInstr(instrDead, this);
  14082. }
  14083. IR::Instr *instrNextBlock = instrDead->m_next;
  14084. this->func->m_fg->RemoveInstr(instrDead, this);
  14085. this->currentBlock->SetLastInstr(instrNextBlock->m_prev);
  14086. }
  14087. // Cleanup dead successors
  14088. FOREACH_SUCCESSOR_BLOCK_EDITING(deadBlock, this->currentBlock, iter)
  14089. {
  14090. this->currentBlock->RemoveDeadSucc(deadBlock, this->func->m_fg);
  14091. if (this->currentBlock->GetDataUseCount() > 0)
  14092. {
  14093. this->currentBlock->DecrementDataUseCount();
  14094. }
  14095. } NEXT_SUCCESSOR_BLOCK_EDITING;
  14096. }
  14097. void
  14098. GlobOpt::ProcessTryHandler(IR::Instr* instr)
  14099. {
  14100. Assert(instr->m_next->IsLabelInstr() && instr->m_next->AsLabelInstr()->GetRegion()->GetType() == RegionType::RegionTypeTry);
  14101. Region* tryRegion = instr->m_next->AsLabelInstr()->GetRegion();
  14102. BVSparse<JitArenaAllocator> * writeThroughSymbolsSet = tryRegion->writeThroughSymbolsSet;
  14103. ToVar(writeThroughSymbolsSet, this->currentBlock);
  14104. }
  14105. bool
  14106. GlobOpt::ProcessExceptionHandlingEdges(IR::Instr* instr)
  14107. {
  14108. Assert(instr->m_opcode == Js::OpCode::BrOnException || instr->m_opcode == Js::OpCode::BrOnNoException);
  14109. if (instr->m_opcode == Js::OpCode::BrOnException)
  14110. {
  14111. if (instr->AsBranchInstr()->GetTarget()->GetRegion()->GetType() == RegionType::RegionTypeCatch)
  14112. {
  14113. // BrOnException was added to model flow from try region to the catch region to assist
  14114. // the backward pass in propagating bytecode upward exposed info from the catch block
  14115. // to the try, and to handle break blocks. Removing it here as it has served its purpose
  14116. // and keeping it around might also have unintended effects while merging block data for
  14117. // the catch block's predecessors.
  14118. // Note that the Deadstore pass will still be able to propagate bytecode upward exposed info
  14119. // because it doesn't skip dead blocks for that.
  14120. this->RemoveFlowEdgeToCatchBlock(instr);
  14121. this->currentBlock->RemoveInstr(instr);
  14122. return true;
  14123. }
  14124. else
  14125. {
  14126. // We add BrOnException from a finally region to early exit, remove that since it has served its purpose
  14127. return this->RemoveFlowEdgeToFinallyOnExceptionBlock(instr);
  14128. }
  14129. }
  14130. else if (instr->m_opcode == Js::OpCode::BrOnNoException)
  14131. {
  14132. if (instr->AsBranchInstr()->GetTarget()->GetRegion()->GetType() == RegionType::RegionTypeCatch)
  14133. {
  14134. this->RemoveFlowEdgeToCatchBlock(instr);
  14135. }
  14136. else
  14137. {
  14138. this->RemoveFlowEdgeToFinallyOnExceptionBlock(instr);
  14139. }
  14140. }
  14141. return false;
  14142. }
  14143. void
  14144. GlobOpt::InsertToVarAtDefInTryRegion(IR::Instr * instr, IR::Opnd * dstOpnd)
  14145. {
  14146. if ((this->currentRegion->GetType() == RegionTypeTry || this->currentRegion->GetType() == RegionTypeFinally) &&
  14147. dstOpnd->IsRegOpnd() && dstOpnd->AsRegOpnd()->m_sym->HasByteCodeRegSlot())
  14148. {
  14149. StackSym * sym = dstOpnd->AsRegOpnd()->m_sym;
  14150. if (sym->IsVar())
  14151. {
  14152. return;
  14153. }
  14154. StackSym * varSym = sym->GetVarEquivSym(nullptr);
  14155. if ((this->currentRegion->GetType() == RegionTypeTry && this->currentRegion->writeThroughSymbolsSet->Test(varSym->m_id)) ||
  14156. ((this->currentRegion->GetType() == RegionTypeFinally && this->currentRegion->GetMatchingTryRegion()->writeThroughSymbolsSet->Test(varSym->m_id))))
  14157. {
  14158. IR::RegOpnd * regOpnd = IR::RegOpnd::New(varSym, IRType::TyVar, instr->m_func);
  14159. this->ToVar(instr->m_next, regOpnd, this->currentBlock, NULL, false);
  14160. }
  14161. }
  14162. }
  14163. void
  14164. GlobOpt::RemoveFlowEdgeToCatchBlock(IR::Instr * instr)
  14165. {
  14166. Assert(instr->IsBranchInstr());
  14167. BasicBlock * catchBlock = nullptr;
  14168. BasicBlock * predBlock = nullptr;
  14169. if (instr->m_opcode == Js::OpCode::BrOnException)
  14170. {
  14171. catchBlock = instr->AsBranchInstr()->GetTarget()->GetBasicBlock();
  14172. predBlock = this->currentBlock;
  14173. }
  14174. else
  14175. {
  14176. Assert(instr->m_opcode == Js::OpCode::BrOnNoException);
  14177. IR::Instr * nextInstr = instr->GetNextRealInstrOrLabel();
  14178. Assert(nextInstr->IsLabelInstr());
  14179. IR::LabelInstr * nextLabel = nextInstr->AsLabelInstr();
  14180. if (nextLabel->GetRegion() && nextLabel->GetRegion()->GetType() == RegionTypeCatch)
  14181. {
  14182. catchBlock = nextLabel->GetBasicBlock();
  14183. predBlock = this->currentBlock;
  14184. }
  14185. else
  14186. {
  14187. Assert(nextLabel->m_next->IsBranchInstr() && nextLabel->m_next->AsBranchInstr()->IsUnconditional());
  14188. BasicBlock * nextBlock = nextLabel->GetBasicBlock();
  14189. IR::BranchInstr * branchToCatchBlock = nextLabel->m_next->AsBranchInstr();
  14190. IR::LabelInstr * catchBlockLabel = branchToCatchBlock->GetTarget();
  14191. Assert(catchBlockLabel->GetRegion()->GetType() == RegionTypeCatch);
  14192. catchBlock = catchBlockLabel->GetBasicBlock();
  14193. predBlock = nextBlock;
  14194. }
  14195. }
  14196. Assert(catchBlock);
  14197. Assert(predBlock);
  14198. if (this->func->m_fg->FindEdge(predBlock, catchBlock))
  14199. {
  14200. predBlock->RemoveDeadSucc(catchBlock, this->func->m_fg);
  14201. if (predBlock == this->currentBlock)
  14202. {
  14203. predBlock->DecrementDataUseCount();
  14204. }
  14205. }
  14206. }
  14207. bool
  14208. GlobOpt::RemoveFlowEdgeToFinallyOnExceptionBlock(IR::Instr * instr)
  14209. {
  14210. Assert(instr->IsBranchInstr());
  14211. if (instr->m_opcode == Js::OpCode::BrOnNoException && instr->AsBranchInstr()->m_brFinallyToEarlyExit)
  14212. {
  14213. // We add edge from finally to early exit block
  14214. // We should not remove this edge
  14215. // If a loop has continue, and we add edge in finally to continue
  14216. // Break block removal can move all continues inside the loop to branch to the continue added within finally
  14217. // If we get rid of this edge, then loop may loose all backedges
  14218. // Ideally, doing tail duplication before globopt would enable us to remove these edges, but since we do it after globopt, keep it this way for now
  14219. // See test1() in core/test/tryfinallytests.js
  14220. return false;
  14221. }
  14222. BasicBlock * finallyBlock = nullptr;
  14223. BasicBlock * predBlock = nullptr;
  14224. if (instr->m_opcode == Js::OpCode::BrOnException)
  14225. {
  14226. finallyBlock = instr->AsBranchInstr()->GetTarget()->GetBasicBlock();
  14227. predBlock = this->currentBlock;
  14228. }
  14229. else
  14230. {
  14231. Assert(instr->m_opcode == Js::OpCode::BrOnNoException);
  14232. IR::Instr * nextInstr = instr->GetNextRealInstrOrLabel();
  14233. Assert(nextInstr->IsLabelInstr());
  14234. IR::LabelInstr * nextLabel = nextInstr->AsLabelInstr();
  14235. if (nextLabel->GetRegion() && nextLabel->GetRegion()->GetType() == RegionTypeFinally)
  14236. {
  14237. finallyBlock = nextLabel->GetBasicBlock();
  14238. predBlock = this->currentBlock;
  14239. }
  14240. else
  14241. {
  14242. if (!(nextLabel->m_next->IsBranchInstr() && nextLabel->m_next->AsBranchInstr()->IsUnconditional()))
  14243. {
  14244. return false;
  14245. }
  14246. BasicBlock * nextBlock = nextLabel->GetBasicBlock();
  14247. IR::BranchInstr * branchTofinallyBlockOrEarlyExit = nextLabel->m_next->AsBranchInstr();
  14248. IR::LabelInstr * finallyBlockLabelOrEarlyExitLabel = branchTofinallyBlockOrEarlyExit->GetTarget();
  14249. finallyBlock = finallyBlockLabelOrEarlyExitLabel->GetBasicBlock();
  14250. predBlock = nextBlock;
  14251. }
  14252. }
  14253. Assert(finallyBlock && predBlock);
  14254. if (this->func->m_fg->FindEdge(predBlock, finallyBlock))
  14255. {
  14256. predBlock->RemoveDeadSucc(finallyBlock, this->func->m_fg);
  14257. if (instr->m_opcode == Js::OpCode::BrOnException)
  14258. {
  14259. this->currentBlock->RemoveInstr(instr);
  14260. }
  14261. if (finallyBlock->GetFirstInstr()->AsLabelInstr()->IsUnreferenced())
  14262. {
  14263. // Traverse predBlocks of finallyBlock, if any of the preds have a different region, set m_hasNonBranchRef to true
  14264. // If not, this label can get eliminated and an incorrect region from the predecessor can get propagated in lowered code
  14265. // See test3() in tryfinallytests.js
  14266. Region * finallyRegion = finallyBlock->GetFirstInstr()->AsLabelInstr()->GetRegion();
  14267. FOREACH_PREDECESSOR_BLOCK(pred, finallyBlock)
  14268. {
  14269. Region * predRegion = pred->GetFirstInstr()->AsLabelInstr()->GetRegion();
  14270. if (predRegion != finallyRegion)
  14271. {
  14272. finallyBlock->GetFirstInstr()->AsLabelInstr()->m_hasNonBranchRef = true;
  14273. }
  14274. } NEXT_PREDECESSOR_BLOCK;
  14275. }
  14276. if (predBlock == this->currentBlock)
  14277. {
  14278. predBlock->DecrementDataUseCount();
  14279. }
  14280. }
  14281. return true;
  14282. }
  14283. IR::Instr *
  14284. GlobOpt::OptPeep(IR::Instr *instr, Value *src1Val, Value *src2Val)
  14285. {
  14286. IR::Opnd *dst, *src1, *src2;
  14287. if (this->IsLoopPrePass())
  14288. {
  14289. return instr;
  14290. }
  14291. switch (instr->m_opcode)
  14292. {
  14293. case Js::OpCode::DeadBrEqual:
  14294. case Js::OpCode::DeadBrRelational:
  14295. case Js::OpCode::DeadBrSrEqual:
  14296. src1 = instr->GetSrc1();
  14297. src2 = instr->GetSrc2();
  14298. // These branches were turned into dead branches because they were unnecessary (branch to next, ...).
  14299. // The DeadBr are necessary in case the evaluation of the sources have side-effects.
  14300. // If we know for sure the srcs are primitive or have been type specialized, we don't need these instructions
  14301. if (((src1Val && src1Val->GetValueInfo()->IsPrimitive()) || (src1->IsRegOpnd() && CurrentBlockData()->IsTypeSpecialized(src1->AsRegOpnd()->m_sym))) &&
  14302. ((src2Val && src2Val->GetValueInfo()->IsPrimitive()) || (src2->IsRegOpnd() && CurrentBlockData()->IsTypeSpecialized(src2->AsRegOpnd()->m_sym))))
  14303. {
  14304. this->CaptureByteCodeSymUses(instr);
  14305. instr->m_opcode = Js::OpCode::Nop;
  14306. }
  14307. break;
  14308. case Js::OpCode::DeadBrOnHasProperty:
  14309. src1 = instr->GetSrc1();
  14310. if (((src1Val && src1Val->GetValueInfo()->IsPrimitive()) || (src1->IsRegOpnd() && CurrentBlockData()->IsTypeSpecialized(src1->AsRegOpnd()->m_sym))))
  14311. {
  14312. this->CaptureByteCodeSymUses(instr);
  14313. instr->m_opcode = Js::OpCode::Nop;
  14314. }
  14315. break;
  14316. case Js::OpCode::Ld_A:
  14317. case Js::OpCode::Ld_I4:
  14318. src1 = instr->GetSrc1();
  14319. dst = instr->GetDst();
  14320. if (dst->IsRegOpnd() && dst->IsEqual(src1))
  14321. {
  14322. dst = instr->UnlinkDst();
  14323. if (!dst->GetIsJITOptimizedReg())
  14324. {
  14325. IR::ByteCodeUsesInstr *bytecodeUse = IR::ByteCodeUsesInstr::New(instr);
  14326. bytecodeUse->SetDst(dst);
  14327. instr->InsertAfter(bytecodeUse);
  14328. }
  14329. instr->FreeSrc1();
  14330. instr->m_opcode = Js::OpCode::Nop;
  14331. }
  14332. break;
  14333. }
  14334. return instr;
  14335. }
  14336. void
  14337. GlobOpt::OptimizeIndirUses(IR::IndirOpnd *indirOpnd, IR::Instr * *pInstr, Value **indirIndexValRef)
  14338. {
  14339. IR::Instr * &instr = *pInstr;
  14340. Assert(!indirIndexValRef || !*indirIndexValRef);
  14341. // Update value types and copy-prop the base
  14342. OptSrc(indirOpnd->GetBaseOpnd(), &instr, nullptr, indirOpnd);
  14343. IR::RegOpnd *indexOpnd = indirOpnd->GetIndexOpnd();
  14344. if (!indexOpnd)
  14345. {
  14346. return;
  14347. }
  14348. // Update value types and copy-prop the index
  14349. Value *indexVal = OptSrc(indexOpnd, &instr, nullptr, indirOpnd);
  14350. if(indirIndexValRef)
  14351. {
  14352. *indirIndexValRef = indexVal;
  14353. }
  14354. }
  14355. bool
  14356. GlobOpt::IsPREInstrCandidateLoad(Js::OpCode opcode)
  14357. {
  14358. switch (opcode)
  14359. {
  14360. case Js::OpCode::LdFld:
  14361. case Js::OpCode::LdFldForTypeOf:
  14362. case Js::OpCode::LdRootFld:
  14363. case Js::OpCode::LdRootFldForTypeOf:
  14364. case Js::OpCode::LdMethodFld:
  14365. case Js::OpCode::LdRootMethodFld:
  14366. case Js::OpCode::LdSlot:
  14367. case Js::OpCode::LdSlotArr:
  14368. return true;
  14369. }
  14370. return false;
  14371. }
  14372. bool
  14373. GlobOpt::IsPREInstrSequenceCandidateLoad(Js::OpCode opcode)
  14374. {
  14375. switch (opcode)
  14376. {
  14377. default:
  14378. return IsPREInstrCandidateLoad(opcode);
  14379. case Js::OpCode::Ld_A:
  14380. case Js::OpCode::BytecodeArgOutCapture:
  14381. return true;
  14382. }
  14383. }
  14384. bool
  14385. GlobOpt::IsPREInstrCandidateStore(Js::OpCode opcode)
  14386. {
  14387. switch (opcode)
  14388. {
  14389. case Js::OpCode::StFld:
  14390. case Js::OpCode::StRootFld:
  14391. case Js::OpCode::StSlot:
  14392. return true;
  14393. }
  14394. return false;
  14395. }
  14396. bool
  14397. GlobOpt::ImplicitCallFlagsAllowOpts(Loop *loop)
  14398. {
  14399. return loop->GetImplicitCallFlags() != Js::ImplicitCall_HasNoInfo &&
  14400. (((loop->GetImplicitCallFlags() & ~Js::ImplicitCall_Accessor) | Js::ImplicitCall_None) == Js::ImplicitCall_None);
  14401. }
  14402. bool
  14403. GlobOpt::ImplicitCallFlagsAllowOpts(Func const *func)
  14404. {
  14405. return func->m_fg->implicitCallFlags != Js::ImplicitCall_HasNoInfo &&
  14406. (((func->m_fg->implicitCallFlags & ~Js::ImplicitCall_Accessor) | Js::ImplicitCall_None) == Js::ImplicitCall_None);
  14407. }
  14408. #if DBG_DUMP
  14409. void
  14410. GlobOpt::Dump() const
  14411. {
  14412. this->DumpSymToValueMap();
  14413. }
  14414. void
  14415. GlobOpt::DumpSymToValueMap(BasicBlock const * block) const
  14416. {
  14417. Output::Print(_u("\n*** SymToValueMap ***\n"));
  14418. block->globOptData.DumpSymToValueMap();
  14419. }
  14420. void
  14421. GlobOpt::DumpSymToValueMap() const
  14422. {
  14423. DumpSymToValueMap(this->currentBlock);
  14424. }
  14425. void
  14426. GlobOpt::DumpSymVal(int index)
  14427. {
  14428. SymID id = index;
  14429. extern Func *CurrentFunc;
  14430. Sym *sym = this->func->m_symTable->Find(id);
  14431. AssertMsg(sym, "Sym not found!!!");
  14432. Output::Print(_u("Sym: "));
  14433. sym->Dump();
  14434. Output::Print(_u("\t\tValueNumber: "));
  14435. Value * pValue = CurrentBlockData()->FindValueFromMapDirect(sym->m_id);
  14436. pValue->Dump();
  14437. Output::Print(_u("\n"));
  14438. }
  14439. void
  14440. GlobOpt::Trace(BasicBlock * block, bool before) const
  14441. {
  14442. bool globOptTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::GlobOptPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14443. bool typeSpecTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::TypeSpecPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14444. bool floatTypeSpecTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::FloatTypeSpecPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14445. bool fieldCopyPropTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldCopyPropPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14446. bool objTypeSpecTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::ObjTypeSpecPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14447. bool valueTableTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::ValueTablePhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14448. bool fieldPRETrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldPREPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14449. bool anyTrace = globOptTrace || typeSpecTrace || floatTypeSpecTrace || fieldCopyPropTrace || objTypeSpecTrace || valueTableTrace || fieldPRETrace;
  14450. if (!anyTrace)
  14451. {
  14452. return;
  14453. }
  14454. if (fieldPRETrace && this->IsLoopPrePass())
  14455. {
  14456. if (block->isLoopHeader && before)
  14457. {
  14458. Output::Print(_u("==== Loop Prepass block header #%-3d, Visiting Loop block head #%-3d\n"),
  14459. this->prePassLoop->GetHeadBlock()->GetBlockNum(), block->GetBlockNum());
  14460. }
  14461. }
  14462. if (!typeSpecTrace && !floatTypeSpecTrace && !valueTableTrace && !Js::Configuration::Global.flags.Verbose)
  14463. {
  14464. return;
  14465. }
  14466. if (before)
  14467. {
  14468. Output::Print(_u("========================================================================\n"));
  14469. Output::Print(_u("Begin OptBlock: Block #%-3d"), block->GetBlockNum());
  14470. if (block->loop)
  14471. {
  14472. Output::Print(_u(" Loop block header:%-3d currentLoop block head:%-3d %s"),
  14473. block->loop->GetHeadBlock()->GetBlockNum(),
  14474. this->prePassLoop ? this->prePassLoop->GetHeadBlock()->GetBlockNum() : 0,
  14475. this->IsLoopPrePass() ? _u("PrePass") : _u(""));
  14476. }
  14477. Output::Print(_u("\n"));
  14478. }
  14479. else
  14480. {
  14481. Output::Print(_u("-----------------------------------------------------------------------\n"));
  14482. Output::Print(_u("After OptBlock: Block #%-3d\n"), block->GetBlockNum());
  14483. }
  14484. if ((typeSpecTrace || floatTypeSpecTrace) && !block->globOptData.liveVarSyms->IsEmpty())
  14485. {
  14486. Output::Print(_u(" Live var syms: "));
  14487. block->globOptData.liveVarSyms->Dump();
  14488. }
  14489. if (typeSpecTrace && !block->globOptData.liveInt32Syms->IsEmpty())
  14490. {
  14491. Assert(this->tempBv->IsEmpty());
  14492. this->tempBv->Minus(block->globOptData.liveInt32Syms, block->globOptData.liveLossyInt32Syms);
  14493. if(!this->tempBv->IsEmpty())
  14494. {
  14495. Output::Print(_u(" Int32 type specialized (lossless) syms: "));
  14496. this->tempBv->Dump();
  14497. }
  14498. this->tempBv->ClearAll();
  14499. if(!block->globOptData.liveLossyInt32Syms->IsEmpty())
  14500. {
  14501. Output::Print(_u(" Int32 converted (lossy) syms: "));
  14502. block->globOptData.liveLossyInt32Syms->Dump();
  14503. }
  14504. }
  14505. if (floatTypeSpecTrace && !block->globOptData.liveFloat64Syms->IsEmpty())
  14506. {
  14507. Output::Print(_u(" Float64 type specialized syms: "));
  14508. block->globOptData.liveFloat64Syms->Dump();
  14509. }
  14510. if ((fieldCopyPropTrace || objTypeSpecTrace) && this->DoFieldCopyProp(block->loop) && !block->globOptData.liveFields->IsEmpty())
  14511. {
  14512. Output::Print(_u(" Live field syms: "));
  14513. block->globOptData.liveFields->Dump();
  14514. }
  14515. if (objTypeSpecTrace || valueTableTrace)
  14516. {
  14517. Output::Print(_u(" Value table:\n"));
  14518. block->globOptData.DumpSymToValueMap();
  14519. }
  14520. if (before)
  14521. {
  14522. Output::Print(_u("-----------------------------------------------------------------------\n")); \
  14523. }
  14524. Output::Flush();
  14525. }
  14526. void
  14527. GlobOpt::TraceSettings() const
  14528. {
  14529. Output::Print(_u("GlobOpt Settings:\r\n"));
  14530. Output::Print(_u(" FloatTypeSpec: %s\r\n"), this->DoFloatTypeSpec() ? _u("enabled") : _u("disabled"));
  14531. Output::Print(_u(" AggressiveIntTypeSpec: %s\r\n"), this->DoAggressiveIntTypeSpec() ? _u("enabled") : _u("disabled"));
  14532. Output::Print(_u(" LossyIntTypeSpec: %s\r\n"), this->DoLossyIntTypeSpec() ? _u("enabled") : _u("disabled"));
  14533. Output::Print(_u(" ArrayCheckHoist: %s\r\n"), this->func->IsArrayCheckHoistDisabled() ? _u("disabled") : _u("enabled"));
  14534. Output::Print(_u(" ImplicitCallFlags: %s\r\n"), Js::DynamicProfileInfo::GetImplicitCallFlagsString(this->func->m_fg->implicitCallFlags));
  14535. for (Loop * loop = this->func->m_fg->loopList; loop != NULL; loop = loop->next)
  14536. {
  14537. Output::Print(_u(" loop: %d, ImplicitCallFlags: %s\r\n"), loop->GetLoopNumber(),
  14538. Js::DynamicProfileInfo::GetImplicitCallFlagsString(loop->GetImplicitCallFlags()));
  14539. }
  14540. Output::Flush();
  14541. }
  14542. #endif // DBG_DUMP
  14543. IR::Instr *
  14544. GlobOpt::TrackMarkTempObject(IR::Instr * instrStart, IR::Instr * instrLast)
  14545. {
  14546. if (!this->func->GetHasMarkTempObjects())
  14547. {
  14548. return instrLast;
  14549. }
  14550. IR::Instr * instr = instrStart;
  14551. IR::Instr * instrEnd = instrLast->m_next;
  14552. IR::Instr * lastInstr = nullptr;
  14553. GlobOptBlockData& globOptData = *CurrentBlockData();
  14554. do
  14555. {
  14556. bool mayNeedBailOnImplicitCallsPreOp = !this->IsLoopPrePass()
  14557. && instr->HasAnyImplicitCalls()
  14558. && globOptData.maybeTempObjectSyms != nullptr;
  14559. if (mayNeedBailOnImplicitCallsPreOp)
  14560. {
  14561. IR::Opnd * src1 = instr->GetSrc1();
  14562. if (src1)
  14563. {
  14564. instr = GenerateBailOutMarkTempObjectIfNeeded(instr, src1, false);
  14565. IR::Opnd * src2 = instr->GetSrc2();
  14566. if (src2)
  14567. {
  14568. instr = GenerateBailOutMarkTempObjectIfNeeded(instr, src2, false);
  14569. }
  14570. }
  14571. }
  14572. IR::Opnd *dst = instr->GetDst();
  14573. if (dst)
  14574. {
  14575. if (dst->IsRegOpnd())
  14576. {
  14577. TrackTempObjectSyms(instr, dst->AsRegOpnd());
  14578. }
  14579. else if (mayNeedBailOnImplicitCallsPreOp)
  14580. {
  14581. instr = GenerateBailOutMarkTempObjectIfNeeded(instr, dst, true);
  14582. }
  14583. }
  14584. lastInstr = instr;
  14585. instr = instr->m_next;
  14586. }
  14587. while (instr != instrEnd);
  14588. return lastInstr;
  14589. }
  14590. void
  14591. GlobOpt::TrackTempObjectSyms(IR::Instr * instr, IR::RegOpnd * opnd)
  14592. {
  14593. // If it is marked as dstIsTempObject, we should have mark temped it, or type specialized it to Ld_I4.
  14594. Assert(!instr->dstIsTempObject || ObjectTempVerify::CanMarkTemp(instr, nullptr));
  14595. GlobOptBlockData& globOptData = *CurrentBlockData();
  14596. bool canStoreTemp = false;
  14597. bool maybeTemp = false;
  14598. if (OpCodeAttr::TempObjectProducing(instr->m_opcode))
  14599. {
  14600. maybeTemp = instr->dstIsTempObject;
  14601. // We have to make sure that lower will always generate code to do stack allocation
  14602. // before we can store any other stack instance onto it. Otherwise, we would not
  14603. // walk object to box the stack property.
  14604. canStoreTemp = instr->dstIsTempObject && ObjectTemp::CanStoreTemp(instr);
  14605. }
  14606. else if (OpCodeAttr::TempObjectTransfer(instr->m_opcode))
  14607. {
  14608. // Need to check both sources, GetNewScObject has two srcs for transfer.
  14609. // No need to get var equiv sym here as transfer of type spec value does not transfer a mark temp object.
  14610. maybeTemp = globOptData.maybeTempObjectSyms && (
  14611. (instr->GetSrc1()->IsRegOpnd() && globOptData.maybeTempObjectSyms->Test(instr->GetSrc1()->AsRegOpnd()->m_sym->m_id))
  14612. || (instr->GetSrc2() && instr->GetSrc2()->IsRegOpnd() && globOptData.maybeTempObjectSyms->Test(instr->GetSrc2()->AsRegOpnd()->m_sym->m_id)));
  14613. canStoreTemp = globOptData.canStoreTempObjectSyms && (
  14614. (instr->GetSrc1()->IsRegOpnd() && globOptData.canStoreTempObjectSyms->Test(instr->GetSrc1()->AsRegOpnd()->m_sym->m_id))
  14615. && (!instr->GetSrc2() || (instr->GetSrc2()->IsRegOpnd() && globOptData.canStoreTempObjectSyms->Test(instr->GetSrc2()->AsRegOpnd()->m_sym->m_id))));
  14616. AssertOrFailFast(!canStoreTemp || instr->dstIsTempObject);
  14617. AssertOrFailFast(!maybeTemp || instr->dstIsTempObject);
  14618. }
  14619. // Need to get the var equiv sym as assignment of type specialized sym kill the var sym value anyway.
  14620. StackSym * sym = opnd->m_sym;
  14621. if (!sym->IsVar())
  14622. {
  14623. sym = sym->GetVarEquivSym(nullptr);
  14624. if (sym == nullptr)
  14625. {
  14626. return;
  14627. }
  14628. }
  14629. SymID symId = sym->m_id;
  14630. if (maybeTemp)
  14631. {
  14632. // Only var sym should be temp objects
  14633. Assert(opnd->m_sym == sym);
  14634. if (globOptData.maybeTempObjectSyms == nullptr)
  14635. {
  14636. globOptData.maybeTempObjectSyms = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  14637. }
  14638. globOptData.maybeTempObjectSyms->Set(symId);
  14639. if (canStoreTemp)
  14640. {
  14641. if (instr->m_opcode == Js::OpCode::NewScObjectLiteral && !this->IsLoopPrePass())
  14642. {
  14643. // For object literal, we install the final type up front.
  14644. // If there are bailout before we finish initializing all the fields, we need to
  14645. // zero out the rest if we stack allocate the literal, so that the boxing would not
  14646. // try to box trash pointer in the properties.
  14647. // Although object Literal initialization can be done lexically, BailOnNoProfile may cause some path
  14648. // to disappear. Do it is flow base make it easier to stop propagate those entries.
  14649. IR::IntConstOpnd * propertyArrayIdOpnd = instr->GetSrc1()->AsIntConstOpnd();
  14650. const Js::PropertyIdArray * propIds = instr->m_func->GetJITFunctionBody()->ReadPropertyIdArrayFromAuxData(propertyArrayIdOpnd->AsUint32());
  14651. // Duplicates are removed by parser
  14652. Assert(!propIds->hadDuplicates);
  14653. if (globOptData.stackLiteralInitFldDataMap == nullptr)
  14654. {
  14655. globOptData.stackLiteralInitFldDataMap = JitAnew(alloc, StackLiteralInitFldDataMap, alloc);
  14656. }
  14657. else
  14658. {
  14659. Assert(!globOptData.stackLiteralInitFldDataMap->ContainsKey(sym));
  14660. }
  14661. StackLiteralInitFldData data = { propIds, 0};
  14662. globOptData.stackLiteralInitFldDataMap->AddNew(sym, data);
  14663. }
  14664. if (globOptData.canStoreTempObjectSyms == nullptr)
  14665. {
  14666. globOptData.canStoreTempObjectSyms = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  14667. }
  14668. globOptData.canStoreTempObjectSyms->Set(symId);
  14669. }
  14670. else if (globOptData.canStoreTempObjectSyms)
  14671. {
  14672. globOptData.canStoreTempObjectSyms->Clear(symId);
  14673. }
  14674. }
  14675. else
  14676. {
  14677. Assert(!canStoreTemp);
  14678. if (globOptData.maybeTempObjectSyms)
  14679. {
  14680. if (globOptData.canStoreTempObjectSyms)
  14681. {
  14682. globOptData.canStoreTempObjectSyms->Clear(symId);
  14683. }
  14684. globOptData.maybeTempObjectSyms->Clear(symId);
  14685. }
  14686. else
  14687. {
  14688. Assert(!globOptData.canStoreTempObjectSyms);
  14689. }
  14690. // The symbol is being assigned to, the sym shouldn't still be in the stackLiteralInitFldDataMap
  14691. Assert(this->IsLoopPrePass() ||
  14692. globOptData.stackLiteralInitFldDataMap == nullptr
  14693. || globOptData.stackLiteralInitFldDataMap->Count() == 0
  14694. || !globOptData.stackLiteralInitFldDataMap->ContainsKey(sym));
  14695. }
  14696. }
  14697. IR::Instr *
  14698. GlobOpt::GenerateBailOutMarkTempObjectIfNeeded(IR::Instr * instr, IR::Opnd * opnd, bool isDst)
  14699. {
  14700. Assert(opnd);
  14701. Assert(isDst == (opnd == instr->GetDst()));
  14702. Assert(opnd != instr->GetDst() || !opnd->IsRegOpnd());
  14703. Assert(!this->IsLoopPrePass());
  14704. Assert(instr->HasAnyImplicitCalls());
  14705. // Only dst reg opnd opcode or ArgOut_A should have dstIsTempObject marked
  14706. Assert(!isDst || !instr->dstIsTempObject || instr->m_opcode == Js::OpCode::ArgOut_A);
  14707. // Post-op implicit call shouldn't have installed yet
  14708. Assert(!instr->HasBailOutInfo() || (instr->GetBailOutKind() & IR::BailOutKindBits) != IR::BailOutOnImplicitCalls);
  14709. GlobOptBlockData& globOptData = *CurrentBlockData();
  14710. Assert(globOptData.maybeTempObjectSyms != nullptr);
  14711. IR::PropertySymOpnd * propertySymOpnd = nullptr;
  14712. StackSym * stackSym = ObjectTemp::GetStackSym(opnd, &propertySymOpnd);
  14713. // It is okay to not get the var equiv sym here, as use of a type specialized sym is not use of the temp object
  14714. // so no need to add mark temp bailout.
  14715. // TempObjectSysm doesn't contain any type spec sym, so we will get false here for all type spec sym.
  14716. if (stackSym && globOptData.maybeTempObjectSyms->Test(stackSym->m_id))
  14717. {
  14718. if (instr->HasBailOutInfo())
  14719. {
  14720. instr->SetBailOutKind(instr->GetBailOutKind() | IR::BailOutMarkTempObject);
  14721. }
  14722. else
  14723. {
  14724. // On insert the pre op bailout if it is not Direct field access do nothing, don't check the dst yet.
  14725. // SetTypeCheckBailout will clear this out if it is direct field access.
  14726. if (isDst
  14727. || (instr->m_opcode == Js::OpCode::FromVar && !opnd->GetValueType().IsPrimitive())
  14728. || propertySymOpnd == nullptr
  14729. || !propertySymOpnd->IsTypeCheckProtected())
  14730. {
  14731. this->GenerateBailAtOperation(&instr, IR::BailOutMarkTempObject);
  14732. }
  14733. }
  14734. if (!opnd->IsRegOpnd() && (!isDst || (globOptData.canStoreTempObjectSyms && globOptData.canStoreTempObjectSyms->Test(stackSym->m_id))))
  14735. {
  14736. // If this opnd is a dst, that means that the object pointer is a stack object,
  14737. // and we can store temp object/number on it.
  14738. // If the opnd is a src, that means that the object pointer may be a stack object
  14739. // so the load may be a temp object/number and we need to track its use.
  14740. // Don't mark start of indir as can store temp, because we don't actually know
  14741. // what it is assigning to.
  14742. if (!isDst || !opnd->IsIndirOpnd())
  14743. {
  14744. opnd->SetCanStoreTemp();
  14745. }
  14746. if (propertySymOpnd)
  14747. {
  14748. // Track initfld of stack literals
  14749. if (isDst && instr->m_opcode == Js::OpCode::InitFld)
  14750. {
  14751. const Js::PropertyId propertyId = propertySymOpnd->m_sym->AsPropertySym()->m_propertyId;
  14752. // We don't need to track numeric properties init
  14753. if (!this->func->GetThreadContextInfo()->IsNumericProperty(propertyId))
  14754. {
  14755. DebugOnly(bool found = false);
  14756. globOptData.stackLiteralInitFldDataMap->RemoveIf(stackSym,
  14757. [&](StackSym * key, StackLiteralInitFldData & data)
  14758. {
  14759. DebugOnly(found = true);
  14760. Assert(key == stackSym);
  14761. Assert(data.currentInitFldCount < data.propIds->count);
  14762. if (data.propIds->elements[data.currentInitFldCount] != propertyId)
  14763. {
  14764. #if DBG
  14765. bool duplicate = false;
  14766. for (uint i = 0; i < data.currentInitFldCount; i++)
  14767. {
  14768. if (data.propIds->elements[i] == propertyId)
  14769. {
  14770. duplicate = true;
  14771. break;
  14772. }
  14773. }
  14774. Assert(duplicate);
  14775. #endif
  14776. // duplicate initialization
  14777. return false;
  14778. }
  14779. bool finished = (++data.currentInitFldCount == data.propIds->count);
  14780. #if DBG
  14781. if (finished)
  14782. {
  14783. // We can still track the finished stack literal InitFld lexically.
  14784. this->finishedStackLiteralInitFld->Set(stackSym->m_id);
  14785. }
  14786. #endif
  14787. return finished;
  14788. });
  14789. // We might still see InitFld even we have finished with all the property Id because
  14790. // of duplicate entries at the end
  14791. Assert(found || finishedStackLiteralInitFld->Test(stackSym->m_id));
  14792. }
  14793. }
  14794. }
  14795. }
  14796. }
  14797. return instr;
  14798. }
  14799. LoopCount *
  14800. GlobOpt::GetOrGenerateLoopCountForMemOp(Loop *loop)
  14801. {
  14802. LoopCount *loopCount = loop->loopCount;
  14803. if (loopCount && !loopCount->HasGeneratedLoopCountSym())
  14804. {
  14805. Assert(loop->bailOutInfo);
  14806. EnsureBailTarget(loop);
  14807. GenerateLoopCountPlusOne(loop, loopCount);
  14808. }
  14809. return loopCount;
  14810. }
  14811. IR::Opnd *
  14812. GlobOpt::GenerateInductionVariableChangeForMemOp(Loop *loop, byte unroll, IR::Instr *insertBeforeInstr)
  14813. {
  14814. LoopCount *loopCount = loop->loopCount;
  14815. IR::Opnd *sizeOpnd = nullptr;
  14816. Assert(loopCount);
  14817. Assert(loop->memOpInfo->inductionVariableOpndPerUnrollMap);
  14818. if (loop->memOpInfo->inductionVariableOpndPerUnrollMap->TryGetValue(unroll, &sizeOpnd))
  14819. {
  14820. return sizeOpnd;
  14821. }
  14822. Func *localFunc = loop->GetFunc();
  14823. const auto InsertInstr = [&](IR::Instr *instr)
  14824. {
  14825. if (insertBeforeInstr == nullptr)
  14826. {
  14827. loop->landingPad->InsertAfter(instr);
  14828. }
  14829. else
  14830. {
  14831. insertBeforeInstr->InsertBefore(instr);
  14832. }
  14833. };
  14834. if (loopCount->LoopCountMinusOneSym())
  14835. {
  14836. IRType type = loopCount->LoopCountSym()->GetType();
  14837. // Loop count is off by one, so add one
  14838. IR::RegOpnd *loopCountOpnd = IR::RegOpnd::New(loopCount->LoopCountSym(), type, localFunc);
  14839. sizeOpnd = loopCountOpnd;
  14840. if (unroll != 1)
  14841. {
  14842. sizeOpnd = IR::RegOpnd::New(TyUint32, this->func);
  14843. IR::Opnd *unrollOpnd = IR::IntConstOpnd::New(unroll, type, localFunc);
  14844. InsertInstr(IR::Instr::New(Js::OpCode::Mul_I4,
  14845. sizeOpnd,
  14846. loopCountOpnd,
  14847. unrollOpnd,
  14848. localFunc));
  14849. }
  14850. }
  14851. else
  14852. {
  14853. uint size = (loopCount->LoopCountMinusOneConstantValue() + 1) * unroll;
  14854. sizeOpnd = IR::IntConstOpnd::New(size, IRType::TyUint32, localFunc);
  14855. }
  14856. loop->memOpInfo->inductionVariableOpndPerUnrollMap->Add(unroll, sizeOpnd);
  14857. return sizeOpnd;
  14858. }
  14859. IR::RegOpnd*
  14860. GlobOpt::GenerateStartIndexOpndForMemop(Loop *loop, IR::Opnd *indexOpnd, IR::Opnd *sizeOpnd, bool isInductionVariableChangeIncremental, bool bIndexAlreadyChanged, IR::Instr *insertBeforeInstr)
  14861. {
  14862. IR::RegOpnd *startIndexOpnd = nullptr;
  14863. Func *localFunc = loop->GetFunc();
  14864. IRType type = indexOpnd->GetType();
  14865. const int cacheIndex = ((int)isInductionVariableChangeIncremental << 1) | (int)bIndexAlreadyChanged;
  14866. if (loop->memOpInfo->startIndexOpndCache[cacheIndex])
  14867. {
  14868. return loop->memOpInfo->startIndexOpndCache[cacheIndex];
  14869. }
  14870. const auto InsertInstr = [&](IR::Instr *instr)
  14871. {
  14872. if (insertBeforeInstr == nullptr)
  14873. {
  14874. loop->landingPad->InsertAfter(instr);
  14875. }
  14876. else
  14877. {
  14878. insertBeforeInstr->InsertBefore(instr);
  14879. }
  14880. };
  14881. startIndexOpnd = IR::RegOpnd::New(type, localFunc);
  14882. // If the 2 are different we can simply use indexOpnd
  14883. if (isInductionVariableChangeIncremental != bIndexAlreadyChanged)
  14884. {
  14885. InsertInstr(IR::Instr::New(Js::OpCode::Ld_A,
  14886. startIndexOpnd,
  14887. indexOpnd,
  14888. localFunc));
  14889. }
  14890. else
  14891. {
  14892. // Otherwise add 1 to it
  14893. InsertInstr(IR::Instr::New(Js::OpCode::Add_I4,
  14894. startIndexOpnd,
  14895. indexOpnd,
  14896. IR::IntConstOpnd::New(1, type, localFunc, true),
  14897. localFunc));
  14898. }
  14899. if (!isInductionVariableChangeIncremental)
  14900. {
  14901. InsertInstr(IR::Instr::New(Js::OpCode::Sub_I4,
  14902. startIndexOpnd,
  14903. startIndexOpnd,
  14904. sizeOpnd,
  14905. localFunc));
  14906. }
  14907. loop->memOpInfo->startIndexOpndCache[cacheIndex] = startIndexOpnd;
  14908. return startIndexOpnd;
  14909. }
  14910. IR::Instr*
  14911. GlobOpt::FindUpperBoundsCheckInstr(IR::Instr* fromInstr)
  14912. {
  14913. IR::Instr *upperBoundCheck = fromInstr;
  14914. do
  14915. {
  14916. upperBoundCheck = upperBoundCheck->m_prev;
  14917. Assert(upperBoundCheck);
  14918. Assert(!upperBoundCheck->IsLabelInstr());
  14919. } while (upperBoundCheck->m_opcode != Js::OpCode::BoundCheck);
  14920. return upperBoundCheck;
  14921. }
  14922. IR::Instr*
  14923. GlobOpt::FindArraySegmentLoadInstr(IR::Instr* fromInstr)
  14924. {
  14925. IR::Instr *headSegmentLengthLoad = fromInstr;
  14926. do
  14927. {
  14928. headSegmentLengthLoad = headSegmentLengthLoad->m_prev;
  14929. Assert(headSegmentLengthLoad);
  14930. Assert(!headSegmentLengthLoad->IsLabelInstr());
  14931. } while (headSegmentLengthLoad->m_opcode != Js::OpCode::LdIndir);
  14932. return headSegmentLengthLoad;
  14933. }
  14934. void
  14935. GlobOpt::RemoveMemOpSrcInstr(IR::Instr* memopInstr, IR::Instr* srcInstr, BasicBlock* block)
  14936. {
  14937. Assert(srcInstr && (srcInstr->m_opcode == Js::OpCode::LdElemI_A || srcInstr->m_opcode == Js::OpCode::StElemI_A || srcInstr->m_opcode == Js::OpCode::StElemI_A_Strict));
  14938. Assert(memopInstr && (memopInstr->m_opcode == Js::OpCode::Memcopy || memopInstr->m_opcode == Js::OpCode::Memset));
  14939. Assert(block);
  14940. const bool isDst = srcInstr->m_opcode == Js::OpCode::StElemI_A || srcInstr->m_opcode == Js::OpCode::StElemI_A_Strict;
  14941. IR::RegOpnd* opnd = (isDst ? memopInstr->GetDst() : memopInstr->GetSrc1())->AsIndirOpnd()->GetBaseOpnd();
  14942. IR::ArrayRegOpnd* arrayOpnd = opnd->IsArrayRegOpnd() ? opnd->AsArrayRegOpnd() : nullptr;
  14943. IR::Instr* topInstr = srcInstr;
  14944. if (srcInstr->extractedUpperBoundCheckWithoutHoisting)
  14945. {
  14946. IR::Instr *upperBoundCheck = FindUpperBoundsCheckInstr(srcInstr);
  14947. Assert(upperBoundCheck && upperBoundCheck != srcInstr);
  14948. topInstr = upperBoundCheck;
  14949. }
  14950. if (srcInstr->loadedArrayHeadSegmentLength && arrayOpnd && arrayOpnd->HeadSegmentLengthSym())
  14951. {
  14952. IR::Instr *arrayLoadSegmentHeadLength = FindArraySegmentLoadInstr(topInstr);
  14953. Assert(arrayLoadSegmentHeadLength);
  14954. topInstr = arrayLoadSegmentHeadLength;
  14955. arrayOpnd->RemoveHeadSegmentLengthSym();
  14956. }
  14957. if (srcInstr->loadedArrayHeadSegment && arrayOpnd && arrayOpnd->HeadSegmentSym())
  14958. {
  14959. IR::Instr *arrayLoadSegmentHead = FindArraySegmentLoadInstr(topInstr);
  14960. Assert(arrayLoadSegmentHead);
  14961. topInstr = arrayLoadSegmentHead;
  14962. arrayOpnd->RemoveHeadSegmentSym();
  14963. }
  14964. // If no bounds check are present, simply look up for instruction added for instrumentation
  14965. if(topInstr == srcInstr)
  14966. {
  14967. bool checkPrev = true;
  14968. while (checkPrev)
  14969. {
  14970. switch (topInstr->m_prev->m_opcode)
  14971. {
  14972. case Js::OpCode::BailOnNotArray:
  14973. case Js::OpCode::NoImplicitCallUses:
  14974. case Js::OpCode::ByteCodeUses:
  14975. topInstr = topInstr->m_prev;
  14976. checkPrev = !!topInstr->m_prev;
  14977. break;
  14978. default:
  14979. checkPrev = false;
  14980. break;
  14981. }
  14982. }
  14983. }
  14984. while (topInstr != srcInstr)
  14985. {
  14986. IR::Instr* removeInstr = topInstr;
  14987. topInstr = topInstr->m_next;
  14988. Assert(
  14989. removeInstr->m_opcode == Js::OpCode::BailOnNotArray ||
  14990. removeInstr->m_opcode == Js::OpCode::NoImplicitCallUses ||
  14991. removeInstr->m_opcode == Js::OpCode::ByteCodeUses ||
  14992. removeInstr->m_opcode == Js::OpCode::LdIndir ||
  14993. removeInstr->m_opcode == Js::OpCode::BoundCheck
  14994. );
  14995. if (removeInstr->m_opcode != Js::OpCode::ByteCodeUses)
  14996. {
  14997. block->RemoveInstr(removeInstr);
  14998. }
  14999. }
  15000. this->ConvertToByteCodeUses(srcInstr);
  15001. }
  15002. void
  15003. GlobOpt::GetMemOpSrcInfo(Loop* loop, IR::Instr* instr, IR::RegOpnd*& base, IR::RegOpnd*& index, IRType& arrayType)
  15004. {
  15005. Assert(instr && (instr->m_opcode == Js::OpCode::LdElemI_A || instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict));
  15006. IR::Opnd* arrayOpnd = instr->m_opcode == Js::OpCode::LdElemI_A ? instr->GetSrc1() : instr->GetDst();
  15007. Assert(arrayOpnd->IsIndirOpnd());
  15008. IR::IndirOpnd* indirArrayOpnd = arrayOpnd->AsIndirOpnd();
  15009. IR::RegOpnd* baseOpnd = (IR::RegOpnd*)indirArrayOpnd->GetBaseOpnd();
  15010. IR::RegOpnd* indexOpnd = (IR::RegOpnd*)indirArrayOpnd->GetIndexOpnd();
  15011. Assert(baseOpnd);
  15012. Assert(indexOpnd);
  15013. // Process Out Params
  15014. base = baseOpnd;
  15015. index = indexOpnd;
  15016. arrayType = indirArrayOpnd->GetType();
  15017. }
  15018. void
  15019. GlobOpt::EmitMemop(Loop * loop, LoopCount *loopCount, const MemOpEmitData* emitData)
  15020. {
  15021. Assert(emitData);
  15022. Assert(emitData->candidate);
  15023. Assert(emitData->stElemInstr);
  15024. Assert(emitData->stElemInstr->m_opcode == Js::OpCode::StElemI_A || emitData->stElemInstr->m_opcode == Js::OpCode::StElemI_A_Strict);
  15025. IR::BailOutKind bailOutKind = emitData->bailOutKind;
  15026. const byte unroll = emitData->inductionVar.unroll;
  15027. Assert(unroll == 1);
  15028. const bool isInductionVariableChangeIncremental = emitData->inductionVar.isIncremental;
  15029. const bool bIndexAlreadyChanged = emitData->candidate->bIndexAlreadyChanged;
  15030. IR::RegOpnd *baseOpnd = nullptr;
  15031. IR::RegOpnd *indexOpnd = nullptr;
  15032. IRType dstType;
  15033. GetMemOpSrcInfo(loop, emitData->stElemInstr, baseOpnd, indexOpnd, dstType);
  15034. Func *localFunc = loop->GetFunc();
  15035. // Handle bailout info
  15036. EnsureBailTarget(loop);
  15037. Assert(bailOutKind != IR::BailOutInvalid);
  15038. // Keep only Array bits bailOuts. Consider handling these bailouts instead of simply ignoring them
  15039. bailOutKind &= IR::BailOutForArrayBits;
  15040. // Add our custom bailout to handle Op_MemCopy return value.
  15041. bailOutKind |= IR::BailOutOnMemOpError;
  15042. BailOutInfo *const bailOutInfo = loop->bailOutInfo;
  15043. Assert(bailOutInfo);
  15044. IR::Instr *insertBeforeInstr = bailOutInfo->bailOutInstr;
  15045. Assert(insertBeforeInstr);
  15046. IR::Opnd *sizeOpnd = GenerateInductionVariableChangeForMemOp(loop, unroll, insertBeforeInstr);
  15047. IR::RegOpnd *startIndexOpnd = GenerateStartIndexOpndForMemop(loop, indexOpnd, sizeOpnd, isInductionVariableChangeIncremental, bIndexAlreadyChanged, insertBeforeInstr);
  15048. IR::IndirOpnd* dstOpnd = IR::IndirOpnd::New(baseOpnd, startIndexOpnd, dstType, localFunc);
  15049. IR::Opnd *src1;
  15050. const bool isMemset = emitData->candidate->IsMemSet();
  15051. // Get the source according to the memop type
  15052. if (isMemset)
  15053. {
  15054. MemSetEmitData* data = (MemSetEmitData*)emitData;
  15055. const Loop::MemSetCandidate* candidate = data->candidate->AsMemSet();
  15056. if (candidate->srcSym)
  15057. {
  15058. IR::RegOpnd* regSrc = IR::RegOpnd::New(candidate->srcSym, candidate->srcSym->GetType(), func);
  15059. regSrc->SetIsJITOptimizedReg(true);
  15060. src1 = regSrc;
  15061. }
  15062. else
  15063. {
  15064. src1 = IR::AddrOpnd::New(candidate->constant.ToVar(localFunc), IR::AddrOpndKindConstantAddress, localFunc);
  15065. }
  15066. }
  15067. else
  15068. {
  15069. Assert(emitData->candidate->IsMemCopy());
  15070. MemCopyEmitData* data = (MemCopyEmitData*)emitData;
  15071. Assert(data->ldElemInstr);
  15072. Assert(data->ldElemInstr->m_opcode == Js::OpCode::LdElemI_A);
  15073. IR::RegOpnd *srcBaseOpnd = nullptr;
  15074. IR::RegOpnd *srcIndexOpnd = nullptr;
  15075. IRType srcType;
  15076. GetMemOpSrcInfo(loop, data->ldElemInstr, srcBaseOpnd, srcIndexOpnd, srcType);
  15077. Assert(GetVarSymID(srcIndexOpnd->GetStackSym()) == GetVarSymID(indexOpnd->GetStackSym()));
  15078. src1 = IR::IndirOpnd::New(srcBaseOpnd, startIndexOpnd, srcType, localFunc);
  15079. }
  15080. // Generate memcopy
  15081. IR::Instr* memopInstr = IR::BailOutInstr::New(isMemset ? Js::OpCode::Memset : Js::OpCode::Memcopy, bailOutKind, bailOutInfo, localFunc);
  15082. memopInstr->SetDst(dstOpnd);
  15083. memopInstr->SetSrc1(src1);
  15084. memopInstr->SetSrc2(sizeOpnd);
  15085. insertBeforeInstr->InsertBefore(memopInstr);
  15086. loop->memOpInfo->instr = memopInstr;
  15087. #if DBG_DUMP
  15088. if (DO_MEMOP_TRACE())
  15089. {
  15090. char valueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  15091. baseOpnd->GetValueType().ToString(valueTypeStr);
  15092. const int loopCountBufSize = 16;
  15093. char16 loopCountBuf[loopCountBufSize];
  15094. if (loopCount->LoopCountMinusOneSym())
  15095. {
  15096. swprintf_s(loopCountBuf, _u("s%u"), loopCount->LoopCountMinusOneSym()->m_id);
  15097. }
  15098. else
  15099. {
  15100. swprintf_s(loopCountBuf, _u("%u"), loopCount->LoopCountMinusOneConstantValue() + 1);
  15101. }
  15102. if (isMemset)
  15103. {
  15104. const Loop::MemSetCandidate* candidate = emitData->candidate->AsMemSet();
  15105. const int constBufSize = 32;
  15106. char16 constBuf[constBufSize];
  15107. if (candidate->srcSym)
  15108. {
  15109. swprintf_s(constBuf, _u("s%u"), candidate->srcSym->m_id);
  15110. }
  15111. else
  15112. {
  15113. switch (candidate->constant.type)
  15114. {
  15115. case TyInt8:
  15116. case TyInt16:
  15117. case TyInt32:
  15118. case TyInt64:
  15119. swprintf_s(constBuf, sizeof(IntConstType) == 8 ? _u("%lld") : _u("%d"), candidate->constant.u.intConst.value);
  15120. break;
  15121. case TyFloat32:
  15122. case TyFloat64:
  15123. swprintf_s(constBuf, _u("%.4f"), candidate->constant.u.floatConst.value);
  15124. break;
  15125. case TyVar:
  15126. swprintf_s(constBuf, sizeof(Js::Var) == 8 ? _u("0x%.16llX") : _u("0x%.8X"), candidate->constant.u.varConst.value);
  15127. break;
  15128. default:
  15129. AssertMsg(false, "Unsupported constant type");
  15130. swprintf_s(constBuf, _u("Unknown"));
  15131. break;
  15132. }
  15133. }
  15134. TRACE_MEMOP_PHASE(MemSet, loop, emitData->stElemInstr,
  15135. _u("ValueType: %S, Base: s%u, Index: s%u, Constant: %s, LoopCount: %s, IsIndexChangedBeforeUse: %d"),
  15136. valueTypeStr,
  15137. candidate->base,
  15138. candidate->index,
  15139. constBuf,
  15140. loopCountBuf,
  15141. bIndexAlreadyChanged);
  15142. }
  15143. else
  15144. {
  15145. const Loop::MemCopyCandidate* candidate = emitData->candidate->AsMemCopy();
  15146. TRACE_MEMOP_PHASE(MemCopy, loop, emitData->stElemInstr,
  15147. _u("ValueType: %S, StBase: s%u, Index: s%u, LdBase: s%u, LoopCount: %s, IsIndexChangedBeforeUse: %d"),
  15148. valueTypeStr,
  15149. candidate->base,
  15150. candidate->index,
  15151. candidate->ldBase,
  15152. loopCountBuf,
  15153. bIndexAlreadyChanged);
  15154. }
  15155. }
  15156. #endif
  15157. RemoveMemOpSrcInstr(memopInstr, emitData->stElemInstr, emitData->block);
  15158. if (!isMemset)
  15159. {
  15160. RemoveMemOpSrcInstr(memopInstr, ((MemCopyEmitData*)emitData)->ldElemInstr, emitData->block);
  15161. }
  15162. }
  15163. bool
  15164. GlobOpt::InspectInstrForMemSetCandidate(Loop* loop, IR::Instr* instr, MemSetEmitData* emitData, bool& errorInInstr)
  15165. {
  15166. Assert(emitData && emitData->candidate && emitData->candidate->IsMemSet());
  15167. Loop::MemSetCandidate* candidate = (Loop::MemSetCandidate*)emitData->candidate;
  15168. if (instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict)
  15169. {
  15170. if (instr->GetDst()->IsIndirOpnd()
  15171. && (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetStackSym()) == candidate->base)
  15172. && (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetIndexOpnd()->GetStackSym()) == candidate->index)
  15173. )
  15174. {
  15175. Assert(instr->IsProfiledInstr());
  15176. emitData->stElemInstr = instr;
  15177. emitData->bailOutKind = instr->GetBailOutKind();
  15178. return true;
  15179. }
  15180. TRACE_MEMOP_PHASE_VERBOSE(MemSet, loop, instr, _u("Orphan StElemI_A detected"));
  15181. errorInInstr = true;
  15182. }
  15183. else if (instr->m_opcode == Js::OpCode::LdElemI_A)
  15184. {
  15185. TRACE_MEMOP_PHASE_VERBOSE(MemSet, loop, instr, _u("Orphan LdElemI_A detected"));
  15186. errorInInstr = true;
  15187. }
  15188. return false;
  15189. }
  15190. bool
  15191. GlobOpt::InspectInstrForMemCopyCandidate(Loop* loop, IR::Instr* instr, MemCopyEmitData* emitData, bool& errorInInstr)
  15192. {
  15193. Assert(emitData && emitData->candidate && emitData->candidate->IsMemCopy());
  15194. Loop::MemCopyCandidate* candidate = (Loop::MemCopyCandidate*)emitData->candidate;
  15195. if (instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict)
  15196. {
  15197. if (
  15198. instr->GetDst()->IsIndirOpnd() &&
  15199. (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetStackSym()) == candidate->base) &&
  15200. (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetIndexOpnd()->GetStackSym()) == candidate->index)
  15201. )
  15202. {
  15203. Assert(instr->IsProfiledInstr());
  15204. emitData->stElemInstr = instr;
  15205. emitData->bailOutKind = instr->GetBailOutKind();
  15206. // Still need to find the LdElem
  15207. return false;
  15208. }
  15209. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Orphan StElemI_A detected"));
  15210. errorInInstr = true;
  15211. }
  15212. else if (instr->m_opcode == Js::OpCode::LdElemI_A)
  15213. {
  15214. if (
  15215. emitData->stElemInstr &&
  15216. instr->GetSrc1()->IsIndirOpnd() &&
  15217. (GetVarSymID(instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetStackSym()) == candidate->ldBase) &&
  15218. (GetVarSymID(instr->GetSrc1()->AsIndirOpnd()->GetIndexOpnd()->GetStackSym()) == candidate->index)
  15219. )
  15220. {
  15221. Assert(instr->IsProfiledInstr());
  15222. emitData->ldElemInstr = instr;
  15223. ValueType stValueType = emitData->stElemInstr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType();
  15224. ValueType ldValueType = emitData->ldElemInstr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType();
  15225. if (stValueType != ldValueType)
  15226. {
  15227. #if DBG_DUMP
  15228. char16 stValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  15229. stValueType.ToString(stValueTypeStr);
  15230. char16 ldValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  15231. ldValueType.ToString(ldValueTypeStr);
  15232. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("for mismatch in Load(%s) and Store(%s) value type"), ldValueTypeStr, stValueTypeStr);
  15233. #endif
  15234. errorInInstr = true;
  15235. return false;
  15236. }
  15237. // We found both instruction for this candidate
  15238. return true;
  15239. }
  15240. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Orphan LdElemI_A detected"));
  15241. errorInInstr = true;
  15242. }
  15243. return false;
  15244. }
  15245. // The caller is responsible to free the memory allocated between inOrderEmitData[iEmitData -> end]
  15246. bool
  15247. GlobOpt::ValidateMemOpCandidates(Loop * loop, _Out_writes_(iEmitData) MemOpEmitData** inOrderEmitData, int& iEmitData)
  15248. {
  15249. AnalysisAssert(iEmitData == (int)loop->memOpInfo->candidates->Count());
  15250. // We iterate over the second block of the loop only. MemOp Works only if the loop has exactly 2 blocks
  15251. Assert(loop->blockList.HasTwo());
  15252. Loop::MemOpList::Iterator iter(loop->memOpInfo->candidates);
  15253. BasicBlock* bblock = loop->blockList.Head()->next;
  15254. Loop::MemOpCandidate* candidate = nullptr;
  15255. MemOpEmitData* emitData = nullptr;
  15256. // Iterate backward because the list of candidate is reversed
  15257. FOREACH_INSTR_BACKWARD_IN_BLOCK(instr, bblock)
  15258. {
  15259. if (!candidate)
  15260. {
  15261. // Time to check next candidate
  15262. if (!iter.Next())
  15263. {
  15264. // We have been through the whole list of candidates, finish
  15265. break;
  15266. }
  15267. candidate = iter.Data();
  15268. if (!candidate)
  15269. {
  15270. continue;
  15271. }
  15272. // Common check for memset and memcopy
  15273. Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 0, 0 };
  15274. // Get the inductionVariable changeInfo
  15275. if (!loop->memOpInfo->inductionVariableChangeInfoMap->TryGetValue(candidate->index, &inductionVariableChangeInfo))
  15276. {
  15277. TRACE_MEMOP_VERBOSE(loop, nullptr, _u("MemOp skipped (s%d): no induction variable"), candidate->base);
  15278. return false;
  15279. }
  15280. if (inductionVariableChangeInfo.unroll != candidate->count)
  15281. {
  15282. TRACE_MEMOP_VERBOSE(loop, nullptr, _u("MemOp skipped (s%d): not matching unroll count"), candidate->base);
  15283. return false;
  15284. }
  15285. if (candidate->IsMemSet())
  15286. {
  15287. Assert(!PHASE_OFF(Js::MemSetPhase, this->func));
  15288. emitData = JitAnew(this->alloc, MemSetEmitData);
  15289. }
  15290. else
  15291. {
  15292. Assert(!PHASE_OFF(Js::MemCopyPhase, this->func));
  15293. // Specific check for memcopy
  15294. Assert(candidate->IsMemCopy());
  15295. Loop::MemCopyCandidate* memcopyCandidate = candidate->AsMemCopy();
  15296. if (memcopyCandidate->base == Js::Constants::InvalidSymID
  15297. || memcopyCandidate->ldBase == Js::Constants::InvalidSymID
  15298. || (memcopyCandidate->ldCount != memcopyCandidate->count))
  15299. {
  15300. TRACE_MEMOP_PHASE(MemCopy, loop, nullptr, _u("(s%d): not matching ldElem and stElem"), candidate->base);
  15301. return false;
  15302. }
  15303. emitData = JitAnew(this->alloc, MemCopyEmitData);
  15304. }
  15305. Assert(emitData);
  15306. emitData->block = bblock;
  15307. emitData->inductionVar = inductionVariableChangeInfo;
  15308. emitData->candidate = candidate;
  15309. }
  15310. bool errorInInstr = false;
  15311. bool candidateFound = candidate->IsMemSet() ?
  15312. InspectInstrForMemSetCandidate(loop, instr, (MemSetEmitData*)emitData, errorInInstr)
  15313. : InspectInstrForMemCopyCandidate(loop, instr, (MemCopyEmitData*)emitData, errorInInstr);
  15314. if (errorInInstr)
  15315. {
  15316. JitAdelete(this->alloc, emitData);
  15317. return false;
  15318. }
  15319. if (candidateFound)
  15320. {
  15321. AnalysisAssert(iEmitData > 0);
  15322. if (iEmitData == 0)
  15323. {
  15324. // Explicit for OACR
  15325. break;
  15326. }
  15327. inOrderEmitData[--iEmitData] = emitData;
  15328. candidate = nullptr;
  15329. emitData = nullptr;
  15330. }
  15331. } NEXT_INSTR_BACKWARD_IN_BLOCK;
  15332. if (iter.IsValid())
  15333. {
  15334. TRACE_MEMOP(loop, nullptr, _u("Candidates not found in loop while validating"));
  15335. return false;
  15336. }
  15337. return true;
  15338. }
  15339. void
  15340. GlobOpt::ProcessMemOp()
  15341. {
  15342. FOREACH_LOOP_IN_FUNC_EDITING(loop, this->func)
  15343. {
  15344. if (HasMemOp(loop))
  15345. {
  15346. const int candidateCount = loop->memOpInfo->candidates->Count();
  15347. Assert(candidateCount > 0);
  15348. LoopCount * loopCount = GetOrGenerateLoopCountForMemOp(loop);
  15349. // If loopCount is not available we can not continue with memop
  15350. if (!loopCount || !(loopCount->LoopCountMinusOneSym() || loopCount->LoopCountMinusOneConstantValue()))
  15351. {
  15352. TRACE_MEMOP(loop, nullptr, _u("MemOp skipped for no loop count"));
  15353. loop->doMemOp = false;
  15354. loop->memOpInfo->candidates->Clear();
  15355. continue;
  15356. }
  15357. // The list is reversed, check them and place them in order in the following array
  15358. MemOpEmitData** inOrderCandidates = JitAnewArray(this->alloc, MemOpEmitData*, candidateCount);
  15359. int i = candidateCount;
  15360. if (ValidateMemOpCandidates(loop, inOrderCandidates, i))
  15361. {
  15362. Assert(i == 0);
  15363. // Process the valid MemOp candidate in order.
  15364. for (; i < candidateCount; ++i)
  15365. {
  15366. // Emit
  15367. EmitMemop(loop, loopCount, inOrderCandidates[i]);
  15368. JitAdelete(this->alloc, inOrderCandidates[i]);
  15369. }
  15370. }
  15371. else
  15372. {
  15373. Assert(i != 0);
  15374. for (; i < candidateCount; ++i)
  15375. {
  15376. JitAdelete(this->alloc, inOrderCandidates[i]);
  15377. }
  15378. // One of the memop candidates did not validate. Do not emit for this loop.
  15379. loop->doMemOp = false;
  15380. loop->memOpInfo->candidates->Clear();
  15381. }
  15382. // Free memory
  15383. JitAdeleteArray(this->alloc, candidateCount, inOrderCandidates);
  15384. }
  15385. } NEXT_LOOP_EDITING;
  15386. }
  15387. void GlobOpt::PRE::FieldPRE(Loop *loop)
  15388. {
  15389. JitArenaAllocator *alloc = this->globOpt->tempAlloc;
  15390. this->FindPossiblePRECandidates(loop, alloc);
  15391. this->PreloadPRECandidates(loop);
  15392. this->RemoveOverlyOptimisticInitialValues(loop);
  15393. }
  15394. bool
  15395. GlobOpt::PRE::InsertSymDefinitionInLandingPad(StackSym * sym, Loop * loop, Sym ** objPtrCopyPropSym)
  15396. {
  15397. Assert(sym->IsSingleDef());
  15398. IR::Instr * symDefInstr = sym->GetInstrDef();
  15399. if (!GlobOpt::IsPREInstrSequenceCandidateLoad(symDefInstr->m_opcode))
  15400. {
  15401. return false;
  15402. }
  15403. IR::Opnd * symDefInstrSrc1 = symDefInstr->GetSrc1();
  15404. if (symDefInstrSrc1->IsSymOpnd())
  15405. {
  15406. Assert(symDefInstrSrc1->AsSymOpnd()->m_sym->IsPropertySym());
  15407. // $L1
  15408. // T1 = o.x (v1|T3)
  15409. // T2 = T1.y (v2|T4) <-- T1 is not live in the loop landing pad
  15410. // jmp $L1
  15411. // Trying to make T1 live in the landing pad
  15412. // o.x
  15413. PropertySym* propSym = symDefInstrSrc1->AsSymOpnd()->m_sym->AsPropertySym();
  15414. if (candidates->candidatesBv->Test(propSym->m_id))
  15415. {
  15416. // If propsym is a PRE candidate, then it must have had the same value on all back edges.
  15417. // So, just look up the value on one of the back edges.
  15418. BasicBlock* loopTail = loop->GetAnyTailBlock();
  15419. Value * valueOnBackEdge = loopTail->globOptData.FindValue(propSym);
  15420. // If o.x is not invariant in the loop, we can't use the preloaded value of o.x.y in the landing pad
  15421. Value * valueInLandingPad = loop->landingPad->globOptData.FindValue(propSym);
  15422. if (valueOnBackEdge->GetValueNumber() != valueInLandingPad->GetValueNumber())
  15423. {
  15424. return false;
  15425. }
  15426. *objPtrCopyPropSym = valueOnBackEdge->GetValueInfo()->GetSymStore();
  15427. if (candidates->candidatesToProcess->Test(propSym->m_id))
  15428. {
  15429. GlobHashBucket bucket;
  15430. bucket.element = valueOnBackEdge;
  15431. bucket.value = propSym;
  15432. if (!PreloadPRECandidate(loop, &bucket))
  15433. {
  15434. return false;
  15435. }
  15436. Assert(!candidates->candidatesToProcess->Test(propSym->m_id));
  15437. Assert(loop->landingPad->globOptData.IsLive(valueOnBackEdge->GetValueInfo()->GetSymStore()));
  15438. // Inserted T3 = o.x
  15439. // Now, we want to
  15440. // 1. Insert T1 = o.x
  15441. // 2. Insert T4 = T1.y
  15442. // 3. Indentify T3 as the objptr copy prop sym for T1, and make T3.y live on the back-edges
  15443. // #1 is done next. #2 and #3 are done as part of preloading T1.y
  15444. // Insert T1 = o.x
  15445. if (!InsertPropertySymPreloadInLandingPad(symDefInstr->Copy(), loop, propSym))
  15446. {
  15447. return false;
  15448. }
  15449. return true;
  15450. }
  15451. else
  15452. {
  15453. // o.x was already processed as a PRE candidate. If we were successful in preloading o.x,
  15454. // we can now insert T1 = o.x
  15455. if (loop->landingPad->globOptData.IsLive(*objPtrCopyPropSym))
  15456. {
  15457. // insert T1 = o.x
  15458. if (!InsertPropertySymPreloadInLandingPad(symDefInstr->Copy(), loop, propSym))
  15459. {
  15460. return false;
  15461. }
  15462. return true;
  15463. }
  15464. else
  15465. {
  15466. return false;
  15467. }
  15468. }
  15469. }
  15470. else
  15471. {
  15472. return false;
  15473. }
  15474. }
  15475. else if (symDefInstrSrc1->IsRegOpnd())
  15476. {
  15477. // T2 = T1
  15478. // T3 = T2.y
  15479. // trying to insert def of T2
  15480. // T1
  15481. StackSym * symDefInstrSrc1Sym = symDefInstrSrc1->AsRegOpnd()->GetStackSym();
  15482. if (!loop->landingPad->globOptData.IsLive(symDefInstrSrc1Sym))
  15483. {
  15484. if (symDefInstrSrc1Sym->IsSingleDef())
  15485. {
  15486. if (!InsertSymDefinitionInLandingPad(symDefInstrSrc1Sym, loop, objPtrCopyPropSym))
  15487. {
  15488. return false;
  15489. }
  15490. }
  15491. }
  15492. else
  15493. {
  15494. *objPtrCopyPropSym = symDefInstrSrc1Sym;
  15495. }
  15496. if (!(OpCodeAttr::TempNumberTransfer(symDefInstr->m_opcode) && OpCodeAttr::TempObjectTransfer(symDefInstr->m_opcode)))
  15497. {
  15498. *objPtrCopyPropSym = sym;
  15499. }
  15500. IR::Instr * instr = symDefInstr->Copy();
  15501. if (instr->m_opcode == Js::OpCode::BytecodeArgOutCapture)
  15502. {
  15503. instr->m_opcode = Js::OpCode::Ld_A;
  15504. }
  15505. InsertInstrInLandingPad(instr, loop);
  15506. return true;
  15507. }
  15508. else
  15509. {
  15510. return false;
  15511. }
  15512. }
  15513. void
  15514. GlobOpt::PRE::InsertInstrInLandingPad(IR::Instr * instr, Loop * loop)
  15515. {
  15516. instr->GetSrc1()->SetIsJITOptimizedReg(true);
  15517. if (instr->GetDst())
  15518. {
  15519. instr->GetDst()->SetIsJITOptimizedReg(true);
  15520. loop->landingPad->globOptData.liveVarSyms->Set(instr->GetDst()->GetStackSym()->m_id);
  15521. }
  15522. if (instr->HasAnyImplicitCalls())
  15523. {
  15524. IR::Instr * bailInstr = globOpt->EnsureDisableImplicitCallRegion(loop);
  15525. bailInstr->InsertBefore(instr);
  15526. }
  15527. else if (loop->endDisableImplicitCall)
  15528. {
  15529. loop->endDisableImplicitCall->InsertBefore(instr);
  15530. }
  15531. else
  15532. {
  15533. loop->landingPad->InsertAfter(instr);
  15534. }
  15535. instr->ClearByteCodeOffset();
  15536. instr->SetByteCodeOffset(loop->landingPad->GetFirstInstr());
  15537. }
  15538. IR::Instr *
  15539. GlobOpt::PRE::InsertPropertySymPreloadInLandingPad(IR::Instr * ldInstr, Loop * loop, PropertySym * propertySym)
  15540. {
  15541. IR::SymOpnd *ldSrc = ldInstr->GetSrc1()->AsSymOpnd();
  15542. if (ldSrc->m_sym != propertySym)
  15543. {
  15544. // It's possible that the property syms are different but have equivalent objPtrs. Verify their values.
  15545. Value *val1 = globOpt->CurrentBlockData()->FindValue(ldSrc->m_sym->AsPropertySym()->m_stackSym);
  15546. Value *val2 = globOpt->CurrentBlockData()->FindValue(propertySym->m_stackSym);
  15547. if (!val1 || !val2 || val1->GetValueNumber() != val2->GetValueNumber())
  15548. {
  15549. return nullptr;
  15550. }
  15551. }
  15552. // Consider: Shouldn't be necessary once we have copy-prop in prepass...
  15553. ldInstr->GetSrc1()->AsSymOpnd()->m_sym = propertySym;
  15554. ldSrc = ldInstr->GetSrc1()->AsSymOpnd();
  15555. if (ldSrc->IsPropertySymOpnd())
  15556. {
  15557. IR::PropertySymOpnd *propSymOpnd = ldSrc->AsPropertySymOpnd();
  15558. IR::PropertySymOpnd *newPropSymOpnd;
  15559. newPropSymOpnd = propSymOpnd->AsPropertySymOpnd()->CopyWithoutFlowSensitiveInfo(this->globOpt->func);
  15560. ldInstr->ReplaceSrc1(newPropSymOpnd);
  15561. }
  15562. if (ldInstr->GetDst())
  15563. {
  15564. loop->landingPad->globOptData.liveVarSyms->Set(ldInstr->GetDst()->GetStackSym()->m_id);
  15565. }
  15566. InsertInstrInLandingPad(ldInstr, loop);
  15567. return ldInstr;
  15568. }
  15569. void
  15570. GlobOpt::PRE::MakePropertySymLiveOnBackEdges(PropertySym * propertySym, Loop * loop, Value * valueToAdd)
  15571. {
  15572. BasicBlock * loopHeader = loop->GetHeadBlock();
  15573. FOREACH_PREDECESSOR_BLOCK(blockPred, loopHeader)
  15574. {
  15575. if (!loop->IsDescendentOrSelf(blockPred->loop))
  15576. {
  15577. // Not a loop back-edge
  15578. continue;
  15579. }
  15580. // Insert it in the value table
  15581. blockPred->globOptData.SetValue(valueToAdd, propertySym);
  15582. // Make it a live field
  15583. blockPred->globOptData.liveFields->Set(propertySym->m_id);
  15584. } NEXT_PREDECESSOR_BLOCK;
  15585. }
  15586. void GlobOpt::PRE::RemoveOverlyOptimisticInitialValues(Loop * loop)
  15587. {
  15588. BasicBlock * landingPad = loop->landingPad;
  15589. // For a property sym whose obj ptr sym wasn't live in the landing pad, we can optmistically (if the obj ptr sym was
  15590. // single def) insert an initial value in the landing pad, with the hope that PRE could make the obj ptr sym live.
  15591. // But, if PRE couldn't make the obj ptr sym live, we need to clear the value for the property sym from the landing pad
  15592. for (auto it = loop->initialValueFieldMap.GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext())
  15593. {
  15594. PropertySym * propertySym = it.CurrentKey();
  15595. StackSym * objPtrSym = propertySym->m_stackSym;
  15596. if (!landingPad->globOptData.IsLive(objPtrSym))
  15597. {
  15598. Value * landingPadPropSymValue = landingPad->globOptData.FindValue(propertySym);
  15599. Assert(landingPadPropSymValue);
  15600. Assert(landingPadPropSymValue->GetValueNumber() == it.CurrentValue()->GetValueNumber());
  15601. Assert(landingPadPropSymValue->GetValueInfo()->GetSymStore() == propertySym);
  15602. landingPad->globOptData.ClearSymValue(propertySym);
  15603. it.RemoveCurrent();
  15604. }
  15605. }
  15606. }
  15607. #if DBG_DUMP
  15608. void GlobOpt::PRE::TraceFailedPreloadInLandingPad(const Loop *const loop, PropertySym * propertySym, const char16* reason) const
  15609. {
  15610. if (PHASE_TRACE(Js::FieldPREPhase, this->globOpt->func))
  15611. {
  15612. int32 propertyId = propertySym->m_propertyId;
  15613. SymID objectSymId = propertySym->m_stackSym->m_id;
  15614. char16 propSymStr[32];
  15615. switch (propertySym->m_fieldKind)
  15616. {
  15617. case PropertyKindData:
  15618. if (JITManager::GetJITManager()->IsOOPJITEnabled())
  15619. {
  15620. swprintf_s(propSymStr, _u("s%d->#%d"), objectSymId, propertyId);
  15621. }
  15622. else
  15623. {
  15624. Js::PropertyRecord const* fieldName = propertySym->m_func->GetInProcThreadContext()->GetPropertyRecord(propertyId);
  15625. swprintf_s(propSymStr, _u("s%d->%s"), objectSymId, fieldName->GetBuffer());
  15626. }
  15627. break;
  15628. case PropertyKindSlots:
  15629. case PropertyKindSlotArray:
  15630. swprintf_s(propSymStr, _u("s%d[%d]"), objectSymId, propertyId);
  15631. break;
  15632. case PropertyKindLocalSlots:
  15633. swprintf_s(propSymStr, _u("s%dl[%d]"), objectSymId, propertyId);
  15634. break;
  15635. default:
  15636. AssertMsg(0, "Unknown field kind");
  15637. break;
  15638. }
  15639. Output::Print(_u("** TRACE: Field PRE: "));
  15640. this->globOpt->func->DumpFullFunctionName();
  15641. Output::Print(_u(": Failed to pre-load (%s) in landing pad of loop #%d. Reason: %s "), propSymStr, loop->GetLoopNumber(), reason);
  15642. Output::Print(_u("\n"));
  15643. }
  15644. }
  15645. #endif